diff --git a/.github/skills/azsdk-common-apiview-feedback-resolution/SKILL.md b/.github/skills/azsdk-common-apiview-feedback-resolution/SKILL.md new file mode 100644 index 000000000000..b3be1cf3c1f5 --- /dev/null +++ b/.github/skills/azsdk-common-apiview-feedback-resolution/SKILL.md @@ -0,0 +1,43 @@ +--- +name: azsdk-common-apiview-feedback-resolution +license: MIT +metadata: + version: "1.0.0" + distribution: shared +description: "Analyze and resolve APIView review feedback on Azure SDK PRs. **UTILITY SKILL**. USE FOR: APIView comments, API review feedback, SDK API surface changes. DO NOT USE FOR: general code review, non-APIView feedback. INVOKES: azure-sdk-mcp:azsdk_apiview_get_comments, azure-sdk-mcp:azsdk_customized_code_update." +compatibility: + requires: "azure-sdk-mcp server, SDK pull request with APIView review link" +--- + +# APIView Feedback Resolution + +**Prerequisites:** azure-sdk-mcp server required; no CLI fallback. Without MCP, this skill cannot retrieve APIView comments or apply TypeSpec changes. Connect the `azure-sdk-mcp` server before use. + +## MCP Tools + +| Tool | Purpose | +|------|---------| +| `azure-sdk-mcp:azsdk_apiview_get_comments` | Get APIView comments | +| `azure-sdk-mcp:azsdk_customized_code_update` | Apply TypeSpec & code customization changes locally | +| `azure-sdk-mcp:azsdk_typespec_delegate_apiview_feedback` | Delegate to CCA pipeline | +| `azure-sdk-mcp:azsdk_run_typespec_validation` | Validate TypeSpec | +| `azure-sdk-mcp:azsdk_package_generate_code` | Regenerate SDK | + +## Steps + +1. **Retrieve** — Get APIView URL from SDK PR, run `azsdk_apiview_get_comments`. +2. **Categorize** — Group as Critical/Suggestions/Informational per [feedback steps](references/feedback-resolution-steps.md). +3. **Resolve** — Use `azsdk_customized_code_update` for TypeSpec changes; delegate via `azsdk_typespec_delegate_apiview_feedback` for complex cases. +4. **Validate** — Run validation, regenerate SDK, build and test. +5. **Confirm** — Verify all items addressed. If delegated, follow [post-delegation follow-up](references/feedback-resolution-steps.md#post-delegation-follow-up). Request re-review. + +## Examples + +- "Resolve the APIView comments on my SDK pull request" +- "What feedback did the API reviewer leave?" + +## Troubleshooting + +- **No comments**: Verify PR has APIView link and MCP server is connected. +- **Validation fails**: Re-run after fixing TypeSpec errors. +- **MCP unavailable**: Requires `azure-sdk-mcp` server; no CLI fallback. Connect the server and retry. diff --git a/.github/skills/azsdk-common-apiview-feedback-resolution/eval.yaml b/.github/skills/azsdk-common-apiview-feedback-resolution/eval.yaml new file mode 100644 index 000000000000..a91c6eaae971 --- /dev/null +++ b/.github/skills/azsdk-common-apiview-feedback-resolution/eval.yaml @@ -0,0 +1,29 @@ +name: azsdk-common-apiview-feedback-resolution-eval +description: Evaluation suite for APIView feedback resolution skill. +skill: azsdk-common-apiview-feedback-resolution +version: '1.0' +config: + model: gpt-4o + executor: mock + trials_per_task: 1 + timeout_seconds: 300 + parallel: false +metrics: + - name: task_completion + weight: 0.7 + threshold: 0.8 + description: Did the skill complete trigger and anti-trigger checks? + - name: efficiency + weight: 0.3 + threshold: 0.7 + description: Did the skill stay within behavior limits? +graders: + - type: text + name: no_fatal_errors + config: + regex_not_match: + - (?i)fatal error + - (?i)unhandled exception + - '(?i)panic:' +tasks: + - "tasks/*.yaml" diff --git a/.github/skills/azsdk-common-apiview-feedback-resolution/evals/eval.yaml b/.github/skills/azsdk-common-apiview-feedback-resolution/evals/eval.yaml new file mode 100644 index 000000000000..d71941857959 --- /dev/null +++ b/.github/skills/azsdk-common-apiview-feedback-resolution/evals/eval.yaml @@ -0,0 +1,29 @@ +name: azsdk-common-apiview-feedback-resolution-evals +description: Evaluations for the APIView feedback resolution skill +skill: azsdk-common-apiview-feedback-resolution +version: "1.0" +config: + model: gpt-4o + executor: mock + trials_per_task: 1 + timeout_seconds: 300 + parallel: false +metrics: + - name: task_completion + weight: 0.7 + threshold: 0.8 + description: Did the skill complete trigger and anti-trigger checks? + - name: efficiency + weight: 0.3 + threshold: 0.7 + description: Did the skill stay within behavior limits? +graders: + - type: text + name: no_fatal_errors + config: + regex_not_match: + - (?i)fatal error + - (?i)unhandled exception + - '(?i)panic:' +tasks: + - "tasks/*.yaml" diff --git a/.github/skills/azsdk-common-apiview-feedback-resolution/evals/tasks/anti-trigger.yaml b/.github/skills/azsdk-common-apiview-feedback-resolution/evals/tasks/anti-trigger.yaml new file mode 100644 index 000000000000..b45655a15378 --- /dev/null +++ b/.github/skills/azsdk-common-apiview-feedback-resolution/evals/tasks/anti-trigger.yaml @@ -0,0 +1,16 @@ +id: anti-trigger +name: anti-trigger +description: Should not trigger on unrelated prompts +inputs: + prompt: "How do I set up a new Azure Storage account?" +expected: + skill_activated: false +graders: + - type: text + name: no-apiview-advice + config: + not_contains: + - "APIView" + - "API review" + - "apiview.dev" + - "apiviewstagingtest.com" diff --git a/.github/skills/azsdk-common-apiview-feedback-resolution/evals/tasks/basic-trigger.yaml b/.github/skills/azsdk-common-apiview-feedback-resolution/evals/tasks/basic-trigger.yaml new file mode 100644 index 000000000000..e94dcc5c2a19 --- /dev/null +++ b/.github/skills/azsdk-common-apiview-feedback-resolution/evals/tasks/basic-trigger.yaml @@ -0,0 +1,13 @@ +id: basic-trigger +name: basic-trigger +description: Should trigger on APIView feedback resolution requests +inputs: + prompt: "I have APIView comments on my SDK PR that need to be resolved. Can you help?" +expected: + skill_activated: true +graders: + - type: text + name: mentions-apiview + config: + contains: + - "APIView" diff --git a/.github/skills/azsdk-common-apiview-feedback-resolution/evals/tasks/url-trigger.yaml b/.github/skills/azsdk-common-apiview-feedback-resolution/evals/tasks/url-trigger.yaml new file mode 100644 index 000000000000..34836fd27e3b --- /dev/null +++ b/.github/skills/azsdk-common-apiview-feedback-resolution/evals/tasks/url-trigger.yaml @@ -0,0 +1,13 @@ +id: url-trigger +name: url-trigger +description: Should trigger when user provides an APIView URL without explicitly saying APIView +inputs: + prompt: "I have review comments on my SDK PR from this page: https://spa.apiview.dev/review/abc123?activeApiRevisionId=def456 — can you help me resolve them?" +expected: + skill_activated: true +graders: + - type: text + name: mentions-apiview-domain + config: + contains: + - "apiview" diff --git a/.github/skills/azsdk-common-apiview-feedback-resolution/evals/trigger_tests.yaml b/.github/skills/azsdk-common-apiview-feedback-resolution/evals/trigger_tests.yaml new file mode 100644 index 000000000000..ba3641050842 --- /dev/null +++ b/.github/skills/azsdk-common-apiview-feedback-resolution/evals/trigger_tests.yaml @@ -0,0 +1,29 @@ +skill: azsdk-common-apiview-feedback-resolution +should_trigger_prompts: + - prompt: "resolve APIView feedback on my PR" + reason: "Should activate skill" + - prompt: "fix APIView comments" + reason: "Should activate skill" + - prompt: "address API review feedback" + reason: "Should activate skill" + - prompt: "APIView shows issues with my SDK" + reason: "Should activate skill" + - prompt: "help me with APIView suggestions" + reason: "Should activate skill" + - prompt: "Help me fix these comments: https://spa.apiview.dev/review/0qw98r9qwur9q823r?activeApiRevisionId=afiq948tquajf8jq9f" + reason: "URL-only prompt without explicit APIView mention should activate skill" + - prompt: "Address this feedback: https://spa.apiview.dev/review/akldfja9qjta498tjaejg0?activeApiRevisionId=odiufa9sefighj98439" + reason: "URL-only prompt without explicit APIView mention should activate skill" + - prompt: "Resolve these comments: https://apiviewstagingtest.com/review/akldfja9qjta498tjaejg0?activeApiRevisionId=odiufa9sefighj98439" + reason: "URL-only prompt without explicit APIView mention should activate skill" +should_not_trigger_prompts: + - prompt: "deploy my Azure Function" + reason: "Unrelated to skill scope" + - prompt: "fix my pipeline build error" + reason: "Unrelated to skill scope" + - prompt: "create a new TypeSpec definition" + reason: "Unrelated to skill scope" + - prompt: "how do I release an SDK package" + reason: "Unrelated to skill scope" + - prompt: "set up Azure Storage account" + reason: "Unrelated to skill scope" diff --git a/.github/skills/azsdk-common-apiview-feedback-resolution/fixtures/apiview-comment.json b/.github/skills/azsdk-common-apiview-feedback-resolution/fixtures/apiview-comment.json new file mode 100644 index 000000000000..d8ca2b1ce8b4 --- /dev/null +++ b/.github/skills/azsdk-common-apiview-feedback-resolution/fixtures/apiview-comment.json @@ -0,0 +1,18 @@ +{ + "comments": [ + { + "id": "c1", + "text": "Consider renaming 'ComputeProvisioningState' to 'ProvisioningState' for consistency.", + "severity": "suggestion", + "language": "python", + "resolved": false + }, + { + "id": "c2", + "text": "Missing documentation on the 'createOrUpdate' method parameters.", + "severity": "critical", + "language": "python", + "resolved": false + } + ] +} diff --git a/.github/skills/azsdk-common-apiview-feedback-resolution/references/feedback-resolution-steps.md b/.github/skills/azsdk-common-apiview-feedback-resolution/references/feedback-resolution-steps.md new file mode 100644 index 000000000000..2ce0c7b24148 --- /dev/null +++ b/.github/skills/azsdk-common-apiview-feedback-resolution/references/feedback-resolution-steps.md @@ -0,0 +1,37 @@ +# APIView Feedback Resolution Details + +## Feedback Categories + +- **Critical**: Breaking changes, security issues - must resolve +- **Suggestions**: Naming improvements, documentation - should resolve +- **Informational**: Style notes - optional + +## Resolution Approaches + +**If TypeSpec change needed:** + +1. Run `azsdk_typespec_delegate_apiview_feedback` to create a GH issue with all actionable comments in the specs repo and assign to Copilot. + +**If code-only fix needed:** + +1. Apply the fix directly in the SDK repository +2. Regenerate SDK if needed + +## Post-Resolution Steps + +1. Run `azsdk_run_typespec_validation` to verify TypeSpec changes +2. Run `azsdk_package_generate_code` to regenerate SDK +3. Build and test the updated SDK +4. Update the SDK PR with changes +5. Re-check APIView for any new comments +6. Inform user to request re-review if needed + +## Post-Delegation Follow-Up + +When `azsdk_typespec_delegate_apiview_feedback` is used to delegate changes, follow these steps to close the loop with the user: + +1. **Share the issue link** — Immediately share the created GitHub issue URL with the user so they can track progress. +2. **Explain the pipeline chain** — Let the user know the expected flow: GitHub issue → spec PR (created by Copilot) → SDK generation pipeline → SDK PRs → updated APIView revision. +3. **Monitor pipeline status** — Offer to check pipeline progress using `azure-sdk-mcp:azsdk_get_pipeline_status`. +4. **Track SDK PRs** — Once the spec PR is created, use `azure-sdk-mcp:azsdk_get_sdk_pull_request_link` to find the resulting SDK pull requests. +5. **Present the full link chain** — When available, share the complete chain with the user: issue → spec PR → SDK PRs → APIView revision link. diff --git a/.github/skills/azsdk-common-apiview-feedback-resolution/tasks/basic-usage.yaml b/.github/skills/azsdk-common-apiview-feedback-resolution/tasks/basic-usage.yaml new file mode 100644 index 000000000000..b4138f454155 --- /dev/null +++ b/.github/skills/azsdk-common-apiview-feedback-resolution/tasks/basic-usage.yaml @@ -0,0 +1,20 @@ +id: apiview-basic-001 +name: Retrieve and Review APIView Comments +description: | + Test that the skill retrieves APIView comments for an SDK package + and presents them organized by priority. +tags: + - basic + - happy-path +inputs: + prompt: "My Python SDK PR has APIView comments that I need to address. Can you help me retrieve and resolve them?" +expected: + output_contains: + - "APIView" + - "comment" + outcomes: + - type: tool_called + tool_name: azsdk_apiview_get_comments + behavior: + max_tool_calls: 5 + max_response_time_ms: 30000 diff --git a/.github/skills/azsdk-common-apiview-feedback-resolution/tasks/edge-case.yaml b/.github/skills/azsdk-common-apiview-feedback-resolution/tasks/edge-case.yaml new file mode 100644 index 000000000000..c2dbad941117 --- /dev/null +++ b/.github/skills/azsdk-common-apiview-feedback-resolution/tasks/edge-case.yaml @@ -0,0 +1,19 @@ +id: apiview-edge-001 +name: APIView Requires TypeSpec Change +description: | + Test that the skill handles feedback requiring TypeSpec modifications + and triggers regeneration. +tags: + - edge-case + - typespec-change +inputs: + prompt: "An APIView reviewer says I need to rename 'ComputeProvisioningState' to 'ProvisioningState' in my SDK. This needs a TypeSpec change." +expected: + output_contains: + - "client.tsp" + - "clientName" + output_excludes: + - "cannot help" + behavior: + max_tool_calls: 5 + max_response_time_ms: 30000 diff --git a/.github/skills/azsdk-common-apiview-feedback-resolution/tasks/no-feedback.yaml b/.github/skills/azsdk-common-apiview-feedback-resolution/tasks/no-feedback.yaml new file mode 100644 index 000000000000..18768825deda --- /dev/null +++ b/.github/skills/azsdk-common-apiview-feedback-resolution/tasks/no-feedback.yaml @@ -0,0 +1,16 @@ +id: apiview-no-feedback-001 +name: No APIView Feedback Found +description: | + Test that the skill handles the case where no APIView comments exist + and advises the user to proceed. +tags: + - basic + - no-feedback +inputs: + prompt: "Check if there are any APIView comments on my .NET SDK PR that I need to address before release." +expected: + output_contains: + - "APIView" + behavior: + max_tool_calls: 3 + max_response_time_ms: 15000 diff --git a/.github/skills/azsdk-common-apiview-feedback-resolution/tasks/should-not-trigger.yaml b/.github/skills/azsdk-common-apiview-feedback-resolution/tasks/should-not-trigger.yaml new file mode 100644 index 000000000000..5469392a8d5b --- /dev/null +++ b/.github/skills/azsdk-common-apiview-feedback-resolution/tasks/should-not-trigger.yaml @@ -0,0 +1,14 @@ +id: apiview-negative-001 +name: Should Not Trigger - Create TypeSpec Project +description: | + Test that the skill does NOT activate when user wants to create + a TypeSpec project from scratch. +tags: + - negative-test + - anti-trigger +inputs: + prompt: "I need to create a new TypeSpec project for my Azure Storage service." +expected: + output_not_contains: + - "azsdk_apiview_get_comments" + - "APIView" diff --git a/.github/skills/azsdk-common-generate-sdk-locally/SKILL.md b/.github/skills/azsdk-common-generate-sdk-locally/SKILL.md new file mode 100644 index 000000000000..8f23faadd8cc --- /dev/null +++ b/.github/skills/azsdk-common-generate-sdk-locally/SKILL.md @@ -0,0 +1,69 @@ +--- +name: azsdk-common-generate-sdk-locally +license: MIT +metadata: + version: "1.1.0" + distribution: shared +description: "Generate, build, and test Azure SDKs locally from TypeSpec with automatic customization. WHEN: \"generate SDK locally\", \"build SDK\", \"run SDK tests\", \"update changelog\", \"fix SDK build errors\", \"fix breaking changes\", \"resolve SDK generation errors\", \"customize TypeSpec\", \"rename SDK client\", \"rename SDK model\", \"hide operation from SDK\", \"fix analyzer errors\", \"resolve customization drift\", \"create subclient\", \"update metadata\", \"update version\". DO NOT USE FOR: publishing to package registries, CI pipeline configuration, API design review. INVOKES: azsdk_verify_setup, azsdk_package_generate_code, azsdk_package_build_code, azsdk_package_run_check, azsdk_package_run_tests, azsdk_customized_code_update, azsdk_package_update_changelog_content, azsdk_package_update_metadata, azsdk_package_update_version." +compatibility: + requires: "azure-sdk-mcp server, local azure-sdk-for-{language} clone, language build tools" +--- + +# Generate SDK Locally + +## MCP Tools + +| Tool | Purpose | +|------|---------| +| `azure-sdk-mcp:azsdk_verify_setup` | Verify environment | +| `azure-sdk-mcp:azsdk_package_generate_code` | Generate SDK | +| `azure-sdk-mcp:azsdk_package_build_code` | Build package | +| `azure-sdk-mcp:azsdk_package_run_check` | Validate package | +| `azure-sdk-mcp:azsdk_package_run_tests` | Run tests | +| `azure-sdk-mcp:azsdk_customized_code_update` | Apply customizations (includes regeneration and build) | +| `azure-sdk-mcp:azsdk_package_update_changelog_content` | Update changelog | +| `azure-sdk-mcp:azsdk_package_update_metadata` | Update metadata including ci.yml | +| `azure-sdk-mcp:azsdk_package_update_version` | Update version | + +Prerequisites: azure-sdk-mcp server must be running. Without MCP, use `npx tsp-client` CLI. + +## Steps + +1. **Select language** — Confirm target language: .NET, Java, JavaScript, Python, Go, or Rust. +2. **Verify repo** — Ensure the user has a local clone of the correct [SDK repo](references/sdk-repos.md). If not cloned, instruct user to clone it. +3. **Identify config file** — Determine the path to the TypeSpec configuration file. See [config file details](references/detailed-workflow.md). + - From `azure-rest-api-specs` repo: use path to `tspconfig.yaml`. + - From an SDK language repo: use path to `tsp-location.yaml`. +4. **Verify setup** — Run `azure-sdk-mcp:azsdk_verify_setup` to confirm environment. +5. **Generate** — Run `azure-sdk-mcp:azsdk_package_generate_code` with the config file path. +6. **Build** — Run `azure-sdk-mcp:azsdk_package_build_code`. If build succeeds, proceed to step 8. +7. **Customize** — If build fails, or if user requests SDK modifications, run `azure-sdk-mcp:azsdk_customized_code_update` with the build errors or user request. The tool handles the full workflow internally: it classifies the issue, applies TypeSpec decorators and/or code patches, regenerates the SDK, and builds — all in one call. See [customization workflow](references/customization-workflow.md). +8. **Commit checkpoint** — Prompt the user to commit generated changes before proceeding. See [commit checkpoint details](references/detailed-workflow.md). +9. **Validate** — Run `azure-sdk-mcp:azsdk_package_run_check` and `azure-sdk-mcp:azsdk_package_run_tests`. +10. **Metadata** — Run `azure-sdk-mcp:azsdk_package_update_changelog_content`, `azure-sdk-mcp:azsdk_package_update_metadata`, and `azure-sdk-mcp:azsdk_package_update_version`. _(Note: For .NET data plane, skip this step — metadata, changelog, and version updates are per-commit tasks, not part of the generate/build/test workflow.)_ +11. **Final commit** — Prompt the user to commit final changes (changelog, metadata, version). See [commit checkpoint details](references/detailed-workflow.md). + +[SDK repos](references/sdk-repos.md) | [Customization workflow](references/customization-workflow.md) | [Detailed workflow](references/detailed-workflow.md) + +## Examples + +- "Generate the SDK locally for my TypeSpec service" +- "Build and test the Python SDK package" +- "Fix the SDK build errors on this PR" +- "The SDK generation has breaking changes, resolve them" +- "Rename FooClient to BarClient for .NET" +- "Hide the internal polling operation from the Python SDK" +- "Fix .NET analyzer errors AZC0030 and AZC0012" +- "The build is failing because a customization references a renamed property" +- "Create a subclient architecture for the Python SDK" +- "Apply TypeSpec customizations to fix compilation errors" +- "Update the changelog for this SDK package" +- "Update the package version" +- "Update the package metadata and ci.yml" + +## Troubleshooting + +- Run `azure-sdk-mcp:azsdk_verify_setup` to confirm MCP and tools. +- If build fails with type conflicts, breaking changes, analyzer errors, or customization drift, use `azure-sdk-mcp:azsdk_customized_code_update` to apply customizations. +- The customization tool uses a two-phase approach: TypeSpec decorators first (Phase A), then code repairs if needed (Phase B). +- Without MCP, use `npx tsp-client` CLI. diff --git a/.github/skills/azsdk-common-generate-sdk-locally/eval.yaml b/.github/skills/azsdk-common-generate-sdk-locally/eval.yaml new file mode 100644 index 000000000000..57a864e4d22b --- /dev/null +++ b/.github/skills/azsdk-common-generate-sdk-locally/eval.yaml @@ -0,0 +1,28 @@ +name: azsdk-common-generate-sdk-locally-eval +description: Evaluation suite for local SDK generation skill. +skill: azsdk-common-generate-sdk-locally +version: '1.0' +config: + model: gpt-4o + executor: mock + trials_per_task: 1 + timeout_seconds: 600 + parallel: false +metrics: + - name: task_completion + weight: 0.7 + threshold: 0.8 + description: Did the skill complete trigger and anti-trigger checks? + - name: efficiency + weight: 0.3 + threshold: 0.7 + description: Did the skill stay within behavior limits? +graders: + - type: text + name: no_fatal_errors + config: + regex_not_match: + - (?i)fatal error + - (?i)unhandled exception +tasks: + - "tasks/*.yaml" diff --git a/.github/skills/azsdk-common-generate-sdk-locally/fixtures/tspconfig.yaml b/.github/skills/azsdk-common-generate-sdk-locally/fixtures/tspconfig.yaml new file mode 100644 index 000000000000..59b7c1b3b72f --- /dev/null +++ b/.github/skills/azsdk-common-generate-sdk-locally/fixtures/tspconfig.yaml @@ -0,0 +1,12 @@ +parameters: + service-dir: + default: sdk/compute +emit: + - "@azure-tools/typespec-autorest" +options: + "@azure-tools/typespec-python": + package-dir: azure-mgmt-compute + flavor: azure + "@azure-tools/typespec-java": + package-dir: azure-resourcemanager-compute + flavor: azure diff --git a/.github/skills/azsdk-common-generate-sdk-locally/references/customization-workflow.md b/.github/skills/azsdk-common-generate-sdk-locally/references/customization-workflow.md new file mode 100644 index 000000000000..e0a8588f7f2f --- /dev/null +++ b/.github/skills/azsdk-common-generate-sdk-locally/references/customization-workflow.md @@ -0,0 +1,76 @@ +# TypeSpec Customization Workflow + +The customization workflow is an AI-assisted process that applies TypeSpec decorators and code repairs to ensure SDK functionality. It uses a two-phase approach: **Phase A** applies `client.tsp` decorators (~80% of issues), **Phase B** applies code-level repairs when builds still fail (~10%), and the remaining ~10% receive manual guidance. + +## Entry Points + +The customization tool (`azure-sdk-mcp:azsdk_customized_code_update`) can be triggered from multiple sources: + +| Entry Point | Description | Example | +|-------------|-------------|---------| +| **Build failures** | Compilation errors, analyzer violations, linting failures after SDK generation | `error CS0246: The type or namespace name 'FooModel' could not be found` | +| **Breaking changes** | Output from breaking changes analysis tools detecting renamed/removed properties, changed types | `Breaking changes detected: FooOptions.timeout property type changed from int to Duration` | +| **User prompts** | Natural language requests to modify SDK behavior | "Rename FooClient to BarClient for .NET" | +| **API review feedback** | Feedback from APIView or PR comments on SDK naming/structure | "Model name doesn't follow .NET casing conventions" | +| **.NET analyzer errors** | AZC0030 (naming violations), AZC0012 (generic type names), etc. | `AZC0030: Model name ends with 'Parameters'` | +| **Customization drift** | Existing customization code references renamed/removed TypeSpec entities | `cannot find symbol: method getField(String)` | +| **Duplicate field conflicts** | TypeSpec adds a property that already exists in manual customization code | `variable operationId is already defined in class AnalyzeOperationDetails` | + +## When to Use + +- Build fails after `azure-sdk-mcp:azsdk_package_build_code` with compilation errors +- Type name conflicts with reserved keywords or existing types +- Breaking changes from spec updates (renamed/removed properties, changed types) +- API surface changes that require `client.tsp` customizations +- .NET analyzer violations (AZC0030, AZC0012, etc.) +- Renaming clients, models, or operations for specific language SDKs +- Hiding internal operations from public SDK APIs +- Restructuring client architecture (e.g., creating subclients) +- Customization files reference entities that no longer exist after TypeSpec regeneration +- Duplicate fields between generated code and manual customization code + +## Customization Steps + +1. **Capture context** — Collect the build error output, user request, or API review feedback. +2. **Apply customization** — Run `azure-sdk-mcp:azsdk_customized_code_update` with the error/request context. The tool handles the full workflow internally: + - Classifies the request (TypeSpec fix, code patch, or manual guidance) + - Applies `client.tsp` decorators (Phase A) + - Regenerates the SDK automatically + - Builds to validate + - If build still fails and customization files exist, applies code patches (Phase B) + - Regenerates again (Java) and rebuilds +3. **Validate** — Run `azure-sdk-mcp:azsdk_package_run_check` and `azure-sdk-mcp:azsdk_package_run_tests` to verify no regressions. +4. **Review changes** — The tool leaves all changes uncommitted. Review with `git status`/`git diff` across both repos. + +## Common Scenarios + +| Scenario | Phase | Customization | +|----------|-------|--------------| +| Type name conflict with reserved keyword | A | Rename via `@@clientName` in `client.tsp` | +| Property renamed in new API version | A | Add `@@clientName` to preserve backward compatibility | +| .NET analyzer error (AZC0030, AZC0012) | A | Apply scoped `@@clientName` decorators to fix naming violations | +| Hide internal operation from SDK | A | Apply `@@access` decorator with language scope | +| Create subclient architecture | A | Use `@client` and `@clientInitialization` decorators | +| API review naming feedback | A | Apply scoped `@@clientName` for specific language | +| Duplicate field from customization conflict | B | Remove duplicate `addField()` from customization class | +| Customization references renamed property | B | Update references in `_patch.py`, `*Customization.java`, or partial classes | +| Feature request with no TypeSpec solution | Manual | Tool provides guidance to create customization infrastructure | + +## Two-Phase Workflow + +**Phase A — TypeSpec Customizations:** +Apply `client.tsp` decorators (e.g., `@@clientName`, `@@access`, `@client`), regenerate SDK, validate build. Handles ~80% of issues. + +**Phase B — Code Customizations:** +Activates automatically when Phase A build fails AND customization files exist (Java: `*Customization.java`, Python: `*_patch.py`, .NET: partial classes). Applies mechanical code repairs: duplicate removal, reference updates, import fixes. Handles ~10% of issues. + +**Manual Guidance:** +When neither phase resolves the issue, or no customization files exist, the tool returns structured guidance for manual implementation. + +## Retry Logic + +The tool handles retries internally with a two-pass classification approach: +1. First pass: classify feedback → apply TypeSpec fixes → regenerate → build +2. Second pass: re-classify remaining items with build error context → apply code patches → rebuild +3. If the tool response indicates build still failing, you can re-run `azure-sdk-mcp:azsdk_customized_code_update` with the updated error output +4. Max 2 attempts per phase (4 total iterations within the tool) diff --git a/.github/skills/azsdk-common-generate-sdk-locally/references/detailed-workflow.md b/.github/skills/azsdk-common-generate-sdk-locally/references/detailed-workflow.md new file mode 100644 index 000000000000..87232bd04a42 --- /dev/null +++ b/.github/skills/azsdk-common-generate-sdk-locally/references/detailed-workflow.md @@ -0,0 +1,32 @@ +# Detailed Workflow + +## Config File Identification + +Determine the correct path to the TypeSpec configuration file based on the working context. + +**Scenario A: Working in `azure-rest-api-specs`** +- Identify the path to `tspconfig.yaml` (local path or HTTPS URL). +- The local folder name can be arbitrary; MCP validates that the remote origin points to the official repo. +- Example paths: + - `/home/usr/azure-rest-api-specs/specification/contosowidgetmanager/Contoso.Management/tspconfig.yaml` + - `https://github.com/Azure/azure-rest-api-specs/blob/main/specification/contosowidgetmanager/Contoso.Management/tspconfig.yaml` + +**Scenario B: Working in an Azure SDK language repo** +- Identify the path to `tsp-location.yaml`. +- Example: `/home/usr/azure-sdk-for-net/sdk/contoso/Azure.ResourceManager.Contoso/tsp-location.yaml` + +## Commit Checkpoints + +There are two commit checkpoints in the workflow. At each one: + +1. Inform the user that the preceding steps completed successfully. +2. **Prompt the user** to decide whether to commit now. Do NOT skip this prompt. +3. If user chooses to commit: + - Check if on the `main` branch. If so, prompt: *"You are currently on the main branch. Please create a new branch using `git checkout -b ` before proceeding."* Suggest a default branch name (e.g., `sdk//`). + - Stage changed files with `git add`. + - Prompt for a commit message, then run `git commit -m ""`. +4. If user skips, acknowledge and proceed. + +**Checkpoint 1 (after step 7/8):** Commit generated + built SDK changes before running validation and tests. + +**Checkpoint 2 (after step 10/11):** Commit changelog, metadata, and version updates. diff --git a/.github/skills/azsdk-common-generate-sdk-locally/references/sdk-repos.md b/.github/skills/azsdk-common-generate-sdk-locally/references/sdk-repos.md new file mode 100644 index 000000000000..beac40503207 --- /dev/null +++ b/.github/skills/azsdk-common-generate-sdk-locally/references/sdk-repos.md @@ -0,0 +1,38 @@ +# SDK Repository Details + +## Language Repository Mapping + +| Language | Repository | +| ---------- | ---------------------- | +| .NET | `azure-sdk-for-net` | +| Java | `azure-sdk-for-java` | +| JavaScript | `azure-sdk-for-js` | +| Python | `azure-sdk-for-python` | +| Go | `azure-sdk-for-go` | +| Rust | `azure-sdk-for-rust` | + +## Configuration File Paths + +- **From azure-rest-api-specs repo:** Use path to `tspconfig.yaml` +- **From SDK language repo:** Use path to `tsp-location.yaml` + +## MCP Tools + +| Tool | Purpose | +|------|---------| +| `azure-sdk-mcp:azsdk_verify_setup` | Verify local environment for selected language | +| `azure-sdk-mcp:azsdk_package_generate_code` | Generate SDK from TypeSpec | +| `azure-sdk-mcp:azsdk_package_build_code` | Build package | +| `azure-sdk-mcp:azsdk_package_run_check` | Validate package | +| `azure-sdk-mcp:azsdk_package_run_tests` | Run tests | +| `azure-sdk-mcp:azsdk_customized_code_update` | Apply TypeSpec and code customizations to resolve build errors, breaking changes, or SDK modification requests (includes regeneration and build internally) | +| `azure-sdk-mcp:azsdk_package_update_changelog_content` | Update changelog | +| `azure-sdk-mcp:azsdk_package_update_metadata` | Update package metadata including ci.yml | +| `azure-sdk-mcp:azsdk_package_update_version` | Update package version | + +## Build Failure Resolution + +If build fails: + +- Run `azure-sdk-mcp:azsdk_customized_code_update` — it handles classification, TypeSpec fixes, regeneration, and build internally +- See [customization workflow](customization-workflow.md) for full details diff --git a/.github/skills/azsdk-common-generate-sdk-locally/tasks/analyzer-errors.yaml b/.github/skills/azsdk-common-generate-sdk-locally/tasks/analyzer-errors.yaml new file mode 100644 index 000000000000..0e380bdd6b71 --- /dev/null +++ b/.github/skills/azsdk-common-generate-sdk-locally/tasks/analyzer-errors.yaml @@ -0,0 +1,25 @@ +id: sdk-local-analyzer-errors-001 +name: Fix .NET Analyzer Errors via Customization +description: | + Test that the skill triggers customization when .NET build fails + with analyzer errors like AZC0030 (naming violations) or AZC0012 + (generic type names). +tags: + - customization + - analyzer + - dotnet +inputs: + prompt: "The .NET SDK build is failing with analyzer errors: AZC0030 says the model name ends with 'Parameters' and AZC0012 says type name 'Tasks' is too generic. Can you fix these?" +expected: + output_contains: + - "customize" + - "customization" + - "clientName" + output_excludes: + - "cannot help" + outcomes: + - type: tool_called + tool_name: azsdk_customized_code_update + behavior: + max_tool_calls: 10 + max_response_time_ms: 60000 diff --git a/.github/skills/azsdk-common-generate-sdk-locally/tasks/anti-trigger.yaml b/.github/skills/azsdk-common-generate-sdk-locally/tasks/anti-trigger.yaml new file mode 100644 index 000000000000..84b84bd5cb7e --- /dev/null +++ b/.github/skills/azsdk-common-generate-sdk-locally/tasks/anti-trigger.yaml @@ -0,0 +1,14 @@ +id: sdk-local-negative-001 +name: Anti-Trigger - Pipeline Generation +description: | + Test that the skill does NOT activate when user wants pipeline-based + SDK generation rather than local generation. +tags: + - negative-test + - anti-trigger +inputs: + prompt: "I want to use the Azure DevOps pipeline to generate SDKs automatically and have PRs created for me. I don't want to set up local tools." +expected: + output_not_contains: + - "azsdk_verify_setup" + - "azsdk_package_generate_code" diff --git a/.github/skills/azsdk-common-generate-sdk-locally/tasks/basic-usage.yaml b/.github/skills/azsdk-common-generate-sdk-locally/tasks/basic-usage.yaml new file mode 100644 index 000000000000..b8c476b60f88 --- /dev/null +++ b/.github/skills/azsdk-common-generate-sdk-locally/tasks/basic-usage.yaml @@ -0,0 +1,23 @@ +id: sdk-local-basic-001 +name: Generate Python SDK Locally +description: | + Test that the skill guides through local SDK generation for Python, + including environment verification and build steps. +tags: + - basic + - happy-path + - python +inputs: + prompt: "I want to generate the Python SDK locally from my TypeSpec project at specification/compute/resource-manager/Microsoft.Compute/ComputeRP/tspconfig.yaml. My azure-sdk-for-python repo is at ~/repos/azure-sdk-for-python." +expected: + output_contains: + - "Python" + - "azure-sdk-for-python" + outcomes: + - type: tool_called + tool_name: azsdk_verify_setup + - type: tool_called + tool_name: azsdk_package_generate_code + behavior: + max_tool_calls: 10 + max_response_time_ms: 60000 diff --git a/.github/skills/azsdk-common-generate-sdk-locally/tasks/breaking-changes.yaml b/.github/skills/azsdk-common-generate-sdk-locally/tasks/breaking-changes.yaml new file mode 100644 index 000000000000..fe45c2ed1877 --- /dev/null +++ b/.github/skills/azsdk-common-generate-sdk-locally/tasks/breaking-changes.yaml @@ -0,0 +1,27 @@ +id: sdk-local-breaking-changes-001 +name: Resolve Breaking Changes After Spec Update +description: | + Test that the skill triggers customization when breaking changes + are detected after a TypeSpec specification update — properties + renamed, types changed, or customization code drifts. +tags: + - customization + - breaking-changes + - full-workflow +inputs: + prompt: "After regenerating the Java SDK, the build is failing because the service team renamed 'displayName' to 'name' in the TypeSpec. Our customization class still references getField('displayName') and it can't find the symbol. Can you fix this?" +expected: + output_contains: + - "customize" + - "customization" + - "TypeSpec" + output_excludes: + - "cannot help" + outcomes: + - type: tool_called + tool_name: azsdk_customized_code_update + - type: tool_called + tool_name: azsdk_package_build_code + behavior: + max_tool_calls: 12 + max_response_time_ms: 120000 diff --git a/.github/skills/azsdk-common-generate-sdk-locally/tasks/customization-workflow.yaml b/.github/skills/azsdk-common-generate-sdk-locally/tasks/customization-workflow.yaml new file mode 100644 index 000000000000..b3f8ef4461e0 --- /dev/null +++ b/.github/skills/azsdk-common-generate-sdk-locally/tasks/customization-workflow.yaml @@ -0,0 +1,27 @@ +id: sdk-local-customization-001 +name: Duplicate Field Conflict After TypeSpec Addition +description: | + Test that the skill triggers TypeSpec customization when SDK build + fails because TypeSpec now generates a field that already exists + in manual customization code, causing a duplicate field error. +tags: + - customization + - error-handling + - full-workflow +inputs: + prompt: "The Java SDK build is failing after regeneration. The error says 'variable operationId is already defined in class AnalyzeOperationDetails'. It looks like the service team added operationId to the TypeSpec, but our DocumentIntelligenceCustomizations.java already injects that field manually. Can you fix this?" +expected: + output_contains: + - "customize" + - "customization" + - "duplicate" + output_excludes: + - "cannot help" + outcomes: + - type: tool_called + tool_name: azsdk_customized_code_update + - type: tool_called + tool_name: azsdk_package_build_code + behavior: + max_tool_calls: 12 + max_response_time_ms: 120000 diff --git a/.github/skills/azsdk-common-generate-sdk-locally/tasks/edge-case.yaml b/.github/skills/azsdk-common-generate-sdk-locally/tasks/edge-case.yaml new file mode 100644 index 000000000000..1ea18f262045 --- /dev/null +++ b/.github/skills/azsdk-common-generate-sdk-locally/tasks/edge-case.yaml @@ -0,0 +1,23 @@ +id: sdk-local-edge-001 +name: Build Failure Recovery +description: | + Test that the skill handles SDK build failures gracefully + and uses TypeSpec customization to resolve errors. +tags: + - edge-case + - error-handling +inputs: + prompt: "I generated the .NET SDK locally but the build is failing with compilation errors. The error says a type name conflicts with a reserved keyword." +expected: + output_contains: + - "customize" + - "customization" + - "client.tsp" + output_excludes: + - "cannot help" + outcomes: + - type: tool_called + tool_name: azsdk_customized_code_update + behavior: + max_tool_calls: 8 + max_response_time_ms: 60000 diff --git a/.github/skills/azsdk-common-generate-sdk-locally/tasks/full-workflow.yaml b/.github/skills/azsdk-common-generate-sdk-locally/tasks/full-workflow.yaml new file mode 100644 index 000000000000..f976521c536d --- /dev/null +++ b/.github/skills/azsdk-common-generate-sdk-locally/tasks/full-workflow.yaml @@ -0,0 +1,36 @@ +id: sdk-local-full-001 +name: Full Local Generation Workflow +description: | + Test that the skill covers the complete local workflow: + generate, build, validate, test, update metadata. +tags: + - basic + - full-workflow +inputs: + prompt: "Walk me through the complete process of generating the Java SDK locally, building it, running tests, and preparing it for a PR." +expected: + output_contains: + - "generate" + - "build" + - "test" + - "changelog" + outcomes: + - type: tool_called + tool_name: azsdk_verify_setup + - type: tool_called + tool_name: azsdk_package_generate_code + - type: tool_called + tool_name: azsdk_package_build_code + - type: tool_called + tool_name: azsdk_package_run_check + - type: tool_called + tool_name: azsdk_package_run_tests + - type: tool_called + tool_name: azsdk_package_update_changelog_content + - type: tool_called + tool_name: azsdk_package_update_metadata + - type: tool_called + tool_name: azsdk_package_update_version + behavior: + max_tool_calls: 15 + max_response_time_ms: 120000 diff --git a/.github/skills/azsdk-common-generate-sdk-locally/tasks/hide-operation.yaml b/.github/skills/azsdk-common-generate-sdk-locally/tasks/hide-operation.yaml new file mode 100644 index 000000000000..2f4ef26d688b --- /dev/null +++ b/.github/skills/azsdk-common-generate-sdk-locally/tasks/hide-operation.yaml @@ -0,0 +1,23 @@ +id: sdk-local-hide-operation-001 +name: Hide Internal Operation from SDK +description: | + Test that the skill triggers customization when user requests + hiding an internal operation from a specific language SDK. +tags: + - customization + - user-prompt + - access-control +inputs: + prompt: "Remove the get_create_project_status polling operation from the Python SDK public API. It's an internal operation that shouldn't be exposed to users." +expected: + output_contains: + - "access" + - "client.tsp" + output_excludes: + - "cannot help" + outcomes: + - type: tool_called + tool_name: azsdk_customized_code_update + behavior: + max_tool_calls: 10 + max_response_time_ms: 60000 diff --git a/.github/skills/azsdk-common-generate-sdk-locally/tasks/rename-client.yaml b/.github/skills/azsdk-common-generate-sdk-locally/tasks/rename-client.yaml new file mode 100644 index 000000000000..1ffd07caf523 --- /dev/null +++ b/.github/skills/azsdk-common-generate-sdk-locally/tasks/rename-client.yaml @@ -0,0 +1,23 @@ +id: sdk-local-rename-client-001 +name: Rename SDK Client via TypeSpec Customization +description: | + Test that the skill triggers TypeSpec customization when user + requests renaming a client or model for a specific language SDK. +tags: + - customization + - user-prompt + - typespec +inputs: + prompt: "Rename the AIProjectConnectionEntraIDCredential model to use 'Id' instead of 'ID' in the .NET SDK. This was flagged in API review." +expected: + output_contains: + - "clientName" + - "client.tsp" + output_excludes: + - "cannot help" + outcomes: + - type: tool_called + tool_name: azsdk_customized_code_update + behavior: + max_tool_calls: 10 + max_response_time_ms: 60000 diff --git a/.github/skills/azsdk-common-generate-sdk-locally/tasks/update-changelog.yaml b/.github/skills/azsdk-common-generate-sdk-locally/tasks/update-changelog.yaml new file mode 100644 index 000000000000..2b7d42b7533b --- /dev/null +++ b/.github/skills/azsdk-common-generate-sdk-locally/tasks/update-changelog.yaml @@ -0,0 +1,20 @@ +id: sdk-local-update-changelog-001 +name: Update SDK Package Changelog +description: | + Test that the skill triggers changelog update for a generated SDK package. +tags: + - metadata + - changelog +inputs: + prompt: "Update the changelog for the Python SDK package at sdk/compute/azure-mgmt-compute." +expected: + output_contains: + - "changelog" + output_excludes: + - "cannot help" + outcomes: + - type: tool_called + tool_name: azsdk_package_update_changelog_content + behavior: + max_tool_calls: 5 + max_response_time_ms: 30000 diff --git a/.github/skills/azsdk-common-generate-sdk-locally/tasks/update-metadata.yaml b/.github/skills/azsdk-common-generate-sdk-locally/tasks/update-metadata.yaml new file mode 100644 index 000000000000..5db6a88d8a00 --- /dev/null +++ b/.github/skills/azsdk-common-generate-sdk-locally/tasks/update-metadata.yaml @@ -0,0 +1,21 @@ +id: sdk-local-update-metadata-001 +name: Update SDK Package Metadata and ci.yml +description: | + Test that the skill triggers metadata update (including ci.yml) + for a generated SDK package. +tags: + - metadata + - ci +inputs: + prompt: "Update the package metadata and ci.yml for the .NET SDK at sdk/contoso/Azure.ResourceManager.Contoso." +expected: + output_contains: + - "metadata" + output_excludes: + - "cannot help" + outcomes: + - type: tool_called + tool_name: azsdk_package_update_metadata + behavior: + max_tool_calls: 5 + max_response_time_ms: 30000 diff --git a/.github/skills/azsdk-common-generate-sdk-locally/tasks/update-version.yaml b/.github/skills/azsdk-common-generate-sdk-locally/tasks/update-version.yaml new file mode 100644 index 000000000000..a64e8ace181f --- /dev/null +++ b/.github/skills/azsdk-common-generate-sdk-locally/tasks/update-version.yaml @@ -0,0 +1,20 @@ +id: sdk-local-update-version-001 +name: Update SDK Package Version +description: | + Test that the skill triggers version update for a generated SDK package. +tags: + - metadata + - version +inputs: + prompt: "Update the package version for the Java SDK at sdk/contoso/azure-resourcemanager-contoso." +expected: + output_contains: + - "version" + output_excludes: + - "cannot help" + outcomes: + - type: tool_called + tool_name: azsdk_package_update_version + behavior: + max_tool_calls: 5 + max_response_time_ms: 30000 diff --git a/.github/skills/azsdk-common-pipeline-troubleshooting/SKILL.md b/.github/skills/azsdk-common-pipeline-troubleshooting/SKILL.md new file mode 100644 index 000000000000..5dd6b764bb6c --- /dev/null +++ b/.github/skills/azsdk-common-pipeline-troubleshooting/SKILL.md @@ -0,0 +1,41 @@ +--- +name: azsdk-common-pipeline-troubleshooting +license: MIT +metadata: + version: "1.0.0" + distribution: shared +description: "Diagnose and resolve failures in Azure SDK CI and generation pipelines. **UTILITY SKILL**. USE FOR: \"pipeline failed\", \"build failure\", \"CI check failing\", \"SDK generation error\", \"reproduce pipeline locally\", \"debug SDK pipeline\". DO NOT USE FOR: local build issues without pipeline context, API design review, SDK publishing. INVOKES: azure-sdk-mcp:azsdk_analyze_pipeline, azure-sdk-mcp:azsdk_package_build_code, azure-sdk-mcp:azsdk_package_run_check." +compatibility: + requires: "azure-sdk-mcp server, Azure DevOps pipeline build ID" +--- + +# Pipeline Troubleshooting + +## MCP Tools + +| Tool | Purpose | +| -------------------------- | ------------------------ | +| `azure-sdk-mcp:azsdk_analyze_pipeline` | Analyze pipeline failure | +| `azure-sdk-mcp:azsdk_verify_setup` | Verify local environment | +| `azure-sdk-mcp:azsdk_package_build_code` | Reproduce build locally | +| `azure-sdk-mcp:azsdk_package_run_check` | Run validation checks | +| `azure-sdk-mcp:azsdk_package_pack` | Create SDK packages | + +**Prerequisites:** azure-sdk-mcp server required. Without MCP, view pipeline logs in Azure DevOps UI. + +## Steps + +1. **Identify** — Get build ID, run `azure-sdk-mcp:azsdk_analyze_pipeline`. Categorize failure. +2. **Analyze** — See [failure patterns](references/failure-patterns.md) for common causes. +3. **Reproduce** — Verify setup, then run `azure-sdk-mcp:azsdk_package_build_code` or `azure-sdk-mcp:azsdk_package_run_check`. +4. **Fix** — Apply fixes for code or TypeSpec issues. +5. **Verify** — Confirm fix locally, push, monitor re-run. + +## Examples + +- "My pipeline build 12345 failed, help debug it" +- "Reproduce CI failure locally" + +## Troubleshooting + +If `azure-sdk-mcp:azsdk_analyze_pipeline` returns no data, verify build ID and MCP. Without MCP, check pipeline logs in Azure DevOps UI. diff --git a/.github/skills/azsdk-common-pipeline-troubleshooting/eval.yaml b/.github/skills/azsdk-common-pipeline-troubleshooting/eval.yaml new file mode 100644 index 000000000000..772285117d44 --- /dev/null +++ b/.github/skills/azsdk-common-pipeline-troubleshooting/eval.yaml @@ -0,0 +1,29 @@ +name: azsdk-common-pipeline-troubleshooting-eval +description: Evaluation suite for pipeline troubleshooting skill. +skill: azsdk-common-pipeline-troubleshooting +version: '1.0' +config: + model: gpt-4o + executor: mock + trials_per_task: 1 + timeout_seconds: 300 + parallel: false +metrics: + - name: task_completion + weight: 0.7 + threshold: 0.8 + description: Did the skill complete trigger and anti-trigger checks? + - name: efficiency + weight: 0.3 + threshold: 0.7 + description: Did the skill stay within behavior limits? +graders: + - type: text + name: no_fatal_errors + config: + regex_not_match: + - (?i)fatal error + - (?i)unhandled exception + - '(?i)panic:' +tasks: + - "tasks/*.yaml" diff --git a/.github/skills/azsdk-common-pipeline-troubleshooting/evals/eval.yaml b/.github/skills/azsdk-common-pipeline-troubleshooting/evals/eval.yaml new file mode 100644 index 000000000000..9f1d0626332c --- /dev/null +++ b/.github/skills/azsdk-common-pipeline-troubleshooting/evals/eval.yaml @@ -0,0 +1,29 @@ +name: azsdk-common-pipeline-troubleshooting-evals +description: Evaluations for the pipeline troubleshooting skill +skill: azsdk-common-pipeline-troubleshooting +version: "1.0" +config: + model: gpt-4o + executor: mock + trials_per_task: 1 + timeout_seconds: 300 + parallel: false +metrics: + - name: task_completion + weight: 0.7 + threshold: 0.8 + description: Did the skill complete trigger and anti-trigger checks? + - name: efficiency + weight: 0.3 + threshold: 0.7 + description: Did the skill stay within behavior limits? +graders: + - type: text + name: no_fatal_errors + config: + regex_not_match: + - (?i)fatal error + - (?i)unhandled exception + - '(?i)panic:' +tasks: + - "tasks/*.yaml" diff --git a/.github/skills/azsdk-common-pipeline-troubleshooting/evals/tasks/anti-trigger.yaml b/.github/skills/azsdk-common-pipeline-troubleshooting/evals/tasks/anti-trigger.yaml new file mode 100644 index 000000000000..c7e5fd8b6f08 --- /dev/null +++ b/.github/skills/azsdk-common-pipeline-troubleshooting/evals/tasks/anti-trigger.yaml @@ -0,0 +1,14 @@ +id: anti-trigger +name: anti-trigger +description: Should not trigger on unrelated prompts +inputs: + prompt: "How do I write a TypeSpec definition for a new Azure service?" +expected: + skill_activated: false +graders: + - type: text + name: no-pipeline-advice + config: + not_contains: + - "pipeline" + - "CI failure" diff --git a/.github/skills/azsdk-common-pipeline-troubleshooting/evals/tasks/basic-trigger.yaml b/.github/skills/azsdk-common-pipeline-troubleshooting/evals/tasks/basic-trigger.yaml new file mode 100644 index 000000000000..71291005e71c --- /dev/null +++ b/.github/skills/azsdk-common-pipeline-troubleshooting/evals/tasks/basic-trigger.yaml @@ -0,0 +1,13 @@ +id: basic-trigger +name: basic-trigger +description: Should trigger on pipeline troubleshooting requests +inputs: + prompt: "My Azure SDK CI pipeline is failing. Can you help me debug it?" +expected: + skill_activated: true +graders: + - type: text + name: mentions-pipeline + config: + contains: + - "pipeline" diff --git a/.github/skills/azsdk-common-pipeline-troubleshooting/evals/trigger_tests.yaml b/.github/skills/azsdk-common-pipeline-troubleshooting/evals/trigger_tests.yaml new file mode 100644 index 000000000000..2ca93f179ae8 --- /dev/null +++ b/.github/skills/azsdk-common-pipeline-troubleshooting/evals/trigger_tests.yaml @@ -0,0 +1,23 @@ +skill: azsdk-common-pipeline-troubleshooting +should_trigger_prompts: + - prompt: "my CI pipeline is failing" + reason: "Should activate skill" + - prompt: "debug pipeline build error" + reason: "Should activate skill" + - prompt: "pipeline troubleshooting help" + reason: "Should activate skill" + - prompt: "fix failing SDK pipeline" + reason: "Should activate skill" + - prompt: "CI build broke after my change" + reason: "Should activate skill" +should_not_trigger_prompts: + - prompt: "write a TypeSpec definition" + reason: "Unrelated to skill scope" + - prompt: "create a new skill" + reason: "Unrelated to skill scope" + - prompt: "optimize markdown tokens" + reason: "Unrelated to skill scope" + - prompt: "how do I release an SDK package" + reason: "Unrelated to skill scope" + - prompt: "resolve APIView feedback" + reason: "Unrelated to skill scope" diff --git a/.github/skills/azsdk-common-pipeline-troubleshooting/references/failure-patterns.md b/.github/skills/azsdk-common-pipeline-troubleshooting/references/failure-patterns.md new file mode 100644 index 000000000000..36b7c16602c0 --- /dev/null +++ b/.github/skills/azsdk-common-pipeline-troubleshooting/references/failure-patterns.md @@ -0,0 +1,45 @@ +# Pipeline Failure Patterns and Resolution + +## Build Failures + +- Check for TypeSpec customization needs (missing client.tsp changes) +- Verify tspconfig.yaml emitter configuration is correct +- Check for breaking changes in the generated API surface + +## Test Failures + +- Check if tests need playback recording updates +- Verify test fixtures are present and valid + +## Validation Failures + +- Run `azsdk_package_run_check` locally to reproduce +- Check changelog format and content +- Verify API compatibility (no unintended breaking changes) + +## Common Failure Patterns + +| Pattern | Likely Cause | Fix | +| ----------------------------- | ------------------------------ | ---------------------------------- | +| Missing types/models | TypeSpec compilation issue | Fix TypeSpec, regenerate | +| Breaking change detected | API surface changed | Add `@clientName` or revert change | +| Test playback failure | Recorded responses outdated | Re-record test sessions | +| Changelog validation error | Missing or malformed changelog | Update changelog content | +| Dependency resolution failure | Package version conflict | Check dependency versions | + +## Fix Application + +**If TypeSpec change is needed:** + +- Use TypeSpec customization to apply changes +- Regenerate SDK after TypeSpec fixes + +**If code fix is needed:** + +- Apply fix directly in SDK repo +- Re-run build and validation locally + +**If pipeline/infrastructure issue:** + +- Suggest re-running the pipeline +- Report infrastructure issues to engineering systems team diff --git a/.github/skills/azsdk-common-pipeline-troubleshooting/tasks/basic-usage.yaml b/.github/skills/azsdk-common-pipeline-troubleshooting/tasks/basic-usage.yaml new file mode 100644 index 000000000000..c906ab9baf77 --- /dev/null +++ b/.github/skills/azsdk-common-pipeline-troubleshooting/tasks/basic-usage.yaml @@ -0,0 +1,20 @@ +id: pipeline-basic-001 +name: Analyze Pipeline Build Failure +description: | + Test that the skill analyzes a failing pipeline and provides + actionable diagnosis. +tags: + - basic + - happy-path +inputs: + prompt: "My SDK generation pipeline failed for the Python language. The pipeline URL is https://dev.azure.com/azure-sdk/internal/_build/results?buildId=12345. Can you help me figure out what went wrong?" +expected: + output_contains: + - "pipeline" + - "Python" + outcomes: + - type: tool_called + tool_name: azsdk_analyze_pipeline + behavior: + max_tool_calls: 5 + max_response_time_ms: 30000 diff --git a/.github/skills/azsdk-common-pipeline-troubleshooting/tasks/edge-case.yaml b/.github/skills/azsdk-common-pipeline-troubleshooting/tasks/edge-case.yaml new file mode 100644 index 000000000000..25c2d246a32f --- /dev/null +++ b/.github/skills/azsdk-common-pipeline-troubleshooting/tasks/edge-case.yaml @@ -0,0 +1,19 @@ +id: pipeline-edge-001 +name: Multiple Languages Failed +description: | + Test that the skill handles failures across multiple languages + and prioritizes troubleshooting. +tags: + - edge-case + - multi-language +inputs: + prompt: "My SDK generation pipeline failed for both Python and Java but succeeded for .NET and JavaScript. How do I fix the failing languages?" +expected: + output_contains: + - "Python" + - "Java" + output_excludes: + - "cannot help" + behavior: + max_tool_calls: 5 + max_response_time_ms: 30000 diff --git a/.github/skills/azsdk-common-pipeline-troubleshooting/tasks/local-reproduction.yaml b/.github/skills/azsdk-common-pipeline-troubleshooting/tasks/local-reproduction.yaml new file mode 100644 index 000000000000..abd995b326cc --- /dev/null +++ b/.github/skills/azsdk-common-pipeline-troubleshooting/tasks/local-reproduction.yaml @@ -0,0 +1,17 @@ +id: pipeline-repro-001 +name: Reproduce Pipeline Failure Locally +description: | + Test that the skill guides through local reproduction of a pipeline failure. +tags: + - basic + - local-repro +inputs: + prompt: "The CI pipeline for my Go SDK PR is failing with a compilation error. I want to reproduce this locally to debug it." +expected: + output_contains: + - "local" + - "Go" + - "build" + behavior: + max_tool_calls: 5 + max_response_time_ms: 30000 diff --git a/.github/skills/azsdk-common-pipeline-troubleshooting/tasks/should-not-trigger.yaml b/.github/skills/azsdk-common-pipeline-troubleshooting/tasks/should-not-trigger.yaml new file mode 100644 index 000000000000..afa31d7f613d --- /dev/null +++ b/.github/skills/azsdk-common-pipeline-troubleshooting/tasks/should-not-trigger.yaml @@ -0,0 +1,13 @@ +id: pipeline-negative-001 +name: Should Not Trigger - TypeSpec Authoring Request +description: | + Test that the skill does NOT activate for TypeSpec authoring questions. +tags: + - negative-test + - anti-trigger +inputs: + prompt: "How do I define a new REST operation in my TypeSpec project?" +expected: + output_not_contains: + - "azsdk_analyze_pipeline" + - "pipeline failure" diff --git a/.github/skills/azsdk-common-prepare-release-plan/SKILL.md b/.github/skills/azsdk-common-prepare-release-plan/SKILL.md new file mode 100644 index 000000000000..e73a947fe5e0 --- /dev/null +++ b/.github/skills/azsdk-common-prepare-release-plan/SKILL.md @@ -0,0 +1,44 @@ +--- +name: azsdk-common-prepare-release-plan +license: MIT +metadata: + version: "1.0.0" + distribution: shared +description: "Create and manage release plan work items for Azure SDK releases across languages. **UTILITY SKILL**. USE FOR: \"create release plan\", \"update release plan\", \"link SDK PR to plan\", \"namespace approval\", \"check release plan status\". DO NOT USE FOR: SDK code generation, pipeline troubleshooting, API review feedback. INVOKES: azure-sdk-mcp:azsdk_create_release_plan, azure-sdk-mcp:azsdk_get_release_plan, azure-sdk-mcp:azsdk_link_sdk_pull_request_to_release_plan." +compatibility: + requires: "azure-sdk-mcp server, API spec PR in Azure/azure-rest-api-specs" +--- + +# Prepare Release Plan + +> Do not display Azure DevOps work item URLs. Only provide Release Plan Link and ID. + +## MCP Tools + +| Tool | Purpose | +|------|---------| +| `azure-sdk-mcp:azsdk_create_release_plan` | Create plan | +| `azure-sdk-mcp:azsdk_get_release_plan` | Get details | +| `azure-sdk-mcp:azsdk_get_release_plan_for_spec_pr` | Find by spec PR | +| `azure-sdk-mcp:azsdk_update_sdk_details_in_release_plan` | Update SDK info | +| `azure-sdk-mcp:azsdk_link_sdk_pull_request_to_release_plan` | Link SDK PR | +| `azure-sdk-mcp:azsdk_link_namespace_approval_issue` | Link namespace | + +## Steps + +1. **Prerequisites** — Check for API spec PR; prompt if unavailable. +2. **Check Existing** — Query by plan number or spec PR link. +3. **Gather Info** — Collect Service Tree IDs, timeline. See [details](references/release-plan-details.md). +4. **Create** — Run `azure-sdk-mcp:azsdk_create_release_plan`. +5. **Namespace** — For mgmt plane first releases, link approval issue. +6. **Link PRs** — Link SDK PRs to plan. + +## Examples + +- "Create a release plan for my spec PR" +- "Link my SDK PR to release plan" + +## Troubleshooting + +- Requires `azure-sdk-mcp` server; no CLI fallback. +- If creation fails, verify spec PR URL and Service Tree IDs. diff --git a/.github/skills/azsdk-common-prepare-release-plan/eval.yaml b/.github/skills/azsdk-common-prepare-release-plan/eval.yaml new file mode 100644 index 000000000000..429f50843baf --- /dev/null +++ b/.github/skills/azsdk-common-prepare-release-plan/eval.yaml @@ -0,0 +1,29 @@ +name: azsdk-common-prepare-release-plan-eval +description: Evaluation suite for release plan management skill. +skill: azsdk-common-prepare-release-plan +version: '1.0' +config: + model: gpt-4o + executor: mock + trials_per_task: 1 + timeout_seconds: 300 + parallel: false +metrics: + - name: task_completion + weight: 0.7 + threshold: 0.8 + description: Did the skill complete trigger and anti-trigger checks? + - name: efficiency + weight: 0.3 + threshold: 0.7 + description: Did the skill stay within behavior limits? +graders: + - type: text + name: no_fatal_errors + config: + regex_not_match: + - (?i)fatal error + - (?i)unhandled exception + - '(?i)panic:' +tasks: + - "tasks/*.yaml" diff --git a/.github/skills/azsdk-common-prepare-release-plan/evals/eval.yaml b/.github/skills/azsdk-common-prepare-release-plan/evals/eval.yaml new file mode 100644 index 000000000000..17ce1eb7aef1 --- /dev/null +++ b/.github/skills/azsdk-common-prepare-release-plan/evals/eval.yaml @@ -0,0 +1,29 @@ +name: azsdk-common-prepare-release-plan-evals +description: Evaluations for the release plan preparation skill +skill: azsdk-common-prepare-release-plan +version: "1.0" +config: + model: gpt-4o + executor: mock + trials_per_task: 1 + timeout_seconds: 300 + parallel: false +metrics: + - name: task_completion + weight: 0.7 + threshold: 0.8 + description: Did the skill complete trigger and anti-trigger checks? + - name: efficiency + weight: 0.3 + threshold: 0.7 + description: Did the skill stay within behavior limits? +graders: + - type: text + name: no_fatal_errors + config: + regex_not_match: + - (?i)fatal error + - (?i)unhandled exception + - '(?i)panic:' +tasks: + - "tasks/*.yaml" diff --git a/.github/skills/azsdk-common-prepare-release-plan/evals/tasks/anti-trigger.yaml b/.github/skills/azsdk-common-prepare-release-plan/evals/tasks/anti-trigger.yaml new file mode 100644 index 000000000000..a10ce18190e6 --- /dev/null +++ b/.github/skills/azsdk-common-prepare-release-plan/evals/tasks/anti-trigger.yaml @@ -0,0 +1,14 @@ +id: anti-trigger +name: anti-trigger +description: Should not trigger on unrelated prompts +inputs: + prompt: "How do I configure logging in my Azure Function?" +expected: + skill_activated: false +graders: + - type: text + name: no-release-advice + config: + not_contains: + - "release plan" + - "changelog" diff --git a/.github/skills/azsdk-common-prepare-release-plan/evals/tasks/basic-trigger.yaml b/.github/skills/azsdk-common-prepare-release-plan/evals/tasks/basic-trigger.yaml new file mode 100644 index 000000000000..b4e8da95f43c --- /dev/null +++ b/.github/skills/azsdk-common-prepare-release-plan/evals/tasks/basic-trigger.yaml @@ -0,0 +1,13 @@ +id: basic-trigger +name: basic-trigger +description: Should trigger on release plan preparation requests +inputs: + prompt: "I need to prepare a release plan for the new Azure SDK package version" +expected: + skill_activated: true +graders: + - type: text + name: mentions-release + config: + contains: + - "release" diff --git a/.github/skills/azsdk-common-prepare-release-plan/evals/trigger_tests.yaml b/.github/skills/azsdk-common-prepare-release-plan/evals/trigger_tests.yaml new file mode 100644 index 000000000000..b884c8496b08 --- /dev/null +++ b/.github/skills/azsdk-common-prepare-release-plan/evals/trigger_tests.yaml @@ -0,0 +1,23 @@ +skill: azsdk-common-prepare-release-plan +should_trigger_prompts: + - prompt: "prepare a release plan" + reason: "Should activate skill" + - prompt: "plan the next SDK release" + reason: "Should activate skill" + - prompt: "create release plan for my package" + reason: "Should activate skill" + - prompt: "what goes in a release plan" + reason: "Should activate skill" + - prompt: "help me prepare for SDK release" + reason: "Should activate skill" +should_not_trigger_prompts: + - prompt: "deploy my Azure Function" + reason: "Unrelated to skill scope" + - prompt: "fix my pipeline build error" + reason: "Unrelated to skill scope" + - prompt: "create a new TypeSpec definition" + reason: "Unrelated to skill scope" + - prompt: "configure Azure logging" + reason: "Unrelated to skill scope" + - prompt: "resolve APIView feedback" + reason: "Unrelated to skill scope" diff --git a/.github/skills/azsdk-common-prepare-release-plan/fixtures/tspconfig.yaml b/.github/skills/azsdk-common-prepare-release-plan/fixtures/tspconfig.yaml new file mode 100644 index 000000000000..2f05df5c4245 --- /dev/null +++ b/.github/skills/azsdk-common-prepare-release-plan/fixtures/tspconfig.yaml @@ -0,0 +1,15 @@ +parameters: + service-dir: + default: sdk/compute +options: + "@azure-tools/typespec-python": + package-dir: azure-mgmt-compute + "@azure-tools/typespec-csharp": + namespace: Azure.ResourceManager.Compute + "@azure-tools/typespec-java": + package-dir: azure-resourcemanager-compute + "@azure-tools/typespec-ts": + packageDetails: + name: "@azure/arm-compute" + "@azure-tools/typespec-go": + module: "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" diff --git a/.github/skills/azsdk-common-prepare-release-plan/references/release-plan-details.md b/.github/skills/azsdk-common-prepare-release-plan/references/release-plan-details.md new file mode 100644 index 000000000000..49435bcbb4d4 --- /dev/null +++ b/.github/skills/azsdk-common-prepare-release-plan/references/release-plan-details.md @@ -0,0 +1,33 @@ +# Release Plan Detailed Steps + +> **CRITICAL**: Do not mention or display Azure DevOps work item links/URLs. Only provide Release Plan Link and Release Plan ID to the user. All manual updates must be made through the Release Planner Tool (https://aka.ms/sdk-release-planner). + +## Required Information + +Collect these details (do not use temporary values): + +- **Service Tree ID**: GUID format - confirm with user +- **Product Service Tree ID**: GUID format - confirm with user +- **Expected Release Timeline**: "Month YYYY" format +- **SDK Release Type**: "beta" (preview) or "stable" (GA) + +## SDK Details Update + +To update SDK details in the release plan: + +- Run `azsdk_update_sdk_details_in_release_plan` with the release plan work item ID and TypeSpec project path + +## Namespace Approval (Management Plane Only) + +For first release of management plane SDK: + +1. Check if namespace approval issue already exists +2. If not, collect GitHub issue from Azure/azure-sdk repo +3. Run `azsdk_link_namespace_approval_issue` + +## Linking SDK Pull Requests + +If SDK PRs exist: + +1. Ensure GitHub CLI authentication (`gh auth login`) +2. Run `azsdk_link_sdk_pull_request_to_release_plan` for each PR \ No newline at end of file diff --git a/.github/skills/azsdk-common-prepare-release-plan/tasks/basic-usage.yaml b/.github/skills/azsdk-common-prepare-release-plan/tasks/basic-usage.yaml new file mode 100644 index 000000000000..f0d8eba2867e --- /dev/null +++ b/.github/skills/azsdk-common-prepare-release-plan/tasks/basic-usage.yaml @@ -0,0 +1,17 @@ +id: release-plan-basic-001 +name: Create New Release Plan +description: | + Test that the skill guides creation of a new release plan + with all required information gathered from the user. +tags: + - basic + - happy-path +inputs: + prompt: "I need to create a release plan for my API spec PR https://github.com/Azure/azure-rest-api-specs/pull/12345. The API version is 2024-06-01 and it's a stable release." +expected: + output_contains: + - "Service Tree" + - "release plan" + behavior: + max_tool_calls: 5 + max_response_time_ms: 30000 diff --git a/.github/skills/azsdk-common-prepare-release-plan/tasks/edge-case.yaml b/.github/skills/azsdk-common-prepare-release-plan/tasks/edge-case.yaml new file mode 100644 index 000000000000..d864c5d7e96d --- /dev/null +++ b/.github/skills/azsdk-common-prepare-release-plan/tasks/edge-case.yaml @@ -0,0 +1,16 @@ +id: release-plan-edge-001 +name: Invalid Package Name Format +description: | + Test that the skill validates package name formats + and auto-corrects invalid names. +tags: + - edge-case + - validation +inputs: + prompt: "Update the release plan with these SDK details: JavaScript package 'arm-compute' and Go package 'sdk/armcompute'." +expected: + output_contains: + - "@azure/" + behavior: + max_tool_calls: 3 + max_response_time_ms: 15000 diff --git a/.github/skills/azsdk-common-prepare-release-plan/tasks/link-sdk-prs.yaml b/.github/skills/azsdk-common-prepare-release-plan/tasks/link-sdk-prs.yaml new file mode 100644 index 000000000000..7f7aeb2f6a93 --- /dev/null +++ b/.github/skills/azsdk-common-prepare-release-plan/tasks/link-sdk-prs.yaml @@ -0,0 +1,16 @@ +id: release-plan-link-001 +name: Link SDK PRs to Release Plan +description: | + Test that the skill links SDK pull requests to an existing release plan. +tags: + - basic + - linking +inputs: + prompt: "I have release plan 67890 and SDK PRs for Python (PR #100) and .NET (PR #200) in the azure-sdk repos. Link them to the release plan." +expected: + output_contains: + - "link" + - "release plan" + behavior: + max_tool_calls: 5 + max_response_time_ms: 30000 diff --git a/.github/skills/azsdk-common-prepare-release-plan/tasks/should-not-trigger.yaml b/.github/skills/azsdk-common-prepare-release-plan/tasks/should-not-trigger.yaml new file mode 100644 index 000000000000..179ae8376861 --- /dev/null +++ b/.github/skills/azsdk-common-prepare-release-plan/tasks/should-not-trigger.yaml @@ -0,0 +1,14 @@ +id: release-plan-negative-001 +name: Should Not Trigger - Package Release Request +description: | + Test that the skill does NOT activate when user wants to release a package + rather than create/manage a release plan. +tags: + - negative-test + - anti-trigger +inputs: + prompt: "Release the azure-mgmt-compute Python package to PyPI now." +expected: + output_not_contains: + - "azsdk_create_release_plan" + - "Service Tree ID" diff --git a/.github/skills/azsdk-common-sdk-release/SKILL.md b/.github/skills/azsdk-common-sdk-release/SKILL.md new file mode 100644 index 000000000000..03d86381d4e2 --- /dev/null +++ b/.github/skills/azsdk-common-sdk-release/SKILL.md @@ -0,0 +1,36 @@ +--- +name: azsdk-common-sdk-release +license: MIT +metadata: + version: "1.0.0" + distribution: shared +description: "Check release readiness and trigger the release pipeline for Azure SDK packages. **UTILITY SKILL**. USE FOR: \"release SDK\", \"trigger release\", \"check release readiness\", \"release pipeline\", \"publish package\", \"ship SDK\". DO NOT USE FOR: SDK development, code generation, pipeline debugging, release plan creation. INVOKES: azure-sdk-mcp:azsdk_release_sdk." +compatibility: + requires: "azure-sdk-mcp server, SDK package merged on release branch" + supports: ".NET, Java, JavaScript, Python, Go" +--- + +# SDK Release + +## MCP Tools + +| Tool | Purpose | +| ------------------- | ----------------------------------------------------------- | +| `azure-sdk-mcp:azsdk_release_sdk` | Check release readiness and/or trigger the release pipeline | + +## Steps + +1. **Collect Info** — Get `packageName` and `language` from the user. Optionally get `branch` (defaults to main). +2. **Check Readiness** — Run `azure-sdk-mcp:azsdk_release_sdk` with `checkReady: true` to verify API review approval, changelog, package name approval, and release date. +3. **Review Results** — If not ready, display failing checks and guide user to resolve. +4. **Trigger Release** — Once ready, run `azure-sdk-mcp:azsdk_release_sdk` with `checkReady: false`. Show pipeline link and inform user they must approve the release stage. + +## Examples + +- "Check if azure-storage-blob is ready for release" +- "Trigger release for @azure/core-rest-pipeline on JavaScript" + +## Troubleshooting + +- If readiness check fails, review each failing prerequisite individually before re-triggering. +- Requires `azure-sdk-mcp` server. No CLI fallback — prompt user to configure MCP if unavailable. diff --git a/.github/skills/azsdk-common-sdk-release/eval.yaml b/.github/skills/azsdk-common-sdk-release/eval.yaml new file mode 100644 index 000000000000..6e56f0310a33 --- /dev/null +++ b/.github/skills/azsdk-common-sdk-release/eval.yaml @@ -0,0 +1,29 @@ +name: azsdk-common-sdk-release-eval +description: Evaluation suite for SDK release skill. +skill: azsdk-common-sdk-release +version: '1.0' +config: + model: gpt-4o + executor: mock + trials_per_task: 1 + timeout_seconds: 300 + parallel: false +metrics: + - name: task_completion + weight: 0.7 + threshold: 0.8 + description: Did the skill complete trigger and anti-trigger checks? + - name: efficiency + weight: 0.3 + threshold: 0.7 + description: Did the skill stay within behavior limits? +graders: + - type: text + name: no_fatal_errors + config: + regex_not_match: + - (?i)fatal error + - (?i)unhandled exception + - '(?i)panic:' +tasks: + - "tasks/*.yaml" diff --git a/.github/skills/azsdk-common-sdk-release/evals/eval.yaml b/.github/skills/azsdk-common-sdk-release/evals/eval.yaml new file mode 100644 index 000000000000..5ee394c86cc8 --- /dev/null +++ b/.github/skills/azsdk-common-sdk-release/evals/eval.yaml @@ -0,0 +1,29 @@ +name: azsdk-common-sdk-release-evals +description: Evaluations for the SDK release skill +skill: azsdk-common-sdk-release +version: "1.0" +config: + model: gpt-4o + executor: mock + trials_per_task: 1 + timeout_seconds: 300 + parallel: false +metrics: + - name: task_completion + weight: 0.7 + threshold: 0.8 + description: Did the skill complete trigger and anti-trigger checks? + - name: efficiency + weight: 0.3 + threshold: 0.7 + description: Did the skill stay within behavior limits? +graders: + - type: text + name: no_fatal_errors + config: + regex_not_match: + - (?i)fatal error + - (?i)unhandled exception + - '(?i)panic:' +tasks: + - "tasks/*.yaml" diff --git a/.github/skills/azsdk-common-sdk-release/evals/tasks/anti-trigger.yaml b/.github/skills/azsdk-common-sdk-release/evals/tasks/anti-trigger.yaml new file mode 100644 index 000000000000..9bfeaa58e415 --- /dev/null +++ b/.github/skills/azsdk-common-sdk-release/evals/tasks/anti-trigger.yaml @@ -0,0 +1,14 @@ +id: anti-trigger +name: anti-trigger +description: Should not trigger on unrelated prompts +inputs: + prompt: "How do I write unit tests for my Azure SDK client?" +expected: + skill_activated: false +graders: + - type: text + name: no-release-advice + config: + not_contains: + - "release process" + - "publish package" diff --git a/.github/skills/azsdk-common-sdk-release/evals/tasks/basic-trigger.yaml b/.github/skills/azsdk-common-sdk-release/evals/tasks/basic-trigger.yaml new file mode 100644 index 000000000000..eb65512130d6 --- /dev/null +++ b/.github/skills/azsdk-common-sdk-release/evals/tasks/basic-trigger.yaml @@ -0,0 +1,13 @@ +id: basic-trigger +name: basic-trigger +description: Should trigger on SDK release requests +inputs: + prompt: "I need to release a new version of my Azure SDK package. Walk me through the process." +expected: + skill_activated: true +graders: + - type: text + name: mentions-release + config: + contains: + - "release" diff --git a/.github/skills/azsdk-common-sdk-release/evals/trigger_tests.yaml b/.github/skills/azsdk-common-sdk-release/evals/trigger_tests.yaml new file mode 100644 index 000000000000..13764c99ed05 --- /dev/null +++ b/.github/skills/azsdk-common-sdk-release/evals/trigger_tests.yaml @@ -0,0 +1,23 @@ +skill: azsdk-common-sdk-release +should_trigger_prompts: + - prompt: "release my SDK package" + reason: "Should activate skill" + - prompt: "how to publish a new SDK version" + reason: "Should activate skill" + - prompt: "walk me through SDK release process" + reason: "Should activate skill" + - prompt: "release a new version of azure-sdk" + reason: "Should activate skill" + - prompt: "SDK release checklist" + reason: "Should activate skill" +should_not_trigger_prompts: + - prompt: "write unit tests" + reason: "Unrelated to skill scope" + - prompt: "fix my pipeline build error" + reason: "Unrelated to skill scope" + - prompt: "create a new TypeSpec definition" + reason: "Unrelated to skill scope" + - prompt: "optimize markdown tokens" + reason: "Unrelated to skill scope" + - prompt: "resolve APIView feedback" + reason: "Unrelated to skill scope" diff --git a/.github/skills/azsdk-common-sdk-release/tasks/release-basic-001.yaml b/.github/skills/azsdk-common-sdk-release/tasks/release-basic-001.yaml new file mode 100644 index 000000000000..3adce69205b8 --- /dev/null +++ b/.github/skills/azsdk-common-sdk-release/tasks/release-basic-001.yaml @@ -0,0 +1,20 @@ +id: release-basic-001 +name: Release SDK Package - Full Flow +description: | + Test that the skill guides through the full release flow: + readiness check then trigger. +tags: + - basic + - happy-path +inputs: + prompt: "I want to release my Azure Event Hubs Python SDK package." +expected: + output_contains: + - "release" + - "azsdk_release_sdk" + outcomes: + - type: tool_called + tool_name: azsdk_release_sdk + behavior: + max_tool_calls: 5 + max_response_time_ms: 30000 diff --git a/.github/skills/azsdk-common-sdk-release/tasks/release-negative-001.yaml b/.github/skills/azsdk-common-sdk-release/tasks/release-negative-001.yaml new file mode 100644 index 000000000000..610db444eca9 --- /dev/null +++ b/.github/skills/azsdk-common-sdk-release/tasks/release-negative-001.yaml @@ -0,0 +1,14 @@ +id: release-negative-001 +name: Should Not Trigger - Unrelated Request +description: | + Test that the skill does NOT trigger for non-release requests. +tags: + - negative +inputs: + prompt: "I need to validate my CODEOWNERS file for the Python SDK repo." +expected: + output_contains: + - "CODEOWNERS" + behavior: + max_tool_calls: 2 + max_response_time_ms: 30000 diff --git a/.github/skills/azsdk-common-sdk-release/tasks/release-readiness-001.yaml b/.github/skills/azsdk-common-sdk-release/tasks/release-readiness-001.yaml new file mode 100644 index 000000000000..2ec9655683ec --- /dev/null +++ b/.github/skills/azsdk-common-sdk-release/tasks/release-readiness-001.yaml @@ -0,0 +1,18 @@ +id: release-readiness-001 +name: Check Release Readiness Only +description: | + Test that the skill checks readiness without triggering release. +tags: + - basic +inputs: + prompt: "Check if azure-messaging-eventhubs is ready for release in Java." +expected: + output_contains: + - "ready" + - "azsdk_release_sdk" + outcomes: + - type: tool_called + tool_name: azsdk_release_sdk + behavior: + max_tool_calls: 3 + max_response_time_ms: 30000 diff --git a/.github/skills/azsdk-common-sdk-release/tasks/release-trigger-001.yaml b/.github/skills/azsdk-common-sdk-release/tasks/release-trigger-001.yaml new file mode 100644 index 000000000000..d3db5c96f24b --- /dev/null +++ b/.github/skills/azsdk-common-sdk-release/tasks/release-trigger-001.yaml @@ -0,0 +1,19 @@ +id: release-trigger-001 +name: Trigger Release Pipeline +description: | + Test that the skill triggers the release pipeline and mentions approval. +tags: + - basic +inputs: + prompt: "The readiness check passed. Go ahead and trigger the release pipeline for @azure/event-hubs JavaScript SDK." +expected: + output_contains: + - "release" + - "pipeline" + - "approve" + outcomes: + - type: tool_called + tool_name: azsdk_release_sdk + behavior: + max_tool_calls: 3 + max_response_time_ms: 30000