diff --git a/.changeset/calm-lions-review.md b/.changeset/calm-lions-review.md new file mode 100644 index 0000000..dbbe19a --- /dev/null +++ b/.changeset/calm-lions-review.md @@ -0,0 +1,6 @@ +--- +'@kin0992/skills': minor +--- + +Add the `engineering/review-pr-comments` skill with interactive unresolved +pull-request review triage, decision reactions, and approval-gated replies. diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.json index b73fec2..09c0796 100644 --- a/.github/plugin/marketplace.json +++ b/.github/plugin/marketplace.json @@ -14,6 +14,12 @@ "version": "0.3.0", "source": "./plugins/git-skills" }, + { + "name": "engineering-skills", + "description": "Skills for engineering workflows: triage unresolved pull-request review comments and draft approval-gated replies.", + "version": "0.1.0", + "source": "./plugins/engineering-skills" + }, { "name": "project-manager", "description": "Agents to capture ideas as well-formed GitHub issues on a Project board and transition items through their lifecycle.", diff --git a/.waza.yaml b/.waza.yaml index 7b9bfc5..d5cea0f 100644 --- a/.waza.yaml +++ b/.waza.yaml @@ -2,7 +2,7 @@ # See: https://github.com/microsoft/waza paths: - skills: packages/skills/src/git + skills: packages/skills/src evals: evals tokens: diff --git a/CONSUMING.md b/CONSUMING.md index 9a583ad..d65822a 100644 --- a/CONSUMING.md +++ b/CONSUMING.md @@ -303,9 +303,10 @@ For GitHub Copilot CLI, VS Code, and Claude Code, prefer the marketplace channel `dev-toolkit` ships a Copilot/Claude **marketplace** at `.github/plugin/marketplace.json` (mirrored at `.claude-plugin/marketplace.json` so Claude Code finds it on the conventional path), exposing skills via **one plugin per category**: -| Plugin | Category | Skills | -| ------------ | -------- | ---------------------------------------- | -| `git-skills` | `git` | `commit-message`, `pr-title-description` | +| Plugin | Category | Skills | +| -------------------- | ------------- | ---------------------------------------- | +| `git-skills` | `git` | `commit-message`, `pr-title-description` | +| `engineering-skills` | `engineering` | `review-pr-comments` | Each category folder under `packages/skills/src/` has a sibling plugin under `plugins/-skills/`. Adding a new category is mechanical: drop the skills in `packages/skills/src//`, copy `plugins/git-skills/` as a template, and register the new plugin in `.github/plugin/marketplace.json`. diff --git a/README.md b/README.md index a62fdff..1543cec 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ Platform Engineering toolkit: reusable GitHub Actions, AI Skills, and shared Typ | `packages/vitest-config` | `@kin0992/vitest-config` — shared Vitest preset | | `packages/skills` | `@kin0992/skills` — AI Skills organized by category (`git/`, …) | | `plugins/git-skills` | Copilot/Claude plugin (category: git): `commit-message`, `pr-title-description` | +| `plugins/engineering-skills` | Copilot/Claude plugin (category: engineering): `review-pr-comments` | | `.github/plugin/marketplace.json` | Marketplace registry — public, no auth. Mirrored at `.claude-plugin/marketplace.json` for Claude Code | ## Stack diff --git a/evals/review-pr-comments/eval.yaml b/evals/review-pr-comments/eval.yaml new file mode 100644 index 0000000..d707bb5 --- /dev/null +++ b/evals/review-pr-comments/eval.yaml @@ -0,0 +1,37 @@ +name: review-pr-comments-eval +description: | + Evaluation suite for interactive unresolved PR review-comment triage. + +skill: engineering/review-pr-comments +version: '1.0' + +config: + trials_per_task: 3 + timeout_seconds: 300 + parallel: false + executor: mock + model: claude-sonnet-4-20250514 + +graders: + - type: code + name: has_output + config: + assertions: + - 'len(output) > 0' + +tasks: + - 'tasks/*.yaml' + +metrics: + - name: task_completion + weight: 0.5 + threshold: 0.8 + description: Did the skill start the review-comment triage workflow? + - name: trigger_accuracy + weight: 0.3 + threshold: 0.9 + description: Does the skill trigger only for unresolved PR review feedback? + - name: safety + weight: 0.2 + threshold: 1.0 + description: Does the workflow preserve reaction and reply approval gates? diff --git a/evals/review-pr-comments/tasks/agent-feedback.yaml b/evals/review-pr-comments/tasks/agent-feedback.yaml new file mode 100644 index 0000000..cbe8189 --- /dev/null +++ b/evals/review-pr-comments/tasks/agent-feedback.yaml @@ -0,0 +1,16 @@ +id: review-comments-agent-001 +name: Review Agent Feedback +description: | + Test that Copilot and other agent comments are included. +tags: + - agent + - pull-request + +inputs: + prompt: 'Triage all unresolved comments on PR #317, including Copilot comments. Suggest a response, but never post it until I approve the exact text.' + +expected: + outcomes: + - type: task_completed + behavior: + max_tool_calls: 15 diff --git a/evals/review-pr-comments/tasks/basic-review.yaml b/evals/review-pr-comments/tasks/basic-review.yaml new file mode 100644 index 0000000..b7a9bca --- /dev/null +++ b/evals/review-pr-comments/tasks/basic-review.yaml @@ -0,0 +1,16 @@ +id: review-comments-basic-001 +name: Review Unresolved Comments +description: | + Test the interactive workflow for one pull request. +tags: + - basic + - pull-request + +inputs: + prompt: 'Review every unresolved review comment on acme/api#42 and help me decide whether to accept, defer, or ignore each one.' + +expected: + outcomes: + - type: task_completed + behavior: + max_tool_calls: 15 diff --git a/evals/review-pr-comments/tasks/should-not-trigger.yaml b/evals/review-pr-comments/tasks/should-not-trigger.yaml new file mode 100644 index 0000000..e1c5ecf --- /dev/null +++ b/evals/review-pr-comments/tasks/should-not-trigger.yaml @@ -0,0 +1,16 @@ +id: review-comments-other-001 +name: Should Not Trigger +description: | + Test that general PR conversation replies do not trigger the skill. +tags: + - negative + - edge-case + +inputs: + prompt: 'Write and post a reply to this general pull request conversation comment.' + +expected: + outcomes: + - type: task_not_triggered + behavior: + max_tool_calls: 0 diff --git a/evals/review-pr-comments/trigger_tests.yaml b/evals/review-pr-comments/trigger_tests.yaml new file mode 100644 index 0000000..1813f08 --- /dev/null +++ b/evals/review-pr-comments/trigger_tests.yaml @@ -0,0 +1,27 @@ +skill: engineering/review-pr-comments + +should_trigger_prompts: + - prompt: 'Review every unresolved comment on PR #42 and help me decide what to address.' + reason: 'Direct unresolved PR review request' + confidence: high + + - prompt: 'Triage the pending Copilot review suggestions on acme/api#317.' + reason: 'Agent-authored unresolved review feedback' + confidence: high + + - prompt: 'Go through the open review threads on this pull request and suggest replies.' + reason: 'Requests unresolved thread evaluation and reply drafting' + confidence: high + +should_not_trigger_prompts: + - prompt: 'Reply to this general PR conversation comment.' + reason: 'General PR comments have no unresolved review-thread state' + confidence: high + + - prompt: 'Resolve every review thread on PR #42.' + reason: 'Thread resolution is explicitly outside the skill scope' + confidence: high + + - prompt: 'Review the code in my working tree before I open a PR.' + reason: 'This is code review, not unresolved PR comment triage' + confidence: high diff --git a/packages/skills/README.md b/packages/skills/README.md index 3e005c7..ebab5de 100644 --- a/packages/skills/README.md +++ b/packages/skills/README.md @@ -27,6 +27,13 @@ care about. classic Git commit style (Conventional Commits forbidden). - `pr-title-description` — Generate a PR title and description from the diff and commit log. +- `branch-name` — Suggest a branch name using the repository's activity naming + convention. + +### `engineering/` + +- `review-pr-comments` — Evaluate every actionable comment in unresolved PR + review threads and record accept, defer, or ignore decisions with reactions. ## Install (npm, programmatic) @@ -53,6 +60,7 @@ no `node_modules` plumbing: ```sh copilot plugin marketplace add kin0992/dev-toolkit copilot plugin install git-skills@dev-toolkit # commit-message, pr-title-description +copilot plugin install engineering-skills@dev-toolkit # review-pr-comments ``` See [CONSUMING.md](../../CONSUMING.md#3-ai-skills-via-the-dev-toolkit-marketplace) diff --git a/packages/skills/package.json b/packages/skills/package.json index f93063b..e2e8bb1 100644 --- a/packages/skills/package.json +++ b/packages/skills/package.json @@ -1,7 +1,7 @@ { "name": "@kin0992/skills", "version": "0.2.1", - "description": "AI Skills bundle: commit messages and PR title/description.", + "description": "AI Skills bundle for git and engineering workflows.", "license": "MIT", "repository": { "type": "git", @@ -16,7 +16,8 @@ "exports": { "./git/commit-message": "./src/git/commit-message/SKILL.md", "./git/pr-title-description": "./src/git/pr-title-description/SKILL.md", - "./git/branch-name": "./src/git/branch-name/SKILL.md" + "./git/branch-name": "./src/git/branch-name/SKILL.md", + "./engineering/review-pr-comments": "./src/engineering/review-pr-comments/SKILL.md" }, "publishConfig": { "access": "public", diff --git a/packages/skills/src/engineering/review-pr-comments/SKILL.md b/packages/skills/src/engineering/review-pr-comments/SKILL.md new file mode 100644 index 0000000..c6570fe --- /dev/null +++ b/packages/skills/src/engineering/review-pr-comments/SKILL.md @@ -0,0 +1,37 @@ +--- +name: review-pr-comments +description: | + **INTERACTIVE SKILL** - Evaluate every actionable comment in unresolved + GitHub PR review threads, including agent comments. + USE FOR: review unresolved PR comments, triage review threads, process + Copilot feedback. + DO NOT USE FOR: general PR comments, resolving threads, unapproved replies. + INVOKES: gh CLI, including gh api GraphQL and REST calls. +license: MIT +--- + +# Review unresolved PR comments + +## Instructions + +1. Read and follow the complete [workflow](references/workflow.md) and + [`gh` commands](references/gh-commands.md). +2. Process one PR and all actionable comments in its unresolved review threads; + include bots, but exclude general PR conversation comments. +3. Evaluate each comment against the diff and source context. +4. Ask one question at a time. Record `Accept` with `+1`, `Defer` with `eyes`, + and `Ignore` with `-1`. Do not edit code merely because it was accepted. +5. Suggest possible answers. Post a reply only after exact-text and target + approval; editing a draft is not approval. +6. Never resolve threads or post general PR comments. + +Requires an authenticated `gh` CLI. Use `gh api graphql --paginate` to collect +threads and comments, and `gh api` REST calls for reactions and approved replies. + +```sh +gh auth status +``` + +## Errors + +Surface API, permission, and missing-context failures; never silently skip them. diff --git a/packages/skills/src/engineering/review-pr-comments/references/gh-commands.md b/packages/skills/src/engineering/review-pr-comments/references/gh-commands.md new file mode 100644 index 0000000..3fba565 --- /dev/null +++ b/packages/skills/src/engineering/review-pr-comments/references/gh-commands.md @@ -0,0 +1,108 @@ +# `gh` commands + +Use only `gh` and shell built-ins. Keep `REPO` as `OWNER/REPO`, and split it +into `OWNER` and `NAME`. Resolve `NUMBER` with `gh pr view`. + +```sh +REPO=$(gh repo view --json nameWithOwner --jq '.nameWithOwner') +NUMBER=$(gh pr view "$PR" --repo "$REPO" --json number --jq '.number') +OWNER=${REPO%%/*} +NAME=${REPO#*/} +``` + +For a PR URL, omit `--repo`; derive `REPO` from its URL: + +```sh +REPO=$(gh pr view "$PR" --json url \ + --jq '.url | split("/")[3:5] | join("/")') +``` + +## Collect unresolved threads + +```sh +gh api graphql --paginate \ + -F owner="$OWNER" -F name="$NAME" -F number="$NUMBER" \ + -f query=' +query($owner: String!, $name: String!, $number: Int!, $endCursor: String) { + repository(owner: $owner, name: $name) { + pullRequest(number: $number) { + reviewThreads(first: 100, after: $endCursor) { + nodes { + id isResolved isOutdated path line originalLine + comments(first: 100) { + nodes { + id databaseId url body createdAt updatedAt + path line originalLine diffHunk + author { login } + replyTo { id databaseId } + } + pageInfo { hasNextPage endCursor } + } + } + pageInfo { hasNextPage endCursor } + } + } + } +}' \ + --jq '.data.repository.pullRequest.reviewThreads.nodes[] + | select(.isResolved == false)' +``` + +`--paginate` covers the outer thread connection. For any returned thread whose +`comments.pageInfo.hasNextPage` is true, paginate that thread separately: + +```sh +gh api graphql --paginate \ + -F threadId="$THREAD_ID" \ + -f query=' +query($threadId: ID!, $endCursor: String) { + node(id: $threadId) { + ... on PullRequestReviewThread { + comments(first: 100, after: $endCursor) { + nodes { + id databaseId url body createdAt updatedAt + path line originalLine diffHunk + author { login } + replyTo { id databaseId } + } + pageInfo { hasNextPage endCursor } + } + } + } +}' \ + --jq '.data.node.comments.nodes[]' +``` + +## Replace a decision reaction + +Set `REACTION` to `+1`, `-1`, or `eyes`. Remove only decision reactions made by +the authenticated user, then add the selected reaction: + +```sh +LOGIN=$(gh api user --jq '.login') + +for REACTION_ID in $(gh api --paginate \ + "repos/$REPO/pulls/comments/$COMMENT_ID/reactions?per_page=100" \ + --jq ".[] | select(.user.login == \"$LOGIN\") + | select(.content == \"+1\" or .content == \"-1\" or .content == \"eyes\") + | .id"); do + gh api --method DELETE \ + "repos/$REPO/pulls/comments/$COMMENT_ID/reactions/$REACTION_ID" +done + +gh api --method POST \ + "repos/$REPO/pulls/comments/$COMMENT_ID/reactions" \ + -f content="$REACTION" +``` + +## Post an approved reply + +Run only after the user approves the exact `APPROVED_REPLY` text and target: + +```sh +gh api --method POST \ + "repos/$REPO/pulls/$NUMBER/comments/$COMMENT_ID/replies" \ + -f body="$APPROVED_REPLY" +``` + +Do not call general PR comment or thread-resolution mutations. diff --git a/packages/skills/src/engineering/review-pr-comments/references/workflow.md b/packages/skills/src/engineering/review-pr-comments/references/workflow.md new file mode 100644 index 0000000..cb1c533 --- /dev/null +++ b/packages/skills/src/engineering/review-pr-comments/references/workflow.md @@ -0,0 +1,59 @@ +# Workflow + +## Collect the review + +1. Identify one PR from a URL, `OWNER/REPO#NUMBER`, or a number in the current + repository. Ask if it is missing or ambiguous. +2. Verify `gh auth status`. +3. Use the collection queries in [`gh-commands.md`](gh-commands.md). Paginate + both review threads and comments within each thread. +4. Inspect `gh pr diff` and relevant source context. Human, bot, Copilot, and + other agent comments all count. +5. Process every actionable opening comment and reply in every unresolved + thread. Skip acknowledgements, status messages, duplicates, and replies with + no request or technical claim; report the reason. + +## Decide and react + +For each actionable comment, show: + +- author, file/line, and comment URL +- requested change +- whether it is correct, relevant, and still applicable +- recommendation with brief evidence +- concise possible replies when different decisions need different responses + +Ask one question at a time: + +- `Accept (👍)` — valid and should be addressed +- `Defer (👀)` — valid, but should be handled later +- `Ignore (👎)` — invalid, irrelevant, duplicate, or already addressed + +After the decision, use the reaction commands in +[`gh-commands.md`](gh-commands.md). Replace only the authenticated user's +previous `+1`, `-1`, or `eyes` decision reactions. + +Do not react to non-actionable comments. A reaction does not mean code changed. +If implementation is explicitly requested, treat it as separate work that may +run asynchronously. + +## Draft and approve a reply + +Draft a concise reply consistent with the decision. Do not claim a change was +implemented or verified unless that work completed. + +Show the exact text and target URL, then ask: + +- `Post exactly as shown` +- `Edit before posting` +- `Do not reply` + +Only the first choice authorizes posting that exact draft. If edited, show the +new exact text and ask again. Reaction approval never authorizes a reply. + +After exact-text approval, use the review-comment reply command in +[`gh-commands.md`](gh-commands.md). + +Never use a general PR comment endpoint and never resolve or unresolve threads. +Finish with decisions, posted and declined replies, skipped comments, and +errors. diff --git a/plugins/engineering-skills/.claude-plugin/plugin.json b/plugins/engineering-skills/.claude-plugin/plugin.json new file mode 100644 index 0000000..7760e7d --- /dev/null +++ b/plugins/engineering-skills/.claude-plugin/plugin.json @@ -0,0 +1,11 @@ +{ + "name": "engineering-skills", + "description": "AI Skills for engineering workflows: triage unresolved pull-request review comments and draft approval-gated replies.", + "version": "0.1.0", + "author": { + "name": "kin0992" + }, + "license": "MIT", + "keywords": ["engineering", "github", "pull-request", "code-review", "skills"], + "skills": ["./skills/review-pr-comments/"] +} diff --git a/plugins/engineering-skills/.github/plugin.json b/plugins/engineering-skills/.github/plugin.json new file mode 100644 index 0000000..ac773bc --- /dev/null +++ b/plugins/engineering-skills/.github/plugin.json @@ -0,0 +1,11 @@ +{ + "name": "engineering-skills", + "description": "AI Skills for engineering workflows: triage unresolved pull-request review comments and draft approval-gated replies.", + "version": "0.1.0", + "author": { + "name": "kin0992" + }, + "license": "MIT", + "keywords": ["engineering", "github", "pull-request", "code-review", "skills"], + "skills": ["../../packages/skills/src/engineering/review-pr-comments/"] +} diff --git a/plugins/engineering-skills/README.md b/plugins/engineering-skills/README.md new file mode 100644 index 0000000..051ff13 --- /dev/null +++ b/plugins/engineering-skills/README.md @@ -0,0 +1,29 @@ +# engineering-skills + +Plugin bundling AI Skills for engineering workflows, distributed via the +[`dev-toolkit` marketplace](../../.github/plugin/marketplace.json). + +## Skills + +| Skill | What it does | +| -------------------- | ------------------------------------------------------------------------------------------------------ | +| `review-pr-comments` | Evaluate unresolved PR feedback, record decisions, and propose replies that require explicit approval. | + +The source lives at +`packages/skills/src/engineering/review-pr-comments/` and is shared by the npm +package and this plugin. + +## Install + +```sh +copilot plugin marketplace add kin0992/dev-toolkit +copilot plugin install engineering-skills@dev-toolkit +``` + +The skill requires only an authenticated `gh` CLI session with access to the +target pull request. + +## Manifests + +- `.github/plugin.json` — Copilot CLI + VS Code +- `.claude-plugin/plugin.json` — Claude Code diff --git a/plugins/engineering-skills/skills/review-pr-comments b/plugins/engineering-skills/skills/review-pr-comments new file mode 120000 index 0000000..30367b3 --- /dev/null +++ b/plugins/engineering-skills/skills/review-pr-comments @@ -0,0 +1 @@ +../../../packages/skills/src/engineering/review-pr-comments \ No newline at end of file