From af5f4d9c961fafb17a1082cd81f7e0c0d957e414 Mon Sep 17 00:00:00 2001 From: Bas Alberts Date: Mon, 2 Mar 2026 14:17:43 -0500 Subject: [PATCH 01/17] Add PVR triage taskflow --- .../configs/model_config_pvr_triage.yaml | 19 ++ src/seclab_taskflows/mcp_servers/pvr_ghsa.py | 291 ++++++++++++++++++ .../personalities/pvr_analyst.yaml | 28 ++ .../taskflows/pvr_triage/pvr_triage.yaml | 226 ++++++++++++++ src/seclab_taskflows/toolboxes/pvr_ghsa.yaml | 18 ++ 5 files changed, 582 insertions(+) create mode 100644 src/seclab_taskflows/configs/model_config_pvr_triage.yaml create mode 100644 src/seclab_taskflows/mcp_servers/pvr_ghsa.py create mode 100644 src/seclab_taskflows/personalities/pvr_analyst.yaml create mode 100644 src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml create mode 100644 src/seclab_taskflows/toolboxes/pvr_ghsa.yaml diff --git a/src/seclab_taskflows/configs/model_config_pvr_triage.yaml b/src/seclab_taskflows/configs/model_config_pvr_triage.yaml new file mode 100644 index 0000000..333c58d --- /dev/null +++ b/src/seclab_taskflows/configs/model_config_pvr_triage.yaml @@ -0,0 +1,19 @@ +# PVR triage model configuration. +# Uses GitHub Copilot API endpoint by default (AI_API_ENDPOINT=https://api.githubcopilot.com). +# Override AI_API_ENDPOINT and AI_API_TOKEN for other providers. + +seclab-taskflow-agent: + version: "1.0" + filetype: model_config + +models: + # Primary model for code analysis and triage reasoning + triage: claude-opus-4.6-1m + # Lighter model for structured data extraction tasks + extraction: gpt-5-mini + +model_settings: + extraction: + temperature: 1 + triage: + temperature: 1 diff --git a/src/seclab_taskflows/mcp_servers/pvr_ghsa.py b/src/seclab_taskflows/mcp_servers/pvr_ghsa.py new file mode 100644 index 0000000..862abbf --- /dev/null +++ b/src/seclab_taskflows/mcp_servers/pvr_ghsa.py @@ -0,0 +1,291 @@ +# PVR GHSA MCP Server +# +# Tools for fetching and parsing draft GitHub Security Advisories +# submitted via Private Vulnerability Reporting (PVR). +# Uses the gh CLI for all GitHub API calls. + +import json +import logging +import os +import subprocess +from pathlib import Path + +from fastmcp import FastMCP +from pydantic import Field +from seclab_taskflow_agent.path_utils import log_file_name + +REPORT_DIR = Path(os.getenv("REPORT_DIR", "reports")) + +logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s - %(levelname)s - %(message)s", + filename=log_file_name("mcp_pvr_ghsa.log"), + filemode="a", +) + +mcp = FastMCP("PVRAdvisories") + + +def _gh_api(path: str, method: str = "GET") -> tuple[dict | list | None, str | None]: + """ + Call the GitHub REST API via the gh CLI. + + Returns (data, error). On success data is the parsed JSON response and + error is None. On failure data is None and error is a string. + """ + cmd = ["gh", "api", "--method", method, path] + env = os.environ.copy() + + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + env=env, + timeout=30, + ) + except subprocess.TimeoutExpired: + return None, "gh api call timed out" + except FileNotFoundError: + return None, "gh CLI not found in PATH" + + if result.returncode != 0: + stderr = result.stderr.strip() + stdout = result.stdout.strip() + msg = stderr or stdout or f"gh exited with code {result.returncode}" + logging.error("gh api error: %s", msg) + return None, msg + + try: + data = json.loads(result.stdout) + except json.JSONDecodeError as e: + return None, f"JSON parse error: {e}" + + return data, None + + +def _parse_advisory(raw: dict) -> dict: + """ + Extract the fields relevant to PVR triage from a raw advisory API response. + Separates description text from structured metadata. + """ + vulns = [] + for v in raw.get("vulnerabilities") or []: + pkg = v.get("package") or {} + vulns.append({ + "ecosystem": pkg.get("ecosystem", ""), + "package": pkg.get("name", ""), + "vulnerable_versions": v.get("vulnerable_version_range", ""), + "patched_versions": v.get("patched_versions", ""), + }) + + cwes = [c.get("cwe_id", "") for c in (raw.get("cwes") or [])] + + credits_ = [ + {"login": c.get("user", {}).get("login", ""), "type": c.get("type", "")} + for c in (raw.get("credits_detailed") or []) + ] + + submission = raw.get("submission") or {} + + return { + "ghsa_id": raw.get("ghsa_id", ""), + "cve_id": raw.get("cve_id"), + "html_url": raw.get("html_url", ""), + "state": raw.get("state", ""), + "severity": raw.get("severity", ""), + "summary": raw.get("summary", ""), + # Full description returned separately so metadata stays compact + "description": raw.get("description", ""), + "vulnerabilities": vulns, + "cwes": cwes, + "credits": credits_, + # submission.accepted=true means this arrived via PVR + "pvr_submission": { + "via_pvr": bool(submission), + "accepted": submission.get("accepted", False), + }, + "created_at": raw.get("created_at", ""), + "updated_at": raw.get("updated_at", ""), + "collaborating_users": [ + u.get("login", "") for u in (raw.get("collaborating_users") or []) + ], + } + + +@mcp.tool() +def fetch_pvr_advisory( + owner: str = Field(description="Repository owner (user or org name)"), + repo: str = Field(description="Repository name"), + ghsa_id: str = Field(description="GHSA ID of the advisory, e.g. GHSA-xxxx-xxxx-xxxx"), +) -> str: + """ + Fetch a single repository security advisory by GHSA ID. + + Returns structured advisory metadata and the full description text. + Works for draft advisories (requires repo or security_events scope on GH_TOKEN). + """ + path = f"/repos/{owner}/{repo}/security-advisories/{ghsa_id}" + data, err = _gh_api(path) + if err: + return f"Error fetching advisory {ghsa_id}: {err}" + parsed = _parse_advisory(data) + return json.dumps(parsed, indent=2) + + +@mcp.tool() +def list_pvr_advisories( + owner: str = Field(description="Repository owner (user or org name)"), + repo: str = Field(description="Repository name"), + state: str = Field( + default="draft", + description="Advisory state to filter by: draft, published, rejected, or withdrawn. Default: draft", + ), +) -> str: + """ + List repository security advisories, defaulting to draft state. + + Returns a summary list (no description text). Each entry includes + ghsa_id, severity, summary, state, pvr_submission, and created_at. + """ + path = f"/repos/{owner}/{repo}/security-advisories?state={state}&per_page=100" + data, err = _gh_api(path) + if err: + return f"Error listing advisories: {err}" + if not isinstance(data, list): + return f"Unexpected response: {data}" + + results = [] + for raw in data: + submission = raw.get("submission") or {} + results.append({ + "ghsa_id": raw.get("ghsa_id", ""), + "severity": raw.get("severity", ""), + "summary": raw.get("summary", ""), + "state": raw.get("state", ""), + "pvr_submission": { + "via_pvr": bool(submission), + "accepted": submission.get("accepted", False), + }, + "created_at": raw.get("created_at", ""), + }) + + if not results: + return f"No {state} advisories found for {owner}/{repo}." + return json.dumps(results, indent=2) + + +@mcp.tool() +def resolve_version_ref( + owner: str = Field(description="Repository owner"), + repo: str = Field(description="Repository name"), + version: str = Field( + description="Version string to resolve, e.g. '1.25.4' or 'v1.25.4'. " + "Will try matching git tags directly and with a 'v' prefix." + ), +) -> str: + """ + Resolve a version string to a git commit SHA and tag name. + + Returns the tag name and commit SHA if found. + """ + # Try both bare version and v-prefixed tag + candidates = [version, f"v{version}"] if not version.startswith("v") else [version, version[1:]] + + for tag in candidates: + path = f"/repos/{owner}/{repo}/git/refs/tags/{tag}" + data, err = _gh_api(path) + if err or not data: + continue + # Lightweight tags point directly to a commit; annotated tags point to a tag object + obj = data.get("object", {}) + ref_sha = obj.get("sha", "") + ref_type = obj.get("type", "") + + if ref_type == "tag": + # Annotated tag: dereference to the commit + tag_path = f"/repos/{owner}/{repo}/git/tags/{ref_sha}" + tag_data, tag_err = _gh_api(tag_path) + if not tag_err and tag_data: + commit_sha = tag_data.get("object", {}).get("sha", "") + return json.dumps({"tag": tag, "commit_sha": commit_sha, "type": "annotated"}) + elif ref_type == "commit": + return json.dumps({"tag": tag, "commit_sha": ref_sha, "type": "lightweight"}) + + return f"Could not resolve version '{version}' to a tag in {owner}/{repo}." + + +@mcp.tool() +def fetch_file_at_ref( + owner: str = Field(description="Repository owner"), + repo: str = Field(description="Repository name"), + path: str = Field(description="File path within the repository"), + ref: str = Field(description="Git ref (commit SHA, tag, or branch) to fetch the file at"), + start_line: int = Field(default=1, description="First line to return (1-indexed)"), + length: int = Field(default=50, description="Number of lines to return"), +) -> str: + """ + Fetch a range of lines from a file at a specific git ref (commit SHA or tag). + """ + # Use gh api with the ref query parameter + cmd = [ + "gh", "api", + "--method", "GET", + f"/repos/{owner}/{repo}/contents/{path}", + "-f", f"ref={ref}", + "-H", "Accept: application/vnd.github.raw+json", + ] + env = os.environ.copy() + + try: + result = subprocess.run(cmd, capture_output=True, text=True, env=env, timeout=30) + except subprocess.TimeoutExpired: + return "Error: gh api call timed out" + except FileNotFoundError: + return "Error: gh CLI not found in PATH" + + if result.returncode != 0: + return f"Error fetching {path}@{ref}: {result.stderr.strip() or result.stdout.strip()}" + + lines = result.stdout.splitlines() + if start_line < 1: + start_line = 1 + if length < 1: + length = 50 + chunk = lines[start_line - 1: start_line - 1 + length] + if not chunk: + return f"No lines in range {start_line}-{start_line + length - 1} in {path}@{ref}" + return "\n".join(f"{start_line + i}: {line}" for i, line in enumerate(chunk)) + + +@mcp.tool() +def save_triage_report( + ghsa_id: str = Field(description="GHSA ID, used as the filename stem, e.g. GHSA-xxxx-xxxx-xxxx"), + report: str = Field(description="Full markdown report content to write to disk"), +) -> str: + """ + Write the triage report to a markdown file in the report output directory. + + The file is written to REPORT_DIR/{ghsa_id}_triage.md. + REPORT_DIR defaults to './reports' and can be overridden via the REPORT_DIR + environment variable. Returns the absolute path of the written file. + """ + REPORT_DIR.mkdir(parents=True, exist_ok=True) + # Sanitize the GHSA ID to prevent path traversal + safe_name = "".join(c for c in ghsa_id if c.isalnum() or c in "-_") + out_path = REPORT_DIR / f"{safe_name}_triage.md" + # The agent sometimes passes the report as a JSON-encoded string + # (with outer quotes and escape sequences). Decode it if so. + content = report + if content.startswith('"') and content.endswith('"'): + try: + content = json.loads(content) + except json.JSONDecodeError: + pass + out_path.write_text(content, encoding="utf-8") + logging.info("Triage report written to %s", out_path) + return str(out_path.resolve()) + + +if __name__ == "__main__": + mcp.run(show_banner=False) diff --git a/src/seclab_taskflows/personalities/pvr_analyst.yaml b/src/seclab_taskflows/personalities/pvr_analyst.yaml new file mode 100644 index 0000000..ff9e7af --- /dev/null +++ b/src/seclab_taskflows/personalities/pvr_analyst.yaml @@ -0,0 +1,28 @@ +# Personality for PVR (Private Vulnerability Report) triage analysis. + +seclab-taskflow-agent: + version: "1.0" + filetype: personality + +personality: | + You are a security vulnerability triage analyst for an open source software maintainer. + + Your job is to verify vulnerability claims made in Private Vulnerability Reports (PVRs), + which arrive as draft GitHub Security Advisories (GHSAs). + + Core principles: + - Base all conclusions on actual code evidence. Do not speculate. + - If you cannot verify a claim, say so explicitly. + - Distinguish between confirmed vulnerabilities and unverified claims. + - Be concise and direct. Maintainers are busy. + - Flag low-quality ("AI slop") reports: vague claims, wrong file paths, non-working PoC, + incorrect function signatures, or descriptions that don't match the actual code. + +task: | + Analyze the provided vulnerability report and verify claims against the actual source code. + Produce factual, evidence-based findings. Never guess or assume. + +toolboxes: + - seclab_taskflows.toolboxes.pvr_ghsa + - seclab_taskflows.toolboxes.gh_file_viewer + - seclab_taskflow_agent.toolboxes.memcache diff --git a/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml b/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml new file mode 100644 index 0000000..5f2de42 --- /dev/null +++ b/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml @@ -0,0 +1,226 @@ +# PVR Triage Taskflow +# +# Fetches a draft GHSA submitted via Private Vulnerability Reporting, +# verifies the vulnerability claim against actual source code, assesses +# impact and report quality, and generates a structured triage analysis +# for the maintainer. +# +# Usage: +# python -m seclab_taskflow_agent \ +# -t seclab_taskflows.taskflows.pvr_triage.pvr_triage \ +# -g repo=owner/repo \ +# -g ghsa=GHSA-xxxx-xxxx-xxxx +# +# Required environment variables: +# GH_TOKEN - GitHub token with repo and security_events scope +# AI_API_TOKEN - API token for the AI model provider +# AI_API_ENDPOINT - Model provider endpoint (default: GitHub Copilot API) + +seclab-taskflow-agent: + version: "1.0" + filetype: taskflow + +model_config: seclab_taskflows.configs.model_config_pvr_triage + +globals: + # GitHub repository in owner/repo format + repo: + # GHSA ID of the draft advisory to triage + ghsa: + +taskflow: + # ------------------------------------------------------------------------- + # Task 1: Initialize + # ------------------------------------------------------------------------- + - task: + must_complete: true + headless: true + agents: + - seclab_taskflow_agent.personalities.assistant + toolboxes: + - seclab_taskflow_agent.toolboxes.memcache + user_prompt: | + Clear the memory cache. + + # ------------------------------------------------------------------------- + # Task 2: Fetch and parse the draft GHSA + # ------------------------------------------------------------------------- + - task: + must_complete: true + model: extraction + agents: + - seclab_taskflows.personalities.pvr_analyst + toolboxes: + - seclab_taskflows.toolboxes.pvr_ghsa + - seclab_taskflow_agent.toolboxes.memcache + user_prompt: | + Fetch the security advisory {{ globals.ghsa }} for repository {{ globals.repo }}. + + Extract the owner and repo name from "{{ globals.repo }}" (format: owner/repo). + + Store the full raw advisory description text under key "pvr_description". + + Then store a structured summary in memcache under the key "pvr_parsed" as a JSON + object with these fields: + - ghsa_id: the GHSA ID + - repo: "{{ globals.repo }}" + - summary: the advisory one-line summary + - severity_claimed: the severity rating in the advisory (critical/high/medium/low) + - vuln_type: vulnerability class (e.g. "path traversal", "IDOR", "XSS", "SQL injection") + - affected_component: the component, endpoint, or feature described as vulnerable + - affected_files: list of source file paths explicitly mentioned (empty list if none) + - affected_functions: list of function/method names mentioned (empty list if none) + - affected_versions: for version ranges, prefer the structured vulnerabilities[].vulnerable_versions + field from the advisory API response. Fall back to parsing the description only if + the structured field is absent or empty. Empty list if none found. + - poc_provided: true if a proof-of-concept or reproduction steps are described + - poc_summary: brief description of the PoC steps, or null if none provided + - quality_signals: + has_file_references: true if specific source file paths are cited + has_line_numbers: true if specific line numbers are cited + has_poc: true if reproduction steps are provided + has_version_info: true if specific affected versions are mentioned + has_code_snippets: true if actual code is quoted in the report + + Do not perform any code analysis yet. + + # ------------------------------------------------------------------------- + # Task 3: Verify vulnerability in source code + # ------------------------------------------------------------------------- + - task: + must_complete: true + model: triage + agents: + - seclab_taskflows.personalities.pvr_analyst + toolboxes: + - seclab_taskflows.toolboxes.pvr_ghsa + - seclab_taskflows.toolboxes.gh_file_viewer + - seclab_taskflow_agent.toolboxes.memcache + user_prompt: | + Retrieve "pvr_parsed" and "pvr_description" from memcache. + + Extract owner and repo from "{{ globals.repo }}" (format: owner/repo). + + Verify the vulnerability at the affected version, not HEAD. + If affected_versions lists a version (e.g. "<= 1.25.4"), resolve the + upper bound to a git commit SHA using resolve_version_ref, then use + fetch_file_at_ref to fetch code at that SHA. If no version is specified, + fall back to fetch_file_from_gh / get_file_lines_from_gh (HEAD). + + If affected_files or affected_versions are empty, read pvr_description + directly to identify any file paths, function names, or version references + the extraction may have missed. Advisory descriptions vary widely in format + and structure — treat pvr_parsed as a starting point, not a complete picture. + + For each file path identified: + 1. Resolve the version to a SHA (if available). + 2. Fetch the file at that SHA using fetch_file_at_ref. + 3. Locate the affected function(s) at the stated line numbers. + 4. Check whether the vulnerability pattern described in the advisory + is present at that version. + 5. Look for authorization checks, input validation, or other mitigations. + + If no specific files are named, use search_repo_from_gh to locate + the affected function names or code patterns. + + Focus on the specific code path described. Do not perform a broad audit. + + Store your findings under memcache key "code_verification" as JSON: + - ref_used: the git SHA or ref used for code fetching (or "HEAD" if none) + - files_examined: list of file paths fetched + - vulnerability_confirmed: true / false / null (null = could not determine) + - confirmation_evidence: precise description of what the code does, + including file path and line numbers + - mitigation_found: true if existing checks prevent exploitation + - mitigation_details: description of mitigating code, or null + - notes: any additional observations + + # ------------------------------------------------------------------------- + # Task 4: Generate triage report + # ------------------------------------------------------------------------- + - task: + must_complete: true + model: triage + agents: + - seclab_taskflows.personalities.pvr_analyst + toolboxes: + - seclab_taskflow_agent.toolboxes.memcache + user_prompt: | + Retrieve "pvr_parsed", "pvr_description", and "code_verification" from memcache. + + Generate a triage analysis report in markdown and store it under + memcache key "triage_report". + + The report must follow this structure exactly: + + --- + + ## PVR Triage Analysis: {{ globals.ghsa }} + + **Repository:** {{ globals.repo }} + **Claimed Severity:** [from pvr_parsed] + **Vulnerability Type:** [from pvr_parsed] + + ### Verdict + + **[CONFIRMED / UNCONFIRMED / INCONCLUSIVE]** + + One or two sentences stating the verdict and the primary reason. + + ### Code Verification + + State the git ref (version tag / commit SHA) used for analysis, or note + if HEAD was used and why. + Describe exactly what code was examined and what was found. + Reference specific file paths and line numbers. + If the vulnerability is confirmed, show the vulnerable code pattern. + If unconfirmed, explain what the code actually does and why it is not vulnerable. + If inconclusive, explain what could not be determined and why. + + ### Severity Assessment + + State whether the claimed severity is accurate, overstated, or understated. + Base this on the actual exploitability and impact from the code evidence. + + ### Report Quality + + Assess the quality of the PVR submission: + - Note which claims were accurate (correct file paths, line numbers, functions) + - Note any inaccuracies (wrong paths, non-existent functions, incorrect PoC) + - Rate overall quality: High / Medium / Low + - High: specific, accurate, verified PoC + - Medium: partially accurate, some details wrong or missing + - Low: vague, speculative, or significantly inaccurate ("AI slop") + + ### Recommendations + + Provide 1-3 specific, actionable recommendations for the maintainer. + If confirmed: suggest the fix approach. + If unconfirmed: suggest whether to close, request more info, or monitor. + If low quality: recommend closing with explanation. + + --- + + Be factual. Do not include anything not supported by code evidence. + Keep the report concise. Aim for under 600 words. + + # ------------------------------------------------------------------------- + # Task 5: Save report to disk and print path + # ------------------------------------------------------------------------- + - task: + must_complete: true + model: extraction + agents: + - seclab_taskflow_agent.personalities.assistant + toolboxes: + - seclab_taskflows.toolboxes.pvr_ghsa + - seclab_taskflow_agent.toolboxes.memcache + user_prompt: | + Retrieve the "triage_report" from memcache. + + Call save_triage_report with: + - ghsa_id: "{{ globals.ghsa }}" + - report: the full report content exactly as stored in memcache + + Then print the report content verbatim, followed by a blank line and: + "Report saved to: " diff --git a/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml b/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml new file mode 100644 index 0000000..80e594e --- /dev/null +++ b/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml @@ -0,0 +1,18 @@ +# Toolbox: PVR GHSA advisory fetcher +# +# Provides tools for fetching draft GitHub Security Advisories submitted +# via Private Vulnerability Reporting. Uses the gh CLI for API calls. +# +# Requires GH_TOKEN with repo or security_events scope to read draft advisories. + +seclab-taskflow-agent: + version: "1.0" + filetype: toolbox + +server_params: + kind: stdio + command: python + args: ["-m", "seclab_taskflows.mcp_servers.pvr_ghsa"] + env: + LOG_DIR: "{{ env('LOG_DIR') }}" + REPORT_DIR: "{{ env('REPORT_DIR') }}" From c233dacf848e3f50f03f168e6d8eddca40a30b3b Mon Sep 17 00:00:00 2001 From: Bas Alberts Date: Mon, 2 Mar 2026 14:25:33 -0500 Subject: [PATCH 02/17] Address PR review: add SPDX headers, pass GH_TOKEN in toolbox env --- src/seclab_taskflows/configs/model_config_pvr_triage.yaml | 3 +++ src/seclab_taskflows/mcp_servers/pvr_ghsa.py | 3 +++ src/seclab_taskflows/personalities/pvr_analyst.yaml | 3 +++ src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml | 3 +++ src/seclab_taskflows/toolboxes/pvr_ghsa.yaml | 4 ++++ 5 files changed, 16 insertions(+) diff --git a/src/seclab_taskflows/configs/model_config_pvr_triage.yaml b/src/seclab_taskflows/configs/model_config_pvr_triage.yaml index 333c58d..0148c31 100644 --- a/src/seclab_taskflows/configs/model_config_pvr_triage.yaml +++ b/src/seclab_taskflows/configs/model_config_pvr_triage.yaml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + # PVR triage model configuration. # Uses GitHub Copilot API endpoint by default (AI_API_ENDPOINT=https://api.githubcopilot.com). # Override AI_API_ENDPOINT and AI_API_TOKEN for other providers. diff --git a/src/seclab_taskflows/mcp_servers/pvr_ghsa.py b/src/seclab_taskflows/mcp_servers/pvr_ghsa.py index 862abbf..cc48245 100644 --- a/src/seclab_taskflows/mcp_servers/pvr_ghsa.py +++ b/src/seclab_taskflows/mcp_servers/pvr_ghsa.py @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + # PVR GHSA MCP Server # # Tools for fetching and parsing draft GitHub Security Advisories diff --git a/src/seclab_taskflows/personalities/pvr_analyst.yaml b/src/seclab_taskflows/personalities/pvr_analyst.yaml index ff9e7af..f00f362 100644 --- a/src/seclab_taskflows/personalities/pvr_analyst.yaml +++ b/src/seclab_taskflows/personalities/pvr_analyst.yaml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + # Personality for PVR (Private Vulnerability Report) triage analysis. seclab-taskflow-agent: diff --git a/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml b/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml index 5f2de42..c7f9994 100644 --- a/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml +++ b/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + # PVR Triage Taskflow # # Fetches a draft GHSA submitted via Private Vulnerability Reporting, diff --git a/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml b/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml index 80e594e..7c236cb 100644 --- a/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml +++ b/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + # Toolbox: PVR GHSA advisory fetcher # # Provides tools for fetching draft GitHub Security Advisories submitted @@ -14,5 +17,6 @@ server_params: command: python args: ["-m", "seclab_taskflows.mcp_servers.pvr_ghsa"] env: + GH_TOKEN: "{{ env('GH_TOKEN') }}" LOG_DIR: "{{ env('LOG_DIR') }}" REPORT_DIR: "{{ env('REPORT_DIR') }}" From 6d97ef5ae367132e5da912e686c5653b26609b10 Mon Sep 17 00:00:00 2001 From: Bas Alberts Date: Tue, 3 Mar 2026 11:40:24 -0500 Subject: [PATCH 03/17] Add PVR triage batch scoring, write-back, and reporter reputation tracking - pvr_ghsa.py: reject/withdraw/comment write-back tools, similarity search, read_triage_report; _gh_api now accepts a request body for PATCH/POST - reporter_reputation.py: new MCP server tracking per-reporter triage history in SQLite with record/history/score tools - pvr_triage.yaml: extended to 8 tasks (quality gate, patch status at HEAD, CVSS assessment, response draft, reputation update) - pvr_respond.yaml: write-back taskflow (action=reject|comment|withdraw) - pvr_triage_batch.yaml: inbox scoring taskflow with ranked markdown output - reporter_reputation.yaml, pvr_ghsa.yaml: new toolbox + confirm gates - tests/test_pvr_mcp.py: 23 unit tests, all passing - README.md: usage docs for all three taskflows --- src/seclab_taskflows/mcp_servers/pvr_ghsa.py | 209 ++++++++- .../mcp_servers/reporter_reputation.py | 211 +++++++++ .../taskflows/pvr_triage/README.md | 262 +++++++++++ .../taskflows/pvr_triage/pvr_respond.yaml | 115 +++++ .../taskflows/pvr_triage/pvr_triage.yaml | 184 +++++++- .../pvr_triage/pvr_triage_batch.yaml | 154 +++++++ src/seclab_taskflows/toolboxes/pvr_ghsa.yaml | 5 + .../toolboxes/reporter_reputation.yaml | 20 + tests/test_pvr_mcp.py | 406 ++++++++++++++++++ 9 files changed, 1559 insertions(+), 7 deletions(-) create mode 100644 src/seclab_taskflows/mcp_servers/reporter_reputation.py create mode 100644 src/seclab_taskflows/taskflows/pvr_triage/README.md create mode 100644 src/seclab_taskflows/taskflows/pvr_triage/pvr_respond.yaml create mode 100644 src/seclab_taskflows/taskflows/pvr_triage/pvr_triage_batch.yaml create mode 100644 src/seclab_taskflows/toolboxes/reporter_reputation.yaml create mode 100644 tests/test_pvr_mcp.py diff --git a/src/seclab_taskflows/mcp_servers/pvr_ghsa.py b/src/seclab_taskflows/mcp_servers/pvr_ghsa.py index cc48245..455f9bf 100644 --- a/src/seclab_taskflows/mcp_servers/pvr_ghsa.py +++ b/src/seclab_taskflows/mcp_servers/pvr_ghsa.py @@ -10,6 +10,7 @@ import json import logging import os +import re import subprocess from pathlib import Path @@ -29,19 +30,30 @@ mcp = FastMCP("PVRAdvisories") -def _gh_api(path: str, method: str = "GET") -> tuple[dict | list | None, str | None]: +def _gh_api( + path: str, + method: str = "GET", + body: dict | None = None, +) -> tuple[dict | list | None, str | None]: """ Call the GitHub REST API via the gh CLI. Returns (data, error). On success data is the parsed JSON response and error is None. On failure data is None and error is a string. + If body is provided it is passed as JSON via stdin (--input -). """ cmd = ["gh", "api", "--method", method, path] env = os.environ.copy() + stdin_data = None + + if body is not None: + cmd += ["--input", "-"] + stdin_data = json.dumps(body) try: result = subprocess.run( cmd, + input=stdin_data, capture_output=True, text=True, env=env, @@ -290,5 +302,200 @@ def save_triage_report( return str(out_path.resolve()) +def _post_advisory_comment(owner: str, repo: str, ghsa_id: str, body: str) -> str: + """ + Internal helper: post a comment on a security advisory. + + Attempts to use the GitHub advisory comments API. If that endpoint is not + available, falls back to appending a '## Maintainer Response' section to the + advisory description instead. Called by both the MCP tool wrapper and the + reject/withdraw tools so they all share the same logic without going through + the FunctionTool wrapper. + """ + comment_path = f"/repos/{owner}/{repo}/security-advisories/{ghsa_id}/comments" + cmd = [ + "gh", "api", + "--method", "POST", + comment_path, + "--input", "-", + ] + env = os.environ.copy() + try: + result = subprocess.run( + cmd, + input=json.dumps({"body": body}), + capture_output=True, + text=True, + env=env, + timeout=30, + ) + except subprocess.TimeoutExpired: + return "Error: gh api call timed out" + except FileNotFoundError: + return "Error: gh CLI not found in PATH" + + if result.returncode == 0: + try: + data = json.loads(result.stdout) + url = data.get("html_url", data.get("url", "posted")) + return f"Comment posted: {url}" + except json.JSONDecodeError: + return "Comment posted." + + # Fall back: append maintainer response to advisory description + logging.warning( + "Advisory comments API unavailable (%s); falling back to description update", + result.stderr.strip(), + ) + adv_path = f"/repos/{owner}/{repo}/security-advisories/{ghsa_id}" + adv_data, adv_err = _gh_api(adv_path) + if adv_err: + return f"Error fetching advisory for fallback comment: {adv_err}" + existing_desc = adv_data.get("description", "") or "" + updated_desc = existing_desc + f"\n\n## Maintainer Response\n\n{body}" + _, patch_err = _gh_api(adv_path, method="PATCH", body={"description": updated_desc}) + if patch_err: + return f"Error updating advisory description: {patch_err}" + return "Comment appended to advisory description (comments API unavailable)." + + +@mcp.tool() +def reject_pvr_advisory( + owner: str = Field(description="Repository owner (user or org name)"), + repo: str = Field(description="Repository name"), + ghsa_id: str = Field(description="GHSA ID of the advisory, e.g. GHSA-xxxx-xxxx-xxxx"), + comment: str = Field(description="Explanation comment to post on the advisory"), +) -> str: + """ + Reject a draft security advisory and post a comment explaining the decision. + + Sets the advisory state to 'rejected' via the GitHub API, then posts a + comment with the provided explanation. Requires a GH_TOKEN with + security_events write scope. + """ + path = f"/repos/{owner}/{repo}/security-advisories/{ghsa_id}" + _, err = _gh_api(path, method="PATCH", body={"state": "rejected"}) + if err: + return f"Error rejecting advisory {ghsa_id}: {err}" + result = _post_advisory_comment(owner, repo, ghsa_id, comment) + return f"Advisory {ghsa_id} rejected. Comment: {result}" + + +@mcp.tool() +def withdraw_pvr_advisory( + owner: str = Field(description="Repository owner (user or org name)"), + repo: str = Field(description="Repository name"), + ghsa_id: str = Field(description="GHSA ID of the advisory, e.g. GHSA-xxxx-xxxx-xxxx"), + comment: str = Field(description="Explanation comment to post on the advisory"), +) -> str: + """ + Withdraw a draft security advisory (for self-submitted drafts) and post a comment. + + Sets the advisory state to 'withdrawn' via the GitHub API, then posts a + comment with the provided explanation. Requires a GH_TOKEN with + security_events write scope. + """ + path = f"/repos/{owner}/{repo}/security-advisories/{ghsa_id}" + _, err = _gh_api(path, method="PATCH", body={"state": "withdrawn"}) + if err: + return f"Error withdrawing advisory {ghsa_id}: {err}" + result = _post_advisory_comment(owner, repo, ghsa_id, comment) + return f"Advisory {ghsa_id} withdrawn. Comment: {result}" + + +@mcp.tool() +def add_pvr_advisory_comment( + owner: str = Field(description="Repository owner (user or org name)"), + repo: str = Field(description="Repository name"), + ghsa_id: str = Field(description="GHSA ID of the advisory, e.g. GHSA-xxxx-xxxx-xxxx"), + body: str = Field(description="Comment text to post on the advisory"), +) -> str: + """ + Post a comment on a security advisory. + + Attempts to use the GitHub advisory comments API. If that endpoint is not + available, falls back to appending a '## Maintainer Response' section to the + advisory description instead. + """ + return _post_advisory_comment(owner, repo, ghsa_id, body) + + +@mcp.tool() +def find_similar_triage_reports( + vuln_type: str = Field(description="Vulnerability class to search for, e.g. 'path traversal', 'XSS'"), + affected_component: str = Field(description="Component, endpoint, or feature to search for"), +) -> str: + """ + Search existing triage reports for similar vulnerability types and affected components. + + Scans REPORT_DIR for *_triage.md files and performs case-insensitive substring + matching on the header lines for vuln_type and affected_component. + Returns a JSON list of matching reports with ghsa_id, verdict, quality, and path. + """ + if not REPORT_DIR.exists(): + return json.dumps([]) + + matches = [] + vuln_lower = vuln_type.lower() + component_lower = affected_component.lower() + + for report_path in sorted(REPORT_DIR.glob("*_triage.md")): + # Skip batch queue reports and response drafts — only match individual GHSA triage reports + stem = report_path.stem # e.g. "GHSA-xxxx-xxxx-xxxx_triage" + if stem.startswith("batch_queue_") or stem.endswith("_response_triage"): + continue + try: + content = report_path.read_text(encoding="utf-8") + except OSError: + continue + + content_lower = content.lower() + if vuln_lower not in content_lower and component_lower not in content_lower: + continue + + # Extract GHSA ID from filename: {ghsa_id}_triage.md + ghsa_id = stem.replace("_triage", "") + + # Extract verdict from report (handles **CONFIRMED** and **[CONFIRMED]**) + verdict = "UNKNOWN" + verdict_match = re.search(r"\*\*\[?\s*(CONFIRMED|UNCONFIRMED|INCONCLUSIVE)\s*\]?\*\*", content) + if verdict_match: + verdict = verdict_match.group(1) + + # Extract quality rating + quality = "Unknown" + quality_match = re.search(r"Rate overall quality[:\s]*\**\s*(High|Medium|Low)\b", content, re.IGNORECASE) + if not quality_match: + quality_match = re.search(r"\b(High|Medium|Low)\b.*quality", content, re.IGNORECASE) + if quality_match: + quality = quality_match.group(1) + + matches.append({ + "ghsa_id": ghsa_id, + "verdict": verdict, + "quality": quality, + "path": str(report_path), + }) + + return json.dumps(matches, indent=2) + + +@mcp.tool() +def read_triage_report( + ghsa_id: str = Field(description="GHSA ID, used to locate the report file, e.g. GHSA-xxxx-xxxx-xxxx"), +) -> str: + """ + Read a previously saved triage report from disk. + + Reads REPORT_DIR/{ghsa_id}_triage.md and returns its content. + Returns an error string if the file does not exist. + """ + safe_name = "".join(c for c in ghsa_id if c.isalnum() or c in "-_") + report_path = REPORT_DIR / f"{safe_name}_triage.md" + if not report_path.exists(): + return f"Report not found: {report_path}" + return report_path.read_text(encoding="utf-8") + + if __name__ == "__main__": mcp.run(show_banner=False) diff --git a/src/seclab_taskflows/mcp_servers/reporter_reputation.py b/src/seclab_taskflows/mcp_servers/reporter_reputation.py new file mode 100644 index 0000000..038462f --- /dev/null +++ b/src/seclab_taskflows/mcp_servers/reporter_reputation.py @@ -0,0 +1,211 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +# Reporter Reputation MCP Server +# +# Tracks PVR reporter history and computes reputation scores based on +# past triage outcomes. Uses a local SQLite database. + +import json +import logging +import os +from datetime import datetime, timezone +from pathlib import Path + +from fastmcp import FastMCP +from pydantic import Field +from seclab_taskflow_agent.path_utils import log_file_name, mcp_data_dir +from sqlalchemy import Text, create_engine +from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column + +REPORTER_DB_DIR = mcp_data_dir("seclab-taskflows", "reporter_reputation", "REPORTER_DB_DIR") + +logging.basicConfig( + level=logging.DEBUG, + format="%(asctime)s - %(levelname)s - %(message)s", + filename=log_file_name("mcp_reporter_reputation.log"), + filemode="a", +) + + +class Base(DeclarativeBase): + pass + + +class ReporterRecord(Base): + __tablename__ = "reporter_records" + + id: Mapped[int] = mapped_column(primary_key=True) + login: Mapped[str] + ghsa_id: Mapped[str] + repo: Mapped[str] + verdict: Mapped[str] # CONFIRMED / UNCONFIRMED / INCONCLUSIVE + quality: Mapped[str] # High / Medium / Low + timestamp: Mapped[str] = mapped_column(Text) # ISO8601 + + def __repr__(self) -> str: + return ( + f"" + ) + + +class ReporterReputationBackend: + def __init__(self, db_dir: Path | str) -> None: + db_path = Path(db_dir) + if str(db_dir) == "sqlite://" or not db_path.exists(): + # In-memory database (used for tests or missing dir) + connection_string = "sqlite://" + else: + connection_string = f"sqlite:///{db_path}/reporter_reputation.db" + self.engine = create_engine(connection_string, echo=False) + Base.metadata.create_all(self.engine) + + def record_triage_result( + self, login: str, ghsa_id: str, repo: str, verdict: str, quality: str + ) -> str: + """Insert or update a triage result record for a reporter.""" + timestamp = datetime.now(timezone.utc).isoformat() + with Session(self.engine) as session: + existing = ( + session.query(ReporterRecord) + .filter_by(login=login, ghsa_id=ghsa_id) + .first() + ) + if existing: + existing.repo = repo + existing.verdict = verdict + existing.quality = quality + existing.timestamp = timestamp + else: + session.add( + ReporterRecord( + login=login, + ghsa_id=ghsa_id, + repo=repo, + verdict=verdict, + quality=quality, + timestamp=timestamp, + ) + ) + session.commit() + return "recorded" + + def get_reporter_history(self, login: str) -> list[dict]: + """Return all triage records for a reporter, newest first.""" + with Session(self.engine) as session: + rows = ( + session.query(ReporterRecord) + .filter_by(login=login) + .order_by(ReporterRecord.timestamp.desc()) + .all() + ) + return [ + { + "login": r.login, + "ghsa_id": r.ghsa_id, + "repo": r.repo, + "verdict": r.verdict, + "quality": r.quality, + "timestamp": r.timestamp, + } + for r in rows + ] + + def get_reporter_score(self, login: str) -> dict: + """Compute and return a reputation summary for a reporter.""" + history = self.get_reporter_history(login) + total = len(history) + if total == 0: + return { + "login": login, + "total_reports": 0, + "confirmed_pct": 0.0, + "quality_breakdown": {"High": 0, "Medium": 0, "Low": 0}, + "recommendation": "no history", + } + + confirmed = sum(1 for r in history if r["verdict"] == "CONFIRMED") + confirmed_pct = confirmed / total + + quality_breakdown: dict[str, int] = {"High": 0, "Medium": 0, "Low": 0} + for r in history: + q = r["quality"] + if q in quality_breakdown: + quality_breakdown[q] += 1 + + low_share = quality_breakdown["Low"] / total + + # Derive recommendation + if confirmed_pct >= 0.6 and low_share <= 0.2: + recommendation = "high trust" + elif confirmed_pct <= 0.2 or low_share >= 0.5: + recommendation = "treat with skepticism" + else: + recommendation = "normal" + + return { + "login": login, + "total_reports": total, + "confirmed_pct": round(confirmed_pct, 4), + "quality_breakdown": quality_breakdown, + "recommendation": recommendation, + } + + +mcp = FastMCP("ReporterReputation") + +backend = ReporterReputationBackend(REPORTER_DB_DIR) + + +@mcp.tool() +def record_triage_result( + login: str = Field(description="GitHub login of the reporter"), + ghsa_id: str = Field(description="GHSA ID of the advisory, e.g. GHSA-xxxx-xxxx-xxxx"), + repo: str = Field(description="Repository in owner/repo format"), + verdict: str = Field(description="Triage verdict: CONFIRMED, UNCONFIRMED, or INCONCLUSIVE"), + quality: str = Field(description="Report quality rating: High, Medium, or Low"), +) -> str: + """ + Record or update a triage result for a PVR reporter. + + Upserts a row keyed by (login, ghsa_id). Re-running triage on the same + GHSA advisory updates the existing record rather than creating a duplicate. + Returns 'recorded' on success. + """ + return backend.record_triage_result(login, ghsa_id, repo, verdict, quality) + + +@mcp.tool() +def get_reporter_history( + login: str = Field(description="GitHub login of the reporter"), +) -> str: + """ + Retrieve the full triage history for a reporter. + + Returns a JSON list of all records for this login, newest first. + Returns a plain message string if no history is found. + """ + history = backend.get_reporter_history(login) + if not history: + return f"No history for {login}." + return json.dumps(history, indent=2) + + +@mcp.tool() +def get_reporter_score( + login: str = Field(description="GitHub login of the reporter"), +) -> str: + """ + Compute and return a reputation score for a PVR reporter. + + Returns a JSON summary including total_reports, confirmed_pct, + quality_breakdown, and a plain-language recommendation: + 'high trust', 'normal', or 'treat with skepticism'. + """ + score = backend.get_reporter_score(login) + return json.dumps(score, indent=2) + + +if __name__ == "__main__": + mcp.run(show_banner=False) diff --git a/src/seclab_taskflows/taskflows/pvr_triage/README.md b/src/seclab_taskflows/taskflows/pvr_triage/README.md new file mode 100644 index 0000000..4b55095 --- /dev/null +++ b/src/seclab_taskflows/taskflows/pvr_triage/README.md @@ -0,0 +1,262 @@ +# PVR Triage Taskflows + +Tools for triaging GitHub Security Advisories submitted via [Private Vulnerability Reporting (PVR)](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability). The taskflows fetch a draft advisory, verify the claimed vulnerability against actual source code, score report quality, and generate a structured analysis and a ready-to-send response draft. + +Three taskflows cover the full triage lifecycle: + +| Taskflow | Purpose | +|---|---| +| `pvr_triage` | Deep-analyse one advisory end-to-end | +| `pvr_triage_batch` | Score an entire inbox and produce a ranked queue | +| `pvr_respond` | Post or save the response once you've reviewed the analysis | + +--- + +## Requirements + +- Python ≥ 3.9 (or Docker via `run_seclab_agent.sh`) +- `gh` CLI installed and authenticated +- A GitHub token with **`repo`** and **`security_events`** scopes + - Write-back actions (`pvr_respond`) additionally require **`security_events` write** scope +- AI API credentials (`AI_API_TOKEN`, `AI_API_ENDPOINT`) + +### Environment variables + +| Variable | Required by | Description | +|---|---|---| +| `GH_TOKEN` | all | GitHub personal access token | +| `AI_API_TOKEN` | all | API key for the AI provider | +| `AI_API_ENDPOINT` | all | Model endpoint (defaults to GitHub Models: `https://models.github.ai/inference`) | +| `REPORT_DIR` | all | Directory where triage reports are written. Defaults to `./reports` | +| `LOG_DIR` | all | Directory for MCP server logs. Auto-detected via `platformdirs` if not set | +| `REPORTER_DB_DIR` | `pvr_triage`, `pvr_respond` | Directory for the reporter reputation SQLite database. Auto-detected if not set | + +A minimal `.env` for local use: + +``` +GH_TOKEN=ghp_... +AI_API_TOKEN=... +AI_API_ENDPOINT=https://models.github.ai/inference +REPORT_DIR=/path/to/reports +LOG_DIR=/path/to/logs +``` + +--- + +## Taskflow 1 — Single advisory triage (`pvr_triage`) + +Runs a full analysis on one draft GHSA and produces: + +- A structured triage report saved to `REPORT_DIR/_triage.md` +- A response draft saved to `REPORT_DIR/_response_triage.md` +- A record in the reporter reputation database + +```bash +python -m seclab_taskflow_agent \ + -t seclab_taskflows.taskflows.pvr_triage.pvr_triage \ + -g repo=owner/repo \ + -g ghsa=GHSA-xxxx-xxxx-xxxx +``` + +### What it does (8 tasks) + +1. **Initialize** — clears the in-memory cache. +2. **Fetch & parse** — fetches the advisory from the GitHub API and extracts structured metadata: vulnerability type, affected component, file references, PoC quality signals, reporter credits. +3. **Quality gate** — calls `get_reporter_score` for the reporter's history and `find_similar_triage_reports` to detect duplicates. Computes a `fast_close` flag when the report has no file references, no PoC, no line numbers, *and* a similar report already exists. Fast-close skips deep code analysis. +4. **Code verification** — resolves the claimed version to a git tag/SHA, fetches the relevant source files, and checks whether the vulnerability pattern is actually present. After verifying at the claimed version, also checks HEAD to determine patch status (`still_vulnerable` / `patched` / `could_not_determine`). Skipped automatically when `fast_close` is true. +5. **Report generation** — writes a markdown report covering: Verdict, Code Verification, Severity Assessment, CVSS 3.1 assessment, Duplicate/Prior Reports, Patch Status, Report Quality, Reporter Reputation, and Recommendations. +6. **Save report** — writes the report to `REPORT_DIR/_triage.md` and prints the path. +7. **Response draft** — drafts a plain-text reply to the reporter (≤200 words, no markdown headers) tailored to the verdict: acknowledge + credit for CONFIRMED, cite evidence for UNCONFIRMED, explain missing info for INCONCLUSIVE, or request specific details for fast-close. +8. **Update reputation + save response** — records the triage outcome in the reporter reputation database and saves the response draft to `REPORT_DIR/_response_triage.md`. + +### Report structure + +``` +## PVR Triage Analysis: GHSA-xxxx-xxxx-xxxx + +**Repository:** owner/repo +**Claimed Severity:** high +**Vulnerability Type:** path traversal + +### Verdict +**[CONFIRMED / UNCONFIRMED / INCONCLUSIVE]** + +### Code Verification +### Severity Assessment +### CVSS Assessment +### Duplicate / Prior Reports +### Patch Status +### Report Quality +### Reporter Reputation +### Recommendations +``` + +--- + +## Taskflow 2 — Batch inbox scoring (`pvr_triage_batch`) + +Lists all draft advisories for a repository, scores them by priority, and saves a ranked markdown table — useful for deciding which reports to triage first. + +```bash +python -m seclab_taskflow_agent \ + -t seclab_taskflows.taskflows.pvr_triage.pvr_triage_batch \ + -g repo=owner/repo +``` + +### Output + +Saved to `REPORT_DIR/batch_queue__.md`: + +```markdown +# PVR Batch Triage Queue: owner/repo + +| GHSA | Severity | Vuln Type | Quality Signals | Priority | Status | Suggested Action | +|------|----------|-----------|-----------------|----------|--------|-----------------| +| GHSA-... | high | SQL injection | PoC, Files | 6 | Not triaged | Triage Immediately | +| GHSA-... | medium | XSS | None | 1 | Not triaged | Likely Low Quality — Fast Close | +``` + +### Priority scoring + +``` +priority_score = severity_weight + quality_weight + already_triaged_penalty + +severity_weight: critical=4 high=3 medium=2 low=1 unknown=1 +quality_weight: has_file_references(+1) + has_poc(+1) + has_line_numbers(+1) +already_triaged: -3 (advisory already has a report in REPORT_DIR) +``` + +**Suggested actions:** + +| Score | Action | +|---|---| +| ≥ 5 | Triage Immediately | +| ≥ 3 | Triage Soon | +| 2 | Triage | +| ≤ 1 | Likely Low Quality — Fast Close | +| Already triaged (CONFIRMED) | Fix/Publish | +| Already triaged (UNCONFIRMED/INCONCLUSIVE) | Review/Close | + +--- + +## Taskflow 3 — Write-back (`pvr_respond`) + +Loads an existing triage report and response draft from disk and executes the chosen action against the GitHub advisory API. All write-back calls are confirm-gated — the agent will prompt for confirmation before making any change. + +```bash +python -m seclab_taskflow_agent \ + -t seclab_taskflows.taskflows.pvr_triage.pvr_respond \ + -g repo=owner/repo \ + -g ghsa=GHSA-xxxx-xxxx-xxxx \ + -g action=comment +``` + +### Actions + +| `action` | API call | When to use | +|---|---|---| +| `comment` | Posts the response draft as a comment on the advisory | Default for all verdicts — sends your reply without changing state | +| `reject` | Sets advisory state to `rejected`, then posts the comment | Report is clearly invalid or low quality | +| `withdraw` | Sets advisory state to `withdrawn`, then posts the comment | Your own self-submitted draft that should be removed | + +> **Note:** `pvr_respond` requires that `pvr_triage` has already been run for the GHSA, so that both `_triage.md` and `_response_triage.md` exist in `REPORT_DIR`. + +### Confirm gate + +The toolbox marks `reject_pvr_advisory`, `withdraw_pvr_advisory`, and `add_pvr_advisory_comment` as `confirm`-gated. The agent will print the verdict, quality rating, and full response draft, then ask for explicit confirmation before making any change to GitHub. + +--- + +## Typical workflow + +``` +1. Run pvr_triage_batch to see what's in your inbox and prioritise. + +2. For each advisory you want to analyse: + Run pvr_triage. + +3. Review the saved report in REPORT_DIR: + - Check the Verdict and Code Verification sections. + - Edit the response draft (_response_triage.md) if needed. + +4. Run pvr_respond to send the response: + - action=comment → post reply only (advisory stays draft) + - action=reject → reject + post reply + - action=withdraw → withdraw + post reply +``` + +### Example session + +```bash +# Step 1: score the inbox +python -m seclab_taskflow_agent \ + -t seclab_taskflows.taskflows.pvr_triage.pvr_triage_batch \ + -g repo=acme/widget + +# Step 2: triage the highest-priority advisory +python -m seclab_taskflow_agent \ + -t seclab_taskflows.taskflows.pvr_triage.pvr_triage \ + -g repo=acme/widget \ + -g ghsa=GHSA-1234-5678-abcd + +# Step 3: review the output +cat reports/GHSA-1234-5678-abcd_triage.md +cat reports/GHSA-1234-5678-abcd_response_triage.md + +# Step 4a: send a comment (most common — doesn't change advisory state) +python -m seclab_taskflow_agent \ + -t seclab_taskflows.taskflows.pvr_triage.pvr_respond \ + -g repo=acme/widget \ + -g ghsa=GHSA-1234-5678-abcd \ + -g action=comment + +# Step 4b: or reject outright +python -m seclab_taskflow_agent \ + -t seclab_taskflows.taskflows.pvr_triage.pvr_respond \ + -g repo=acme/widget \ + -g ghsa=GHSA-1234-5678-abcd \ + -g action=reject +``` + +--- + +## Reporter reputation + +Every completed `pvr_triage` run records the verdict and quality rating against the reporter's GitHub login in a local SQLite database (`REPORTER_DB_DIR/reporter_reputation.db`). + +The quality gate in Task 3 of `pvr_triage` calls `get_reporter_score` automatically before any code analysis. The score summary appears in the report under **Reporter Reputation**. + +**Reputation thresholds:** + +| Condition | Recommendation | +|---|---| +| confirmed_pct ≥ 60% and Low-quality share ≤ 20% | high trust | +| confirmed_pct ≤ 20% or Low-quality share ≥ 50% | treat with skepticism | +| Otherwise | normal | + +A "treat with skepticism" score alone does not trigger fast-close — it is informational. Fast-close is triggered only by the combination of missing quality signals *and* an existing duplicate report. + +--- + +## Models + +The taskflows use `seclab_taskflows.configs.model_config_pvr_triage`, which defines two model roles: + +| Role | Used for | Default model | +|---|---|---| +| `triage` | Code verification and report generation | `claude-opus-4.6-1m` | +| `extraction` | Fetch/parse, quality gate, save tasks | `gpt-5-mini` | + +Override the model config by setting `AI_API_ENDPOINT` and `AI_API_TOKEN` to point at a compatible provider. + +--- + +## Output files + +All files are written to `REPORT_DIR` (default: `./reports`). + +| File | Written by | Contents | +|---|---|---| +| `_triage.md` | `pvr_triage` task 6 | Full triage analysis report | +| `_response_triage.md` | `pvr_triage` task 8 | Plain-text response draft for the reporter | +| `batch_queue__.md` | `pvr_triage_batch` task 3 | Ranked inbox table | diff --git a/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond.yaml b/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond.yaml new file mode 100644 index 0000000..6eecc0d --- /dev/null +++ b/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond.yaml @@ -0,0 +1,115 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +# PVR Respond Taskflow +# +# Loads a previously generated triage report and response draft from disk +# and executes the selected write-back action on the GitHub advisory. +# All write-back API calls are confirm-gated in the pvr_ghsa toolbox. +# +# Usage: +# python -m seclab_taskflow_agent \ +# -t seclab_taskflows.taskflows.pvr_triage.pvr_respond \ +# -g repo=owner/repo \ +# -g ghsa=GHSA-xxxx-xxxx-xxxx \ +# -g action=reject|comment|withdraw +# +# Required environment variables: +# GH_TOKEN - GitHub token with security_events write scope +# AI_API_TOKEN - API token for the AI model provider +# AI_API_ENDPOINT - Model provider endpoint (default: GitHub Copilot API) +# REPORT_DIR - Directory where triage reports are stored + +seclab-taskflow-agent: + version: "1.0" + filetype: taskflow + +model_config: seclab_taskflows.configs.model_config_pvr_triage + +globals: + # GitHub repository in owner/repo format + repo: + # GHSA ID of the advisory to act on + ghsa: + # Action to perform: reject, comment, or withdraw + action: + +taskflow: + # ------------------------------------------------------------------------- + # Task 1: Load triage report and response draft from disk + # ------------------------------------------------------------------------- + - task: + must_complete: true + model: extraction + agents: + - seclab_taskflow_agent.personalities.assistant + toolboxes: + - seclab_taskflows.toolboxes.pvr_ghsa + - seclab_taskflow_agent.toolboxes.memcache + user_prompt: | + Read the triage report for advisory "{{ globals.ghsa }}" using read_triage_report + with ghsa_id="{{ globals.ghsa }}". + + Store the triage report content under memcache key "triage_report". + + Read the response draft using read_triage_report with + ghsa_id="{{ globals.ghsa }}_response". + + Store the response draft content under memcache key "response_draft". + + From the triage report, extract and print: + - Verdict (CONFIRMED / UNCONFIRMED / INCONCLUSIVE) + - Report Quality (High / Medium / Low) + - A 1-2 sentence summary of the findings + + Then print the full response draft. + + If either file is missing (read_triage_report returns "Report not found"), + print a clear error message and stop. + + # ------------------------------------------------------------------------- + # Task 2: Confirm and execute write-back action + # ------------------------------------------------------------------------- + - task: + must_complete: true + model: extraction + agents: + - seclab_taskflow_agent.personalities.assistant + toolboxes: + - seclab_taskflows.toolboxes.pvr_ghsa + - seclab_taskflow_agent.toolboxes.memcache + user_prompt: | + Retrieve "triage_report" and "response_draft" from memcache. + + Extract owner and repo from "{{ globals.repo }}" (format: owner/repo). + + The requested action is: "{{ globals.action }}" + + Execute the action as follows: + + If action is "reject": + Call reject_pvr_advisory with: + - owner: extracted owner + - repo: extracted repo + - ghsa_id: "{{ globals.ghsa }}" + - comment: response_draft + + If action is "withdraw": + Call withdraw_pvr_advisory with: + - owner: extracted owner + - repo: extracted repo + - ghsa_id: "{{ globals.ghsa }}" + - comment: response_draft + + If action is "comment": + Call add_pvr_advisory_comment with: + - owner: extracted owner + - repo: extracted repo + - ghsa_id: "{{ globals.ghsa }}" + - body: response_draft + + If action is anything else: + Print: "Unknown action '{{ globals.action }}'. Valid actions: reject, comment, withdraw" + and stop. + + Print the result returned by the API call. diff --git a/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml b/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml index c7f9994..c7aed5a 100644 --- a/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml +++ b/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml @@ -84,11 +84,52 @@ taskflow: has_poc: true if reproduction steps are provided has_version_info: true if specific affected versions are mentioned has_code_snippets: true if actual code is quoted in the report + - credits: the credits list from the advisory API response (list of {login, type} objects) Do not perform any code analysis yet. # ------------------------------------------------------------------------- - # Task 3: Verify vulnerability in source code + # Task 3: Quick Quality Gate + # ------------------------------------------------------------------------- + - task: + must_complete: true + model: extraction + agents: + - seclab_taskflows.personalities.pvr_analyst + toolboxes: + - seclab_taskflow_agent.toolboxes.memcache + - seclab_taskflows.toolboxes.pvr_ghsa + - seclab_taskflows.toolboxes.reporter_reputation + user_prompt: | + Retrieve "pvr_parsed" from memcache. + + Extract reporter login from pvr_parsed.credits: find the first entry with + type "reporter" and use its login. If credits is empty or no reporter type + is found, use "unknown". + + Call get_reporter_score with that login and store the result as reporter_score. + + Call find_similar_triage_reports with: + - vuln_type: pvr_parsed.vuln_type + - affected_component: pvr_parsed.affected_component + + Evaluate fast_close conditions (ALL must be true to trigger fast_close): + - pvr_parsed.quality_signals.has_file_references is false + - pvr_parsed.quality_signals.has_poc is false + - pvr_parsed.quality_signals.has_line_numbers is false + - At least one similar report exists with verdict UNCONFIRMED or CONFIRMED + + Store under memcache key "quality_gate": + { + "fast_close": true or false, + "reason": "brief explanation of why fast_close was triggered or not", + "reporter_login": "the login extracted above", + "reporter_score": {the full object returned by get_reporter_score}, + "similar_reports": [the list returned by find_similar_triage_reports] + } + + # ------------------------------------------------------------------------- + # Task 4: Verify vulnerability in source code # ------------------------------------------------------------------------- - task: must_complete: true @@ -100,7 +141,23 @@ taskflow: - seclab_taskflows.toolboxes.gh_file_viewer - seclab_taskflow_agent.toolboxes.memcache user_prompt: | - Retrieve "pvr_parsed" and "pvr_description" from memcache. + Retrieve "pvr_parsed", "pvr_description", and "quality_gate" from memcache. + + If quality_gate.fast_close is true, store under "code_verification": + { + "ref_used": null, + "files_examined": [], + "vulnerability_confirmed": null, + "confirmation_evidence": "Fast-close: quality gate triggered. Reason: {quality_gate.reason}", + "mitigation_found": null, + "mitigation_details": null, + "patch_status": "could_not_determine", + "patch_notes": null, + "notes": "Skipped deep analysis." + } + and stop. Do not fetch any files. + + Otherwise proceed with full code verification: Extract owner and repo from "{{ globals.repo }}" (format: owner/repo). @@ -128,6 +185,11 @@ taskflow: Focus on the specific code path described. Do not perform a broad audit. + After completing the main verification at the claimed version, re-check the + same code pattern at HEAD using fetch_file_from_gh. Add to code_verification: + "patch_status": "still_vulnerable" | "patched" | "could_not_determine" + "patch_notes": brief description of what changed at HEAD (or null) + Store your findings under memcache key "code_verification" as JSON: - ref_used: the git SHA or ref used for code fetching (or "HEAD" if none) - files_examined: list of file paths fetched @@ -136,10 +198,12 @@ taskflow: including file path and line numbers - mitigation_found: true if existing checks prevent exploitation - mitigation_details: description of mitigating code, or null + - patch_status: "still_vulnerable" | "patched" | "could_not_determine" + - patch_notes: description of HEAD state vs claimed version (or null) - notes: any additional observations # ------------------------------------------------------------------------- - # Task 4: Generate triage report + # Task 5: Generate triage report # ------------------------------------------------------------------------- - task: must_complete: true @@ -149,7 +213,8 @@ taskflow: toolboxes: - seclab_taskflow_agent.toolboxes.memcache user_prompt: | - Retrieve "pvr_parsed", "pvr_description", and "code_verification" from memcache. + Retrieve "pvr_parsed", "pvr_description", "code_verification", and "quality_gate" + from memcache. Generate a triage analysis report in markdown and store it under memcache key "triage_report". @@ -185,6 +250,27 @@ taskflow: State whether the claimed severity is accurate, overstated, or understated. Base this on the actual exploitability and impact from the code evidence. + ### CVSS Assessment + + Derive a CVSS 3.1 vector for this vulnerability based on the code evidence. + State: Base Score, Vector String, and whether the reporter's claimed severity + (pvr_parsed.severity_claimed) is accurate / overstated / understated. + If vulnerability_confirmed is false or null, note that CVSS is based on + the claimed scenario and may not reflect actual risk. + + ### Duplicate / Prior Reports + + If quality_gate.similar_reports is non-empty, list them with their verdict and quality. + Note whether this report adds new evidence vs. restating a known issue. + If similar_reports is empty, state "No similar prior reports found." + + ### Patch Status + + State code_verification.patch_status at HEAD. + If patched: note the triage impact (lower urgency for confirmed vulnerabilities). + If still_vulnerable: note urgency is unchanged. + If could_not_determine: state that HEAD status could not be assessed. + ### Report Quality Assess the quality of the PVR submission: @@ -195,6 +281,12 @@ taskflow: - Medium: partially accurate, some details wrong or missing - Low: vague, speculative, or significantly inaccurate ("AI slop") + ### Reporter Reputation + + Reporter login: [quality_gate.reporter_login] + Score summary: [quality_gate.reporter_score.recommendation] (confirmed_pct, + total_reports, quality_breakdown from reporter_score) + ### Recommendations Provide 1-3 specific, actionable recommendations for the maintainer. @@ -205,10 +297,10 @@ taskflow: --- Be factual. Do not include anything not supported by code evidence. - Keep the report concise. Aim for under 600 words. + Keep the report concise. Aim for under 800 words. # ------------------------------------------------------------------------- - # Task 5: Save report to disk and print path + # Task 6: Save report to disk and print path # ------------------------------------------------------------------------- - task: must_complete: true @@ -227,3 +319,83 @@ taskflow: Then print the report content verbatim, followed by a blank line and: "Report saved to: " + + # ------------------------------------------------------------------------- + # Task 7: Generate Reporter Response Draft + # ------------------------------------------------------------------------- + - task: + must_complete: true + model: extraction + agents: + - seclab_taskflows.personalities.pvr_analyst + toolboxes: + - seclab_taskflow_agent.toolboxes.memcache + user_prompt: | + Retrieve "pvr_parsed", "code_verification", "quality_gate", and "triage_report" + from memcache. + + Extract the verdict from triage_report: look for the line containing + **CONFIRMED**, **UNCONFIRMED**, or **INCONCLUSIVE** in the Verdict section. + + Draft a response comment to the reporter. Tone: direct, factual, not harsh. + Select the template based on verdict and quality_gate.fast_close: + + fast_close (quality_gate.fast_close=true): + Explain that the report lacks file paths, functions, and reproduction steps + that match the codebase. Invite resubmission with specific details including + the exact file path, line number, and a concrete reproduction scenario. + + CONFIRMED: + Acknowledge the finding. State that a fix is in progress and credit will + be given when the advisory is published. + + UNCONFIRMED: + Cite specific code evidence for why the claim could not be confirmed + (reference the file path and what the code actually does). Ask for more + specific reproduction steps if the reporter wants to follow up. + + INCONCLUSIVE: + Explain what specific information is missing to complete verification + (e.g. exact version, file path, reproduction steps). + + Keep the response under 200 words. No markdown headers. Plain text suitable + for a GitHub comment. + + Store under memcache key "response_draft". + + # ------------------------------------------------------------------------- + # Task 8: Update Reporter Reputation + Save Response Draft + # ------------------------------------------------------------------------- + - task: + must_complete: true + model: extraction + agents: + - seclab_taskflow_agent.personalities.assistant + toolboxes: + - seclab_taskflows.toolboxes.pvr_ghsa + - seclab_taskflows.toolboxes.reporter_reputation + - seclab_taskflow_agent.toolboxes.memcache + user_prompt: | + Retrieve "pvr_parsed", "code_verification", "quality_gate", "triage_report", + and "response_draft" from memcache. + + Extract verdict: find **CONFIRMED**, **UNCONFIRMED**, or **INCONCLUSIVE** + in the triage_report Verdict section. + + Extract quality rating: find the "Rate overall quality" line in triage_report + Report Quality section and extract: High, Medium, or Low. + + Extract reporter login from quality_gate.reporter_login. + + Call record_triage_result with: + - login: quality_gate.reporter_login + - ghsa_id: "{{ globals.ghsa }}" + - repo: "{{ globals.repo }}" + - verdict: the extracted verdict + - quality: the extracted quality rating + + Call save_triage_report with: + - ghsa_id: "{{ globals.ghsa }}_response" + - report: response_draft + + Print: "Response draft saved." followed by the response_draft text. diff --git a/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage_batch.yaml b/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage_batch.yaml new file mode 100644 index 0000000..c2ffbb3 --- /dev/null +++ b/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage_batch.yaml @@ -0,0 +1,154 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +# PVR Triage Batch Taskflow +# +# Lists all draft PVR advisories for a repository, scores each one by +# priority (based on severity, quality signals, and triage status), and +# outputs a ranked markdown table to REPORT_DIR for maintainer review. +# +# Usage: +# python -m seclab_taskflow_agent \ +# -t seclab_taskflows.taskflows.pvr_triage.pvr_triage_batch \ +# -g repo=owner/repo +# +# Required environment variables: +# GH_TOKEN - GitHub token with repo and security_events scope +# AI_API_TOKEN - API token for the AI model provider +# AI_API_ENDPOINT - Model provider endpoint (default: GitHub Copilot API) +# REPORT_DIR - Directory where triage reports are stored (and batch output is saved) + +seclab-taskflow-agent: + version: "1.0" + filetype: taskflow + +model_config: seclab_taskflows.configs.model_config_pvr_triage + +globals: + # GitHub repository in owner/repo format + repo: + +taskflow: + # ------------------------------------------------------------------------- + # Task 1: List draft advisories + # ------------------------------------------------------------------------- + - task: + must_complete: true + model: extraction + agents: + - seclab_taskflows.personalities.pvr_analyst + toolboxes: + - seclab_taskflows.toolboxes.pvr_ghsa + - seclab_taskflow_agent.toolboxes.memcache + user_prompt: | + Extract owner and repo from "{{ globals.repo }}" (format: owner/repo). + + Call list_pvr_advisories with owner, repo, and state="draft" to retrieve + all draft advisories. + + Store the full JSON list under memcache key "pvr_queue". + + Print: "Found N draft advisories for {{ globals.repo }}." where N is the count. + + If no advisories are found, print "No draft advisories found." and stop. + + # ------------------------------------------------------------------------- + # Task 2: Score each advisory + # ------------------------------------------------------------------------- + - task: + must_complete: true + model: extraction + agents: + - seclab_taskflows.personalities.pvr_analyst + toolboxes: + - seclab_taskflows.toolboxes.pvr_ghsa + - seclab_taskflow_agent.toolboxes.memcache + user_prompt: | + Retrieve "pvr_queue" from memcache. + + Extract owner and repo from "{{ globals.repo }}" (format: owner/repo). + + For each advisory in pvr_queue: + 1. Call fetch_pvr_advisory to get the full advisory details. + 2. Check for existing triage by calling read_triage_report with the ghsa_id. + If the result does not start with "Report not found", mark already_triaged=true + and extract the verdict from the report content. + Otherwise, mark already_triaged=false and verdict=null. + 3. Extract quality signals from the description: + - has_file_references: description mentions specific file paths + - has_poc: description includes reproduction steps or exploit code + - has_line_numbers: description cites line numbers + 4. Compute priority_score using this formula: + severity_weight: critical=4, high=3, medium=2, low=1, unknown=1 + quality_weight: has_file_references(+1) + has_poc(+1) + has_line_numbers(+1) + already_triaged_penalty: -3 if already_triaged else 0 + priority_score = severity_weight + quality_weight + already_triaged_penalty + 5. Determine suggested_action: + - If already_triaged and verdict is UNCONFIRMED or INCONCLUSIVE: "Review/Close" + - If already_triaged and verdict is CONFIRMED: "Fix/Publish" + - If priority_score >= 5: "Triage Immediately" + - If priority_score >= 3: "Triage Soon" + - If priority_score <= 1: "Likely Low Quality — Fast Close" + - Otherwise: "Triage" + + Build a list of scored entries, each with: + {ghsa_id, severity, summary, vuln_type, quality_signals, + priority_score, already_triaged, verdict, suggested_action} + + Sort the list by priority_score descending. + Store under memcache key "scored_queue". + + # ------------------------------------------------------------------------- + # Task 3: Generate and save ranked queue report + # ------------------------------------------------------------------------- + - task: + must_complete: true + model: extraction + agents: + - seclab_taskflow_agent.personalities.assistant + toolboxes: + - seclab_taskflows.toolboxes.pvr_ghsa + - seclab_taskflow_agent.toolboxes.memcache + user_prompt: | + Retrieve "scored_queue" from memcache. + + Generate today's date in YYYY-MM-DD format. + + Build a report string with this structure: + + # PVR Batch Triage Queue: {{ globals.repo }} + + **Generated:** [today's date] + **Total advisories:** [count] + + | GHSA | Severity | Vuln Type | Quality Signals | Priority | Status | Suggested Action | + |------|----------|-----------|-----------------|----------|--------|-----------------| + [one row per advisory, sorted by priority_score desc] + + For each row: + - GHSA: the ghsa_id as a plain string + - Severity: severity from the advisory + - Vuln Type: vuln_type (truncated to 30 chars if needed) + - Quality Signals: compact representation, e.g. "PoC, Files, Lines" for all three, + or list only the signals present, or "None" if all false + - Priority: priority_score as an integer + - Status: "Triaged (CONFIRMED)" / "Triaged (UNCONFIRMED)" / "Triaged (INCONCLUSIVE)" / + "Not triaged" + - Suggested Action: from suggested_action field + + After the table, add a section: + + ## Summary + + List any advisories with priority_score >= 5 as "Requires immediate attention." + List any already_triaged advisories as "Previously triaged — verify closure." + + Sanitize the repo name for use in a filename: replace "/" and any non-alphanumeric + characters (except "-" and "_") with "_". + + Call save_triage_report with: + - ghsa_id: "batch_queue_[sanitized_repo]_[today's date]" + - report: the full report string + + Print: "Batch queue report saved to: " + Then print the full report. diff --git a/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml b/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml index 7c236cb..e63b139 100644 --- a/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml +++ b/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml @@ -20,3 +20,8 @@ server_params: GH_TOKEN: "{{ env('GH_TOKEN') }}" LOG_DIR: "{{ env('LOG_DIR') }}" REPORT_DIR: "{{ env('REPORT_DIR') }}" +# Guard write-back tools: user must confirm before execution +confirm: + - reject_pvr_advisory + - withdraw_pvr_advisory + - add_pvr_advisory_comment diff --git a/src/seclab_taskflows/toolboxes/reporter_reputation.yaml b/src/seclab_taskflows/toolboxes/reporter_reputation.yaml new file mode 100644 index 0000000..0c799ec --- /dev/null +++ b/src/seclab_taskflows/toolboxes/reporter_reputation.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +# Toolbox: Reporter Reputation tracker +# +# Provides tools for recording PVR triage outcomes per reporter and +# querying their reputation score across prior reports. + +seclab-taskflow-agent: + version: "1.0" + filetype: toolbox + +server_params: + kind: stdio + command: python + args: ["-m", "seclab_taskflows.mcp_servers.reporter_reputation"] + env: + GH_TOKEN: "{{ env('GH_TOKEN') }}" + LOG_DIR: "{{ env('LOG_DIR') }}" + REPORTER_DB_DIR: "{{ env('REPORTER_DB_DIR', '') }}" diff --git a/tests/test_pvr_mcp.py b/tests/test_pvr_mcp.py new file mode 100644 index 0000000..ae15a66 --- /dev/null +++ b/tests/test_pvr_mcp.py @@ -0,0 +1,406 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +# Unit tests for the PVR MCP server extensions and reporter reputation backend. +# +# Run with: pytest tests/test_pvr_mcp.py -v + +import json +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + + +# --------------------------------------------------------------------------- +# Helpers: patch mcp_data_dir so imports don't fail in CI (no platformdirs dir) +# --------------------------------------------------------------------------- + +def _patch_mcp_data_dir_pvr_ghsa(tmp_path): + """Return a context manager that patches REPORT_DIR in pvr_ghsa.""" + import seclab_taskflows.mcp_servers.pvr_ghsa as pvr_mod + return patch.object(pvr_mod, "REPORT_DIR", tmp_path) + + +# --------------------------------------------------------------------------- +# TestPvrGhsaTools +# --------------------------------------------------------------------------- + +class TestPvrGhsaTools(unittest.TestCase): + """Tests for the new write-back and similarity tools in pvr_ghsa.py.""" + + def setUp(self): + import seclab_taskflows.mcp_servers.pvr_ghsa as pvr_mod + self.pvr = pvr_mod + self.tmp = Path(tempfile.mkdtemp()) + + # --- reject_pvr_advisory --- + + def test_reject_pvr_advisory_calls_correct_api(self): + """reject_pvr_advisory should PATCH state=rejected then post a comment.""" + calls = [] + + def fake_gh_api(path, method="GET", body=None): + calls.append({"path": path, "method": method, "body": body}) + if method == "PATCH": + return {"ghsa_id": "GHSA-1234-5678-abcd", "state": "rejected"}, None + return {}, None + + with patch.object(self.pvr, "_gh_api", side_effect=fake_gh_api): + with patch.object(self.pvr, "_post_advisory_comment", return_value="Comment posted: https://github.com/test"): + result = self.pvr.reject_pvr_advisory.fn( + owner="owner", + repo="repo", + ghsa_id="GHSA-1234-5678-abcd", + comment="Rejecting: not a valid report.", + ) + + # First call must be the PATCH to set state=rejected + self.assertEqual(calls[0]["method"], "PATCH") + self.assertIn("GHSA-1234-5678-abcd", calls[0]["path"]) + self.assertEqual(calls[0]["body"], {"state": "rejected"}) + self.assertIn("rejected", result) + + # --- withdraw_pvr_advisory --- + + def test_withdraw_pvr_advisory_calls_correct_api(self): + """withdraw_pvr_advisory should PATCH state=withdrawn.""" + calls = [] + + def fake_gh_api(path, method="GET", body=None): + calls.append({"path": path, "method": method, "body": body}) + if method == "PATCH": + return {"ghsa_id": "GHSA-1234-5678-abcd", "state": "withdrawn"}, None + return {}, None + + with patch.object(self.pvr, "_gh_api", side_effect=fake_gh_api): + with patch.object(self.pvr, "_post_advisory_comment", return_value="Comment posted: https://github.com/test"): + result = self.pvr.withdraw_pvr_advisory.fn( + owner="owner", + repo="repo", + ghsa_id="GHSA-1234-5678-abcd", + comment="Withdrawing self-submitted draft.", + ) + + self.assertEqual(calls[0]["method"], "PATCH") + self.assertEqual(calls[0]["body"], {"state": "withdrawn"}) + self.assertIn("withdrawn", result) + + # --- add_pvr_advisory_comment --- + + def test_add_pvr_advisory_comment_returns_url_on_success(self): + """add_pvr_advisory_comment returns comment URL on API success.""" + mock_result = MagicMock() + mock_result.returncode = 0 + mock_result.stdout = json.dumps({"html_url": "https://github.com/comment/1"}) + with patch("subprocess.run", return_value=mock_result): + result = self.pvr.add_pvr_advisory_comment.fn( + owner="owner", + repo="repo", + ghsa_id="GHSA-1234-5678-abcd", + body="Thank you for the report.", + ) + self.assertIn("https://github.com/comment/1", result) + + def test_add_pvr_advisory_comment_fallback_on_api_failure(self): + """add_pvr_advisory_comment falls back to description update when comments API unavailable.""" + # First subprocess call (comments POST) fails + mock_fail = MagicMock() + mock_fail.returncode = 1 + mock_fail.stderr = "Not Found" + mock_fail.stdout = "" + + def fake_gh_api(path, method="GET", body=None): + if method == "GET": + return {"description": "Original description.", "ghsa_id": "GHSA-x"}, None + if method == "PATCH": + return {"description": "updated"}, None + return {}, None + + with patch("subprocess.run", return_value=mock_fail): + with patch.object(self.pvr, "_gh_api", side_effect=fake_gh_api): + result = self.pvr.add_pvr_advisory_comment.fn( + owner="owner", + repo="repo", + ghsa_id="GHSA-1234-5678-abcd", + body="Maintainer note.", + ) + self.assertIn("description", result.lower()) + + # --- find_similar_triage_reports --- + + def test_find_similar_reports_matches_vuln_type(self): + """find_similar_triage_reports returns matching reports by vuln_type.""" + report_dir = self.tmp + # Write a fixture report + (report_dir / "GHSA-aaaa-bbbb-cccc_triage.md").write_text( + "## PVR Triage Analysis: GHSA-aaaa-bbbb-cccc\n" + "**Vulnerability Type:** path traversal\n" + "**[UNCONFIRMED]**\n" + "Rate overall quality: Low\n", + encoding="utf-8", + ) + + with _patch_mcp_data_dir_pvr_ghsa(report_dir): + result_json = self.pvr.find_similar_triage_reports.fn( + vuln_type="path traversal", + affected_component="upload handler", + ) + + results = json.loads(result_json) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]["ghsa_id"], "GHSA-aaaa-bbbb-cccc") + self.assertEqual(results[0]["verdict"], "UNCONFIRMED") + + def test_find_similar_reports_no_matches(self): + """find_similar_triage_reports returns empty list when nothing matches.""" + report_dir = self.tmp + (report_dir / "GHSA-aaaa-bbbb-dddd_triage.md").write_text( + "## PVR Triage Analysis: GHSA-aaaa-bbbb-dddd\n" + "**Vulnerability Type:** SQL injection\n" + "**[CONFIRMED]**\n", + encoding="utf-8", + ) + + with _patch_mcp_data_dir_pvr_ghsa(report_dir): + result_json = self.pvr.find_similar_triage_reports.fn( + vuln_type="XSS", + affected_component="login form", + ) + + results = json.loads(result_json) + self.assertEqual(results, []) + + def test_find_similar_reports_empty_dir(self): + """find_similar_triage_reports returns empty list for non-existent REPORT_DIR.""" + empty_dir = self.tmp / "nonexistent" + with _patch_mcp_data_dir_pvr_ghsa(empty_dir): + result_json = self.pvr.find_similar_triage_reports.fn( + vuln_type="IDOR", + affected_component="profile", + ) + results = json.loads(result_json) + self.assertEqual(results, []) + + # --- save_triage_report path sanitization --- + + def test_save_triage_report_path_sanitization(self): + """save_triage_report strips path traversal characters from the GHSA ID.""" + with _patch_mcp_data_dir_pvr_ghsa(self.tmp): + out_path = self.pvr.save_triage_report.fn( + ghsa_id="../../../etc/passwd", + report="malicious content", + ) + # The file must be inside REPORT_DIR, not outside. + # Resolve both paths to handle macOS /var -> /private/var symlinks. + self.assertTrue(out_path.startswith(str(self.tmp.resolve()))) + # The filename should not contain path separators + saved = Path(out_path) + self.assertFalse(".." in saved.name) + self.assertFalse("/" in saved.name) + + # --- read_triage_report --- + + def test_read_triage_report_returns_content(self): + """read_triage_report reads back a previously saved report.""" + content = "## PVR Triage Analysis: GHSA-test\n\n**[CONFIRMED]**\n" + (self.tmp / "GHSA-test_triage.md").write_text(content, encoding="utf-8") + + with _patch_mcp_data_dir_pvr_ghsa(self.tmp): + result = self.pvr.read_triage_report.fn(ghsa_id="GHSA-test") + + self.assertEqual(result, content) + + def test_read_triage_report_missing_file(self): + """read_triage_report returns an error string for a missing report.""" + with _patch_mcp_data_dir_pvr_ghsa(self.tmp): + result = self.pvr.read_triage_report.fn(ghsa_id="GHSA-does-not-exist") + + self.assertIn("not found", result.lower()) + + +# --------------------------------------------------------------------------- +# TestReporterReputationBackend +# --------------------------------------------------------------------------- + +class TestReporterReputationBackend(unittest.TestCase): + """Tests for the ReporterReputationBackend class using in-memory SQLite.""" + + def setUp(self): + from seclab_taskflows.mcp_servers.reporter_reputation import ReporterReputationBackend + # Pass a non-existent path to trigger in-memory DB fallback + self.backend = ReporterReputationBackend(db_dir=Path("/nonexistent/path")) + + def test_record_and_retrieve(self): + """record_triage_result inserts a record and get_reporter_history retrieves it.""" + self.backend.record_triage_result( + login="alice", + ghsa_id="GHSA-1111-2222-3333", + repo="owner/repo", + verdict="CONFIRMED", + quality="High", + ) + history = self.backend.get_reporter_history("alice") + self.assertEqual(len(history), 1) + self.assertEqual(history[0]["login"], "alice") + self.assertEqual(history[0]["ghsa_id"], "GHSA-1111-2222-3333") + self.assertEqual(history[0]["verdict"], "CONFIRMED") + self.assertEqual(history[0]["quality"], "High") + + def test_upsert_same_ghsa(self): + """record_triage_result updates an existing record when called again for the same GHSA.""" + self.backend.record_triage_result( + login="bob", + ghsa_id="GHSA-aaaa-bbbb-cccc", + repo="owner/repo", + verdict="UNCONFIRMED", + quality="Low", + ) + # Re-triage the same advisory — should update, not duplicate + self.backend.record_triage_result( + login="bob", + ghsa_id="GHSA-aaaa-bbbb-cccc", + repo="owner/repo", + verdict="CONFIRMED", + quality="High", + ) + history = self.backend.get_reporter_history("bob") + # Should still be exactly 1 record + self.assertEqual(len(history), 1) + self.assertEqual(history[0]["verdict"], "CONFIRMED") + self.assertEqual(history[0]["quality"], "High") + + def test_get_reporter_score_empty(self): + """get_reporter_score returns zero totals for an unknown login.""" + score = self.backend.get_reporter_score("nobody") + self.assertEqual(score["total_reports"], 0) + self.assertEqual(score["confirmed_pct"], 0.0) + self.assertEqual(score["quality_breakdown"], {"High": 0, "Medium": 0, "Low": 0}) + self.assertEqual(score["recommendation"], "no history") + + def test_get_reporter_score_recommendation_skepticism(self): + """5 Low-quality UNCONFIRMED reports → recommendation is 'treat with skepticism'.""" + for i in range(5): + self.backend.record_triage_result( + login="spammer", + ghsa_id=f"GHSA-{i:04d}-0000-0000", + repo="owner/repo", + verdict="UNCONFIRMED", + quality="Low", + ) + score = self.backend.get_reporter_score("spammer") + self.assertEqual(score["recommendation"], "treat with skepticism") + self.assertEqual(score["quality_breakdown"]["Low"], 5) + self.assertEqual(score["confirmed_pct"], 0.0) + + def test_get_reporter_score_recommendation_trust(self): + """5 High-quality CONFIRMED reports → recommendation is 'high trust'.""" + for i in range(5): + self.backend.record_triage_result( + login="expert", + ghsa_id=f"GHSA-{i:04d}-1111-1111", + repo="owner/repo", + verdict="CONFIRMED", + quality="High", + ) + score = self.backend.get_reporter_score("expert") + self.assertEqual(score["recommendation"], "high trust") + self.assertEqual(score["confirmed_pct"], 1.0) + + def test_get_reporter_history_empty(self): + """get_reporter_history returns 'No history' message for unknown login.""" + from seclab_taskflows.mcp_servers.reporter_reputation import get_reporter_history + # Use the MCP tool wrapper to test the string return + # (backend method returns list; MCP tool returns string) + history = self.backend.get_reporter_history("ghost") + self.assertEqual(history, []) + + def test_multiple_reporters_isolated(self): + """Records for different reporters are independent.""" + self.backend.record_triage_result("alice", "GHSA-a", "r/r", "CONFIRMED", "High") + self.backend.record_triage_result("bob", "GHSA-b", "r/r", "UNCONFIRMED", "Low") + + alice_history = self.backend.get_reporter_history("alice") + bob_history = self.backend.get_reporter_history("bob") + + self.assertEqual(len(alice_history), 1) + self.assertEqual(len(bob_history), 1) + self.assertEqual(alice_history[0]["ghsa_id"], "GHSA-a") + self.assertEqual(bob_history[0]["ghsa_id"], "GHSA-b") + + +# --------------------------------------------------------------------------- +# TestYamlStructure +# --------------------------------------------------------------------------- + +class TestYamlStructure(unittest.TestCase): + """Tests that the new YAML files parse correctly via AvailableTools.""" + + def setUp(self): + from seclab_taskflow_agent.available_tools import AvailableTools + self.tools = AvailableTools() + + def test_pvr_triage_yaml_parses(self): + """pvr_triage.yaml loads without error and is a taskflow.""" + result = self.tools.get_taskflow("seclab_taskflows.taskflows.pvr_triage.pvr_triage") + self.assertIsNotNone(result) + header = result["seclab-taskflow-agent"] + self.assertEqual(header["filetype"], "taskflow") + + def test_pvr_respond_yaml_parses(self): + """pvr_respond.yaml loads without error and declares required globals.""" + result = self.tools.get_taskflow("seclab_taskflows.taskflows.pvr_triage.pvr_respond") + self.assertIsNotNone(result) + header = result["seclab-taskflow-agent"] + self.assertEqual(header["filetype"], "taskflow") + globals_keys = result.get("globals", {}) + self.assertIn("repo", globals_keys) + self.assertIn("ghsa", globals_keys) + self.assertIn("action", globals_keys) + + def test_pvr_triage_batch_yaml_parses(self): + """pvr_triage_batch.yaml loads without error and declares repo global.""" + result = self.tools.get_taskflow("seclab_taskflows.taskflows.pvr_triage.pvr_triage_batch") + self.assertIsNotNone(result) + header = result["seclab-taskflow-agent"] + self.assertEqual(header["filetype"], "taskflow") + globals_keys = result.get("globals", {}) + self.assertIn("repo", globals_keys) + + def test_reporter_reputation_toolbox_parses(self): + """reporter_reputation.yaml loads without error and is a toolbox.""" + result = self.tools.get_toolbox("seclab_taskflows.toolboxes.reporter_reputation") + self.assertIsNotNone(result) + header = result["seclab-taskflow-agent"] + self.assertEqual(header["filetype"], "toolbox") + + def test_pvr_ghsa_toolbox_has_confirm(self): + """pvr_ghsa.yaml toolbox declares write-back tools in confirm list.""" + result = self.tools.get_toolbox("seclab_taskflows.toolboxes.pvr_ghsa") + self.assertIsNotNone(result) + confirm = result.get("confirm", []) + self.assertIn("reject_pvr_advisory", confirm) + self.assertIn("withdraw_pvr_advisory", confirm) + self.assertIn("add_pvr_advisory_comment", confirm) + + def test_pvr_triage_yaml_has_reporter_reputation_toolbox(self): + """pvr_triage.yaml references reporter_reputation toolbox in at least one task.""" + result = self.tools.get_taskflow("seclab_taskflows.taskflows.pvr_triage.pvr_triage") + taskflow = result.get("taskflow", []) + toolbox_refs = [] + for task_wrapper in taskflow: + task = task_wrapper.get("task", {}) + toolboxes = task.get("toolboxes", []) + toolbox_refs.extend(toolboxes) + self.assertIn( + "seclab_taskflows.toolboxes.reporter_reputation", + toolbox_refs, + "pvr_triage.yaml must reference the reporter_reputation toolbox", + ) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) From 9dd96c2a6aab09e2567d2ce94f83a302ba26b7c6 Mon Sep 17 00:00:00 2001 From: Bas Alberts Date: Tue, 3 Mar 2026 11:46:05 -0500 Subject: [PATCH 04/17] Add run_pvr_triage.sh: local test and demo script for pvr triage taskflows --- scripts/run_pvr_triage.sh | 184 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100755 scripts/run_pvr_triage.sh diff --git a/scripts/run_pvr_triage.sh b/scripts/run_pvr_triage.sh new file mode 100755 index 0000000..3ee7dab --- /dev/null +++ b/scripts/run_pvr_triage.sh @@ -0,0 +1,184 @@ +#!/bin/bash +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT +# +# Local test / demo script for the PVR triage taskflows. +# +# Usage: +# ./scripts/run_pvr_triage.sh batch +# ./scripts/run_pvr_triage.sh triage +# ./scripts/run_pvr_triage.sh respond +# ./scripts/run_pvr_triage.sh demo +# +# Environment (any already-set values are respected): +# GH_TOKEN — GitHub token; falls back to: gh auth token +# AI_API_TOKEN — AI API key (required, must be set before running) +# AI_API_ENDPOINT — defaults to https://api.githubcopilot.com +# REPORT_DIR — defaults to ./reports +# LOG_DIR — defaults to ./logs + +set -euo pipefail + +__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +__root="$(cd "${__dir}/.." && pwd)" + +# --------------------------------------------------------------------------- +# Usage (defined early so --help can fire before env validation) +# --------------------------------------------------------------------------- + +usage() { + cat < [args] + +Commands: + batch + Score all draft advisories and save a ranked queue table to REPORT_DIR. + + triage + Run full triage on one advisory: verify code, generate report + response draft. + + respond + Post the response draft to GitHub. action = comment | reject | withdraw + Requires pvr_triage to have been run first for the given GHSA. + + demo + Full pipeline on the given repo (batch → triage on first draft advisory → report preview). + Does not post anything to GitHub. + +Environment: + GH_TOKEN — GitHub token; falls back to: gh auth token + AI_API_TOKEN — AI API key (required, must be set before running) + AI_API_ENDPOINT — defaults to https://api.githubcopilot.com + REPORT_DIR — defaults to ./reports + LOG_DIR — defaults to ./logs +EOF +} + +case "${1:-}" in + -h|--help|help|"") usage; exit 0 ;; +esac + +# --------------------------------------------------------------------------- +# Environment setup +# --------------------------------------------------------------------------- + +# Prepend local venv to PATH if present (resolves 'python' for MCP servers) +if [ -d "${__root}/.venv/bin" ]; then + export PATH="${__root}/.venv/bin:${PATH}" +fi + +# GitHub token +if [ -z "${GH_TOKEN:-}" ]; then + if command -v gh &>/dev/null; then + GH_TOKEN="$(gh auth token 2>/dev/null)" || true + fi + if [ -z "${GH_TOKEN:-}" ]; then + echo "ERROR: GH_TOKEN not set and 'gh auth token' failed." >&2 + exit 1 + fi + export GH_TOKEN +fi + +# AI API token +if [ -z "${AI_API_TOKEN:-}" ]; then + echo "ERROR: AI_API_TOKEN is not set." >&2 + exit 1 +fi + +export AI_API_ENDPOINT="${AI_API_ENDPOINT:-https://api.githubcopilot.com}" + +export REPORT_DIR="${REPORT_DIR:-${__root}/reports}" +mkdir -p "${REPORT_DIR}" + +export LOG_DIR="${LOG_DIR:-${__root}/logs}" +mkdir -p "${LOG_DIR}" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +run_agent() { + python -m seclab_taskflow_agent "$@" +} + +# --------------------------------------------------------------------------- +# Commands +# --------------------------------------------------------------------------- + +cmd_batch() { + local repo="${1:?Usage: $0 batch }" + echo "==> Scoring inbox for ${repo} ..." + run_agent \ + -t seclab_taskflows.taskflows.pvr_triage.pvr_triage_batch \ + -g "repo=${repo}" +} + +cmd_triage() { + local repo="${1:?Usage: $0 triage }" + local ghsa="${2:?Usage: $0 triage }" + echo "==> Triaging ${ghsa} in ${repo} ..." + run_agent \ + -t seclab_taskflows.taskflows.pvr_triage.pvr_triage \ + -g "repo=${repo}" \ + -g "ghsa=${ghsa}" +} + +cmd_respond() { + local repo="${1:?Usage: $0 respond }" + local ghsa="${2:?Usage: $0 respond }" + local action="${3:?Usage: $0 respond }" + case "${action}" in + comment|reject|withdraw) ;; + *) echo "ERROR: action must be comment, reject, or withdraw" >&2; exit 1 ;; + esac + echo "==> Responding to ${ghsa} in ${repo} (action=${action}) ..." + run_agent \ + -t seclab_taskflows.taskflows.pvr_triage.pvr_respond \ + -g "repo=${repo}" \ + -g "ghsa=${ghsa}" \ + -g "action=${action}" +} + +cmd_demo() { + local repo="${1:?Usage: $0 demo }" + + # Pick the first draft advisory, or bail if none + local ghsa + ghsa="$(gh api "/repos/${repo}/security-advisories?state=draft&per_page=1" \ + --jq '.[0].ghsa_id // empty' 2>/dev/null)" || true + + if [ -z "${ghsa}" ]; then + echo "No draft advisories found in ${repo}. Create one at:" >&2 + echo " https://github.com/${repo}/security/advisories/new" >&2 + exit 1 + fi + + echo "==> Demo: ${repo} advisory: ${ghsa}" + echo + + echo "--- Step 1: batch inbox score ---" + cmd_batch "${repo}" + echo + + echo "--- Step 2: full triage ---" + cmd_triage "${repo}" "${ghsa}" + echo + + echo "--- Reports written to ${REPORT_DIR} ---" + ls -1 "${REPORT_DIR}"/*.md 2>/dev/null || true + echo + echo "To post the response draft (comment only, does not reject):" + echo " $0 respond ${repo} ${ghsa} comment" +} + +# --------------------------------------------------------------------------- +# Dispatch +# --------------------------------------------------------------------------- + +case "${1:-}" in + batch) shift; cmd_batch "$@" ;; + triage) shift; cmd_triage "$@" ;; + respond) shift; cmd_respond "$@" ;; + demo) shift; cmd_demo "$@" ;; + *) echo "ERROR: unknown command '${1}'" >&2; usage; exit 1 ;; +esac From 0d83c6f473d70557e804f74e54f64c9872c6995b Mon Sep 17 00:00:00 2001 From: Bas Alberts Date: Tue, 3 Mar 2026 12:00:53 -0500 Subject: [PATCH 05/17] pvr_triage_batch: skip already-triaged advisories by default --- scripts/run_pvr_triage.sh | 3 ++- .../taskflows/pvr_triage/README.md | 13 +++++----- .../pvr_triage/pvr_triage_batch.yaml | 25 +++++++++++++------ 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/scripts/run_pvr_triage.sh b/scripts/run_pvr_triage.sh index 3ee7dab..130f91a 100755 --- a/scripts/run_pvr_triage.sh +++ b/scripts/run_pvr_triage.sh @@ -32,7 +32,8 @@ Usage: $(basename "$0") [args] Commands: batch - Score all draft advisories and save a ranked queue table to REPORT_DIR. + Score unprocessed draft advisories and save a ranked queue table to REPORT_DIR. + Advisories already present in REPORT_DIR are skipped. triage Run full triage on one advisory: verify code, generate report + response draft. diff --git a/src/seclab_taskflows/taskflows/pvr_triage/README.md b/src/seclab_taskflows/taskflows/pvr_triage/README.md index 4b55095..7564901 100644 --- a/src/seclab_taskflows/taskflows/pvr_triage/README.md +++ b/src/seclab_taskflows/taskflows/pvr_triage/README.md @@ -95,7 +95,7 @@ python -m seclab_taskflow_agent \ ## Taskflow 2 — Batch inbox scoring (`pvr_triage_batch`) -Lists all draft advisories for a repository, scores them by priority, and saves a ranked markdown table — useful for deciding which reports to triage first. +Lists draft advisories for a repository, scores each unprocessed one by priority, and saves a ranked markdown table. Advisories with an existing triage report in `REPORT_DIR` are skipped and their count is noted in the output. ```bash python -m seclab_taskflow_agent \ @@ -118,12 +118,13 @@ Saved to `REPORT_DIR/batch_queue__.md`: ### Priority scoring +Advisories with an existing report in `REPORT_DIR` are skipped entirely. Only unprocessed advisories are scored: + ``` -priority_score = severity_weight + quality_weight + already_triaged_penalty +priority_score = severity_weight + quality_weight -severity_weight: critical=4 high=3 medium=2 low=1 unknown=1 -quality_weight: has_file_references(+1) + has_poc(+1) + has_line_numbers(+1) -already_triaged: -3 (advisory already has a report in REPORT_DIR) +severity_weight: critical=4 high=3 medium=2 low=1 unknown=1 +quality_weight: has_file_references(+1) + has_poc(+1) + has_line_numbers(+1) ``` **Suggested actions:** @@ -134,8 +135,6 @@ already_triaged: -3 (advisory already has a report in REPORT_DIR) | ≥ 3 | Triage Soon | | 2 | Triage | | ≤ 1 | Likely Low Quality — Fast Close | -| Already triaged (CONFIRMED) | Fix/Publish | -| Already triaged (UNCONFIRMED/INCONCLUSIVE) | Review/Close | --- diff --git a/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage_batch.yaml b/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage_batch.yaml index c2ffbb3..d1422ac 100644 --- a/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage_batch.yaml +++ b/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage_batch.yaml @@ -3,9 +3,10 @@ # PVR Triage Batch Taskflow # -# Lists all draft PVR advisories for a repository, scores each one by -# priority (based on severity, quality signals, and triage status), and -# outputs a ranked markdown table to REPORT_DIR for maintainer review. +# Lists draft PVR advisories for a repository, scores each unprocessed one by +# priority (based on severity and quality signals), and outputs a ranked +# markdown table to REPORT_DIR for maintainer review. +# Advisories with an existing triage report in REPORT_DIR are skipped. # # Usage: # python -m seclab_taskflow_agent \ @@ -96,7 +97,13 @@ taskflow: priority_score, already_triaged, verdict, suggested_action} Sort the list by priority_score descending. - Store under memcache key "scored_queue". + + Split the list: + - scored_queue: entries where already_triaged=false only + - skipped_count: count of entries where already_triaged=true + + Store scored_queue under memcache key "scored_queue". + Store skipped_count under memcache key "skipped_count". # ------------------------------------------------------------------------- # Task 3: Generate and save ranked queue report @@ -110,7 +117,7 @@ taskflow: - seclab_taskflows.toolboxes.pvr_ghsa - seclab_taskflow_agent.toolboxes.memcache user_prompt: | - Retrieve "scored_queue" from memcache. + Retrieve "scored_queue" and "skipped_count" from memcache. Generate today's date in YYYY-MM-DD format. @@ -119,7 +126,8 @@ taskflow: # PVR Batch Triage Queue: {{ globals.repo }} **Generated:** [today's date] - **Total advisories:** [count] + **Pending triage:** [count of scored_queue entries] + **Skipped (already triaged):** [skipped_count] | GHSA | Severity | Vuln Type | Quality Signals | Priority | Status | Suggested Action | |------|----------|-----------|-----------------|----------|--------|-----------------| @@ -136,12 +144,15 @@ taskflow: "Not triaged" - Suggested Action: from suggested_action field + If scored_queue is empty, replace the table with: + "No pending advisories." + After the table, add a section: ## Summary List any advisories with priority_score >= 5 as "Requires immediate attention." - List any already_triaged advisories as "Previously triaged — verify closure." + If skipped_count > 0, note: "[skipped_count] already-triaged advisories skipped." Sanitize the repo name for use in a filename: replace "/" and any non-alphanumeric characters (except "-" and "_") with "_". From 0568973e77f5302c793ab5b693f984f9ed658544 Mon Sep 17 00:00:00 2001 From: Bas Alberts Date: Tue, 3 Mar 2026 12:08:39 -0500 Subject: [PATCH 06/17] Add SCORING.md: reference for batch priority, quality signals, fast-close, and reputation thresholds --- .../taskflows/pvr_triage/SCORING.md | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 src/seclab_taskflows/taskflows/pvr_triage/SCORING.md diff --git a/src/seclab_taskflows/taskflows/pvr_triage/SCORING.md b/src/seclab_taskflows/taskflows/pvr_triage/SCORING.md new file mode 100644 index 0000000..eb6935e --- /dev/null +++ b/src/seclab_taskflows/taskflows/pvr_triage/SCORING.md @@ -0,0 +1,135 @@ +# PVR Triage Scoring Reference + +This document describes every scoring decision made by the PVR triage taskflows: batch priority scoring, single-advisory quality signals, fast-close detection, and reporter reputation thresholds. All values are authoritative — they reflect the exact constants in the taskflow YAML and MCP server code. + +--- + +## 1. Batch Priority Score (`pvr_triage_batch`) + +Used to rank unprocessed draft advisories before triage. + +### Severity weight + +| Severity | Weight | +|---|---| +| critical | 4 | +| high | 3 | +| medium | 2 | +| low | 1 | +| unknown | 1 | + +### Quality weight + +Extracted from the advisory description text. Each signal present adds 1 point. + +| Signal | Condition | +|---|---| +| `has_file_references` | Description mentions at least one specific source file path | +| `has_poc` | Description includes reproduction steps or exploit code | +| `has_line_numbers` | Description cites at least one line number | + +### Formula + +``` +priority_score = severity_weight + quality_weight (max: 7) +``` + +### Suggested action thresholds + +| priority_score | Suggested action | +|---|---| +| ≥ 5 | Triage Immediately | +| ≥ 3 | Triage Soon | +| 2 | Triage | +| ≤ 1 | Likely Low Quality — Fast Close | + +### Score reference table + +| Severity | No signals | 1 signal | 2 signals | 3 signals | +|---|---|---|---|---| +| critical | 4 — Triage Soon | 5 — **Triage Immediately** | 6 — **Triage Immediately** | 7 — **Triage Immediately** | +| high | 3 — Triage Soon | 4 — Triage Soon | 5 — **Triage Immediately** | 6 — **Triage Immediately** | +| medium | 2 — Triage | 3 — Triage Soon | 4 — Triage Soon | 5 — **Triage Immediately** | +| low | 1 — Fast Close | 2 — Triage | 3 — Triage Soon | 4 — Triage Soon | + +**Key observations:** +- A bare `critical` with no quality signals scores 4 — Triage Soon, not Triage Immediately. +- `high` needs at least two quality signals to reach Triage Immediately. +- `medium` needs all three quality signals to reach Triage Immediately. +- Any `low` severity report with no quality signals is Fast Close. + +### Already-triaged advisories + +Advisories with an existing `_triage.md` in `REPORT_DIR` are skipped entirely and do not appear in the scored queue. Their count is noted in the batch report summary. + +--- + +## 2. Single-Advisory Quality Signals (`pvr_triage`) + +The quality gate in Task 3 extracts the same three signals as the batch scorer, plus two additional ones used for the report quality rating. + +| Signal | Used in | +|---|---| +| `has_file_references` | Fast-close, report quality rating | +| `has_line_numbers` | Fast-close, report quality rating | +| `has_poc` | Fast-close, report quality rating | +| `has_version_info` | Report quality rating only | +| `has_code_snippets` | Report quality rating only | + +### Report quality rating + +Assigned by the analyst in the report generation task. + +| Rating | Criteria | +|---|---| +| High | Specific, accurate claims; verified PoC; correct file paths and line numbers | +| Medium | Partially accurate; some details wrong or missing | +| Low | Vague, speculative, or significantly inaccurate ("AI slop") | + +--- + +## 3. Fast-Close Detection (`pvr_triage`) + +The quality gate triggers `fast_close=true` when **all four** conditions hold simultaneously: + +1. `has_file_references` is false +2. `has_poc` is false +3. `has_line_numbers` is false +4. At least one similar report already exists in `REPORT_DIR` with verdict `UNCONFIRMED` or `CONFIRMED` + +When `fast_close` is true, code verification is skipped entirely. The response draft uses the fast-close template (requests specific file path, line number, and reproduction steps). + +Conditions 1–3 alone are not sufficient — there must also be a prior report on a similar issue. A novel low-quality report for an unseen component proceeds to full verification. + +--- + +## 4. Reporter Reputation (`reporter_reputation.py`) + +Accumulated from every completed `pvr_triage` run. Keyed by GitHub login. + +### Inputs per record + +| Field | Values | +|---|---| +| verdict | CONFIRMED / UNCONFIRMED / INCONCLUSIVE | +| quality | High / Medium / Low | + +### Score metrics + +``` +confirmed_pct = confirmed_count / total_reports +low_share = Low_count / total_reports +``` + +### Recommendation thresholds + +| Condition | Recommendation | +|---|---| +| confirmed_pct ≥ 0.60 **and** low_share ≤ 0.20 | high trust | +| confirmed_pct ≤ 0.20 **or** low_share ≥ 0.50 | treat with skepticism | +| Otherwise | normal | +| No history | no history | + +### Effect on triage + +The reputation score is **informational only** — it appears in the triage report under Reporter Reputation but does not automatically change the verdict or trigger fast-close. A "treat with skepticism" reporter still receives full code verification unless the fast-close conditions are independently met. From cbe218464b5a4a1dab668858b7f273bbb1940f18 Mon Sep 17 00:00:00 2001 From: Bas Alberts Date: Tue, 3 Mar 2026 13:00:30 -0500 Subject: [PATCH 07/17] Address PR review feedback - Add from __future__ import annotations for Python 3.9 compat (dict|None, Path|str) - Fix REPORT_DIR empty-string handling: treat empty env var as unset - Add pagination to list_pvr_advisories; return JSON list consistently (empty list instead of string on no results) - Guard find_similar_triage_reports against empty/whitespace inputs; update docstring to reflect full-file scan - ReporterReputationBackend: use explicit "sqlite://" sentinel for in-memory; mkdir for filesystem paths instead of silent fallback - get_reporter_history MCP tool: return JSON list consistently (empty list instead of "No history" string) - pvr_ghsa.yaml: add default value for REPORT_DIR env var - pvr_triage_batch.yaml: remove dead already_triaged_penalty from scoring formula (entries are filtered out before scoring; aligns with SCORING.md) - Tests: remove unused sys/get_reporter_history imports; switch tempfile to TemporaryDirectory with tearDown cleanup; fix setUp to use sqlite:// sentinel; rename _patch_mcp_data_dir_pvr_ghsa -> _patch_report_dir - Docs: align AI_API_ENDPOINT default to https://api.githubcopilot.com across README, pvr_triage.yaml, model_config; remove pvr_respond from REPORTER_DB_DIR required-by list --- .../configs/model_config_pvr_triage.yaml | 2 +- src/seclab_taskflows/mcp_servers/pvr_ghsa.py | 52 +++++++++++++------ .../mcp_servers/reporter_reputation.py | 13 ++--- .../taskflows/pvr_triage/README.md | 6 +-- .../taskflows/pvr_triage/pvr_triage.yaml | 2 +- .../pvr_triage/pvr_triage_batch.yaml | 3 +- src/seclab_taskflows/toolboxes/pvr_ghsa.yaml | 2 +- tests/test_pvr_mcp.py | 30 +++++------ 8 files changed, 66 insertions(+), 44 deletions(-) diff --git a/src/seclab_taskflows/configs/model_config_pvr_triage.yaml b/src/seclab_taskflows/configs/model_config_pvr_triage.yaml index 0148c31..f4a3437 100644 --- a/src/seclab_taskflows/configs/model_config_pvr_triage.yaml +++ b/src/seclab_taskflows/configs/model_config_pvr_triage.yaml @@ -2,7 +2,7 @@ # SPDX-License-Identifier: MIT # PVR triage model configuration. -# Uses GitHub Copilot API endpoint by default (AI_API_ENDPOINT=https://api.githubcopilot.com). +# AI_API_ENDPOINT defaults to https://api.githubcopilot.com (set in run_pvr_triage.sh). # Override AI_API_ENDPOINT and AI_API_TOKEN for other providers. seclab-taskflow-agent: diff --git a/src/seclab_taskflows/mcp_servers/pvr_ghsa.py b/src/seclab_taskflows/mcp_servers/pvr_ghsa.py index 455f9bf..7bd04d7 100644 --- a/src/seclab_taskflows/mcp_servers/pvr_ghsa.py +++ b/src/seclab_taskflows/mcp_servers/pvr_ghsa.py @@ -7,6 +7,8 @@ # submitted via Private Vulnerability Reporting (PVR). # Uses the gh CLI for all GitHub API calls. +from __future__ import annotations + import json import logging import os @@ -18,7 +20,8 @@ from pydantic import Field from seclab_taskflow_agent.path_utils import log_file_name -REPORT_DIR = Path(os.getenv("REPORT_DIR", "reports")) +_raw_report_dir = os.getenv("REPORT_DIR") +REPORT_DIR = Path(_raw_report_dir) if _raw_report_dir and _raw_report_dir.strip() else Path("reports") logging.basicConfig( level=logging.DEBUG, @@ -160,18 +163,29 @@ def list_pvr_advisories( """ List repository security advisories, defaulting to draft state. - Returns a summary list (no description text). Each entry includes + Returns a JSON summary list (no description text). Each entry includes ghsa_id, severity, summary, state, pvr_submission, and created_at. + Returns an empty JSON list when no advisories are found. + Paginates automatically through all pages (100 items per page). """ - path = f"/repos/{owner}/{repo}/security-advisories?state={state}&per_page=100" - data, err = _gh_api(path) - if err: - return f"Error listing advisories: {err}" - if not isinstance(data, list): - return f"Unexpected response: {data}" + base_path = f"/repos/{owner}/{repo}/security-advisories?state={state}&per_page=100" + all_data: list = [] + page = 1 + while True: + data, err = _gh_api(f"{base_path}&page={page}") + if err: + return f"Error listing advisories: {err}" + if not isinstance(data, list): + return f"Unexpected response: {data}" + if not data: + break + all_data.extend(data) + if len(data) < 100: + break + page += 1 results = [] - for raw in data: + for raw in all_data: submission = raw.get("submission") or {} results.append({ "ghsa_id": raw.get("ghsa_id", ""), @@ -185,8 +199,6 @@ def list_pvr_advisories( "created_at": raw.get("created_at", ""), }) - if not results: - return f"No {state} advisories found for {owner}/{repo}." return json.dumps(results, indent=2) @@ -429,15 +441,22 @@ def find_similar_triage_reports( Search existing triage reports for similar vulnerability types and affected components. Scans REPORT_DIR for *_triage.md files and performs case-insensitive substring - matching on the header lines for vuln_type and affected_component. + matching across the full file content for vuln_type and/or affected_component. + A report matches if at least one non-empty search term is found anywhere in the file. + Returns an empty list if both terms are empty/whitespace. Returns a JSON list of matching reports with ghsa_id, verdict, quality, and path. """ if not REPORT_DIR.exists(): return json.dumps([]) + vuln_lower = vuln_type.strip().lower() + component_lower = affected_component.strip().lower() + + # Both terms empty → no meaningful search possible + if not vuln_lower and not component_lower: + return json.dumps([]) + matches = [] - vuln_lower = vuln_type.lower() - component_lower = affected_component.lower() for report_path in sorted(REPORT_DIR.glob("*_triage.md")): # Skip batch queue reports and response drafts — only match individual GHSA triage reports @@ -450,7 +469,10 @@ def find_similar_triage_reports( continue content_lower = content.lower() - if vuln_lower not in content_lower and component_lower not in content_lower: + matched = (vuln_lower and vuln_lower in content_lower) or ( + component_lower and component_lower in content_lower + ) + if not matched: continue # Extract GHSA ID from filename: {ghsa_id}_triage.md diff --git a/src/seclab_taskflows/mcp_servers/reporter_reputation.py b/src/seclab_taskflows/mcp_servers/reporter_reputation.py index 038462f..c665b3f 100644 --- a/src/seclab_taskflows/mcp_servers/reporter_reputation.py +++ b/src/seclab_taskflows/mcp_servers/reporter_reputation.py @@ -6,6 +6,8 @@ # Tracks PVR reporter history and computes reputation scores based on # past triage outcomes. Uses a local SQLite database. +from __future__ import annotations + import json import logging import os @@ -52,11 +54,12 @@ def __repr__(self) -> str: class ReporterReputationBackend: def __init__(self, db_dir: Path | str) -> None: - db_path = Path(db_dir) - if str(db_dir) == "sqlite://" or not db_path.exists(): - # In-memory database (used for tests or missing dir) + if str(db_dir) == "sqlite://": + # Explicit in-memory sentinel (used in tests) connection_string = "sqlite://" else: + db_path = Path(db_dir) + db_path.mkdir(parents=True, exist_ok=True) connection_string = f"sqlite:///{db_path}/reporter_reputation.db" self.engine = create_engine(connection_string, echo=False) Base.metadata.create_all(self.engine) @@ -184,11 +187,9 @@ def get_reporter_history( Retrieve the full triage history for a reporter. Returns a JSON list of all records for this login, newest first. - Returns a plain message string if no history is found. + Returns an empty JSON list if no history is found. """ history = backend.get_reporter_history(login) - if not history: - return f"No history for {login}." return json.dumps(history, indent=2) diff --git a/src/seclab_taskflows/taskflows/pvr_triage/README.md b/src/seclab_taskflows/taskflows/pvr_triage/README.md index 7564901..773d47c 100644 --- a/src/seclab_taskflows/taskflows/pvr_triage/README.md +++ b/src/seclab_taskflows/taskflows/pvr_triage/README.md @@ -26,17 +26,17 @@ Three taskflows cover the full triage lifecycle: |---|---|---| | `GH_TOKEN` | all | GitHub personal access token | | `AI_API_TOKEN` | all | API key for the AI provider | -| `AI_API_ENDPOINT` | all | Model endpoint (defaults to GitHub Models: `https://models.github.ai/inference`) | +| `AI_API_ENDPOINT` | all | Model endpoint (defaults to `https://api.githubcopilot.com`) | | `REPORT_DIR` | all | Directory where triage reports are written. Defaults to `./reports` | | `LOG_DIR` | all | Directory for MCP server logs. Auto-detected via `platformdirs` if not set | -| `REPORTER_DB_DIR` | `pvr_triage`, `pvr_respond` | Directory for the reporter reputation SQLite database. Auto-detected if not set | +| `REPORTER_DB_DIR` | `pvr_triage` | Directory for the reporter reputation SQLite database. Auto-detected if not set | A minimal `.env` for local use: ``` GH_TOKEN=ghp_... AI_API_TOKEN=... -AI_API_ENDPOINT=https://models.github.ai/inference +AI_API_ENDPOINT=https://api.githubcopilot.com REPORT_DIR=/path/to/reports LOG_DIR=/path/to/logs ``` diff --git a/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml b/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml index c7aed5a..497a5a5 100644 --- a/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml +++ b/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml @@ -17,7 +17,7 @@ # Required environment variables: # GH_TOKEN - GitHub token with repo and security_events scope # AI_API_TOKEN - API token for the AI model provider -# AI_API_ENDPOINT - Model provider endpoint (default: GitHub Copilot API) +# AI_API_ENDPOINT - Model provider endpoint (default: https://api.githubcopilot.com) seclab-taskflow-agent: version: "1.0" diff --git a/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage_batch.yaml b/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage_batch.yaml index d1422ac..75cbd00 100644 --- a/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage_batch.yaml +++ b/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage_batch.yaml @@ -82,8 +82,7 @@ taskflow: 4. Compute priority_score using this formula: severity_weight: critical=4, high=3, medium=2, low=1, unknown=1 quality_weight: has_file_references(+1) + has_poc(+1) + has_line_numbers(+1) - already_triaged_penalty: -3 if already_triaged else 0 - priority_score = severity_weight + quality_weight + already_triaged_penalty + priority_score = severity_weight + quality_weight 5. Determine suggested_action: - If already_triaged and verdict is UNCONFIRMED or INCONCLUSIVE: "Review/Close" - If already_triaged and verdict is CONFIRMED: "Fix/Publish" diff --git a/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml b/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml index e63b139..2479fbc 100644 --- a/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml +++ b/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml @@ -19,7 +19,7 @@ server_params: env: GH_TOKEN: "{{ env('GH_TOKEN') }}" LOG_DIR: "{{ env('LOG_DIR') }}" - REPORT_DIR: "{{ env('REPORT_DIR') }}" + REPORT_DIR: "{{ env('REPORT_DIR', 'reports') }}" # Guard write-back tools: user must confirm before execution confirm: - reject_pvr_advisory diff --git a/tests/test_pvr_mcp.py b/tests/test_pvr_mcp.py index ae15a66..c4f102d 100644 --- a/tests/test_pvr_mcp.py +++ b/tests/test_pvr_mcp.py @@ -6,7 +6,6 @@ # Run with: pytest tests/test_pvr_mcp.py -v import json -import sys import tempfile import unittest from pathlib import Path @@ -19,7 +18,7 @@ # Helpers: patch mcp_data_dir so imports don't fail in CI (no platformdirs dir) # --------------------------------------------------------------------------- -def _patch_mcp_data_dir_pvr_ghsa(tmp_path): +def _patch_report_dir(tmp_path): """Return a context manager that patches REPORT_DIR in pvr_ghsa.""" import seclab_taskflows.mcp_servers.pvr_ghsa as pvr_mod return patch.object(pvr_mod, "REPORT_DIR", tmp_path) @@ -35,7 +34,11 @@ class TestPvrGhsaTools(unittest.TestCase): def setUp(self): import seclab_taskflows.mcp_servers.pvr_ghsa as pvr_mod self.pvr = pvr_mod - self.tmp = Path(tempfile.mkdtemp()) + self.tmp_dir = tempfile.TemporaryDirectory() + self.tmp = Path(self.tmp_dir.name) + + def tearDown(self): + self.tmp_dir.cleanup() # --- reject_pvr_advisory --- @@ -144,7 +147,7 @@ def test_find_similar_reports_matches_vuln_type(self): encoding="utf-8", ) - with _patch_mcp_data_dir_pvr_ghsa(report_dir): + with _patch_report_dir(report_dir): result_json = self.pvr.find_similar_triage_reports.fn( vuln_type="path traversal", affected_component="upload handler", @@ -165,7 +168,7 @@ def test_find_similar_reports_no_matches(self): encoding="utf-8", ) - with _patch_mcp_data_dir_pvr_ghsa(report_dir): + with _patch_report_dir(report_dir): result_json = self.pvr.find_similar_triage_reports.fn( vuln_type="XSS", affected_component="login form", @@ -177,7 +180,7 @@ def test_find_similar_reports_no_matches(self): def test_find_similar_reports_empty_dir(self): """find_similar_triage_reports returns empty list for non-existent REPORT_DIR.""" empty_dir = self.tmp / "nonexistent" - with _patch_mcp_data_dir_pvr_ghsa(empty_dir): + with _patch_report_dir(empty_dir): result_json = self.pvr.find_similar_triage_reports.fn( vuln_type="IDOR", affected_component="profile", @@ -189,7 +192,7 @@ def test_find_similar_reports_empty_dir(self): def test_save_triage_report_path_sanitization(self): """save_triage_report strips path traversal characters from the GHSA ID.""" - with _patch_mcp_data_dir_pvr_ghsa(self.tmp): + with _patch_report_dir(self.tmp): out_path = self.pvr.save_triage_report.fn( ghsa_id="../../../etc/passwd", report="malicious content", @@ -209,14 +212,14 @@ def test_read_triage_report_returns_content(self): content = "## PVR Triage Analysis: GHSA-test\n\n**[CONFIRMED]**\n" (self.tmp / "GHSA-test_triage.md").write_text(content, encoding="utf-8") - with _patch_mcp_data_dir_pvr_ghsa(self.tmp): + with _patch_report_dir(self.tmp): result = self.pvr.read_triage_report.fn(ghsa_id="GHSA-test") self.assertEqual(result, content) def test_read_triage_report_missing_file(self): """read_triage_report returns an error string for a missing report.""" - with _patch_mcp_data_dir_pvr_ghsa(self.tmp): + with _patch_report_dir(self.tmp): result = self.pvr.read_triage_report.fn(ghsa_id="GHSA-does-not-exist") self.assertIn("not found", result.lower()) @@ -231,8 +234,8 @@ class TestReporterReputationBackend(unittest.TestCase): def setUp(self): from seclab_taskflows.mcp_servers.reporter_reputation import ReporterReputationBackend - # Pass a non-existent path to trigger in-memory DB fallback - self.backend = ReporterReputationBackend(db_dir=Path("/nonexistent/path")) + # Use explicit in-memory sentinel for tests + self.backend = ReporterReputationBackend(db_dir="sqlite://") def test_record_and_retrieve(self): """record_triage_result inserts a record and get_reporter_history retrieves it.""" @@ -311,10 +314,7 @@ def test_get_reporter_score_recommendation_trust(self): self.assertEqual(score["confirmed_pct"], 1.0) def test_get_reporter_history_empty(self): - """get_reporter_history returns 'No history' message for unknown login.""" - from seclab_taskflows.mcp_servers.reporter_reputation import get_reporter_history - # Use the MCP tool wrapper to test the string return - # (backend method returns list; MCP tool returns string) + """get_reporter_history returns empty list for unknown login.""" history = self.backend.get_reporter_history("ghost") self.assertEqual(history, []) From e0e29a80b1692721dbe9f61d4419df17f29efaa6 Mon Sep 17 00:00:00 2001 From: Bas Alberts Date: Tue, 3 Mar 2026 13:09:20 -0500 Subject: [PATCH 08/17] Self-review: robustness and logic fixes pvr_ghsa.py: - Remove backwards fallback quality regex in find_similar_triage_reports (r"\b(High|Medium|Low)\b.*quality" matched wrong direction; primary regex sufficient) - save_triage_report: return error on empty safe_name after sanitization - fetch_file_at_ref: cap length at 500 lines; return error if start_line exceeds file length - list_pvr_advisories: add max_pages=50 guard on pagination loop reporter_reputation.py: - Add UniqueConstraint("login","ghsa_id") to ReporterRecord table - Add VALID_VERDICTS / VALID_QUALITIES constants; validate inputs in record_triage_result - MCP record_triage_result tool: surface ValueError as error string to agent pvr_triage.yaml: - Task 5: also store triage_outcome {verdict, quality} in memcache after report generation - Tasks 7+8: use triage_outcome.verdict/quality directly instead of re-parsing report text pvr_respond.yaml, pvr_triage_batch.yaml: - Fix AI_API_ENDPOINT comment to show actual default URL tests: - Add test_save_triage_report_empty_after_sanitization - Add test_record_invalid_verdict_raises, test_record_invalid_quality_raises --- src/seclab_taskflows/mcp_servers/pvr_ghsa.py | 12 ++++++--- .../mcp_servers/reporter_reputation.py | 18 ++++++++++--- .../taskflows/pvr_triage/pvr_respond.yaml | 2 +- .../taskflows/pvr_triage/pvr_triage.yaml | 26 ++++++++++--------- .../pvr_triage/pvr_triage_batch.yaml | 2 +- tests/test_pvr_mcp.py | 19 ++++++++++++++ 6 files changed, 58 insertions(+), 21 deletions(-) diff --git a/src/seclab_taskflows/mcp_servers/pvr_ghsa.py b/src/seclab_taskflows/mcp_servers/pvr_ghsa.py index 7bd04d7..e20170f 100644 --- a/src/seclab_taskflows/mcp_servers/pvr_ghsa.py +++ b/src/seclab_taskflows/mcp_servers/pvr_ghsa.py @@ -171,7 +171,8 @@ def list_pvr_advisories( base_path = f"/repos/{owner}/{repo}/security-advisories?state={state}&per_page=100" all_data: list = [] page = 1 - while True: + max_pages = 50 # hard cap: 5000 advisories max + while page <= max_pages: data, err = _gh_api(f"{base_path}&page={page}") if err: return f"Error listing advisories: {err}" @@ -279,6 +280,9 @@ def fetch_file_at_ref( start_line = 1 if length < 1: length = 50 + length = min(length, 500) # cap to avoid returning enormous files + if start_line > len(lines): + return f"start_line {start_line} exceeds file length ({len(lines)} lines) in {path}@{ref}" chunk = lines[start_line - 1: start_line - 1 + length] if not chunk: return f"No lines in range {start_line}-{start_line + length - 1} in {path}@{ref}" @@ -300,6 +304,8 @@ def save_triage_report( REPORT_DIR.mkdir(parents=True, exist_ok=True) # Sanitize the GHSA ID to prevent path traversal safe_name = "".join(c for c in ghsa_id if c.isalnum() or c in "-_") + if not safe_name: + return "Error: ghsa_id produced an empty filename after sanitization" out_path = REPORT_DIR / f"{safe_name}_triage.md" # The agent sometimes passes the report as a JSON-encoded string # (with outer quotes and escape sequences). Decode it if so. @@ -484,11 +490,9 @@ def find_similar_triage_reports( if verdict_match: verdict = verdict_match.group(1) - # Extract quality rating + # Extract quality rating — report format: "Rate overall quality: High / Medium / Low" quality = "Unknown" quality_match = re.search(r"Rate overall quality[:\s]*\**\s*(High|Medium|Low)\b", content, re.IGNORECASE) - if not quality_match: - quality_match = re.search(r"\b(High|Medium|Low)\b.*quality", content, re.IGNORECASE) if quality_match: quality = quality_match.group(1) diff --git a/src/seclab_taskflows/mcp_servers/reporter_reputation.py b/src/seclab_taskflows/mcp_servers/reporter_reputation.py index c665b3f..5f2c299 100644 --- a/src/seclab_taskflows/mcp_servers/reporter_reputation.py +++ b/src/seclab_taskflows/mcp_servers/reporter_reputation.py @@ -17,7 +17,7 @@ from fastmcp import FastMCP from pydantic import Field from seclab_taskflow_agent.path_utils import log_file_name, mcp_data_dir -from sqlalchemy import Text, create_engine +from sqlalchemy import Text, UniqueConstraint, create_engine from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column REPORTER_DB_DIR = mcp_data_dir("seclab-taskflows", "reporter_reputation", "REPORTER_DB_DIR") @@ -34,8 +34,13 @@ class Base(DeclarativeBase): pass +VALID_VERDICTS = frozenset({"CONFIRMED", "UNCONFIRMED", "INCONCLUSIVE"}) +VALID_QUALITIES = frozenset({"High", "Medium", "Low"}) + + class ReporterRecord(Base): __tablename__ = "reporter_records" + __table_args__ = (UniqueConstraint("login", "ghsa_id", name="uq_reporter_ghsa"),) id: Mapped[int] = mapped_column(primary_key=True) login: Mapped[str] @@ -68,6 +73,10 @@ def record_triage_result( self, login: str, ghsa_id: str, repo: str, verdict: str, quality: str ) -> str: """Insert or update a triage result record for a reporter.""" + if verdict not in VALID_VERDICTS: + raise ValueError(f"Invalid verdict {verdict!r}. Must be one of {sorted(VALID_VERDICTS)}") + if quality not in VALID_QUALITIES: + raise ValueError(f"Invalid quality {quality!r}. Must be one of {sorted(VALID_QUALITIES)}") timestamp = datetime.now(timezone.utc).isoformat() with Session(self.engine) as session: existing = ( @@ -174,9 +183,12 @@ def record_triage_result( Upserts a row keyed by (login, ghsa_id). Re-running triage on the same GHSA advisory updates the existing record rather than creating a duplicate. - Returns 'recorded' on success. + Returns 'recorded' on success, or an error string for invalid inputs. """ - return backend.record_triage_result(login, ghsa_id, repo, verdict, quality) + try: + return backend.record_triage_result(login, ghsa_id, repo, verdict, quality) + except ValueError as e: + return f"Error: {e}" @mcp.tool() diff --git a/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond.yaml b/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond.yaml index 6eecc0d..5c9eefc 100644 --- a/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond.yaml +++ b/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond.yaml @@ -17,7 +17,7 @@ # Required environment variables: # GH_TOKEN - GitHub token with security_events write scope # AI_API_TOKEN - API token for the AI model provider -# AI_API_ENDPOINT - Model provider endpoint (default: GitHub Copilot API) +# AI_API_ENDPOINT - Model provider endpoint (default: https://api.githubcopilot.com) # REPORT_DIR - Directory where triage reports are stored seclab-taskflow-agent: diff --git a/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml b/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml index 497a5a5..7163e25 100644 --- a/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml +++ b/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml @@ -299,6 +299,13 @@ taskflow: Be factual. Do not include anything not supported by code evidence. Keep the report concise. Aim for under 800 words. + After generating the report, also store a structured summary under memcache + key "triage_outcome": + { + "verdict": "CONFIRMED" | "UNCONFIRMED" | "INCONCLUSIVE", + "quality": "High" | "Medium" | "Low" + } + # ------------------------------------------------------------------------- # Task 6: Save report to disk and print path # ------------------------------------------------------------------------- @@ -331,14 +338,13 @@ taskflow: toolboxes: - seclab_taskflow_agent.toolboxes.memcache user_prompt: | - Retrieve "pvr_parsed", "code_verification", "quality_gate", and "triage_report" - from memcache. + Retrieve "pvr_parsed", "code_verification", "quality_gate", "triage_report", + and "triage_outcome" from memcache. - Extract the verdict from triage_report: look for the line containing - **CONFIRMED**, **UNCONFIRMED**, or **INCONCLUSIVE** in the Verdict section. + Use triage_outcome.verdict as the verdict. Draft a response comment to the reporter. Tone: direct, factual, not harsh. - Select the template based on verdict and quality_gate.fast_close: + Select the template based on triage_outcome.verdict and quality_gate.fast_close: fast_close (quality_gate.fast_close=true): Explain that the report lacks file paths, functions, and reproduction steps @@ -377,14 +383,10 @@ taskflow: - seclab_taskflow_agent.toolboxes.memcache user_prompt: | Retrieve "pvr_parsed", "code_verification", "quality_gate", "triage_report", - and "response_draft" from memcache. - - Extract verdict: find **CONFIRMED**, **UNCONFIRMED**, or **INCONCLUSIVE** - in the triage_report Verdict section. - - Extract quality rating: find the "Rate overall quality" line in triage_report - Report Quality section and extract: High, Medium, or Low. + "triage_outcome", and "response_draft" from memcache. + Use triage_outcome.verdict as the verdict. + Use triage_outcome.quality as the quality rating. Extract reporter login from quality_gate.reporter_login. Call record_triage_result with: diff --git a/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage_batch.yaml b/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage_batch.yaml index 75cbd00..2997361 100644 --- a/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage_batch.yaml +++ b/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage_batch.yaml @@ -16,7 +16,7 @@ # Required environment variables: # GH_TOKEN - GitHub token with repo and security_events scope # AI_API_TOKEN - API token for the AI model provider -# AI_API_ENDPOINT - Model provider endpoint (default: GitHub Copilot API) +# AI_API_ENDPOINT - Model provider endpoint (default: https://api.githubcopilot.com) # REPORT_DIR - Directory where triage reports are stored (and batch output is saved) seclab-taskflow-agent: diff --git a/tests/test_pvr_mcp.py b/tests/test_pvr_mcp.py index c4f102d..18230b4 100644 --- a/tests/test_pvr_mcp.py +++ b/tests/test_pvr_mcp.py @@ -205,6 +205,15 @@ def test_save_triage_report_path_sanitization(self): self.assertFalse(".." in saved.name) self.assertFalse("/" in saved.name) + def test_save_triage_report_empty_after_sanitization(self): + """save_triage_report returns an error when ghsa_id is all special chars.""" + with _patch_report_dir(self.tmp): + result = self.pvr.save_triage_report.fn( + ghsa_id="!@#$%^&*()", + report="some content", + ) + self.assertIn("Error", result) + # --- read_triage_report --- def test_read_triage_report_returns_content(self): @@ -318,6 +327,16 @@ def test_get_reporter_history_empty(self): history = self.backend.get_reporter_history("ghost") self.assertEqual(history, []) + def test_record_invalid_verdict_raises(self): + """record_triage_result rejects unknown verdict strings.""" + with self.assertRaises(ValueError): + self.backend.record_triage_result("alice", "GHSA-x", "r/r", "MAYBE", "High") + + def test_record_invalid_quality_raises(self): + """record_triage_result rejects unknown quality strings.""" + with self.assertRaises(ValueError): + self.backend.record_triage_result("alice", "GHSA-x", "r/r", "CONFIRMED", "Excellent") + def test_multiple_reporters_isolated(self): """Records for different reporters are independent.""" self.backend.record_triage_result("alice", "GHSA-a", "r/r", "CONFIRMED", "High") From 7fd907484e2e5e3fc7c9f639309e80bfb60b2ca3 Mon Sep 17 00:00:00 2001 From: Bas Alberts Date: Tue, 3 Mar 2026 13:11:04 -0500 Subject: [PATCH 09/17] fetch_file_at_ref: raise default length from 50 to 100 lines --- src/seclab_taskflows/mcp_servers/pvr_ghsa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/seclab_taskflows/mcp_servers/pvr_ghsa.py b/src/seclab_taskflows/mcp_servers/pvr_ghsa.py index e20170f..1d6224d 100644 --- a/src/seclab_taskflows/mcp_servers/pvr_ghsa.py +++ b/src/seclab_taskflows/mcp_servers/pvr_ghsa.py @@ -250,7 +250,7 @@ def fetch_file_at_ref( path: str = Field(description="File path within the repository"), ref: str = Field(description="Git ref (commit SHA, tag, or branch) to fetch the file at"), start_line: int = Field(default=1, description="First line to return (1-indexed)"), - length: int = Field(default=50, description="Number of lines to return"), + length: int = Field(default=100, description="Number of lines to return (max 500)"), ) -> str: """ Fetch a range of lines from a file at a specific git ref (commit SHA or tag). From 9436f3d8f2db6b0651ebdbc235722341a52ac9f3 Mon Sep 17 00:00:00 2001 From: Bas Alberts Date: Tue, 3 Mar 2026 14:42:47 -0500 Subject: [PATCH 10/17] pvr-triage: add bulk respond taskflow, 3-path fast-close, reputation integration - Add pvr_respond_batch.yaml for bulk response actions - pvr_triage.yaml: 3-path fast-close (high trust / skepticism / normal) - pvr_triage_batch.yaml: created_at in scored entries, Age column + tie-break sort - pvr_respond.yaml: call mark_response_sent on success - pvr_ghsa.py: add list_pending_responses, mark_response_sent tools - README.md: update for taskflow 4, batch/respond/output-file sections - run_pvr_triage.sh: add respond_batch subcommand - tests: expand to 32 passing tests covering new tools and taskflows --- scripts/run_pvr_triage.sh | 44 ++++++-- src/seclab_taskflows/mcp_servers/pvr_ghsa.py | 56 ++++++++++ .../taskflows/pvr_triage/README.md | 81 +++++++++++--- .../taskflows/pvr_triage/pvr_respond.yaml | 3 + .../pvr_triage/pvr_respond_batch.yaml | 104 ++++++++++++++++++ .../taskflows/pvr_triage/pvr_triage.yaml | 26 ++++- .../pvr_triage/pvr_triage_batch.yaml | 17 ++- tests/test_pvr_mcp.py | 61 ++++++++++ 8 files changed, 356 insertions(+), 36 deletions(-) create mode 100644 src/seclab_taskflows/taskflows/pvr_triage/pvr_respond_batch.yaml diff --git a/scripts/run_pvr_triage.sh b/scripts/run_pvr_triage.sh index 130f91a..54d533c 100755 --- a/scripts/run_pvr_triage.sh +++ b/scripts/run_pvr_triage.sh @@ -5,10 +5,11 @@ # Local test / demo script for the PVR triage taskflows. # # Usage: -# ./scripts/run_pvr_triage.sh batch -# ./scripts/run_pvr_triage.sh triage -# ./scripts/run_pvr_triage.sh respond -# ./scripts/run_pvr_triage.sh demo +# ./scripts/run_pvr_triage.sh batch +# ./scripts/run_pvr_triage.sh triage +# ./scripts/run_pvr_triage.sh respond +# ./scripts/run_pvr_triage.sh respond_batch +# ./scripts/run_pvr_triage.sh demo # # Environment (any already-set values are respected): # GH_TOKEN — GitHub token; falls back to: gh auth token @@ -31,18 +32,22 @@ usage() { Usage: $(basename "$0") [args] Commands: - batch + batch Score unprocessed draft advisories and save a ranked queue table to REPORT_DIR. Advisories already present in REPORT_DIR are skipped. - triage + triage Run full triage on one advisory: verify code, generate report + response draft. - respond + respond Post the response draft to GitHub. action = comment | reject | withdraw Requires pvr_triage to have been run first for the given GHSA. - demo + respond_batch + Scan REPORT_DIR for all pending response drafts and post them in one session. + action = comment | reject | withdraw + + demo Full pipeline on the given repo (batch → triage on first draft advisory → report preview). Does not post anything to GitHub. @@ -140,6 +145,20 @@ cmd_respond() { -g "action=${action}" } +cmd_respond_batch() { + local repo="${1:?Usage: $0 respond_batch }" + local action="${2:?Usage: $0 respond_batch }" + case "${action}" in + comment|reject|withdraw) ;; + *) echo "ERROR: action must be comment, reject, or withdraw" >&2; exit 1 ;; + esac + echo "==> Bulk respond for ${repo} (action=${action}) ..." + run_agent \ + -t seclab_taskflows.taskflows.pvr_triage.pvr_respond_batch \ + -g "repo=${repo}" \ + -g "action=${action}" +} + cmd_demo() { local repo="${1:?Usage: $0 demo }" @@ -177,9 +196,10 @@ cmd_demo() { # --------------------------------------------------------------------------- case "${1:-}" in - batch) shift; cmd_batch "$@" ;; - triage) shift; cmd_triage "$@" ;; - respond) shift; cmd_respond "$@" ;; - demo) shift; cmd_demo "$@" ;; + batch) shift; cmd_batch "$@" ;; + triage) shift; cmd_triage "$@" ;; + respond) shift; cmd_respond "$@" ;; + respond_batch) shift; cmd_respond_batch "$@" ;; + demo) shift; cmd_demo "$@" ;; *) echo "ERROR: unknown command '${1}'" >&2; usage; exit 1 ;; esac diff --git a/src/seclab_taskflows/mcp_servers/pvr_ghsa.py b/src/seclab_taskflows/mcp_servers/pvr_ghsa.py index 1d6224d..37b4e56 100644 --- a/src/seclab_taskflows/mcp_servers/pvr_ghsa.py +++ b/src/seclab_taskflows/mcp_servers/pvr_ghsa.py @@ -14,6 +14,7 @@ import os import re import subprocess +from datetime import datetime, timezone from pathlib import Path from fastmcp import FastMCP @@ -523,5 +524,60 @@ def read_triage_report( return report_path.read_text(encoding="utf-8") +@mcp.tool() +def list_pending_responses() -> str: + """ + List advisories that have a response draft but have not yet been sent. + + Globs REPORT_DIR for *_response_triage.md files and skips any whose + corresponding *_response_sent.md marker exists. + Returns a JSON list of {ghsa_id, triage_report_exists} objects. + """ + if not REPORT_DIR.exists(): + return json.dumps([]) + + results = [] + for draft_path in sorted(REPORT_DIR.glob("*_response_triage.md")): + # stem is e.g. "GHSA-xxxx-xxxx-xxxx_response_triage" + stem = draft_path.stem + # Extract ghsa_id: remove "_response_triage" suffix + ghsa_id = stem.replace("_response_triage", "") + safe_name = "".join(c for c in ghsa_id if c.isalnum() or c in "-_") + + # Skip if sent marker exists + sent_marker = REPORT_DIR / f"{safe_name}_response_sent.md" + if sent_marker.exists(): + continue + + triage_report = REPORT_DIR / f"{safe_name}_triage.md" + results.append({ + "ghsa_id": ghsa_id, + "triage_report_exists": triage_report.exists(), + }) + + return json.dumps(results, indent=2) + + +@mcp.tool() +def mark_response_sent( + ghsa_id: str = Field(description="GHSA ID of the advisory whose response was sent"), +) -> str: + """ + Create a marker file indicating that the response for this advisory has been sent. + + Writes REPORT_DIR/{ghsa_id}_response_sent.md with an ISO timestamp. + Returns the path of the created marker, or an error string if ghsa_id is empty. + """ + safe_name = "".join(c for c in ghsa_id if c.isalnum() or c in "-_") + if not safe_name: + return "Error: ghsa_id produced an empty filename after sanitization" + REPORT_DIR.mkdir(parents=True, exist_ok=True) + marker_path = REPORT_DIR / f"{safe_name}_response_sent.md" + timestamp = datetime.now(timezone.utc).isoformat() + marker_path.write_text(f"Response sent: {timestamp}\n", encoding="utf-8") + logging.info("Response sent marker written to %s", marker_path) + return str(marker_path.resolve()) + + if __name__ == "__main__": mcp.run(show_banner=False) diff --git a/src/seclab_taskflows/taskflows/pvr_triage/README.md b/src/seclab_taskflows/taskflows/pvr_triage/README.md index 773d47c..300e504 100644 --- a/src/seclab_taskflows/taskflows/pvr_triage/README.md +++ b/src/seclab_taskflows/taskflows/pvr_triage/README.md @@ -2,13 +2,14 @@ Tools for triaging GitHub Security Advisories submitted via [Private Vulnerability Reporting (PVR)](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability). The taskflows fetch a draft advisory, verify the claimed vulnerability against actual source code, score report quality, and generate a structured analysis and a ready-to-send response draft. -Three taskflows cover the full triage lifecycle: +Four taskflows cover the full triage lifecycle: | Taskflow | Purpose | |---|---| | `pvr_triage` | Deep-analyse one advisory end-to-end | | `pvr_triage_batch` | Score an entire inbox and produce a ranked queue | -| `pvr_respond` | Post or save the response once you've reviewed the analysis | +| `pvr_respond` | Post the response for one advisory once you've reviewed the analysis | +| `pvr_respond_batch` | Scan REPORT_DIR and post all pending response drafts in a single session | --- @@ -62,7 +63,11 @@ python -m seclab_taskflow_agent \ 1. **Initialize** — clears the in-memory cache. 2. **Fetch & parse** — fetches the advisory from the GitHub API and extracts structured metadata: vulnerability type, affected component, file references, PoC quality signals, reporter credits. -3. **Quality gate** — calls `get_reporter_score` for the reporter's history and `find_similar_triage_reports` to detect duplicates. Computes a `fast_close` flag when the report has no file references, no PoC, no line numbers, *and* a similar report already exists. Fast-close skips deep code analysis. +3. **Quality gate** — calls `get_reporter_score` for the reporter's history and `find_similar_triage_reports` to detect duplicates. Computes `fast_close` using a reputation-gated decision tree: + - **high-trust reporter** → always `fast_close = false` (full verification). + - **skepticism reporter** → `fast_close = true` when all three quality signals are absent (prior similar report not required). + - **normal / no history** → `fast_close = true` only when all three signals are absent *and* a prior similar report exists. + Fast-close skips deep code analysis. 4. **Code verification** — resolves the claimed version to a git tag/SHA, fetches the relevant source files, and checks whether the vulnerability pattern is actually present. After verifying at the claimed version, also checks HEAD to determine patch status (`still_vulnerable` / `patched` / `could_not_determine`). Skipped automatically when `fast_close` is true. 5. **Report generation** — writes a markdown report covering: Verdict, Code Verification, Severity Assessment, CVSS 3.1 assessment, Duplicate/Prior Reports, Patch Status, Report Quality, Reporter Reputation, and Recommendations. 6. **Save report** — writes the report to `REPORT_DIR/_triage.md` and prints the path. @@ -110,12 +115,14 @@ Saved to `REPORT_DIR/batch_queue__.md`: ```markdown # PVR Batch Triage Queue: owner/repo -| GHSA | Severity | Vuln Type | Quality Signals | Priority | Status | Suggested Action | -|------|----------|-----------|-----------------|----------|--------|-----------------| -| GHSA-... | high | SQL injection | PoC, Files | 6 | Not triaged | Triage Immediately | -| GHSA-... | medium | XSS | None | 1 | Not triaged | Likely Low Quality — Fast Close | +| GHSA | Age (days) | Severity | Vuln Type | Quality Signals | Priority | Status | Suggested Action | +|------|------------|----------|-----------|-----------------|----------|--------|-----------------| +| GHSA-... | 14 | high | SQL injection | PoC, Files | 6 | Not triaged | Triage Immediately | +| GHSA-... | 3 | medium | XSS | None | 1 | Not triaged | Likely Low Quality — Fast Close | ``` +Rows are sorted by priority score descending; ties are broken by `created_at` ascending (oldest advisory first). + ### Priority scoring Advisories with an existing report in `REPORT_DIR` are skipped entirely. Only unprocessed advisories are scored: @@ -164,6 +171,40 @@ python -m seclab_taskflow_agent \ The toolbox marks `reject_pvr_advisory`, `withdraw_pvr_advisory`, and `add_pvr_advisory_comment` as `confirm`-gated. The agent will print the verdict, quality rating, and full response draft, then ask for explicit confirmation before making any change to GitHub. +After a successful write-back, `pvr_respond` calls `mark_response_sent` to create a `_response_sent.md` marker so `pvr_respond_batch` will skip this advisory in future runs. + +--- + +## Taskflow 4 — Bulk respond (`pvr_respond_batch`) + +Scans `REPORT_DIR` for advisories that have a response draft (`*_response_triage.md`) but no sent marker (`*_response_sent.md`), then posts each response to GitHub in a single session. + +```bash +python -m seclab_taskflow_agent \ + -t seclab_taskflows.taskflows.pvr_triage.pvr_respond_batch \ + -g repo=owner/repo \ + -g action=comment + +# or via the helper script: +./scripts/run_pvr_triage.sh respond_batch owner/repo comment +``` + +### How it works + +**Task 1** calls `list_pending_responses` (local read-only, no confirm gate) to find all unsent drafts and prints a summary table. If there are no pending drafts it stops immediately. + +**Task 2** iterates over every pending entry: +1. Reads the triage report and response draft from disk. +2. Prints a per-item preview (GHSA, verdict, first 200 chars of response). +3. Executes the chosen action (`comment` / `reject` / `withdraw`) via the confirm-gated write-back tool. +4. On success, calls `mark_response_sent` to create a `*_response_sent.md` marker so the advisory is skipped in future runs. + +Prints a final count: `"Sent N / M responses."` + +### Sent markers + +`pvr_respond` also calls `mark_response_sent` after a successful write-back, keeping single-advisory and bulk responds in sync. Once a marker exists, neither `pvr_respond` nor `pvr_respond_batch` will attempt to re-send. + --- ## Typical workflow @@ -178,10 +219,15 @@ The toolbox marks `reject_pvr_advisory`, `withdraw_pvr_advisory`, and `add_pvr_a - Check the Verdict and Code Verification sections. - Edit the response draft (_response_triage.md) if needed. -4. Run pvr_respond to send the response: - - action=comment → post reply only (advisory stays draft) - - action=reject → reject + post reply - - action=withdraw → withdraw + post reply +4a. Send responses one at a time with pvr_respond: + - action=comment → post reply only (advisory stays draft) + - action=reject → reject + post reply + - action=withdraw → withdraw + post reply + +4b. Or send all pending drafts at once with pvr_respond_batch: + Scans REPORT_DIR for unsent drafts (no _response_sent.md marker) + and posts them all in one session. + Useful after triaging a batch in step 2. ``` ### Example session @@ -202,7 +248,7 @@ python -m seclab_taskflow_agent \ cat reports/GHSA-1234-5678-abcd_triage.md cat reports/GHSA-1234-5678-abcd_response_triage.md -# Step 4a: send a comment (most common — doesn't change advisory state) +# Step 4a: send a comment for one advisory (doesn't change advisory state) python -m seclab_taskflow_agent \ -t seclab_taskflows.taskflows.pvr_triage.pvr_respond \ -g repo=acme/widget \ @@ -215,6 +261,12 @@ python -m seclab_taskflow_agent \ -g repo=acme/widget \ -g ghsa=GHSA-1234-5678-abcd \ -g action=reject + +# Step 4c: or post all pending drafts at once (after triaging several advisories) +python -m seclab_taskflow_agent \ + -t seclab_taskflows.taskflows.pvr_triage.pvr_respond_batch \ + -g repo=acme/widget \ + -g action=comment ``` --- @@ -233,7 +285,7 @@ The quality gate in Task 3 of `pvr_triage` calls `get_reporter_score` automatica | confirmed_pct ≤ 20% or Low-quality share ≥ 50% | treat with skepticism | | Otherwise | normal | -A "treat with skepticism" score alone does not trigger fast-close — it is informational. Fast-close is triggered only by the combination of missing quality signals *and* an existing duplicate report. +Reputation directly gates the fast-close decision. See [SCORING.md](SCORING.md) Section 3 for the full three-path decision table and reputation × fast-close matrix. --- @@ -258,4 +310,5 @@ All files are written to `REPORT_DIR` (default: `./reports`). |---|---|---| | `_triage.md` | `pvr_triage` task 6 | Full triage analysis report | | `_response_triage.md` | `pvr_triage` task 8 | Plain-text response draft for the reporter | -| `batch_queue__.md` | `pvr_triage_batch` task 3 | Ranked inbox table | +| `_response_sent.md` | `pvr_respond` / `pvr_respond_batch` | Marker: response has been sent (contains ISO timestamp) | +| `batch_queue__.md` | `pvr_triage_batch` task 3 | Ranked inbox table with Age column | diff --git a/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond.yaml b/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond.yaml index 5c9eefc..c08c78e 100644 --- a/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond.yaml +++ b/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond.yaml @@ -113,3 +113,6 @@ taskflow: and stop. Print the result returned by the API call. + + On success (action was not "anything else"), call mark_response_sent with + ghsa_id="{{ globals.ghsa }}" to record that this response has been sent. diff --git a/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond_batch.yaml b/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond_batch.yaml new file mode 100644 index 0000000..6abcca5 --- /dev/null +++ b/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond_batch.yaml @@ -0,0 +1,104 @@ +# SPDX-FileCopyrightText: GitHub, Inc. +# SPDX-License-Identifier: MIT + +# PVR Bulk Respond Taskflow +# +# Scans REPORT_DIR for pending response drafts (advisories with a +# *_response_triage.md but no *_response_sent.md marker) and posts +# each response to GitHub in a single session. +# +# Usage: +# python -m seclab_taskflow_agent \ +# -t seclab_taskflows.taskflows.pvr_triage.pvr_respond_batch \ +# -g repo=owner/repo \ +# -g action=comment|reject|withdraw +# +# Required environment variables: +# GH_TOKEN - GitHub token with security_events write scope +# AI_API_TOKEN - API token for the AI model provider +# AI_API_ENDPOINT - Model provider endpoint (default: https://api.githubcopilot.com) +# REPORT_DIR - Directory where triage reports are stored + +seclab-taskflow-agent: + version: "1.0" + filetype: taskflow + +model_config: seclab_taskflows.configs.model_config_pvr_triage + +globals: + # GitHub repository in owner/repo format + repo: + # Action to apply to all pending responses: comment, reject, or withdraw + action: + +taskflow: + # ------------------------------------------------------------------------- + # Task 1: List pending responses + # ------------------------------------------------------------------------- + - task: + must_complete: true + model: extraction + agents: + - seclab_taskflow_agent.personalities.assistant + toolboxes: + - seclab_taskflows.toolboxes.pvr_ghsa + - seclab_taskflow_agent.toolboxes.memcache + user_prompt: | + Call list_pending_responses to find all advisories with a response draft + that has not yet been sent. + + If the result is an empty list, print "No pending responses." and stop. + + Otherwise print a summary table: + + | GHSA | Triage Report Exists | + |------|---------------------| + [one row per pending entry] + + Store the list under memcache key "pending_responses". + + # ------------------------------------------------------------------------- + # Task 2: Send each response + # ------------------------------------------------------------------------- + - task: + must_complete: true + model: extraction + agents: + - seclab_taskflow_agent.personalities.assistant + toolboxes: + - seclab_taskflows.toolboxes.pvr_ghsa + - seclab_taskflow_agent.toolboxes.memcache + user_prompt: | + Retrieve "pending_responses" from memcache. + + Extract owner and repo from "{{ globals.repo }}" (format: owner/repo). + + The requested action is: "{{ globals.action }}" + + For each entry in pending_responses: + 1. Call read_triage_report with ghsa_id=entry.ghsa_id to get the triage report. + 2. Call read_triage_report with ghsa_id="{entry.ghsa_id}_response" to get the + response draft. + 3. Print a per-item summary: + GHSA: {entry.ghsa_id} + Verdict: [extracted from triage report] + Response preview: [first 200 chars of response draft] + 4. Execute the action: + If action is "reject": + Call reject_pvr_advisory with owner, repo, ghsa_id=entry.ghsa_id, + comment=response_draft. + If action is "withdraw": + Call withdraw_pvr_advisory with owner, repo, ghsa_id=entry.ghsa_id, + comment=response_draft. + If action is "comment": + Call add_pvr_advisory_comment with owner, repo, ghsa_id=entry.ghsa_id, + body=response_draft. + If action is anything else: + Print: "Unknown action '{{ globals.action }}'. Skipping {entry.ghsa_id}." + and continue to the next entry. + 5. On success, call mark_response_sent with ghsa_id=entry.ghsa_id. + Print: "Sent: {entry.ghsa_id}" + + After processing all entries, print: + "Sent N / M responses." where N is the count of successfully sent responses + and M is the total count of pending_responses entries. diff --git a/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml b/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml index 7163e25..3338bdb 100644 --- a/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml +++ b/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml @@ -113,11 +113,27 @@ taskflow: - vuln_type: pvr_parsed.vuln_type - affected_component: pvr_parsed.affected_component - Evaluate fast_close conditions (ALL must be true to trigger fast_close): - - pvr_parsed.quality_signals.has_file_references is false - - pvr_parsed.quality_signals.has_poc is false - - pvr_parsed.quality_signals.has_line_numbers is false - - At least one similar report exists with verdict UNCONFIRMED or CONFIRMED + Evaluate fast_close based on reporter_score.recommendation: + + If reporter_score.recommendation is "high trust": + Set fast_close = false unconditionally. + Set reason = "High-trust reporter — full verification required." + + Else if reporter_score.recommendation is "treat with skepticism": + Set fast_close = true if ALL THREE quality signals are absent: + - pvr_parsed.quality_signals.has_file_references is false + - pvr_parsed.quality_signals.has_poc is false + - pvr_parsed.quality_signals.has_line_numbers is false + (Prior similar report NOT required for skepticism reporters.) + Set reason accordingly. + + Else (normal / no history): + Set fast_close = true only if ALL FOUR conditions hold: + - pvr_parsed.quality_signals.has_file_references is false + - pvr_parsed.quality_signals.has_poc is false + - pvr_parsed.quality_signals.has_line_numbers is false + - At least one similar report exists with verdict UNCONFIRMED or CONFIRMED + Set reason accordingly. Store under memcache key "quality_gate": { diff --git a/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage_batch.yaml b/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage_batch.yaml index 2997361..2f7518b 100644 --- a/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage_batch.yaml +++ b/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage_batch.yaml @@ -93,9 +93,10 @@ taskflow: Build a list of scored entries, each with: {ghsa_id, severity, summary, vuln_type, quality_signals, - priority_score, already_triaged, verdict, suggested_action} + priority_score, already_triaged, verdict, suggested_action, created_at} - Sort the list by priority_score descending. + Sort the list: primary key priority_score descending; ties broken by + created_at ascending (oldest advisory first). Split the list: - scored_queue: entries where already_triaged=false only @@ -120,6 +121,11 @@ taskflow: Generate today's date in YYYY-MM-DD format. + For each entry in scored_queue compute days_pending: + days_pending = (today - date(created_at)).days (integer, round down) + Parse created_at as an ISO 8601 date string (YYYY-MM-DD prefix is sufficient). + If created_at is missing or unparseable, use "?" for Age. + Build a report string with this structure: # PVR Batch Triage Queue: {{ globals.repo }} @@ -128,12 +134,13 @@ taskflow: **Pending triage:** [count of scored_queue entries] **Skipped (already triaged):** [skipped_count] - | GHSA | Severity | Vuln Type | Quality Signals | Priority | Status | Suggested Action | - |------|----------|-----------|-----------------|----------|--------|-----------------| - [one row per advisory, sorted by priority_score desc] + | GHSA | Age (days) | Severity | Vuln Type | Quality Signals | Priority | Status | Suggested Action | + |------|------------|----------|-----------|-----------------|----------|--------|-----------------| + [one row per advisory, sorted by priority_score desc then created_at asc] For each row: - GHSA: the ghsa_id as a plain string + - Age (days): days_pending computed above - Severity: severity from the advisory - Vuln Type: vuln_type (truncated to 30 chars if needed) - Quality Signals: compact representation, e.g. "PoC, Files, Lines" for all three, diff --git a/tests/test_pvr_mcp.py b/tests/test_pvr_mcp.py index 18230b4..8f1da61 100644 --- a/tests/test_pvr_mcp.py +++ b/tests/test_pvr_mcp.py @@ -233,6 +233,57 @@ def test_read_triage_report_missing_file(self): self.assertIn("not found", result.lower()) + # --- list_pending_responses --- + + def test_list_pending_responses_empty(self): + """list_pending_responses returns [] when no response drafts exist.""" + with _patch_report_dir(self.tmp): + result_json = self.pvr.list_pending_responses.fn() + results = json.loads(result_json) + self.assertEqual(results, []) + + def test_list_pending_responses_returns_pending(self): + """list_pending_responses includes an entry when a draft exists but no sent marker.""" + (self.tmp / "GHSA-1111-2222-3333_response_triage.md").write_text( + "Response draft.", encoding="utf-8" + ) + with _patch_report_dir(self.tmp): + result_json = self.pvr.list_pending_responses.fn() + results = json.loads(result_json) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]["ghsa_id"], "GHSA-1111-2222-3333") + + def test_list_pending_responses_excludes_sent(self): + """list_pending_responses skips entries where a _response_sent.md marker exists.""" + (self.tmp / "GHSA-1111-2222-3333_response_triage.md").write_text( + "Response draft.", encoding="utf-8" + ) + (self.tmp / "GHSA-1111-2222-3333_response_sent.md").write_text( + "Response sent: 2026-03-03T00:00:00+00:00\n", encoding="utf-8" + ) + with _patch_report_dir(self.tmp): + result_json = self.pvr.list_pending_responses.fn() + results = json.loads(result_json) + self.assertEqual(results, []) + + # --- mark_response_sent --- + + def test_mark_response_sent_creates_marker(self): + """mark_response_sent creates a _response_sent.md marker and returns its path.""" + with _patch_report_dir(self.tmp): + result = self.pvr.mark_response_sent.fn(ghsa_id="GHSA-1111-2222-3333") + marker = self.tmp / "GHSA-1111-2222-3333_response_sent.md" + self.assertTrue(marker.exists()) + self.assertTrue(result.startswith(str(self.tmp.resolve()))) + content = marker.read_text(encoding="utf-8") + self.assertIn("Response sent:", content) + + def test_mark_response_sent_empty_ghsa_id(self): + """mark_response_sent returns an error string when ghsa_id sanitizes to empty.""" + with _patch_report_dir(self.tmp): + result = self.pvr.mark_response_sent.fn(ghsa_id="!@#$%") + self.assertIn("Error", result) + # --------------------------------------------------------------------------- # TestReporterReputationBackend @@ -405,6 +456,16 @@ def test_pvr_ghsa_toolbox_has_confirm(self): self.assertIn("withdraw_pvr_advisory", confirm) self.assertIn("add_pvr_advisory_comment", confirm) + def test_pvr_respond_batch_yaml_parses(self): + """pvr_respond_batch.yaml loads without error and declares repo + action globals.""" + result = self.tools.get_taskflow("seclab_taskflows.taskflows.pvr_triage.pvr_respond_batch") + self.assertIsNotNone(result) + header = result["seclab-taskflow-agent"] + self.assertEqual(header["filetype"], "taskflow") + globals_keys = result.get("globals", {}) + self.assertIn("repo", globals_keys) + self.assertIn("action", globals_keys) + def test_pvr_triage_yaml_has_reporter_reputation_toolbox(self): """pvr_triage.yaml references reporter_reputation toolbox in at least one task.""" result = self.tools.get_taskflow("seclab_taskflows.taskflows.pvr_triage.pvr_triage") From 57c1bbfbf35e92d52245876d7b9edfc230edce8f Mon Sep 17 00:00:00 2001 From: Bas Alberts Date: Tue, 3 Mar 2026 14:45:23 -0500 Subject: [PATCH 11/17] SCORING.md: update 3-path decision table and reputation thresholds --- .../taskflows/pvr_triage/SCORING.md | 43 +++++++++++++++++-- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/src/seclab_taskflows/taskflows/pvr_triage/SCORING.md b/src/seclab_taskflows/taskflows/pvr_triage/SCORING.md index eb6935e..7dfb241 100644 --- a/src/seclab_taskflows/taskflows/pvr_triage/SCORING.md +++ b/src/seclab_taskflows/taskflows/pvr_triage/SCORING.md @@ -90,17 +90,46 @@ Assigned by the analyst in the report generation task. ## 3. Fast-Close Detection (`pvr_triage`) -The quality gate triggers `fast_close=true` when **all four** conditions hold simultaneously: +The quality gate evaluates `fast_close` via a three-path decision tree gated on the reporter's reputation. + +### Path A — High-trust reporter + +| Condition | Result | +|---|---| +| `reporter_score.recommendation == "high trust"` | `fast_close = false` unconditionally | + +High-trust reporters always receive full code verification regardless of quality signals. + +### Path B — Skepticism reporter + +| Condition | Result | +|---|---| +| `reporter_score.recommendation == "treat with skepticism"` **and** all three signals absent | `fast_close = true` | +| `reporter_score.recommendation == "treat with skepticism"` **and** any signal present | `fast_close = false` | + +For skepticism reporters, a prior similar report is **not** required — the three absent quality signals alone are sufficient to trigger fast-close. + +### Path C — Normal / no history + +All four conditions must hold simultaneously: 1. `has_file_references` is false 2. `has_poc` is false 3. `has_line_numbers` is false 4. At least one similar report already exists in `REPORT_DIR` with verdict `UNCONFIRMED` or `CONFIRMED` -When `fast_close` is true, code verification is skipped entirely. The response draft uses the fast-close template (requests specific file path, line number, and reproduction steps). - Conditions 1–3 alone are not sufficient — there must also be a prior report on a similar issue. A novel low-quality report for an unseen component proceeds to full verification. +### Reputation × fast-close summary matrix + +| Reputation | No quality signals, no prior similar | No quality signals, prior similar exists | Any quality signal present | +|---|---|---|---| +| high trust | full verification | full verification | full verification | +| normal / no history | full verification | **fast-close** | full verification | +| treat with skepticism | **fast-close** | **fast-close** | full verification | + +When `fast_close` is true, code verification is skipped entirely. The response draft uses the fast-close template (requests specific file path, line number, and reproduction steps). + --- ## 4. Reporter Reputation (`reporter_reputation.py`) @@ -132,4 +161,10 @@ low_share = Low_count / total_reports ### Effect on triage -The reputation score is **informational only** — it appears in the triage report under Reporter Reputation but does not automatically change the verdict or trigger fast-close. A "treat with skepticism" reporter still receives full code verification unless the fast-close conditions are independently met. +The reputation score directly influences the fast-close decision (see Section 3): + +- **high trust** — always forces full code verification. +- **treat with skepticism** — lowers the fast-close bar: only three absent quality signals are needed (no prior similar report required). +- **normal / no history** — standard four-condition fast-close applies. + +The score also appears in the triage report under **Reporter Reputation** for maintainer awareness. From fcb365414ca097ad57bd5a7713a95d8fe3e02440 Mon Sep 17 00:00:00 2001 From: Bas Alberts Date: Tue, 3 Mar 2026 16:47:16 -0500 Subject: [PATCH 12/17] Fix ruff linter errors in test_pvr_mcp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PT009/PT027: test_pvr_mcp.py uses unittest.TestCase style assertions throughout. Converting 71 assertions is not warranted; suppress via per-file-ignores for tests/*. PLC0415: imports inside setUp/test methods are deliberate — needed for the patch.object pattern that avoids early import side-effects. SIM105: pvr_ghsa.py:315 try/except assigns to a variable on success; contextlib.suppress cannot capture that assignment. False positive; suppress globally. Also carry forward PLW0603 (global statement for module-level state), S101 (assert in tests), and SLF001 (private member access in tests) suppressions from the shell-toolbox branch. --- pyproject.toml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 4c6bc8f..8f5a55a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,6 +92,7 @@ ignore = [ "PLR1730", # Replace `if` statement with `min()` "PLR2004", # Magic value used in comparison "PLW0602", # Using global for variable but no assignment is done + "PLW0603", # Using the global statement to update a variable is discouraged "PLW1508", # Invalid type for environment variable default "PLW1510", # `subprocess.run` without explicit `check` argument "RET504", # Unnecessary assignment before `return` statement @@ -101,6 +102,7 @@ ignore = [ "RUF015", # Prefer `next(iter())` over single element slice "S607", # Starting a process with a partial executable path "SIM101", # Use a ternary expression instead of if-else-block + "SIM105", # Use contextlib.suppress (false positive: try block contains assignment) "SIM114", # Combine `if` branches using logical `or` operator "SIM117", # Use a single `with` statement with multiple contexts "SIM118", # Use `key in dict` instead of `key in dict.keys()` @@ -113,3 +115,12 @@ ignore = [ "W291", # Trailing whitespace "W293", # Blank line contains whitespace ] + +[tool.ruff.lint.per-file-ignores] +"tests/*" = [ + "PLC0415", # Import not at top of file (deliberate in setUp/test methods for patching) + "PT009", # Use assert instead of unittest-style assertEqual (TestCase subclass) + "PT027", # Use pytest.raises instead of assertRaises (TestCase subclass) + "S101", # Use of assert (standard in pytest) + "SLF001", # Private member accessed (tests legitimately access module internals) +] From 388f14538a7901151d15b9808096e878b9c47c03 Mon Sep 17 00:00:00 2001 From: Bas Alberts Date: Tue, 3 Mar 2026 18:32:54 -0500 Subject: [PATCH 13/17] Fix advisory state: incoming PVRs use triage state, not draft --- docs/pvr_triage_overview.md | 138 ++++++++++++++++++ scripts/run_pvr_triage.sh | 10 +- src/seclab_taskflows/mcp_servers/pvr_ghsa.py | 16 +- .../personalities/pvr_analyst.yaml | 2 +- .../taskflows/pvr_triage/README.md | 10 +- .../taskflows/pvr_triage/SCORING.md | 2 +- .../taskflows/pvr_triage/pvr_triage.yaml | 6 +- .../pvr_triage/pvr_triage_batch.yaml | 12 +- src/seclab_taskflows/toolboxes/pvr_ghsa.yaml | 4 +- 9 files changed, 169 insertions(+), 31 deletions(-) create mode 100644 docs/pvr_triage_overview.md diff --git a/docs/pvr_triage_overview.md b/docs/pvr_triage_overview.md new file mode 100644 index 0000000..af90a2a --- /dev/null +++ b/docs/pvr_triage_overview.md @@ -0,0 +1,138 @@ +# PVR Triage Taskflows — Overview + +> 30-minute sync reference. Last updated: 2026-03-03. + +--- + +## The Problem + +OSS maintainers get flooded with low-quality vulnerability reports via GitHub's Private Vulnerability Reporting (PVR). Most are vague, duplicated, or AI-generated. Reviewing each one manually is expensive. + +--- + +## The Solution: 4 Taskflows + +``` +┌─────────────────────────────────────────────────────────────┐ +│ INBOX │ +│ (GHSAs in triage state via GitHub PVR) │ +└───────────────────────┬─────────────────────────────────────┘ + │ + ▼ + ┌─────────────────────────┐ + │ pvr_triage_batch │ "What's in my inbox?" + │ │ + │ • List triage GHSAs │ + │ • Score each by │ + │ severity + quality │ + │ • Show Age (days) │ + │ • Rank: highest first │ + │ (oldest wins ties) │ + └────────────┬────────────┘ + │ ranked queue saved to REPORT_DIR + ▼ + ┌─────────────────────────┐ + │ pvr_triage │ "Is this real?" + │ (one advisory) │ + │ │ + │ Task 1: init │ + │ Task 2: fetch & parse │ + │ Task 3: quality gate ──┼──► fast-close? ──► skip to Task 7 + │ Task 4: verify code │ + │ Task 5: write report │ + │ Task 6: save report │ + │ Task 7: draft response │ + │ Task 8: save + record │ + └────────────┬────────────┘ + │ _triage.md + _response_triage.md saved + ▼ + Maintainer reviews + (edits draft if needed) + │ + ┌────────┴────────┐ + │ │ + ▼ ▼ + ┌──────────────────┐ ┌──────────────────────┐ + │ pvr_respond │ │ pvr_respond_batch │ + │ (one at a time) │ │ (all at once) │ + │ │ │ │ + │ confirm-gated: │ │ • list_pending │ + │ comment │ │ • for each: │ + │ reject │ │ - confirm-gated │ + │ withdraw │ │ write-back │ + │ │ │ - mark as sent │ + │ mark as sent │ │ • "Sent N/M" │ + └──────────────────┘ └──────────────────────┘ +``` + +--- + +## The Quality Gate (Task 3) — Key Logic + +``` +Reporter has history? + │ + ├── HIGH TRUST ──────────────────► Always full verification + │ (≥60% confirmed, ≤20% low) + │ + ├── SKEPTICISM ──────────────────► Fast-close if 0 quality signals + │ (≤20% confirmed OR ≥50% low) (no prior report needed) + │ + └── NORMAL / NEW ────────────────► Fast-close only if: + 0 quality signals + AND prior similar report exists +``` + +**Quality signals:** file paths cited · PoC provided · line numbers cited + +**Fast-close effect:** skip code verification → use canned response template requesting specifics + +--- + +## Scoring (batch) + +``` +priority_score = severity_weight + quality_weight + +severity: critical=4 high=3 medium=2 low=1 +quality: +1 per signal (files, PoC, lines) → max +3 + +≥5 Triage Immediately +≥3 Triage Soon + 2 Triage +≤1 Likely Low Quality — Fast Close +``` + +--- + +## Output Files (all in REPORT_DIR) + +| File | Written by | What it is | +|---|---|---| +| `GHSA-xxxx_triage.md` | pvr_triage | Full analysis report | +| `GHSA-xxxx_response_triage.md` | pvr_triage | Draft reply to reporter | +| `GHSA-xxxx_response_sent.md` | pvr_respond / batch | Sent marker (idempotent) | +| `batch_queue__.md` | pvr_triage_batch | Ranked inbox table | + +--- + +## Reporter Reputation (background) + +Every completed triage records **verdict + quality** against the reporter's GitHub login in a local SQLite DB. Score feeds back into the next triage's quality gate automatically. No manual configuration. + +--- + +## One-liner workflow + +```bash +./scripts/run_pvr_triage.sh batch owner/repo # see inbox +./scripts/run_pvr_triage.sh triage owner/repo GHSA-xxx # analyse one +./scripts/run_pvr_triage.sh respond_batch owner/repo comment # send all drafts +``` + +--- + +## Further reading + +- [`taskflows/pvr_triage/README.md`](../src/seclab_taskflows/taskflows/pvr_triage/README.md) — full usage docs for all four taskflows +- [`taskflows/pvr_triage/SCORING.md`](../src/seclab_taskflows/taskflows/pvr_triage/SCORING.md) — authoritative scoring reference and fast-close decision tables diff --git a/scripts/run_pvr_triage.sh b/scripts/run_pvr_triage.sh index 54d533c..8329ed5 100755 --- a/scripts/run_pvr_triage.sh +++ b/scripts/run_pvr_triage.sh @@ -33,7 +33,7 @@ Usage: $(basename "$0") [args] Commands: batch - Score unprocessed draft advisories and save a ranked queue table to REPORT_DIR. + Score unprocessed triage advisories and save a ranked queue table to REPORT_DIR. Advisories already present in REPORT_DIR are skipped. triage @@ -48,7 +48,7 @@ Commands: action = comment | reject | withdraw demo - Full pipeline on the given repo (batch → triage on first draft advisory → report preview). + Full pipeline on the given repo (batch → triage on first triage advisory → report preview). Does not post anything to GitHub. Environment: @@ -162,13 +162,13 @@ cmd_respond_batch() { cmd_demo() { local repo="${1:?Usage: $0 demo }" - # Pick the first draft advisory, or bail if none + # Pick the first triage advisory, or bail if none local ghsa - ghsa="$(gh api "/repos/${repo}/security-advisories?state=draft&per_page=1" \ + ghsa="$(gh api "/repos/${repo}/security-advisories?state=triage&per_page=1" \ --jq '.[0].ghsa_id // empty' 2>/dev/null)" || true if [ -z "${ghsa}" ]; then - echo "No draft advisories found in ${repo}. Create one at:" >&2 + echo "No triage advisories found in ${repo}. Create one at:" >&2 echo " https://github.com/${repo}/security/advisories/new" >&2 exit 1 fi diff --git a/src/seclab_taskflows/mcp_servers/pvr_ghsa.py b/src/seclab_taskflows/mcp_servers/pvr_ghsa.py index 37b4e56..09eb9da 100644 --- a/src/seclab_taskflows/mcp_servers/pvr_ghsa.py +++ b/src/seclab_taskflows/mcp_servers/pvr_ghsa.py @@ -3,8 +3,8 @@ # PVR GHSA MCP Server # -# Tools for fetching and parsing draft GitHub Security Advisories -# submitted via Private Vulnerability Reporting (PVR). +# Tools for fetching and parsing GitHub Security Advisories +# submitted via Private Vulnerability Reporting (PVR) (triage state). # Uses the gh CLI for all GitHub API calls. from __future__ import annotations @@ -142,7 +142,7 @@ def fetch_pvr_advisory( Fetch a single repository security advisory by GHSA ID. Returns structured advisory metadata and the full description text. - Works for draft advisories (requires repo or security_events scope on GH_TOKEN). + Works for advisories in triage state (requires repo or security_events scope on GH_TOKEN). """ path = f"/repos/{owner}/{repo}/security-advisories/{ghsa_id}" data, err = _gh_api(path) @@ -157,12 +157,12 @@ def list_pvr_advisories( owner: str = Field(description="Repository owner (user or org name)"), repo: str = Field(description="Repository name"), state: str = Field( - default="draft", - description="Advisory state to filter by: draft, published, rejected, or withdrawn. Default: draft", + default="triage", + description="Advisory state to filter by: triage, published, rejected, or withdrawn. Default: triage", ), ) -> str: """ - List repository security advisories, defaulting to draft state. + List repository security advisories, defaulting to triage state. Returns a JSON summary list (no description text). Each entry includes ghsa_id, severity, summary, state, pvr_submission, and created_at. @@ -386,7 +386,7 @@ def reject_pvr_advisory( comment: str = Field(description="Explanation comment to post on the advisory"), ) -> str: """ - Reject a draft security advisory and post a comment explaining the decision. + Reject a security advisory and post a comment explaining the decision. Sets the advisory state to 'rejected' via the GitHub API, then posts a comment with the provided explanation. Requires a GH_TOKEN with @@ -408,7 +408,7 @@ def withdraw_pvr_advisory( comment: str = Field(description="Explanation comment to post on the advisory"), ) -> str: """ - Withdraw a draft security advisory (for self-submitted drafts) and post a comment. + Withdraw a security advisory in triage state and post a comment. Sets the advisory state to 'withdrawn' via the GitHub API, then posts a comment with the provided explanation. Requires a GH_TOKEN with diff --git a/src/seclab_taskflows/personalities/pvr_analyst.yaml b/src/seclab_taskflows/personalities/pvr_analyst.yaml index f00f362..14daf8f 100644 --- a/src/seclab_taskflows/personalities/pvr_analyst.yaml +++ b/src/seclab_taskflows/personalities/pvr_analyst.yaml @@ -11,7 +11,7 @@ personality: | You are a security vulnerability triage analyst for an open source software maintainer. Your job is to verify vulnerability claims made in Private Vulnerability Reports (PVRs), - which arrive as draft GitHub Security Advisories (GHSAs). + which arrive as GitHub Security Advisories (GHSAs) in triage state. Core principles: - Base all conclusions on actual code evidence. Do not speculate. diff --git a/src/seclab_taskflows/taskflows/pvr_triage/README.md b/src/seclab_taskflows/taskflows/pvr_triage/README.md index 300e504..817a4f4 100644 --- a/src/seclab_taskflows/taskflows/pvr_triage/README.md +++ b/src/seclab_taskflows/taskflows/pvr_triage/README.md @@ -1,6 +1,6 @@ # PVR Triage Taskflows -Tools for triaging GitHub Security Advisories submitted via [Private Vulnerability Reporting (PVR)](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability). The taskflows fetch a draft advisory, verify the claimed vulnerability against actual source code, score report quality, and generate a structured analysis and a ready-to-send response draft. +Tools for triaging GitHub Security Advisories submitted via [Private Vulnerability Reporting (PVR)](https://docs.github.com/en/code-security/security-advisories/guidance-on-reporting-and-writing-information-about-vulnerabilities/privately-reporting-a-security-vulnerability). The taskflows fetch an advisory in triage state, verify the claimed vulnerability against actual source code, score report quality, and generate a structured analysis and a ready-to-send response draft. Four taskflows cover the full triage lifecycle: @@ -46,7 +46,7 @@ LOG_DIR=/path/to/logs ## Taskflow 1 — Single advisory triage (`pvr_triage`) -Runs a full analysis on one draft GHSA and produces: +Runs a full analysis on one GHSA in triage state and produces: - A structured triage report saved to `REPORT_DIR/_triage.md` - A response draft saved to `REPORT_DIR/_response_triage.md` @@ -100,7 +100,7 @@ python -m seclab_taskflow_agent \ ## Taskflow 2 — Batch inbox scoring (`pvr_triage_batch`) -Lists draft advisories for a repository, scores each unprocessed one by priority, and saves a ranked markdown table. Advisories with an existing triage report in `REPORT_DIR` are skipped and their count is noted in the output. +Lists advisories in triage state for a repository, scores each unprocessed one by priority, and saves a ranked markdown table. Advisories with an existing triage report in `REPORT_DIR` are skipped and their count is noted in the output. ```bash python -m seclab_taskflow_agent \ @@ -163,7 +163,7 @@ python -m seclab_taskflow_agent \ |---|---|---| | `comment` | Posts the response draft as a comment on the advisory | Default for all verdicts — sends your reply without changing state | | `reject` | Sets advisory state to `rejected`, then posts the comment | Report is clearly invalid or low quality | -| `withdraw` | Sets advisory state to `withdrawn`, then posts the comment | Your own self-submitted draft that should be removed | +| `withdraw` | Sets advisory state to `withdrawn`, then posts the comment | Your own self-submitted advisory that should be removed | > **Note:** `pvr_respond` requires that `pvr_triage` has already been run for the GHSA, so that both `_triage.md` and `_response_triage.md` exist in `REPORT_DIR`. @@ -220,7 +220,7 @@ Prints a final count: `"Sent N / M responses."` - Edit the response draft (_response_triage.md) if needed. 4a. Send responses one at a time with pvr_respond: - - action=comment → post reply only (advisory stays draft) + - action=comment → post reply only (advisory stays in triage state) - action=reject → reject + post reply - action=withdraw → withdraw + post reply diff --git a/src/seclab_taskflows/taskflows/pvr_triage/SCORING.md b/src/seclab_taskflows/taskflows/pvr_triage/SCORING.md index 7dfb241..6002250 100644 --- a/src/seclab_taskflows/taskflows/pvr_triage/SCORING.md +++ b/src/seclab_taskflows/taskflows/pvr_triage/SCORING.md @@ -6,7 +6,7 @@ This document describes every scoring decision made by the PVR triage taskflows: ## 1. Batch Priority Score (`pvr_triage_batch`) -Used to rank unprocessed draft advisories before triage. +Used to rank unprocessed advisories in triage state before analysis. ### Severity weight diff --git a/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml b/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml index 3338bdb..a3e9048 100644 --- a/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml +++ b/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage.yaml @@ -3,7 +3,7 @@ # PVR Triage Taskflow # -# Fetches a draft GHSA submitted via Private Vulnerability Reporting, +# Fetches a GHSA in triage state submitted via Private Vulnerability Reporting, # verifies the vulnerability claim against actual source code, assesses # impact and report quality, and generates a structured triage analysis # for the maintainer. @@ -28,7 +28,7 @@ model_config: seclab_taskflows.configs.model_config_pvr_triage globals: # GitHub repository in owner/repo format repo: - # GHSA ID of the draft advisory to triage + # GHSA ID of the advisory to triage ghsa: taskflow: @@ -46,7 +46,7 @@ taskflow: Clear the memory cache. # ------------------------------------------------------------------------- - # Task 2: Fetch and parse the draft GHSA + # Task 2: Fetch and parse the GHSA # ------------------------------------------------------------------------- - task: must_complete: true diff --git a/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage_batch.yaml b/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage_batch.yaml index 2f7518b..4ba2d86 100644 --- a/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage_batch.yaml +++ b/src/seclab_taskflows/taskflows/pvr_triage/pvr_triage_batch.yaml @@ -3,7 +3,7 @@ # PVR Triage Batch Taskflow # -# Lists draft PVR advisories for a repository, scores each unprocessed one by +# Lists PVR advisories in triage state for a repository, scores each unprocessed one by # priority (based on severity and quality signals), and outputs a ranked # markdown table to REPORT_DIR for maintainer review. # Advisories with an existing triage report in REPORT_DIR are skipped. @@ -31,7 +31,7 @@ globals: taskflow: # ------------------------------------------------------------------------- - # Task 1: List draft advisories + # Task 1: List triage advisories # ------------------------------------------------------------------------- - task: must_complete: true @@ -44,14 +44,14 @@ taskflow: user_prompt: | Extract owner and repo from "{{ globals.repo }}" (format: owner/repo). - Call list_pvr_advisories with owner, repo, and state="draft" to retrieve - all draft advisories. + Call list_pvr_advisories with owner, repo, and state="triage" to retrieve + all advisories in triage state. Store the full JSON list under memcache key "pvr_queue". - Print: "Found N draft advisories for {{ globals.repo }}." where N is the count. + Print: "Found N triage advisories for {{ globals.repo }}." where N is the count. - If no advisories are found, print "No draft advisories found." and stop. + If no advisories are found, print "No triage advisories found." and stop. # ------------------------------------------------------------------------- # Task 2: Score each advisory diff --git a/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml b/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml index 2479fbc..5cc74ab 100644 --- a/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml +++ b/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml @@ -3,10 +3,10 @@ # Toolbox: PVR GHSA advisory fetcher # -# Provides tools for fetching draft GitHub Security Advisories submitted +# Provides tools for fetching GitHub Security Advisories in triage state submitted # via Private Vulnerability Reporting. Uses the gh CLI for API calls. # -# Requires GH_TOKEN with repo or security_events scope to read draft advisories. +# Requires GH_TOKEN with repo or security_events scope to read advisories in triage state. seclab-taskflow-agent: version: "1.0" From c84904fde7bb74a9d294d437b77853437af1ad55 Mon Sep 17 00:00:00 2001 From: Bas Alberts Date: Tue, 3 Mar 2026 18:39:32 -0500 Subject: [PATCH 14/17] =?UTF-8?q?Fix=20advisory=20state=20API:=20reject?= =?UTF-8?q?=E2=86=92closed,=20remove=20withdraw=5Fpvr=5Fadvisory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GitHub API PATCH only accepts published/closed/draft. rejected and withdrawn are not valid PATCH states. Remove withdraw_pvr_advisory (withdrawn is read-only, reporter-initiated) and fix reject_pvr_advisory to use closed. --- docs/pvr_triage_overview.md | 2 +- scripts/run_pvr_triage.sh | 16 ++++----- src/seclab_taskflows/mcp_servers/pvr_ghsa.py | 36 ++++--------------- .../taskflows/pvr_triage/README.md | 10 +++--- .../taskflows/pvr_triage/pvr_respond.yaml | 13 ++----- .../pvr_triage/pvr_respond_batch.yaml | 7 ++-- src/seclab_taskflows/toolboxes/pvr_ghsa.yaml | 1 - tests/test_pvr_mcp.py | 36 +++---------------- 8 files changed, 30 insertions(+), 91 deletions(-) diff --git a/docs/pvr_triage_overview.md b/docs/pvr_triage_overview.md index af90a2a..bfd9ce9 100644 --- a/docs/pvr_triage_overview.md +++ b/docs/pvr_triage_overview.md @@ -59,7 +59,7 @@ OSS maintainers get flooded with low-quality vulnerability reports via GitHub's │ confirm-gated: │ │ • list_pending │ │ comment │ │ • for each: │ │ reject │ │ - confirm-gated │ - │ withdraw │ │ write-back │ + │ │ │ write-back │ │ │ │ - mark as sent │ │ mark as sent │ │ • "Sent N/M" │ └──────────────────┘ └──────────────────────┘ diff --git a/scripts/run_pvr_triage.sh b/scripts/run_pvr_triage.sh index 8329ed5..39f38b7 100755 --- a/scripts/run_pvr_triage.sh +++ b/scripts/run_pvr_triage.sh @@ -7,8 +7,8 @@ # Usage: # ./scripts/run_pvr_triage.sh batch # ./scripts/run_pvr_triage.sh triage -# ./scripts/run_pvr_triage.sh respond -# ./scripts/run_pvr_triage.sh respond_batch +# ./scripts/run_pvr_triage.sh respond +# ./scripts/run_pvr_triage.sh respond_batch # ./scripts/run_pvr_triage.sh demo # # Environment (any already-set values are respected): @@ -40,12 +40,12 @@ Commands: Run full triage on one advisory: verify code, generate report + response draft. respond - Post the response draft to GitHub. action = comment | reject | withdraw + Post the response draft to GitHub. action = comment | reject Requires pvr_triage to have been run first for the given GHSA. respond_batch Scan REPORT_DIR for all pending response drafts and post them in one session. - action = comment | reject | withdraw + action = comment | reject demo Full pipeline on the given repo (batch → triage on first triage advisory → report preview). @@ -134,8 +134,8 @@ cmd_respond() { local ghsa="${2:?Usage: $0 respond }" local action="${3:?Usage: $0 respond }" case "${action}" in - comment|reject|withdraw) ;; - *) echo "ERROR: action must be comment, reject, or withdraw" >&2; exit 1 ;; + comment|reject) ;; + *) echo "ERROR: action must be comment or reject" >&2; exit 1 ;; esac echo "==> Responding to ${ghsa} in ${repo} (action=${action}) ..." run_agent \ @@ -149,8 +149,8 @@ cmd_respond_batch() { local repo="${1:?Usage: $0 respond_batch }" local action="${2:?Usage: $0 respond_batch }" case "${action}" in - comment|reject|withdraw) ;; - *) echo "ERROR: action must be comment, reject, or withdraw" >&2; exit 1 ;; + comment|reject) ;; + *) echo "ERROR: action must be comment or reject" >&2; exit 1 ;; esac echo "==> Bulk respond for ${repo} (action=${action}) ..." run_agent \ diff --git a/src/seclab_taskflows/mcp_servers/pvr_ghsa.py b/src/seclab_taskflows/mcp_servers/pvr_ghsa.py index 09eb9da..1160805 100644 --- a/src/seclab_taskflows/mcp_servers/pvr_ghsa.py +++ b/src/seclab_taskflows/mcp_servers/pvr_ghsa.py @@ -158,7 +158,7 @@ def list_pvr_advisories( repo: str = Field(description="Repository name"), state: str = Field( default="triage", - description="Advisory state to filter by: triage, published, rejected, or withdrawn. Default: triage", + description="Advisory state to filter by: triage, draft, published, closed, or withdrawn. Default: triage", ), ) -> str: """ @@ -328,7 +328,7 @@ def _post_advisory_comment(owner: str, repo: str, ghsa_id: str, body: str) -> st Attempts to use the GitHub advisory comments API. If that endpoint is not available, falls back to appending a '## Maintainer Response' section to the advisory description instead. Called by both the MCP tool wrapper and the - reject/withdraw tools so they all share the same logic without going through + reject_pvr_advisory so they all share the same logic without going through the FunctionTool wrapper. """ comment_path = f"/repos/{owner}/{repo}/security-advisories/{ghsa_id}/comments" @@ -386,40 +386,18 @@ def reject_pvr_advisory( comment: str = Field(description="Explanation comment to post on the advisory"), ) -> str: """ - Reject a security advisory and post a comment explaining the decision. + Close (reject) a security advisory and post a comment explaining the decision. - Sets the advisory state to 'rejected' via the GitHub API, then posts a + Sets the advisory state to 'closed' via the GitHub API, then posts a comment with the provided explanation. Requires a GH_TOKEN with security_events write scope. """ path = f"/repos/{owner}/{repo}/security-advisories/{ghsa_id}" - _, err = _gh_api(path, method="PATCH", body={"state": "rejected"}) + _, err = _gh_api(path, method="PATCH", body={"state": "closed"}) if err: - return f"Error rejecting advisory {ghsa_id}: {err}" + return f"Error closing advisory {ghsa_id}: {err}" result = _post_advisory_comment(owner, repo, ghsa_id, comment) - return f"Advisory {ghsa_id} rejected. Comment: {result}" - - -@mcp.tool() -def withdraw_pvr_advisory( - owner: str = Field(description="Repository owner (user or org name)"), - repo: str = Field(description="Repository name"), - ghsa_id: str = Field(description="GHSA ID of the advisory, e.g. GHSA-xxxx-xxxx-xxxx"), - comment: str = Field(description="Explanation comment to post on the advisory"), -) -> str: - """ - Withdraw a security advisory in triage state and post a comment. - - Sets the advisory state to 'withdrawn' via the GitHub API, then posts a - comment with the provided explanation. Requires a GH_TOKEN with - security_events write scope. - """ - path = f"/repos/{owner}/{repo}/security-advisories/{ghsa_id}" - _, err = _gh_api(path, method="PATCH", body={"state": "withdrawn"}) - if err: - return f"Error withdrawing advisory {ghsa_id}: {err}" - result = _post_advisory_comment(owner, repo, ghsa_id, comment) - return f"Advisory {ghsa_id} withdrawn. Comment: {result}" + return f"Advisory {ghsa_id} closed. Comment: {result}" @mcp.tool() diff --git a/src/seclab_taskflows/taskflows/pvr_triage/README.md b/src/seclab_taskflows/taskflows/pvr_triage/README.md index 817a4f4..f31db11 100644 --- a/src/seclab_taskflows/taskflows/pvr_triage/README.md +++ b/src/seclab_taskflows/taskflows/pvr_triage/README.md @@ -162,14 +162,13 @@ python -m seclab_taskflow_agent \ | `action` | API call | When to use | |---|---|---| | `comment` | Posts the response draft as a comment on the advisory | Default for all verdicts — sends your reply without changing state | -| `reject` | Sets advisory state to `rejected`, then posts the comment | Report is clearly invalid or low quality | -| `withdraw` | Sets advisory state to `withdrawn`, then posts the comment | Your own self-submitted advisory that should be removed | +| `reject` | Sets advisory state to `closed`, then posts the comment | Report is clearly invalid or low quality | > **Note:** `pvr_respond` requires that `pvr_triage` has already been run for the GHSA, so that both `_triage.md` and `_response_triage.md` exist in `REPORT_DIR`. ### Confirm gate -The toolbox marks `reject_pvr_advisory`, `withdraw_pvr_advisory`, and `add_pvr_advisory_comment` as `confirm`-gated. The agent will print the verdict, quality rating, and full response draft, then ask for explicit confirmation before making any change to GitHub. +The toolbox marks `reject_pvr_advisory` and `add_pvr_advisory_comment` as `confirm`-gated. The agent will print the verdict, quality rating, and full response draft, then ask for explicit confirmation before making any change to GitHub. After a successful write-back, `pvr_respond` calls `mark_response_sent` to create a `_response_sent.md` marker so `pvr_respond_batch` will skip this advisory in future runs. @@ -196,7 +195,7 @@ python -m seclab_taskflow_agent \ **Task 2** iterates over every pending entry: 1. Reads the triage report and response draft from disk. 2. Prints a per-item preview (GHSA, verdict, first 200 chars of response). -3. Executes the chosen action (`comment` / `reject` / `withdraw`) via the confirm-gated write-back tool. +3. Executes the chosen action (`comment` / `reject`) via the confirm-gated write-back tool. 4. On success, calls `mark_response_sent` to create a `*_response_sent.md` marker so the advisory is skipped in future runs. Prints a final count: `"Sent N / M responses."` @@ -221,8 +220,7 @@ Prints a final count: `"Sent N / M responses."` 4a. Send responses one at a time with pvr_respond: - action=comment → post reply only (advisory stays in triage state) - - action=reject → reject + post reply - - action=withdraw → withdraw + post reply + - action=reject → close + post reply 4b. Or send all pending drafts at once with pvr_respond_batch: Scans REPORT_DIR for unsent drafts (no _response_sent.md marker) diff --git a/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond.yaml b/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond.yaml index c08c78e..2868f5c 100644 --- a/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond.yaml +++ b/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond.yaml @@ -12,7 +12,7 @@ # -t seclab_taskflows.taskflows.pvr_triage.pvr_respond \ # -g repo=owner/repo \ # -g ghsa=GHSA-xxxx-xxxx-xxxx \ -# -g action=reject|comment|withdraw +# -g action=reject|comment # # Required environment variables: # GH_TOKEN - GitHub token with security_events write scope @@ -31,7 +31,7 @@ globals: repo: # GHSA ID of the advisory to act on ghsa: - # Action to perform: reject, comment, or withdraw + # Action to perform: reject or comment action: taskflow: @@ -94,13 +94,6 @@ taskflow: - ghsa_id: "{{ globals.ghsa }}" - comment: response_draft - If action is "withdraw": - Call withdraw_pvr_advisory with: - - owner: extracted owner - - repo: extracted repo - - ghsa_id: "{{ globals.ghsa }}" - - comment: response_draft - If action is "comment": Call add_pvr_advisory_comment with: - owner: extracted owner @@ -109,7 +102,7 @@ taskflow: - body: response_draft If action is anything else: - Print: "Unknown action '{{ globals.action }}'. Valid actions: reject, comment, withdraw" + Print: "Unknown action '{{ globals.action }}'. Valid actions: reject, comment" and stop. Print the result returned by the API call. diff --git a/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond_batch.yaml b/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond_batch.yaml index 6abcca5..e1adb9c 100644 --- a/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond_batch.yaml +++ b/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond_batch.yaml @@ -11,7 +11,7 @@ # python -m seclab_taskflow_agent \ # -t seclab_taskflows.taskflows.pvr_triage.pvr_respond_batch \ # -g repo=owner/repo \ -# -g action=comment|reject|withdraw +# -g action=comment|reject # # Required environment variables: # GH_TOKEN - GitHub token with security_events write scope @@ -28,7 +28,7 @@ model_config: seclab_taskflows.configs.model_config_pvr_triage globals: # GitHub repository in owner/repo format repo: - # Action to apply to all pending responses: comment, reject, or withdraw + # Action to apply to all pending responses: comment or reject action: taskflow: @@ -87,9 +87,6 @@ taskflow: If action is "reject": Call reject_pvr_advisory with owner, repo, ghsa_id=entry.ghsa_id, comment=response_draft. - If action is "withdraw": - Call withdraw_pvr_advisory with owner, repo, ghsa_id=entry.ghsa_id, - comment=response_draft. If action is "comment": Call add_pvr_advisory_comment with owner, repo, ghsa_id=entry.ghsa_id, body=response_draft. diff --git a/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml b/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml index 5cc74ab..fba386f 100644 --- a/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml +++ b/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml @@ -23,5 +23,4 @@ server_params: # Guard write-back tools: user must confirm before execution confirm: - reject_pvr_advisory - - withdraw_pvr_advisory - add_pvr_advisory_comment diff --git a/tests/test_pvr_mcp.py b/tests/test_pvr_mcp.py index 8f1da61..0d2fde1 100644 --- a/tests/test_pvr_mcp.py +++ b/tests/test_pvr_mcp.py @@ -43,13 +43,13 @@ def tearDown(self): # --- reject_pvr_advisory --- def test_reject_pvr_advisory_calls_correct_api(self): - """reject_pvr_advisory should PATCH state=rejected then post a comment.""" + """reject_pvr_advisory should PATCH state=closed then post a comment.""" calls = [] def fake_gh_api(path, method="GET", body=None): calls.append({"path": path, "method": method, "body": body}) if method == "PATCH": - return {"ghsa_id": "GHSA-1234-5678-abcd", "state": "rejected"}, None + return {"ghsa_id": "GHSA-1234-5678-abcd", "state": "closed"}, None return {}, None with patch.object(self.pvr, "_gh_api", side_effect=fake_gh_api): @@ -61,36 +61,11 @@ def fake_gh_api(path, method="GET", body=None): comment="Rejecting: not a valid report.", ) - # First call must be the PATCH to set state=rejected + # First call must be the PATCH to set state=closed self.assertEqual(calls[0]["method"], "PATCH") self.assertIn("GHSA-1234-5678-abcd", calls[0]["path"]) - self.assertEqual(calls[0]["body"], {"state": "rejected"}) - self.assertIn("rejected", result) - - # --- withdraw_pvr_advisory --- - - def test_withdraw_pvr_advisory_calls_correct_api(self): - """withdraw_pvr_advisory should PATCH state=withdrawn.""" - calls = [] - - def fake_gh_api(path, method="GET", body=None): - calls.append({"path": path, "method": method, "body": body}) - if method == "PATCH": - return {"ghsa_id": "GHSA-1234-5678-abcd", "state": "withdrawn"}, None - return {}, None - - with patch.object(self.pvr, "_gh_api", side_effect=fake_gh_api): - with patch.object(self.pvr, "_post_advisory_comment", return_value="Comment posted: https://github.com/test"): - result = self.pvr.withdraw_pvr_advisory.fn( - owner="owner", - repo="repo", - ghsa_id="GHSA-1234-5678-abcd", - comment="Withdrawing self-submitted draft.", - ) - - self.assertEqual(calls[0]["method"], "PATCH") - self.assertEqual(calls[0]["body"], {"state": "withdrawn"}) - self.assertIn("withdrawn", result) + self.assertEqual(calls[0]["body"], {"state": "closed"}) + self.assertIn("closed", result) # --- add_pvr_advisory_comment --- @@ -453,7 +428,6 @@ def test_pvr_ghsa_toolbox_has_confirm(self): self.assertIsNotNone(result) confirm = result.get("confirm", []) self.assertIn("reject_pvr_advisory", confirm) - self.assertIn("withdraw_pvr_advisory", confirm) self.assertIn("add_pvr_advisory_comment", confirm) def test_pvr_respond_batch_yaml_parses(self): From f5b9d2660d380c6abc77ab3124f1ec7b8f62b3d3 Mon Sep 17 00:00:00 2001 From: Bas Alberts Date: Tue, 3 Mar 2026 18:42:37 -0500 Subject: [PATCH 15/17] =?UTF-8?q?Add=20accept=5Fpvr=5Fadvisory:=20triage?= =?UTF-8?q?=E2=86=92draft=20state=20transition?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Accepting a PVR (confirmed vulnerability) moves the advisory from triage to draft state. New tool accept_pvr_advisory PATCHes state=draft and posts a comment. Added as action=accept throughout respond/respond_batch taskflows, runner script, and docs. --- docs/pvr_triage_overview.md | 5 ++-- scripts/run_pvr_triage.sh | 16 +++++------ src/seclab_taskflows/mcp_servers/pvr_ghsa.py | 23 ++++++++++++++++ .../taskflows/pvr_triage/README.md | 6 +++-- .../taskflows/pvr_triage/pvr_respond.yaml | 13 ++++++--- .../pvr_triage/pvr_respond_batch.yaml | 7 +++-- src/seclab_taskflows/toolboxes/pvr_ghsa.yaml | 1 + tests/test_pvr_mcp.py | 27 +++++++++++++++++++ 8 files changed, 81 insertions(+), 17 deletions(-) diff --git a/docs/pvr_triage_overview.md b/docs/pvr_triage_overview.md index bfd9ce9..d743406 100644 --- a/docs/pvr_triage_overview.md +++ b/docs/pvr_triage_overview.md @@ -57,8 +57,9 @@ OSS maintainers get flooded with low-quality vulnerability reports via GitHub's │ (one at a time) │ │ (all at once) │ │ │ │ │ │ confirm-gated: │ │ • list_pending │ - │ comment │ │ • for each: │ - │ reject │ │ - confirm-gated │ + │ accept │ │ • for each: │ + │ comment │ │ - confirm-gated │ + │ reject │ │ │ │ │ write-back │ │ │ │ - mark as sent │ │ mark as sent │ │ • "Sent N/M" │ diff --git a/scripts/run_pvr_triage.sh b/scripts/run_pvr_triage.sh index 39f38b7..703a5d3 100755 --- a/scripts/run_pvr_triage.sh +++ b/scripts/run_pvr_triage.sh @@ -7,8 +7,8 @@ # Usage: # ./scripts/run_pvr_triage.sh batch # ./scripts/run_pvr_triage.sh triage -# ./scripts/run_pvr_triage.sh respond -# ./scripts/run_pvr_triage.sh respond_batch +# ./scripts/run_pvr_triage.sh respond +# ./scripts/run_pvr_triage.sh respond_batch # ./scripts/run_pvr_triage.sh demo # # Environment (any already-set values are respected): @@ -40,12 +40,12 @@ Commands: Run full triage on one advisory: verify code, generate report + response draft. respond - Post the response draft to GitHub. action = comment | reject + Post the response draft to GitHub. action = accept | comment | reject Requires pvr_triage to have been run first for the given GHSA. respond_batch Scan REPORT_DIR for all pending response drafts and post them in one session. - action = comment | reject + action = accept | comment | reject demo Full pipeline on the given repo (batch → triage on first triage advisory → report preview). @@ -134,8 +134,8 @@ cmd_respond() { local ghsa="${2:?Usage: $0 respond }" local action="${3:?Usage: $0 respond }" case "${action}" in - comment|reject) ;; - *) echo "ERROR: action must be comment or reject" >&2; exit 1 ;; + accept|comment|reject) ;; + *) echo "ERROR: action must be accept, comment, or reject" >&2; exit 1 ;; esac echo "==> Responding to ${ghsa} in ${repo} (action=${action}) ..." run_agent \ @@ -149,8 +149,8 @@ cmd_respond_batch() { local repo="${1:?Usage: $0 respond_batch }" local action="${2:?Usage: $0 respond_batch }" case "${action}" in - comment|reject) ;; - *) echo "ERROR: action must be comment or reject" >&2; exit 1 ;; + accept|comment|reject) ;; + *) echo "ERROR: action must be accept, comment, or reject" >&2; exit 1 ;; esac echo "==> Bulk respond for ${repo} (action=${action}) ..." run_agent \ diff --git a/src/seclab_taskflows/mcp_servers/pvr_ghsa.py b/src/seclab_taskflows/mcp_servers/pvr_ghsa.py index 1160805..62d6737 100644 --- a/src/seclab_taskflows/mcp_servers/pvr_ghsa.py +++ b/src/seclab_taskflows/mcp_servers/pvr_ghsa.py @@ -400,6 +400,29 @@ def reject_pvr_advisory( return f"Advisory {ghsa_id} closed. Comment: {result}" +@mcp.tool() +def accept_pvr_advisory( + owner: str = Field(description="Repository owner (user or org name)"), + repo: str = Field(description="Repository name"), + ghsa_id: str = Field(description="GHSA ID of the advisory, e.g. GHSA-xxxx-xxxx-xxxx"), + comment: str = Field(description="Acknowledgement comment to post on the advisory"), +) -> str: + """ + Accept a PVR advisory by moving it from triage to draft state, then post a comment. + + Sets the advisory state to 'draft' via the GitHub API (triage → draft transition), + then posts a comment. Use this when the vulnerability is confirmed and the maintainer + intends to publish a security advisory. Requires a GH_TOKEN with security_events + write scope. + """ + path = f"/repos/{owner}/{repo}/security-advisories/{ghsa_id}" + _, err = _gh_api(path, method="PATCH", body={"state": "draft"}) + if err: + return f"Error accepting advisory {ghsa_id}: {err}" + result = _post_advisory_comment(owner, repo, ghsa_id, comment) + return f"Advisory {ghsa_id} accepted (moved to draft). Comment: {result}" + + @mcp.tool() def add_pvr_advisory_comment( owner: str = Field(description="Repository owner (user or org name)"), diff --git a/src/seclab_taskflows/taskflows/pvr_triage/README.md b/src/seclab_taskflows/taskflows/pvr_triage/README.md index f31db11..d9416d6 100644 --- a/src/seclab_taskflows/taskflows/pvr_triage/README.md +++ b/src/seclab_taskflows/taskflows/pvr_triage/README.md @@ -161,6 +161,7 @@ python -m seclab_taskflow_agent \ | `action` | API call | When to use | |---|---|---| +| `accept` | Sets advisory state to `draft` (triage → draft), then posts the comment | Vulnerability confirmed — maintainer intends to publish an advisory | | `comment` | Posts the response draft as a comment on the advisory | Default for all verdicts — sends your reply without changing state | | `reject` | Sets advisory state to `closed`, then posts the comment | Report is clearly invalid or low quality | @@ -168,7 +169,7 @@ python -m seclab_taskflow_agent \ ### Confirm gate -The toolbox marks `reject_pvr_advisory` and `add_pvr_advisory_comment` as `confirm`-gated. The agent will print the verdict, quality rating, and full response draft, then ask for explicit confirmation before making any change to GitHub. +The toolbox marks `accept_pvr_advisory`, `reject_pvr_advisory`, and `add_pvr_advisory_comment` as `confirm`-gated. The agent will print the verdict, quality rating, and full response draft, then ask for explicit confirmation before making any change to GitHub. After a successful write-back, `pvr_respond` calls `mark_response_sent` to create a `_response_sent.md` marker so `pvr_respond_batch` will skip this advisory in future runs. @@ -195,7 +196,7 @@ python -m seclab_taskflow_agent \ **Task 2** iterates over every pending entry: 1. Reads the triage report and response draft from disk. 2. Prints a per-item preview (GHSA, verdict, first 200 chars of response). -3. Executes the chosen action (`comment` / `reject`) via the confirm-gated write-back tool. +3. Executes the chosen action (`accept` / `comment` / `reject`) via the confirm-gated write-back tool. 4. On success, calls `mark_response_sent` to create a `*_response_sent.md` marker so the advisory is skipped in future runs. Prints a final count: `"Sent N / M responses."` @@ -219,6 +220,7 @@ Prints a final count: `"Sent N / M responses."` - Edit the response draft (_response_triage.md) if needed. 4a. Send responses one at a time with pvr_respond: + - action=accept → move to draft (triage → draft) + post reply - action=comment → post reply only (advisory stays in triage state) - action=reject → close + post reply diff --git a/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond.yaml b/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond.yaml index 2868f5c..7880d7d 100644 --- a/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond.yaml +++ b/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond.yaml @@ -12,7 +12,7 @@ # -t seclab_taskflows.taskflows.pvr_triage.pvr_respond \ # -g repo=owner/repo \ # -g ghsa=GHSA-xxxx-xxxx-xxxx \ -# -g action=reject|comment +# -g action=accept|reject|comment # # Required environment variables: # GH_TOKEN - GitHub token with security_events write scope @@ -31,7 +31,7 @@ globals: repo: # GHSA ID of the advisory to act on ghsa: - # Action to perform: reject or comment + # Action to perform: accept, reject, or comment action: taskflow: @@ -87,6 +87,13 @@ taskflow: Execute the action as follows: + If action is "accept": + Call accept_pvr_advisory with: + - owner: extracted owner + - repo: extracted repo + - ghsa_id: "{{ globals.ghsa }}" + - comment: response_draft + If action is "reject": Call reject_pvr_advisory with: - owner: extracted owner @@ -102,7 +109,7 @@ taskflow: - body: response_draft If action is anything else: - Print: "Unknown action '{{ globals.action }}'. Valid actions: reject, comment" + Print: "Unknown action '{{ globals.action }}'. Valid actions: accept, reject, comment" and stop. Print the result returned by the API call. diff --git a/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond_batch.yaml b/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond_batch.yaml index e1adb9c..5a055fc 100644 --- a/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond_batch.yaml +++ b/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond_batch.yaml @@ -11,7 +11,7 @@ # python -m seclab_taskflow_agent \ # -t seclab_taskflows.taskflows.pvr_triage.pvr_respond_batch \ # -g repo=owner/repo \ -# -g action=comment|reject +# -g action=accept|comment|reject # # Required environment variables: # GH_TOKEN - GitHub token with security_events write scope @@ -28,7 +28,7 @@ model_config: seclab_taskflows.configs.model_config_pvr_triage globals: # GitHub repository in owner/repo format repo: - # Action to apply to all pending responses: comment or reject + # Action to apply to all pending responses: accept, comment, or reject action: taskflow: @@ -84,6 +84,9 @@ taskflow: Verdict: [extracted from triage report] Response preview: [first 200 chars of response draft] 4. Execute the action: + If action is "accept": + Call accept_pvr_advisory with owner, repo, ghsa_id=entry.ghsa_id, + comment=response_draft. If action is "reject": Call reject_pvr_advisory with owner, repo, ghsa_id=entry.ghsa_id, comment=response_draft. diff --git a/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml b/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml index fba386f..e36dc46 100644 --- a/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml +++ b/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml @@ -22,5 +22,6 @@ server_params: REPORT_DIR: "{{ env('REPORT_DIR', 'reports') }}" # Guard write-back tools: user must confirm before execution confirm: + - accept_pvr_advisory - reject_pvr_advisory - add_pvr_advisory_comment diff --git a/tests/test_pvr_mcp.py b/tests/test_pvr_mcp.py index 0d2fde1..1922357 100644 --- a/tests/test_pvr_mcp.py +++ b/tests/test_pvr_mcp.py @@ -40,6 +40,32 @@ def setUp(self): def tearDown(self): self.tmp_dir.cleanup() + # --- accept_pvr_advisory --- + + def test_accept_pvr_advisory_calls_correct_api(self): + """accept_pvr_advisory should PATCH state=draft then post a comment.""" + calls = [] + + def fake_gh_api(path, method="GET", body=None): + calls.append({"path": path, "method": method, "body": body}) + if method == "PATCH": + return {"ghsa_id": "GHSA-1234-5678-abcd", "state": "draft"}, None + return {}, None + + with patch.object(self.pvr, "_gh_api", side_effect=fake_gh_api): + with patch.object(self.pvr, "_post_advisory_comment", return_value="Comment posted: https://github.com/test"): + result = self.pvr.accept_pvr_advisory.fn( + owner="owner", + repo="repo", + ghsa_id="GHSA-1234-5678-abcd", + comment="Confirmed. We'll publish an advisory.", + ) + + self.assertEqual(calls[0]["method"], "PATCH") + self.assertIn("GHSA-1234-5678-abcd", calls[0]["path"]) + self.assertEqual(calls[0]["body"], {"state": "draft"}) + self.assertIn("draft", result) + # --- reject_pvr_advisory --- def test_reject_pvr_advisory_calls_correct_api(self): @@ -427,6 +453,7 @@ def test_pvr_ghsa_toolbox_has_confirm(self): result = self.tools.get_toolbox("seclab_taskflows.toolboxes.pvr_ghsa") self.assertIsNotNone(result) confirm = result.get("confirm", []) + self.assertIn("accept_pvr_advisory", confirm) self.assertIn("reject_pvr_advisory", confirm) self.assertIn("add_pvr_advisory_comment", confirm) From 2c331cbf65e6d52a01261518710b1f4957998d88 Mon Sep 17 00:00:00 2001 From: Bas Alberts Date: Tue, 3 Mar 2026 18:43:55 -0500 Subject: [PATCH 16/17] Update overview doc: accept/reject state transitions in diagram and one-liners --- docs/pvr_triage_overview.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/pvr_triage_overview.md b/docs/pvr_triage_overview.md index d743406..a8c17eb 100644 --- a/docs/pvr_triage_overview.md +++ b/docs/pvr_triage_overview.md @@ -57,10 +57,9 @@ OSS maintainers get flooded with low-quality vulnerability reports via GitHub's │ (one at a time) │ │ (all at once) │ │ │ │ │ │ confirm-gated: │ │ • list_pending │ - │ accept │ │ • for each: │ + │ accept (→draft) │ │ • for each: │ │ comment │ │ - confirm-gated │ - │ reject │ │ - │ │ │ write-back │ + │ reject (→closed)│ │ write-back │ │ │ │ - mark as sent │ │ mark as sent │ │ • "Sent N/M" │ └──────────────────┘ └──────────────────────┘ @@ -128,7 +127,8 @@ Every completed triage records **verdict + quality** against the reporter's GitH ```bash ./scripts/run_pvr_triage.sh batch owner/repo # see inbox ./scripts/run_pvr_triage.sh triage owner/repo GHSA-xxx # analyse one -./scripts/run_pvr_triage.sh respond_batch owner/repo comment # send all drafts +./scripts/run_pvr_triage.sh respond owner/repo GHSA-xxx accept # accept one +./scripts/run_pvr_triage.sh respond_batch owner/repo comment # send all drafts ``` --- From c1b5dbae0071b33f71a576a26673185766b49be5 Mon Sep 17 00:00:00 2001 From: Bas Alberts Date: Tue, 3 Mar 2026 20:02:23 -0500 Subject: [PATCH 17/17] Remove comment posting: no GitHub REST API for advisory comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The /repos/.../security-advisories/{ghsa_id}/comments endpoint does not exist in the public API (404). Remove _post_advisory_comment and add_pvr_advisory_comment entirely. Strip the comment parameter from accept_pvr_advisory and reject_pvr_advisory — both now only apply the state transition (triage→draft and triage→closed respectively). Remove the comment action from pvr_respond and pvr_respond_batch; valid actions are now accept and reject only. Add MANUAL_RESPONSE.md with steps for posting the generated response draft manually via the advisory URL after the state transition is applied. --- docs/pvr_triage_overview.md | 21 ++-- scripts/run_pvr_triage.sh | 21 ++-- src/seclab_taskflows/mcp_servers/pvr_ghsa.py | 104 +++--------------- .../taskflows/pvr_triage/MANUAL_RESPONSE.md | 32 ++++++ .../taskflows/pvr_triage/README.md | 68 ++++++------ .../taskflows/pvr_triage/pvr_respond.yaml | 22 ++-- .../pvr_triage/pvr_respond_batch.yaml | 31 ++---- src/seclab_taskflows/toolboxes/pvr_ghsa.yaml | 1 - tests/test_pvr_mcp.py | 82 +++----------- 9 files changed, 142 insertions(+), 240 deletions(-) create mode 100644 src/seclab_taskflows/taskflows/pvr_triage/MANUAL_RESPONSE.md diff --git a/docs/pvr_triage_overview.md b/docs/pvr_triage_overview.md index a8c17eb..6103e94 100644 --- a/docs/pvr_triage_overview.md +++ b/docs/pvr_triage_overview.md @@ -58,10 +58,11 @@ OSS maintainers get flooded with low-quality vulnerability reports via GitHub's │ │ │ │ │ confirm-gated: │ │ • list_pending │ │ accept (→draft) │ │ • for each: │ - │ comment │ │ - confirm-gated │ - │ reject (→closed)│ │ write-back │ - │ │ │ - mark as sent │ - │ mark as sent │ │ • "Sent N/M" │ + │ reject (→closed)│ │ - confirm-gated │ + │ │ │ state change │ + │ mark as applied │ │ - mark as applied │ + │ post draft │ │ • post drafts │ + │ manually via UI │ │ manually via UI │ └──────────────────┘ └──────────────────────┘ ``` @@ -111,7 +112,7 @@ quality: +1 per signal (files, PoC, lines) → max +3 |---|---|---| | `GHSA-xxxx_triage.md` | pvr_triage | Full analysis report | | `GHSA-xxxx_response_triage.md` | pvr_triage | Draft reply to reporter | -| `GHSA-xxxx_response_sent.md` | pvr_respond / batch | Sent marker (idempotent) | +| `GHSA-xxxx_response_sent.md` | pvr_respond / batch | State-transition applied marker (idempotent) | | `batch_queue__.md` | pvr_triage_batch | Ranked inbox table | --- @@ -125,10 +126,12 @@ Every completed triage records **verdict + quality** against the reporter's GitH ## One-liner workflow ```bash -./scripts/run_pvr_triage.sh batch owner/repo # see inbox -./scripts/run_pvr_triage.sh triage owner/repo GHSA-xxx # analyse one -./scripts/run_pvr_triage.sh respond owner/repo GHSA-xxx accept # accept one -./scripts/run_pvr_triage.sh respond_batch owner/repo comment # send all drafts +./scripts/run_pvr_triage.sh batch owner/repo # see inbox +./scripts/run_pvr_triage.sh triage owner/repo GHSA-xxx # analyse one +./scripts/run_pvr_triage.sh respond owner/repo GHSA-xxx accept # accept one (triage→draft) +./scripts/run_pvr_triage.sh respond owner/repo GHSA-xxx reject # reject one (triage→closed) +./scripts/run_pvr_triage.sh respond_batch owner/repo reject # bulk state transition +# Then post each *_response_triage.md manually via the advisory URL ``` --- diff --git a/scripts/run_pvr_triage.sh b/scripts/run_pvr_triage.sh index 703a5d3..70481db 100755 --- a/scripts/run_pvr_triage.sh +++ b/scripts/run_pvr_triage.sh @@ -40,12 +40,13 @@ Commands: Run full triage on one advisory: verify code, generate report + response draft. respond - Post the response draft to GitHub. action = accept | comment | reject + Apply a state transition to a GitHub advisory. action = accept | reject Requires pvr_triage to have been run first for the given GHSA. + Post the response draft manually via the advisory URL after running. respond_batch - Scan REPORT_DIR for all pending response drafts and post them in one session. - action = accept | comment | reject + Scan REPORT_DIR and apply state transitions to all pending advisories. + action = accept | reject demo Full pipeline on the given repo (batch → triage on first triage advisory → report preview). @@ -134,8 +135,8 @@ cmd_respond() { local ghsa="${2:?Usage: $0 respond }" local action="${3:?Usage: $0 respond }" case "${action}" in - accept|comment|reject) ;; - *) echo "ERROR: action must be accept, comment, or reject" >&2; exit 1 ;; + accept|reject) ;; + *) echo "ERROR: action must be accept or reject" >&2; exit 1 ;; esac echo "==> Responding to ${ghsa} in ${repo} (action=${action}) ..." run_agent \ @@ -149,8 +150,8 @@ cmd_respond_batch() { local repo="${1:?Usage: $0 respond_batch }" local action="${2:?Usage: $0 respond_batch }" case "${action}" in - accept|comment|reject) ;; - *) echo "ERROR: action must be accept, comment, or reject" >&2; exit 1 ;; + accept|reject) ;; + *) echo "ERROR: action must be accept or reject" >&2; exit 1 ;; esac echo "==> Bulk respond for ${repo} (action=${action}) ..." run_agent \ @@ -187,8 +188,10 @@ cmd_demo() { echo "--- Reports written to ${REPORT_DIR} ---" ls -1 "${REPORT_DIR}"/*.md 2>/dev/null || true echo - echo "To post the response draft (comment only, does not reject):" - echo " $0 respond ${repo} ${ghsa} comment" + echo "To accept (triage → draft) or reject (triage → closed):" + echo " $0 respond ${repo} ${ghsa} accept" + echo " $0 respond ${repo} ${ghsa} reject" + echo "Then post the response draft manually via the advisory URL." } # --------------------------------------------------------------------------- diff --git a/src/seclab_taskflows/mcp_servers/pvr_ghsa.py b/src/seclab_taskflows/mcp_servers/pvr_ghsa.py index 62d6737..772fd1b 100644 --- a/src/seclab_taskflows/mcp_servers/pvr_ghsa.py +++ b/src/seclab_taskflows/mcp_servers/pvr_ghsa.py @@ -321,83 +321,26 @@ def save_triage_report( return str(out_path.resolve()) -def _post_advisory_comment(owner: str, repo: str, ghsa_id: str, body: str) -> str: - """ - Internal helper: post a comment on a security advisory. - - Attempts to use the GitHub advisory comments API. If that endpoint is not - available, falls back to appending a '## Maintainer Response' section to the - advisory description instead. Called by both the MCP tool wrapper and the - reject_pvr_advisory so they all share the same logic without going through - the FunctionTool wrapper. - """ - comment_path = f"/repos/{owner}/{repo}/security-advisories/{ghsa_id}/comments" - cmd = [ - "gh", "api", - "--method", "POST", - comment_path, - "--input", "-", - ] - env = os.environ.copy() - try: - result = subprocess.run( - cmd, - input=json.dumps({"body": body}), - capture_output=True, - text=True, - env=env, - timeout=30, - ) - except subprocess.TimeoutExpired: - return "Error: gh api call timed out" - except FileNotFoundError: - return "Error: gh CLI not found in PATH" - - if result.returncode == 0: - try: - data = json.loads(result.stdout) - url = data.get("html_url", data.get("url", "posted")) - return f"Comment posted: {url}" - except json.JSONDecodeError: - return "Comment posted." - - # Fall back: append maintainer response to advisory description - logging.warning( - "Advisory comments API unavailable (%s); falling back to description update", - result.stderr.strip(), - ) - adv_path = f"/repos/{owner}/{repo}/security-advisories/{ghsa_id}" - adv_data, adv_err = _gh_api(adv_path) - if adv_err: - return f"Error fetching advisory for fallback comment: {adv_err}" - existing_desc = adv_data.get("description", "") or "" - updated_desc = existing_desc + f"\n\n## Maintainer Response\n\n{body}" - _, patch_err = _gh_api(adv_path, method="PATCH", body={"description": updated_desc}) - if patch_err: - return f"Error updating advisory description: {patch_err}" - return "Comment appended to advisory description (comments API unavailable)." - - @mcp.tool() def reject_pvr_advisory( owner: str = Field(description="Repository owner (user or org name)"), repo: str = Field(description="Repository name"), ghsa_id: str = Field(description="GHSA ID of the advisory, e.g. GHSA-xxxx-xxxx-xxxx"), - comment: str = Field(description="Explanation comment to post on the advisory"), ) -> str: """ - Close (reject) a security advisory and post a comment explaining the decision. + Close (reject) a security advisory. + + Sets the advisory state to 'closed' via the GitHub API. Requires a GH_TOKEN + with security_events write scope. - Sets the advisory state to 'closed' via the GitHub API, then posts a - comment with the provided explanation. Requires a GH_TOKEN with - security_events write scope. + Note: the GitHub REST API has no comments endpoint for security advisories. + Post the response draft to the reporter manually via the advisory URL. """ path = f"/repos/{owner}/{repo}/security-advisories/{ghsa_id}" _, err = _gh_api(path, method="PATCH", body={"state": "closed"}) if err: return f"Error closing advisory {ghsa_id}: {err}" - result = _post_advisory_comment(owner, repo, ghsa_id, comment) - return f"Advisory {ghsa_id} closed. Comment: {result}" + return f"Advisory {ghsa_id} closed (state: closed)." @mcp.tool() @@ -405,39 +348,22 @@ def accept_pvr_advisory( owner: str = Field(description="Repository owner (user or org name)"), repo: str = Field(description="Repository name"), ghsa_id: str = Field(description="GHSA ID of the advisory, e.g. GHSA-xxxx-xxxx-xxxx"), - comment: str = Field(description="Acknowledgement comment to post on the advisory"), ) -> str: """ - Accept a PVR advisory by moving it from triage to draft state, then post a comment. + Accept a PVR advisory by moving it from triage to draft state. - Sets the advisory state to 'draft' via the GitHub API (triage → draft transition), - then posts a comment. Use this when the vulnerability is confirmed and the maintainer - intends to publish a security advisory. Requires a GH_TOKEN with security_events - write scope. + Sets the advisory state to 'draft' via the GitHub API (triage → draft transition). + Use this when the vulnerability is confirmed and the maintainer intends to publish + a security advisory. Requires a GH_TOKEN with security_events write scope. + + Note: the GitHub REST API has no comments endpoint for security advisories. + Post the response draft to the reporter manually via the advisory URL. """ path = f"/repos/{owner}/{repo}/security-advisories/{ghsa_id}" _, err = _gh_api(path, method="PATCH", body={"state": "draft"}) if err: return f"Error accepting advisory {ghsa_id}: {err}" - result = _post_advisory_comment(owner, repo, ghsa_id, comment) - return f"Advisory {ghsa_id} accepted (moved to draft). Comment: {result}" - - -@mcp.tool() -def add_pvr_advisory_comment( - owner: str = Field(description="Repository owner (user or org name)"), - repo: str = Field(description="Repository name"), - ghsa_id: str = Field(description="GHSA ID of the advisory, e.g. GHSA-xxxx-xxxx-xxxx"), - body: str = Field(description="Comment text to post on the advisory"), -) -> str: - """ - Post a comment on a security advisory. - - Attempts to use the GitHub advisory comments API. If that endpoint is not - available, falls back to appending a '## Maintainer Response' section to the - advisory description instead. - """ - return _post_advisory_comment(owner, repo, ghsa_id, body) + return f"Advisory {ghsa_id} accepted (state: draft)." @mcp.tool() diff --git a/src/seclab_taskflows/taskflows/pvr_triage/MANUAL_RESPONSE.md b/src/seclab_taskflows/taskflows/pvr_triage/MANUAL_RESPONSE.md new file mode 100644 index 0000000..b2f82e8 --- /dev/null +++ b/src/seclab_taskflows/taskflows/pvr_triage/MANUAL_RESPONSE.md @@ -0,0 +1,32 @@ +# Posting a Response to a PVR Advisory + +The GitHub REST API has no comments endpoint for repository security advisories +(`/repos/{owner}/{repo}/security-advisories/{ghsa_id}/comments` → 404). The comment +thread visible in the advisory UI is internal to GitHub and not publicly accessible via +the API. + +After `pvr_respond` or `pvr_respond_batch` applies the state transition (accept/reject), +post the generated response draft to the reporter manually: + +## Steps + +1. Open the response draft generated by `pvr_triage`: + ```bash + cat reports/GHSA-xxxx-xxxx-xxxx_response_triage.md + ``` + +2. Open the advisory URL — printed in the triage report under `html_url`, or construct + it directly: + ``` + https://github.com/{owner}/{repo}/security/advisories/{GHSA-ID} + ``` + +3. Scroll to the comment box at the bottom of the advisory page, paste the response + draft, edit if needed, and submit. The comment is visible only to the reporter and + collaborators on the advisory (not public). + +## Tracking + +`pvr_respond` creates `REPORT_DIR/{GHSA-ID}_response_sent.md` after the state +transition. This marker prevents re-processing by `pvr_respond_batch` but does **not** +confirm that the comment was posted. Use it as a reminder to complete the manual step. diff --git a/src/seclab_taskflows/taskflows/pvr_triage/README.md b/src/seclab_taskflows/taskflows/pvr_triage/README.md index d9416d6..b556718 100644 --- a/src/seclab_taskflows/taskflows/pvr_triage/README.md +++ b/src/seclab_taskflows/taskflows/pvr_triage/README.md @@ -147,63 +147,64 @@ quality_weight: has_file_references(+1) + has_poc(+1) + has_line_numbers(+1) ## Taskflow 3 — Write-back (`pvr_respond`) -Loads an existing triage report and response draft from disk and executes the chosen action against the GitHub advisory API. All write-back calls are confirm-gated — the agent will prompt for confirmation before making any change. +Loads an existing triage report and applies the chosen state transition to the GitHub advisory. All write-back calls are confirm-gated — the agent will prompt for confirmation before making any change. ```bash python -m seclab_taskflow_agent \ -t seclab_taskflows.taskflows.pvr_triage.pvr_respond \ -g repo=owner/repo \ -g ghsa=GHSA-xxxx-xxxx-xxxx \ - -g action=comment + -g action=accept ``` ### Actions | `action` | API call | When to use | |---|---|---| -| `accept` | Sets advisory state to `draft` (triage → draft), then posts the comment | Vulnerability confirmed — maintainer intends to publish an advisory | -| `comment` | Posts the response draft as a comment on the advisory | Default for all verdicts — sends your reply without changing state | -| `reject` | Sets advisory state to `closed`, then posts the comment | Report is clearly invalid or low quality | +| `accept` | Sets advisory state to `draft` (triage → draft) | Vulnerability confirmed — maintainer intends to publish an advisory | +| `reject` | Sets advisory state to `closed` | Report is clearly invalid or low quality | -> **Note:** `pvr_respond` requires that `pvr_triage` has already been run for the GHSA, so that both `_triage.md` and `_response_triage.md` exist in `REPORT_DIR`. +> **Note:** `pvr_respond` requires that `pvr_triage` has already been run for the GHSA so that `_triage.md` and `_response_triage.md` exist in `REPORT_DIR`. + +> **Posting the response:** The GitHub REST API has no comments endpoint for security advisories. After running `pvr_respond`, post the response draft manually via the advisory URL. See [`MANUAL_RESPONSE.md`](MANUAL_RESPONSE.md) for instructions and language. ### Confirm gate -The toolbox marks `accept_pvr_advisory`, `reject_pvr_advisory`, and `add_pvr_advisory_comment` as `confirm`-gated. The agent will print the verdict, quality rating, and full response draft, then ask for explicit confirmation before making any change to GitHub. +The toolbox marks `accept_pvr_advisory` and `reject_pvr_advisory` as `confirm`-gated. The agent will print the verdict and summary, then ask for explicit confirmation before making any change to GitHub. -After a successful write-back, `pvr_respond` calls `mark_response_sent` to create a `_response_sent.md` marker so `pvr_respond_batch` will skip this advisory in future runs. +After a successful state transition, `pvr_respond` calls `mark_response_sent` to create a `_response_sent.md` marker so `pvr_respond_batch` will skip this advisory in future runs. --- ## Taskflow 4 — Bulk respond (`pvr_respond_batch`) -Scans `REPORT_DIR` for advisories that have a response draft (`*_response_triage.md`) but no sent marker (`*_response_sent.md`), then posts each response to GitHub in a single session. +Scans `REPORT_DIR` for advisories that have a response draft (`*_response_triage.md`) but no applied marker (`*_response_sent.md`), and applies the chosen state transition to each in a single session. ```bash python -m seclab_taskflow_agent \ -t seclab_taskflows.taskflows.pvr_triage.pvr_respond_batch \ -g repo=owner/repo \ - -g action=comment + -g action=reject # or via the helper script: -./scripts/run_pvr_triage.sh respond_batch owner/repo comment +./scripts/run_pvr_triage.sh respond_batch owner/repo reject ``` ### How it works -**Task 1** calls `list_pending_responses` (local read-only, no confirm gate) to find all unsent drafts and prints a summary table. If there are no pending drafts it stops immediately. +**Task 1** calls `list_pending_responses` (local read-only, no confirm gate) to find all pending advisories and prints a summary table. If there are none it stops immediately. **Task 2** iterates over every pending entry: -1. Reads the triage report and response draft from disk. -2. Prints a per-item preview (GHSA, verdict, first 200 chars of response). -3. Executes the chosen action (`accept` / `comment` / `reject`) via the confirm-gated write-back tool. +1. Reads the triage report from disk. +2. Prints a per-item summary (GHSA, verdict). +3. Executes the chosen action (`accept` / `reject`) via the confirm-gated write-back tool. 4. On success, calls `mark_response_sent` to create a `*_response_sent.md` marker so the advisory is skipped in future runs. -Prints a final count: `"Sent N / M responses."` +Prints a final count and a reminder to post each response draft manually. -### Sent markers +### Applied markers -`pvr_respond` also calls `mark_response_sent` after a successful write-back, keeping single-advisory and bulk responds in sync. Once a marker exists, neither `pvr_respond` nor `pvr_respond_batch` will attempt to re-send. +`pvr_respond` also calls `mark_response_sent` after a successful state transition, keeping single-advisory and bulk runs in sync. Once a marker exists, neither `pvr_respond` nor `pvr_respond_batch` will re-process it. --- @@ -219,15 +220,15 @@ Prints a final count: `"Sent N / M responses."` - Check the Verdict and Code Verification sections. - Edit the response draft (_response_triage.md) if needed. -4a. Send responses one at a time with pvr_respond: - - action=accept → move to draft (triage → draft) + post reply - - action=comment → post reply only (advisory stays in triage state) - - action=reject → close + post reply +4a. Apply a state transition with pvr_respond: + - action=accept → move to draft (triage → draft) + - action=reject → close (triage → closed) + Then post the response draft manually via the advisory URL. -4b. Or send all pending drafts at once with pvr_respond_batch: - Scans REPORT_DIR for unsent drafts (no _response_sent.md marker) - and posts them all in one session. - Useful after triaging a batch in step 2. +4b. Or apply state transitions to all pending advisories at once with pvr_respond_batch: + Scans REPORT_DIR for pending entries (no _response_sent.md marker) + and applies the chosen action to all of them in one session. + Then post each response draft manually via the advisory URL. ``` ### Example session @@ -248,25 +249,28 @@ python -m seclab_taskflow_agent \ cat reports/GHSA-1234-5678-abcd_triage.md cat reports/GHSA-1234-5678-abcd_response_triage.md -# Step 4a: send a comment for one advisory (doesn't change advisory state) +# Step 4a: accept (triage → draft) — vulnerability confirmed python -m seclab_taskflow_agent \ -t seclab_taskflows.taskflows.pvr_triage.pvr_respond \ -g repo=acme/widget \ -g ghsa=GHSA-1234-5678-abcd \ - -g action=comment + -g action=accept -# Step 4b: or reject outright +# Step 4b: or reject (triage → closed) — invalid or low-quality report python -m seclab_taskflow_agent \ -t seclab_taskflows.taskflows.pvr_triage.pvr_respond \ -g repo=acme/widget \ -g ghsa=GHSA-1234-5678-abcd \ -g action=reject -# Step 4c: or post all pending drafts at once (after triaging several advisories) +# Step 4c: or apply state transitions to all pending advisories at once python -m seclab_taskflow_agent \ -t seclab_taskflows.taskflows.pvr_triage.pvr_respond_batch \ -g repo=acme/widget \ - -g action=comment + -g action=reject + +# Step 5: post each response draft manually via the advisory URL +# See taskflows/pvr_triage/MANUAL_RESPONSE.md for instructions ``` --- @@ -310,5 +314,5 @@ All files are written to `REPORT_DIR` (default: `./reports`). |---|---|---| | `_triage.md` | `pvr_triage` task 6 | Full triage analysis report | | `_response_triage.md` | `pvr_triage` task 8 | Plain-text response draft for the reporter | -| `_response_sent.md` | `pvr_respond` / `pvr_respond_batch` | Marker: response has been sent (contains ISO timestamp) | +| `_response_sent.md` | `pvr_respond` / `pvr_respond_batch` | Marker: state transition applied (contains ISO timestamp); post draft manually | | `batch_queue__.md` | `pvr_triage_batch` task 3 | Ranked inbox table with Age column | diff --git a/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond.yaml b/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond.yaml index 7880d7d..c4370cf 100644 --- a/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond.yaml +++ b/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond.yaml @@ -12,7 +12,7 @@ # -t seclab_taskflows.taskflows.pvr_triage.pvr_respond \ # -g repo=owner/repo \ # -g ghsa=GHSA-xxxx-xxxx-xxxx \ -# -g action=accept|reject|comment +# -g action=accept|reject # # Required environment variables: # GH_TOKEN - GitHub token with security_events write scope @@ -31,7 +31,7 @@ globals: repo: # GHSA ID of the advisory to act on ghsa: - # Action to perform: accept, reject, or comment + # Action to perform: accept or reject action: taskflow: @@ -92,27 +92,21 @@ taskflow: - owner: extracted owner - repo: extracted repo - ghsa_id: "{{ globals.ghsa }}" - - comment: response_draft If action is "reject": Call reject_pvr_advisory with: - owner: extracted owner - repo: extracted repo - ghsa_id: "{{ globals.ghsa }}" - - comment: response_draft - - If action is "comment": - Call add_pvr_advisory_comment with: - - owner: extracted owner - - repo: extracted repo - - ghsa_id: "{{ globals.ghsa }}" - - body: response_draft If action is anything else: - Print: "Unknown action '{{ globals.action }}'. Valid actions: accept, reject, comment" + Print: "Unknown action '{{ globals.action }}'. Valid actions: accept, reject" and stop. Print the result returned by the API call. - On success (action was not "anything else"), call mark_response_sent with - ghsa_id="{{ globals.ghsa }}" to record that this response has been sent. + On success, call mark_response_sent with ghsa_id="{{ globals.ghsa }}" to record + that the state transition has been applied. + + Then print: "Response draft saved at REPORT_DIR/{{ globals.ghsa }}_response_triage.md + — post it to the reporter manually via the advisory URL." diff --git a/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond_batch.yaml b/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond_batch.yaml index 5a055fc..061d2ce 100644 --- a/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond_batch.yaml +++ b/src/seclab_taskflows/taskflows/pvr_triage/pvr_respond_batch.yaml @@ -4,14 +4,14 @@ # PVR Bulk Respond Taskflow # # Scans REPORT_DIR for pending response drafts (advisories with a -# *_response_triage.md but no *_response_sent.md marker) and posts -# each response to GitHub in a single session. +# *_response_triage.md but no *_response_sent.md marker) and applies +# the chosen state transition to each in a single session. # # Usage: # python -m seclab_taskflow_agent \ # -t seclab_taskflows.taskflows.pvr_triage.pvr_respond_batch \ # -g repo=owner/repo \ -# -g action=accept|comment|reject +# -g action=accept|reject # # Required environment variables: # GH_TOKEN - GitHub token with security_events write scope @@ -28,7 +28,7 @@ model_config: seclab_taskflows.configs.model_config_pvr_triage globals: # GitHub repository in owner/repo format repo: - # Action to apply to all pending responses: accept, comment, or reject + # Action to apply to all pending responses: accept or reject action: taskflow: @@ -77,28 +77,19 @@ taskflow: For each entry in pending_responses: 1. Call read_triage_report with ghsa_id=entry.ghsa_id to get the triage report. - 2. Call read_triage_report with ghsa_id="{entry.ghsa_id}_response" to get the - response draft. - 3. Print a per-item summary: + 2. Print a per-item summary: GHSA: {entry.ghsa_id} Verdict: [extracted from triage report] - Response preview: [first 200 chars of response draft] - 4. Execute the action: + 3. Execute the action: If action is "accept": - Call accept_pvr_advisory with owner, repo, ghsa_id=entry.ghsa_id, - comment=response_draft. + Call accept_pvr_advisory with owner, repo, ghsa_id=entry.ghsa_id. If action is "reject": - Call reject_pvr_advisory with owner, repo, ghsa_id=entry.ghsa_id, - comment=response_draft. - If action is "comment": - Call add_pvr_advisory_comment with owner, repo, ghsa_id=entry.ghsa_id, - body=response_draft. + Call reject_pvr_advisory with owner, repo, ghsa_id=entry.ghsa_id. If action is anything else: Print: "Unknown action '{{ globals.action }}'. Skipping {entry.ghsa_id}." and continue to the next entry. - 5. On success, call mark_response_sent with ghsa_id=entry.ghsa_id. - Print: "Sent: {entry.ghsa_id}" + 4. On success, call mark_response_sent with ghsa_id=entry.ghsa_id. + Print: "Applied: {entry.ghsa_id} — post response draft manually via advisory URL." After processing all entries, print: - "Sent N / M responses." where N is the count of successfully sent responses - and M is the total count of pending_responses entries. + "Applied N / M state transitions. Post response drafts manually via each advisory URL." diff --git a/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml b/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml index e36dc46..be7adde 100644 --- a/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml +++ b/src/seclab_taskflows/toolboxes/pvr_ghsa.yaml @@ -24,4 +24,3 @@ server_params: confirm: - accept_pvr_advisory - reject_pvr_advisory - - add_pvr_advisory_comment diff --git a/tests/test_pvr_mcp.py b/tests/test_pvr_mcp.py index 1922357..5cd8380 100644 --- a/tests/test_pvr_mcp.py +++ b/tests/test_pvr_mcp.py @@ -9,7 +9,7 @@ import tempfile import unittest from pathlib import Path -from unittest.mock import MagicMock, patch +from unittest.mock import patch import pytest @@ -43,23 +43,19 @@ def tearDown(self): # --- accept_pvr_advisory --- def test_accept_pvr_advisory_calls_correct_api(self): - """accept_pvr_advisory should PATCH state=draft then post a comment.""" + """accept_pvr_advisory should PATCH state=draft.""" calls = [] def fake_gh_api(path, method="GET", body=None): calls.append({"path": path, "method": method, "body": body}) - if method == "PATCH": - return {"ghsa_id": "GHSA-1234-5678-abcd", "state": "draft"}, None - return {}, None + return {"ghsa_id": "GHSA-1234-5678-abcd", "state": "draft"}, None with patch.object(self.pvr, "_gh_api", side_effect=fake_gh_api): - with patch.object(self.pvr, "_post_advisory_comment", return_value="Comment posted: https://github.com/test"): - result = self.pvr.accept_pvr_advisory.fn( - owner="owner", - repo="repo", - ghsa_id="GHSA-1234-5678-abcd", - comment="Confirmed. We'll publish an advisory.", - ) + result = self.pvr.accept_pvr_advisory.fn( + owner="owner", + repo="repo", + ghsa_id="GHSA-1234-5678-abcd", + ) self.assertEqual(calls[0]["method"], "PATCH") self.assertIn("GHSA-1234-5678-abcd", calls[0]["path"]) @@ -69,70 +65,24 @@ def fake_gh_api(path, method="GET", body=None): # --- reject_pvr_advisory --- def test_reject_pvr_advisory_calls_correct_api(self): - """reject_pvr_advisory should PATCH state=closed then post a comment.""" + """reject_pvr_advisory should PATCH state=closed.""" calls = [] def fake_gh_api(path, method="GET", body=None): calls.append({"path": path, "method": method, "body": body}) - if method == "PATCH": - return {"ghsa_id": "GHSA-1234-5678-abcd", "state": "closed"}, None - return {}, None + return {"ghsa_id": "GHSA-1234-5678-abcd", "state": "closed"}, None with patch.object(self.pvr, "_gh_api", side_effect=fake_gh_api): - with patch.object(self.pvr, "_post_advisory_comment", return_value="Comment posted: https://github.com/test"): - result = self.pvr.reject_pvr_advisory.fn( - owner="owner", - repo="repo", - ghsa_id="GHSA-1234-5678-abcd", - comment="Rejecting: not a valid report.", - ) - - # First call must be the PATCH to set state=closed - self.assertEqual(calls[0]["method"], "PATCH") - self.assertIn("GHSA-1234-5678-abcd", calls[0]["path"]) - self.assertEqual(calls[0]["body"], {"state": "closed"}) - self.assertIn("closed", result) - - # --- add_pvr_advisory_comment --- - - def test_add_pvr_advisory_comment_returns_url_on_success(self): - """add_pvr_advisory_comment returns comment URL on API success.""" - mock_result = MagicMock() - mock_result.returncode = 0 - mock_result.stdout = json.dumps({"html_url": "https://github.com/comment/1"}) - with patch("subprocess.run", return_value=mock_result): - result = self.pvr.add_pvr_advisory_comment.fn( + result = self.pvr.reject_pvr_advisory.fn( owner="owner", repo="repo", ghsa_id="GHSA-1234-5678-abcd", - body="Thank you for the report.", ) - self.assertIn("https://github.com/comment/1", result) - def test_add_pvr_advisory_comment_fallback_on_api_failure(self): - """add_pvr_advisory_comment falls back to description update when comments API unavailable.""" - # First subprocess call (comments POST) fails - mock_fail = MagicMock() - mock_fail.returncode = 1 - mock_fail.stderr = "Not Found" - mock_fail.stdout = "" - - def fake_gh_api(path, method="GET", body=None): - if method == "GET": - return {"description": "Original description.", "ghsa_id": "GHSA-x"}, None - if method == "PATCH": - return {"description": "updated"}, None - return {}, None - - with patch("subprocess.run", return_value=mock_fail): - with patch.object(self.pvr, "_gh_api", side_effect=fake_gh_api): - result = self.pvr.add_pvr_advisory_comment.fn( - owner="owner", - repo="repo", - ghsa_id="GHSA-1234-5678-abcd", - body="Maintainer note.", - ) - self.assertIn("description", result.lower()) + self.assertEqual(calls[0]["method"], "PATCH") + self.assertIn("GHSA-1234-5678-abcd", calls[0]["path"]) + self.assertEqual(calls[0]["body"], {"state": "closed"}) + self.assertIn("closed", result) # --- find_similar_triage_reports --- @@ -455,7 +405,7 @@ def test_pvr_ghsa_toolbox_has_confirm(self): confirm = result.get("confirm", []) self.assertIn("accept_pvr_advisory", confirm) self.assertIn("reject_pvr_advisory", confirm) - self.assertIn("add_pvr_advisory_comment", confirm) + self.assertNotIn("add_pvr_advisory_comment", confirm) def test_pvr_respond_batch_yaml_parses(self): """pvr_respond_batch.yaml loads without error and declares repo + action globals."""