diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 9d797a46..fbfb6e82 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -22,7 +22,7 @@ "name": "edge-ocp-ci", "source": "./plugins/edge-ocp-ci", "description": "Edge OCP Payload Monitor — monitor OpenShift nightly payloads for edge topology (SNO/TNA/TNF) failures with AI-enriched analysis", - "version": "1.2.0" + "version": "1.2.1" }, { "name": "edge-scrum", diff --git a/payload-monitor/payload_monitor/__main__.py b/payload-monitor/payload_monitor/__main__.py index b840e70c..985f2265 100644 --- a/payload-monitor/payload_monitor/__main__.py +++ b/payload-monitor/payload_monitor/__main__.py @@ -15,7 +15,7 @@ from .collectors import component_readiness, prow, sippy, timing from .collectors.release_controller import ( collect as collect_payloads, - discover_streams, + configured_stream_names, version_from_stream, ) from .config import Config @@ -154,9 +154,9 @@ def main( logger.info("Starting Edge OCP Payload Monitor") - # Step 1: Discover versions and resolve stream names - logger.info("Step 1: Discovering active versions...") - stream_names = discover_streams(config) + # Step 1: Resolve configured release streams + logger.info("Step 1: Resolving configured release streams...") + stream_names = configured_stream_names(config) active_versions = [version_from_stream(s) for s in stream_names] logger.info(f" Versions: {active_versions}") diff --git a/payload-monitor/payload_monitor/analyzer.py b/payload-monitor/payload_monitor/analyzer.py index eb9a5176..cebaa7e9 100644 --- a/payload-monitor/payload_monitor/analyzer.py +++ b/payload-monitor/payload_monitor/analyzer.py @@ -17,6 +17,7 @@ ) from .collectors import jira as jira_collector from .collectors.jira import has_auth as jira_has_auth +from .collectors.sippy import job_analysis_url logger = logging.getLogger(__name__) @@ -80,6 +81,9 @@ def _find_escalation_risks( for job_name, topology in informing_jobs.items(): consecutive = 0 + latest_prow_url = "" + latest_failure_seen = False + streak_runs: list[dict] = [] for payload in reversed_payloads: job_in_payload = None for job in payload.jobs: @@ -87,21 +91,28 @@ def _find_escalation_risks( job_in_payload = job break if job_in_payload is None: - # Job absent from this payload breaks the streak break if job_in_payload.result == JobResult.FAILURE: consecutive += 1 + streak_runs.append({ + "payload_tag": payload.tag, + "prow_url": job_in_payload.prow_url, + }) + if not latest_failure_seen: + latest_prow_url = job_in_payload.prow_url + latest_failure_seen = True else: break if consecutive >= config.escalation_threshold: - sippy_url = f"https://sippy.dptools.openshift.org/sippy-ng/jobs/{job_name}" risks.append(EscalationRisk( job_name=job_name, topology=topology, version=stream.version, consecutive_failures=consecutive, - sippy_url=sippy_url, + prow_url=latest_prow_url, + triage_url=job_analysis_url(stream.version, job_name), + failing_runs=streak_runs, )) return risks diff --git a/payload-monitor/payload_monitor/collectors/release_controller.py b/payload-monitor/payload_monitor/collectors/release_controller.py index 553edc76..b7c8080e 100644 --- a/payload-monitor/payload_monitor/collectors/release_controller.py +++ b/payload-monitor/payload_monitor/collectors/release_controller.py @@ -90,8 +90,12 @@ def _parse_jobs( -def discover_streams(config: Config) -> list[str]: - """Return list of nightly stream names to monitor.""" +def configured_stream_names(config: Config) -> list[str]: + """Map ``config.versions`` to release-controller stream names. + + This is a static mapping, not a live discovery against the + release-controller index. + """ return [_stream_name(v) for v in config.versions] @@ -204,7 +208,7 @@ def _collect_stream(stream: str, config: Config) -> StreamReport: def collect(config: Config, streams: Optional[list[str]] = None) -> list[StreamReport]: """Collect payload data for all configured streams in parallel.""" if streams is None: - streams = discover_streams(config) + streams = configured_stream_names(config) with ThreadPoolExecutor(max_workers=min(len(streams) or 1, 10)) as pool: futures = { diff --git a/payload-monitor/payload_monitor/collectors/sippy.py b/payload-monitor/payload_monitor/collectors/sippy.py index 277b0489..61366959 100644 --- a/payload-monitor/payload_monitor/collectors/sippy.py +++ b/payload-monitor/payload_monitor/collectors/sippy.py @@ -2,8 +2,10 @@ from __future__ import annotations +import json import logging from concurrent.futures import ThreadPoolExecutor, as_completed +from urllib.parse import quote import requests @@ -48,8 +50,17 @@ def fetch_edge_jobs( return edge_jobs +def job_analysis_url(version: str, name: str) -> str: + """Build the Sippy UI URL for a job's analysis page, filtered by job name.""" + filters = json.dumps({ + "items": [{"columnField": "name", "operatorValue": "equals", "value": name}] + }) + return f"{BASE_URL}/sippy-ng/jobs/{version}/analysis?filters={quote(filters)}" + + def identify_regressions( edge_jobs: list[dict], + version: str = "", min_runs: int = 3, ) -> list[Regression]: """Identify jobs that are getting worse over time. @@ -69,9 +80,9 @@ def identify_regressions( name = job.get("name", "") topology = job.get("_topology", "") jira_component = job.get("jira_component", "") - triage_url = f"{BASE_URL}/sippy-ng/jobs/{name}" + triage_url = job_analysis_url(version, name) - # Skip jobs with too few runs — insufficient data to confirm regression + # Skip jobs with too few runs - insufficient data to confirm regression if current_runs < min_runs: continue @@ -99,7 +110,7 @@ def _collect_version(version: str, config: Config) -> tuple[str, list[Regression session = create_session() try: edge_jobs = fetch_edge_jobs(version, config, session=session) - regressions = identify_regressions(edge_jobs) + regressions = identify_regressions(edge_jobs, version=version) if regressions: logger.info( f" {version}: {len(regressions)} edge job regressions detected" diff --git a/payload-monitor/payload_monitor/collectors/timing.py b/payload-monitor/payload_monitor/collectors/timing.py index 4bd0b936..db5737cc 100644 --- a/payload-monitor/payload_monitor/collectors/timing.py +++ b/payload-monitor/payload_monitor/collectors/timing.py @@ -324,29 +324,14 @@ def compute_stats(runs: list[TimingRun]) -> dict: # --------------------------------------------------------------------------- def fetch_edge_jobs(release: str, config: Config) -> list[dict]: - """Fetch all jobs from Sippy for a release, filter for edge topologies.""" - logger.info(f"Fetching edge topology jobs for release {release}") - try: - resp = _session.get(JOBS_URL, params={"release": release}, timeout=30) - resp.raise_for_status() - all_jobs = resp.json() - except requests_lib.RequestException as e: - logger.error(f"Failed to fetch Sippy jobs for {release}: {e}") - return [] + """Fetch all jobs from Sippy for a release, filter for edge topologies. - if not isinstance(all_jobs, list): - return [] - - edge_jobs = [] - for job in all_jobs: - name = job.get("name", "") - topology = config.classify_topology(name) - if topology in ("SNO", "TNA", "TNF"): - job["_topology"] = topology - edge_jobs.append(job) + Delegates to collectors.sippy.fetch_edge_jobs so topology filtering stays + config-driven and isn't duplicated across collectors. + """ + from . import sippy as _sippy - logger.info(f" Found {len(edge_jobs)} SNO/TNA/TNF jobs for {release}") - return edge_jobs + return _sippy.fetch_edge_jobs(release, config, session=_session) def fetch_job_runs(job_name: str, release: str) -> list[dict]: diff --git a/payload-monitor/payload_monitor/models.py b/payload-monitor/payload_monitor/models.py index fb4e75f2..442e2a23 100644 --- a/payload-monitor/payload_monitor/models.py +++ b/payload-monitor/payload_monitor/models.py @@ -205,7 +205,9 @@ class EscalationRisk: topology: str version: str consecutive_failures: int - sippy_url: str = "" + prow_url: str = "" + triage_url: str = "" + failing_runs: list[dict] = field(default_factory=list) @dataclass @@ -247,17 +249,6 @@ class TimingRun: def is_success(self) -> bool: return self.result == "S" - @property - def duration_minutes(self) -> float: - return self.duration_seconds / 60.0 - - @property - def install_duration_seconds(self) -> float: - """Return install step duration if available, else 0.""" - for key in ("install", "setup"): - if key in self.step_durations: - return self.step_durations[key] - return 0.0 @dataclass diff --git a/payload-monitor/payload_monitor/report/generator.py b/payload-monitor/payload_monitor/report/generator.py index 539014a5..98f19522 100644 --- a/payload-monitor/payload_monitor/report/generator.py +++ b/payload-monitor/payload_monitor/report/generator.py @@ -5,6 +5,7 @@ from typing import Optional +import html as html_mod import json import logging import re @@ -13,6 +14,7 @@ from urllib.parse import urlparse from jinja2 import Environment, FileSystemLoader +from markupsafe import Markup, escape from ..models import ( AttemptAnalysis, @@ -38,14 +40,34 @@ logger = logging.getLogger(__name__) TEMPLATES_DIR = Path(__file__).parent / "templates" +def _js_attr(value): + """Escape a value for use inside an HTML-attribute JavaScript context. + + Produces a JSON-quoted string (JS-safe) with double quotes HTML-escaped + to " so it nests safely inside onclick="..." attributes. + """ + return escape(json.dumps(str(value))) + _JINJA_ENV = Environment( loader=FileSystemLoader(str(TEMPLATES_DIR)), autoescape=True, ) +_JINJA_ENV.filters["js_attr"] = _js_attr _TAG_DATE_RE = re.compile(r"(\d{4}-\d{2}-\d{2})-(\d{2})(\d{2})(\d{2})$") +def _load_json(path: Path, description: str) -> dict: + """Load JSON from *path*, raising a clear, path-aware error on failure.""" + try: + with open(path) as f: + return json.load(f) + except FileNotFoundError: + raise FileNotFoundError(f"{description} not found: {path}") from None + except json.JSONDecodeError as e: + raise ValueError(f"{description} is not valid JSON ({path}): {e}") from e + + def _extract_date(tag: str) -> str: """Extract date and time from a payload tag like '4.19.0-0.nightly-2026-03-25-085944'.""" m = _TAG_DATE_RE.search(tag) @@ -85,13 +107,21 @@ def _fail_sort_key(x): all_failing.sort(key=_fail_sort_key) - # Build blocking job summaries: versions, test names, topology per unique job + # Build blocking job summaries and findings-summary counts in one pass blocking_job_versions: dict[str, list[str]] = {} blocking_job_tests: dict[str, list[str]] = {} + blocking_failures = 0 + informing_failures = 0 + blocking_jobs_seen: set[str] = set() + blocking_jobs_list: list[dict] = [] for item in all_failing: if item["job"].job_type == JobType.BLOCKING: + blocking_failures += 1 name = item["job"].name ver = item["version"] + if name not in blocking_jobs_seen: + blocking_jobs_seen.add(name) + blocking_jobs_list.append(item) if name not in blocking_job_versions: blocking_job_versions[name] = [] blocking_job_tests[name] = [] @@ -100,6 +130,14 @@ def _fail_sort_key(x): for ft in item["job"].failing_tests: if ft.name not in blocking_job_tests[name]: blocking_job_tests[name].append(ft.name) + elif item["job"].job_type == JobType.INFORMING: + informing_failures += 1 + + total_payloads = sum(len(stream.payloads) for stream in report.streams) + rejected_payloads = sum( + 1 for stream in report.streams for p in stream.payloads + if p.status == PayloadStatus.REJECTED + ) # Collect blocking jobs that succeeded only after retry (flaky passes) retried_successes = [] @@ -127,6 +165,28 @@ def _fail_sort_key(x): r.version = stream.version all_regressions.append(r) + # Per-stream views: affected topologies + trend + stream_views = [] + for stream in report.streams: + affected = sorted({ + j.topology for p in stream.payloads for j in p.failing_edge_jobs if j.topology + }) + mid = len(stream.payloads) // 2 + recent_fails = sum( + len(p.blocking_edge_failures) + len(p.informing_edge_failures) + for p in stream.payloads[:mid] + ) + older_fails = sum( + len(p.blocking_edge_failures) + len(p.informing_edge_failures) + for p in stream.payloads[mid:] + ) + stream_views.append({ + "stream": stream, + "affected_topologies": affected, + "recent_fails": recent_fails, + "older_fails": older_fails, + }) + # Unique topology names and versions for filters topologies = sorted(set( job["job"].topology for job in all_failing if job["job"].topology @@ -140,13 +200,6 @@ def _fail_sort_key(x): timing_unavailable = not report.skip_timing and not timing_html timing_errors = [e for e in report.data_errors if e.startswith("Timing:")] if timing_unavailable else [] - # Map blocking job name -> first index in all_failing for stable detail links - blocking_job_first_idx = {} - for idx, item in enumerate(all_failing, 1): - if item["job"].job_type == JobType.BLOCKING: - if item["job"].name not in blocking_job_first_idx: - blocking_job_first_idx[item["job"].name] = idx - return { "report": report, "all_failing": all_failing, @@ -187,13 +240,19 @@ def _fail_sort_key(x): "persistent_threshold": report.persistent_threshold, "escalation_risks": report.escalation_risks, "escalation_risk_jobs": set(er.job_name for er in report.escalation_risks), + "escalation_risk_by_job": {er.job_name: er for er in report.escalation_risks}, "cross_topology": report.cross_topology, "jira_matches_by_job": report.jira_matches, "suggested_bugs_by_job": {b.job_name: b for b in report.suggested_bugs}, "blocking_job_versions": blocking_job_versions, "blocking_job_tests": blocking_job_tests, - "blocking_job_first_idx": blocking_job_first_idx, "retried_successes": retried_successes, + "stream_views": stream_views, + "blocking_failures": blocking_failures, + "informing_failures": informing_failures, + "blocking_jobs_list": blocking_jobs_list, + "total_payloads": total_payloads, + "rejected_payloads": rejected_payloads, } @@ -204,8 +263,14 @@ def generate_html(report: MonitorReport, output_path: Optional[Path] = None) -> also writes to that file. """ # Load CSS and JS to inline - css = (TEMPLATES_DIR / "styles.css").read_text() - js = (TEMPLATES_DIR / "scripts.js").read_text() + try: + css = (TEMPLATES_DIR / "styles.css").read_text() + js = (TEMPLATES_DIR / "scripts.js").read_text() + except FileNotFoundError as e: + raise FileNotFoundError( + f"Report template asset missing ({e.filename}) - " + "reinstall payload_monitor or check TEMPLATES_DIR" + ) from e template = _JINJA_ENV.get_template("dashboard.html") context = _build_template_context(report) @@ -303,7 +368,7 @@ def _safe_urls(urls: list[str]) -> list[str]: return safe -def _render_analysis_card(da: dict, attempt_count: int = 1) -> str: +def _render_analysis_card(da: dict, attempt_count: int = 1, job_name: str = "") -> str: """Render an AI analysis card as an HTML string.""" template = _JINJA_ENV.get_template("_analysis_card.html") return template.render( @@ -315,6 +380,7 @@ def _render_analysis_card(da: dict, attempt_count: int = 1) -> str: same_root_cause=da.get("same_root_cause", True), attempt_analyses=da.get("attempt_analyses", []), attempt_count=attempt_count, + job_name=job_name, ) @@ -327,8 +393,7 @@ def patch_analysis_html(html_path: Path, analysis_path: Path) -> None: - ``data-prow-url`` on ``
`` elements identifies where to add badges - ``data-enrichment-status="data-only"`` marks the header status div """ - with open(analysis_path) as f: - data = json.load(f) + data = _load_json(analysis_path, "Analysis JSON") by_url = data.get("by_prow_url", {}) if not by_url: @@ -339,38 +404,80 @@ def patch_analysis_html(html_path: Path, analysis_path: Path) -> None: patched = 0 badge_html = 'AI Analyzed' - def _insert_badge(m): - summary_content = m.group(2) - if "ai-analyzed" in summary_content: - return m.group(0) - return f'{m.group(1)}\n {summary_content} {badge_html}{m.group(3)}' - - for prow_url, da in by_url.items(): - escaped_url = re.escape(prow_url) - - # Find the claude-suggestion div with matching data-prow-url - suggestion_pattern = ( - rf'
' - r'.*?
\s*' - ) - m = re.search(suggestion_pattern, content, flags=re.DOTALL) - if not m: - continue + _suggestion_re = re.compile( + r'
' + r'.*?', + re.DOTALL, + ) + _details_re = re.compile( + r'(
]*data-prow-url="([^"]*)"[^>]*>)\s*' + r'(.*?)' + r'()', + re.DOTALL, + ) - attempt_count = int(m.group(1)) - card_html = _render_analysis_card(da, attempt_count=attempt_count) + patched_urls: set[str] = set() - content = content[:m.start()] + card_html + content[m.end():] + def _replace_suggestion(m: re.Match) -> str: + nonlocal patched + prow_url = html_mod.unescape(m.group(1)) + da = by_url.get(prow_url) + if da is None: + return m.group(0) + attempt_count = int(m.group(2)) patched += 1 + patched_urls.add(prow_url) + return _render_analysis_card(da, attempt_count=attempt_count, job_name=da.get("job_name", "")) - # Add "AI Analyzed" badge to the
element for this job - details_pattern = ( - rf'(
]*data-prow-url="{escaped_url}"[^>]*>)\s*' - r'(.*?)' - r'()' - ) - content = re.sub(details_pattern, _insert_badge, content, count=1, flags=re.DOTALL) + content = _suggestion_re.sub(_replace_suggestion, content) + + def _insert_badge(m: re.Match) -> str: + if html_mod.unescape(m.group(2)) not in patched_urls: + return m.group(0) + summary_content = m.group(3) + if "ai-analyzed" in summary_content: + return m.group(0) + # Grid layout: insert badge inside the job cell before its closing + job_anchor = 'class="detail-cell-job"' + job_pos = summary_content.find(job_anchor) + if job_pos >= 0: + open_gt = summary_content.find('>', job_pos) + if open_gt >= 0: + close_pos = summary_content.find('', open_gt) + if close_pos >= 0: + summary_content = ( + summary_content[:close_pos] + + badge_html + + summary_content[close_pos:] + ) + return f'{m.group(1)}\n {summary_content}{m.group(4)}' + # Legacy flat layout fallback + for anchor in ('class="detail-date"', 'class="detail-version"'): + anchor_pos = summary_content.find(anchor) + if anchor_pos < 0: + continue + span_start = summary_content.rfind('', 0, span_start) + if last_close >= 0: + ins = last_close + 1 + while ins < span_start and summary_content[ins] in ' \t\n': + ins += 1 + summary_content = ( + summary_content[:ins] + + badge_html + '\n ' + + summary_content[ins:] + ) + else: + summary_content = badge_html + ' ' + summary_content + break + else: + summary_content = summary_content + ' ' + badge_html + return f'{m.group(1)}\n {summary_content}{m.group(4)}' + + content = _details_re.sub(_insert_badge, content) # Update header: replace "Data only" status using data attribute if patched > 0: @@ -407,8 +514,7 @@ def merge_analysis(report: MonitorReport, analysis_path: Path) -> None: } } """ - with open(analysis_path) as f: - data = json.load(f) + data = _load_json(analysis_path, "Analysis JSON") by_url = data.get("by_prow_url", {}) merged = 0 @@ -452,8 +558,7 @@ def _safe_dataclass_init(cls, data: dict): def load_json(json_path: Path) -> MonitorReport: """Load a MonitorReport from a JSON file (potentially enriched with deep analysis).""" - with open(json_path) as f: - data = json.load(f) + data = _load_json(json_path, "Report JSON") streams = [] for s in data.get("streams", []): diff --git a/payload-monitor/payload_monitor/report/templates/_analysis_card.html b/payload-monitor/payload_monitor/report/templates/_analysis_card.html index 0d5fcc7d..349d1aef 100644 --- a/payload-monitor/payload_monitor/report/templates/_analysis_card.html +++ b/payload-monitor/payload_monitor/report/templates/_analysis_card.html @@ -7,16 +7,19 @@ {% endif %} {% endif %}

AI Root Cause Analysis

+ {% if job_name %} +
{{ job_name }}
+ {% endif %}
- Root Cause: + Root Cause: {{ root_cause }}
- Failure Type: + Failure Type: {{ failure_type }}
- Impact: + Impact: {{ impact }}
{% if suspect_prs %} diff --git a/payload-monitor/payload_monitor/report/templates/dashboard.html b/payload-monitor/payload_monitor/report/templates/dashboard.html index f4329be3..69d9c674 100644 --- a/payload-monitor/payload_monitor/report/templates/dashboard.html +++ b/payload-monitor/payload_monitor/report/templates/dashboard.html @@ -12,6 +12,7 @@

Edge OCP Payload Monitor

SNO / TNA / TNF topology health across OpenShift nightly payloads
+
Review the Edge CI Payload Manager Responsibilities
View source on GitHub ↗
@@ -24,7 +25,7 @@

Edge OCP Payload Monitor

{% elif has_blocking %}
○ Data only - — run /edge-ocp-ci:generate-dashboard for AI analysis + - run /edge-ocp-ci:generate-dashboard for AI analysis
{% else %}
@@ -49,13 +50,13 @@

Edge OCP Payload Monitor

Note: This report was generated with limited data. {% if report.skip_prow %} - --skip-prow Prow artifact fetching was skipped — Error column, Failing Tests, and Failure Details will be empty. + --skip-prow Prow artifact fetching was skipped - Error column, Failing Tests, and Failure Details will be empty. {% endif %} {% if report.skip_sippy %} - --skip-sippy Sippy regression check was skipped — Sippy Job Regressions section is not available. + --skip-sippy Sippy regression check was skipped - Sippy Job Regressions section is not available. {% endif %} {% if report.skip_jira %} - JIRA JIRA_TOKEN not set — bug matching and suggested bugs are unavailable. Set JIRA_TOKEN to enable (optionally JIRA_USERNAME for Basic auth). + JIRA JIRA_TOKEN not set - bug matching and suggested bugs are unavailable. Set JIRA_TOKEN to enable (optionally JIRA_USERNAME for Basic auth). {% endif %} {% if report.skip_timing %} Insights Timing insights are disabled by default. Use --with-timing to include install/upgrade timing data. @@ -93,7 +94,7 @@

Edge OCP Payload Monitor

{% if report.payloads_per_stream < report.persistent_threshold %}
- Note: Analyzing only {{ report.payloads_per_stream }} payload{{ 's' if report.payloads_per_stream != 1 else '' }} per stream — + Note: Analyzing only {{ report.payloads_per_stream }} payload{{ 's' if report.payloads_per_stream != 1 else '' }} per stream - {% if report.payloads_per_stream < report.recurring_threshold %}Recurring, {% endif %}Persistent{% if report.payloads_per_stream < 3 %} and Unstable{% endif %} failure labels require more payloads to appear.
{% endif %} @@ -128,30 +129,11 @@

Edge OCP Jobs Health Overview

Payload History older → newer
- {% for stream in report.streams %} + {% for view in stream_views %} + {% set stream = view.stream %} {% set latest = stream.latest_payload %} {% set blocking_count = stream.total_blocking_edge_failures %} {% set informing_count = stream.total_informing_edge_failures %} - {# Collect affected topologies #} - {% set affected_topos = [] %} - {% for p in stream.payloads %} - {% for j in p.failing_edge_jobs %} - {% if j.topology and j.topology not in affected_topos %} - {% if affected_topos.append(j.topology) %}{% endif %} - {% endif %} - {% endfor %} - {% endfor %} - {# Compute trend: compare recent half vs older half #} - {% set mid = (stream.payloads | length // 2) | int %} - {% set recent_fails = namespace(count=0) %} - {% set older_fails = namespace(count=0) %} - {% for p in stream.payloads %} - {% if loop.index0 < mid %} - {% set recent_fails.count = recent_fails.count + (p.blocking_edge_failures | length) + (p.informing_edge_failures | length) %} - {% else %} - {% set older_fails.count = older_fails.count + (p.blocking_edge_failures | length) + (p.informing_edge_failures | length) %} - {% endif %} - {% endfor %}
{{ stream.version }}
Edge OCP Jobs Health Overview {% endif %}
- {% for t in affected_topos | sort %} + {% for t in view.affected_topologies %} {{ t }} {% endfor %} - {% if not affected_topos %} + {% if not view.affected_topologies %} — {% endif %}
{% if blocking_count == 0 and informing_count == 0 %} ▬ - {% elif recent_fails.count > older_fails.count %} - ▼ worsening - {% elif recent_fails.count < older_fails.count %} - ▲ improving + {% elif view.recent_fails > view.older_fails %} + ▼ worsening + {% elif view.recent_fails < view.older_fails %} + ▲ improving {% else %} - ▬ stable + ▬ stable {% endif %}
@@ -240,40 +222,15 @@

Edge OCP Jobs Health Overview

{% set ns = namespace( total_failures=all_failing|length, - blocking_failures=0, - informing_failures=0, - rejected_payloads=0, - total_payloads=0, + blocking_failures=blocking_failures, + informing_failures=informing_failures, + rejected_payloads=rejected_payloads, + total_payloads=total_payloads, regression_count=all_regressions|length, component_regression_count=component_regressions|length, bugs_found=report.jira_bugs|length, bugs_suggested=report.suggested_bugs|length ) %} -{% for item in all_failing %} - {% if item.job.job_type.value == 'blocking' %}{% set ns.blocking_failures = ns.blocking_failures + 1 %}{% endif %} - {% if item.job.job_type.value == 'informing' %}{% set ns.informing_failures = ns.informing_failures + 1 %}{% endif %} -{% endfor %} -{% for stream in report.streams %} - {% for p in stream.payloads %} - {% set ns.total_payloads = ns.total_payloads + 1 %} - {% if p.status.value == 'Rejected' %}{% set ns.rejected_payloads = ns.rejected_payloads + 1 %}{% endif %} - {% endfor %} -{% endfor %} -{% set fs_affected_topos = [] %} -{% for item in all_failing %} - {% if item.job.topology and item.job.topology not in fs_affected_topos %} - {% if fs_affected_topos.append(item.job.topology) %}{% endif %} - {% endif %} -{% endfor %} -{# Collect unique blocking jobs (deduplicated by job name) #} -{% set blocking_jobs_seen = [] %} -{% set blocking_jobs_list = [] %} -{% for item in all_failing %} - {% if item.job.job_type.value == 'blocking' and item.job.name not in blocking_jobs_seen %} - {% if blocking_jobs_seen.append(item.job.name) %}{% endif %} - {% if blocking_jobs_list.append(item) %}{% endif %} - {% endif %} -{% endfor %}

Findings Summary

@@ -311,10 +268,10 @@

Findings Summary

- {% for t in fs_affected_topos | sort %} + {% for t in topologies %} {{ t }} {% endfor %} - {% if not fs_affected_topos %} + {% if not topologies %} — {% endif %}
@@ -322,29 +279,28 @@

Findings Summary

- {# CRITICAL: Blocking failures — show the actual jobs inline #} + {# CRITICAL: Blocking failures - show the actual jobs inline #} {% if ns.blocking_failures > 0 %}
-
!
+
!
-
Critical — Immediate Action Required
+
Critical - Immediate Action Required
{{ ns.blocking_failures }} Blocking Failure{{ 's' if ns.blocking_failures != 1 else '' }}{% if blocking_jobs_list|length < ns.blocking_failures %} ({{ blocking_jobs_list|length }} Unique Job{{ 's' if blocking_jobs_list|length != 1 else '' }}){% endif %}
prevents payload acceptance
Investigate All →
- {# Compact blocking job summary — unique jobs with versions and top failing test #} + {# Compact blocking job summary - unique jobs with versions and top failing test #}
{% for item in blocking_jobs_list %} {% set fc = failure_counts.get(item.job.name, 0) %} {% set versions = blocking_job_versions.get(item.job.name, []) %} {% set tests = blocking_job_tests.get(item.job.name, []) %} -
+
{{ item.job.topology }} - {% set _detail_idx = blocking_job_first_idx.get(item.job.name) %} - {% if _detail_idx is not none %}{{ item.job.name }}{% else %}{{ item.job.name }}{% endif %} + {{ item.job.name }} {% if fc >= recurring_threshold %} {{ fc }}x {% endif %} @@ -367,23 +323,23 @@

Findings Summary {% endif %} - {# RETRIED: Blocking jobs that passed only after retry — flaky but not blocking #} + {# RETRIED: Blocking jobs that passed only after retry - flaky but not blocking #} {% if retried_successes %}
↻
-
Warning — Passed After Retry
+
Warning - Passed After Retry
{{ retried_successes|length }} Blocking Job{{ 's' if retried_successes|length != 1 else '' }} Succeeded on Retry
-
first attempt failed — may indicate flakiness or transient infrastructure issues
+
first attempt failed - may indicate flakiness or transient infrastructure issues
{% for item in retried_successes %} -
+
{{ item.job.topology }} - {{ item.job.name }} + {{ item.job.name }} Succeeded {{ item.version }}
@@ -405,22 +361,22 @@

Findings Summary {% endif %} - {# WARNING: Informing failures — surface escalation risks by name #} + {# WARNING: Informing failures - surface escalation risks by name #} {% if ns.informing_failures > 0 %}
-
▲
+
▲
-
Warning — Monitor for Stability
+
Warning - Monitor for Stability
{{ ns.informing_failures }} Informing Job Failure{{ 's' if ns.informing_failures != 1 else '' }}
-
across {{ ns.total_payloads }} payload{{ 's' if ns.total_payloads != 1 else '' }} — not blocking
+
across {{ ns.total_payloads }} payload{{ 's' if ns.total_payloads != 1 else '' }} - not blocking
View All →
{% if escalation_risks %}
{% for er in escalation_risks %} - + {{ er.job_name }} Unstable @@ -435,13 +391,13 @@

Findings Summary {% endif %} - {# REGRESSIONS: Sippy + Component Readiness — compact #} + {# REGRESSIONS: Sippy + Component Readiness - compact #} {% if ns.regression_count > 0 or ns.component_regression_count > 0 %}
-
i
+
i
-
Info — Triage Recommended
+
Info - Triage Recommended
{{ ns.regression_count + ns.component_regression_count }} Regression{{ 's' if (ns.regression_count + ns.component_regression_count) != 1 else '' }}
{% if ns.regression_count > 0 %}{{ ns.regression_count }} Sippy job regression{{ 's' if ns.regression_count != 1 else '' }}{% endif %} @@ -464,110 +420,135 @@

Findings Summary -{# Badge legend — explains recurring/persistent/unstable badges used in tables below #} -
- Badge Legend - Recurring job failed in {{ recurring_threshold }}+ payloads - Persistent job failed in {{ persistent_threshold }}+ payloads — likely not a flake - Unstable informing job failing 3+ consecutive payloads -
- - -

Nightly Payload — Edge Job Failures ({{ all_failing | length }} total)

+ +

Nightly Payload - Failure Analysis ({{ all_failing | length }} total)

{% if all_failing %}
- Topology: + Topology: {% for topo in topologies %} {% endfor %} - Type: + Type: - Version: + Version: {% for v in versions %} {% endfor %} + + Status: + + + + +
+
+ Badge Legend: + Recurring job failed in {{ recurring_threshold }}+ payloads + Persistent job failed in {{ persistent_threshold }}+ payloads - likely not a flake + Unstable informing job failing 3+ consecutive payloads +
- - - - - - - - - - - - - - - {% for item in all_failing %} - - - - - - - - - - - {% endfor %} - -
JobVersionPayloadTopologyTypeRecurrenceDate
- {{ item.job.name }} - {{ item.version }}{{ item.payload_tag }} - {{ item.job.topology }} - {% if item.job.name in escalation_risk_jobs %} - Unstable - {% endif %} - {% set also_in = cross_topology.get(item.job.name, []) %} - {% if also_in %} - Also in: {% for t in also_in %}{{ t }}{% endfor %} - {% endif %} - {{ item.job.job_type.value }} - {% set fc = failure_counts.get(item.job.name, 0) %} - {% if fc >= persistent_threshold %} - Persistent ({{ fc }}x) - {% elif fc >= recurring_threshold %} - Recurring ({{ fc }}x) - {% endif %} - {{ item.date }}View Details ↓
+
+
+
Job
+
Version
+
Payload
+
Topology
+
Type
+
Recurrence
+
Date
-{% if all_failing | length > 5 %} - -{% endif %} -{% else %} -
No failing edge jobs found across monitored payloads.
-{% endif %} - - -{% if all_failing %} -

Nightly Payload — Failure Analysis

{% for detail in all_failing %} -
+{% set _fc = failure_counts.get(detail.job.name, 0) %} +{% set _statuses = [] %} +{% if _fc >= persistent_threshold %}{% if _statuses.append('persistent') %}{% endif %}{% endif %} +{% if _fc >= recurring_threshold and _fc < persistent_threshold %}{% if _statuses.append('recurring') %}{% endif %}{% endif %} +{% if detail.job.name in escalation_risk_jobs %}{% if _statuses.append('unstable') %}{% endif %}{% endif %} +{% if detail.job.deep_analysis %}{% if _statuses.append('ai-analyzed') %}{% endif %}{% endif %} +
- {{ detail.job.topology }} - {{ detail.job.job_type.value }} - {% if detail.job.name in escalation_risk_jobs %}Unstable{% endif %} - {% if detail.job.deep_analysis %}AI Analyzed{% endif %} - {% set also_in = cross_topology.get(detail.job.name, []) %} - {% if also_in %} - Also in: {% for t in also_in %}{{ t }}{% endfor %} - {% endif %} - {{ detail.job.name }} ({{ detail.version }}) + {{ detail.job.name }}{% if detail.job.deep_analysis %}AI{% endif %}{% set also_in = cross_topology.get(detail.job.name, []) %}{% if also_in %}{% for t in also_in %}{{ t }}{% endfor %}{% endif %} + {{ detail.version }} + {{ detail.payload_tag }} + {{ detail.job.topology }} + {{ detail.job.job_type.value }} + {% set fc = failure_counts.get(detail.job.name, 0) %}{% if fc >= persistent_threshold %}Persistent{% elif fc >= recurring_threshold %}Recurring{% endif %}{% if detail.job.name in escalation_risk_jobs %}Unstable{% endif %}{% if fc < recurring_threshold and detail.job.name not in escalation_risk_jobs %}—{% endif %} + {% if detail.date %}{{ detail.date }}{% endif %}
-

Payload: {{ detail.payload_tag }}

+ + {# AI analysis: rendered inline if deep_analysis exists, or placeholder for post-generation patching #} + {% if detail.job.deep_analysis %} + {% set da = detail.job.deep_analysis %} + {% set root_cause = da.root_cause %} + {% set failure_type = da.failure_type %} + {% set impact = da.impact %} + {% set suspect_prs = da.suspect_prs %} + {% set recommendation = da.recommendation %} + {% set same_root_cause = da.same_root_cause %} + {% set attempt_analyses = da.attempt_analyses %} + {% set attempt_count = detail.job.previous_attempts|length + 1 %} + {% set payload_tag = detail.payload_tag %} + {% set job_name = detail.job.name %} + {% include "_analysis_card.html" %} + {% else %} +
+ For deeper analysis, use Claude directly:
+
+ /ci:prow-job-analysis {{ detail.job.prow_url }} + +
+
+ {% endif %} + + {# Inline JIRA matches for this job #} + {% set job_bugs = jira_matches_by_job.get(detail.job.name, []) %} + {% if job_bugs %} +
+ Related JIRA Bugs: + {% for bug in job_bugs %} + + {{ bug.key }} + {{ bug.status }} + + {% endfor %} +
+ {% elif not report.skip_jira %} + {% set suggested = suggested_bugs_by_job.get(detail.job.name) %} + {% if suggested %} +
+ No existing JIRA bug found for this job. + Create Bug in JIRA +
+ {% endif %} + {% endif %} + + {# Escalation risk: consecutive failure history #} + {% set er_info = escalation_risk_by_job.get(detail.job.name) %} + {% if er_info and er_info.failing_runs %} +
+ Consecutive Failures ({{ er_info.consecutive_failures }}x, newest first) +
+ {% for run in er_info.failing_runs %} + {{ run.payload_tag }} + {% endfor %} + {% if er_info.triage_url %} + View full history in Sippy → + {% endif %} +
+
+ {% endif %} + + {# Attempts and failing tests #} {% if detail.job.previous_attempts %}
{% for attempt in detail.job.previous_attempts %} @@ -612,85 +593,44 @@

Nightly Payload — Failure Analysis

{% else %} -

Prow Job: View in Prow

- {% endif %} - - {# Inline JIRA matches for this job #} - {% set job_bugs = jira_matches_by_job.get(detail.job.name, []) %} - {% if job_bugs %} -
- Related JIRA Bugs: - {% for bug in job_bugs %} - - {{ bug.key }} - {{ bug.status }} - - {% endfor %} -
- {% elif not report.skip_jira %} - {# No JIRA match and JIRA is enabled — show create bug button #} - {% set suggested = suggested_bugs_by_job.get(detail.job.name) %} - {% if suggested %} -
- No existing JIRA bug found for this job. - Create Bug in JIRA -
- {% endif %} - {% endif %} - - {% if detail.job.deep_analysis %} - {% set da = detail.job.deep_analysis %} - {% set root_cause = da.root_cause %} - {% set failure_type = da.failure_type %} - {% set impact = da.impact %} - {% set suspect_prs = da.suspect_prs %} - {% set recommendation = da.recommendation %} - {% set same_root_cause = da.same_root_cause %} - {% set attempt_analyses = da.attempt_analyses %} - {% set attempt_count = detail.job.previous_attempts|length + 1 %} - {% set payload_tag = detail.payload_tag %} - {% include "_analysis_card.html" %} - {% else %} -
- For deeper analysis, use Claude directly:
-
- /ci:prow-job-analyze-test-failure {{ detail.job.prow_url }} - + {# Failing tests for jobs without previous attempts #} + {% if detail.job.failing_tests %} +
+
+

Failing Tests ({{ detail.job.failing_tests|length }})

+ {% for test in detail.job.failing_tests %} +
+ {{ test.name }} + 
{{ test.error_message }}
+
+ {% endfor %}
{% endif %} - - {% if not detail.job.previous_attempts and detail.job.failing_tests %} -

Failing Tests ({{ detail.job.failing_tests | length }})

- {% for test in detail.job.failing_tests %} -
- {{ test.name }} -
- 
{{ test.error_message }}
-
-
- {% endfor %} {% endif %}
{% endfor %} +
{% if all_failing | length > 5 %} {% endif %} +{% else %} +
No failing edge jobs found across monitored payloads.
{% endif %} {% if all_regressions %}

Sippy Job Regressions Status

-
Source: Sippy Jobs — jobs with net improvement drop >10% or pass rate below 80% (≥5 runs). Jobs with fewer than 3 runs are excluded due to insufficient data.
+
Source: Sippy Jobs - jobs with net improvement drop >10% or pass rate below 80% (≥5 runs). Jobs with fewer than 3 runs are excluded due to insufficient data.
- Topology: + Topology: {% for topo in reg_topologies %} {% endfor %} - Version: + Version: {% for v in reg_versions %} @@ -753,26 +693,26 @@

Sippy Job Regressions Status

{% if component_regressions %} -

Component Readiness — HA vs Edge Topologies

+

Component Readiness - HA vs Edge Topologies

- Source: Sippy Component Readiness — components with statistically significant regressions on edge topologies (SNO, TNF) compared to HA (Fisher's exact test, 95% confidence). + Source: Sippy Component Readiness - components with statistically significant regressions on edge topologies (SNO, TNF) compared to HA (Fisher's exact test, 95% confidence).
{% if cr_comparisons | length > 1 %} - Comparison: + Comparison: {% for c in cr_comparisons %} {% endfor %} {% endif %} - Platform: + Platform: {% for p in cr_platforms %} {% endfor %} - Version: + Version: {% for v in cr_versions %} @@ -898,7 +838,7 @@

Suggested New Bugs

diff --git a/payload-monitor/payload_monitor/report/templates/scripts.js b/payload-monitor/payload_monitor/report/templates/scripts.js index 0c44dcba..7a86fabc 100644 --- a/payload-monitor/payload_monitor/report/templates/scripts.js +++ b/payload-monitor/payload_monitor/report/templates/scripts.js @@ -14,7 +14,6 @@ document.addEventListener('DOMContentLoaded', function() { // Filter functionality document.addEventListener('DOMContentLoaded', function() { const filterBtns = document.querySelectorAll('.filter-btn'); - const jobRows = document.querySelectorAll('.job-row'); const detailRows = document.querySelectorAll('.detail-row'); const crRows = document.querySelectorAll('.cr-row'); const regressionRows = document.querySelectorAll('.regression-row'); @@ -63,12 +62,12 @@ document.addEventListener('DOMContentLoaded', function() { const INITIAL_VISIBLE = 5; const sectionVisibleLimit = { anomalies: 10 }; - const sectionsCollapsed = { jobs: true, details: true, regressions: true, cr: true, anomalies: true }; - const sectionSelectors = { jobs: '.job-row', details: '.detail-row', regressions: '.regression-row', cr: '.cr-row', anomalies: '.anomaly-row' }; - const sectionLabels = { jobs: ' failing jobs', details: ' failure details', regressions: ' regressions', cr: ' component regressions', anomalies: ' anomalies' }; + const sectionsCollapsed = { details: true, regressions: true, cr: true, anomalies: true }; + const sectionSelectors = { details: '.detail-row', regressions: '.regression-row', cr: '.cr-row', anomalies: '.anomaly-row' }; + const sectionLabels = { details: ' failure details', regressions: ' regressions', cr: ' component regressions', anomalies: ' anomalies' }; // Count rows that pass the current filter (inline style.display is set by filters, - // collapsed-row class is separate — so style.display !== 'none' means "passes filter") + // collapsed-row class is separate - so style.display !== 'none' means "passes filter") function countFiltered(selector) { let count = 0; document.querySelectorAll(selector).forEach(row => { @@ -81,7 +80,7 @@ document.addEventListener('DOMContentLoaded', function() { Object.entries(sectionSelectors).forEach(([section, selector]) => { let visibleIdx = 0; document.querySelectorAll(selector).forEach(row => { - // Skip rows hidden by filters — don't count them toward the visible limit + // Skip rows hidden by filters - don't count them toward the visible limit if (row.style.display === 'none') { row.classList.remove('collapsed-row'); return; @@ -117,9 +116,9 @@ document.addEventListener('DOMContentLoaded', function() { function applyFilters() { function isVisible(row) { for (const [group, values] of Object.entries(activeFilters)) { + if (!(group in row.dataset)) continue; const rowValue = row.dataset[group]; - if (!rowValue) continue; - const rowValues = rowValue.split(' '); + const rowValues = rowValue ? rowValue.split(' ') : []; if (!rowValues.some(v => values.has(v))) { return false; } @@ -127,10 +126,6 @@ document.addEventListener('DOMContentLoaded', function() { return true; } - jobRows.forEach(row => { - row.style.display = isVisible(row) ? '' : 'none'; - }); - detailRows.forEach(row => { row.style.display = isVisible(row) ? '' : 'none'; }); @@ -162,7 +157,7 @@ document.addEventListener('DOMContentLoaded', function() { updateExpandButtons(); // Update header count - const visibleCount = countFiltered('.job-row'); + const visibleCount = countFiltered('.detail-row'); const countEl = document.getElementById('visible-count'); if (countEl) countEl.textContent = visibleCount; } @@ -178,25 +173,6 @@ document.addEventListener('DOMContentLoaded', function() { updateExpandButtons(); }; - // "View Details" links: expand both sections, open the target detail, scroll to it - document.querySelectorAll('.detail-link').forEach(link => { - link.addEventListener('click', function(e) { - e.preventDefault(); - // Expand both sections so the target is visible - sectionsCollapsed.jobs = false; - sectionsCollapsed.details = false; - applyCollapse(); - updateExpandButtons(); - - const targetId = this.getAttribute('href').substring(1); - const detail = document.getElementById(targetId); - if (detail) { - detail.open = true; - detail.scrollIntoView({ behavior: 'smooth', block: 'start' }); - } - }); - }); - // Column sorting for any table with sortable headers const sortableHeaders = document.querySelectorAll('th.sortable'); sortableHeaders.forEach(th => { @@ -208,7 +184,7 @@ document.addEventListener('DOMContentLoaded', function() { const col = parseInt(this.dataset.col); const rows = Array.from(tbody.querySelectorAll('tr')); - // Toggle sort direction — only clear siblings in the same table + // Toggle sort direction - only clear siblings in the same table const isAsc = this.classList.contains('asc'); table.querySelectorAll('th.sortable').forEach(h => { h.classList.remove('asc', 'desc'); }); this.classList.add(isAsc ? 'desc' : 'asc'); @@ -247,6 +223,43 @@ document.addEventListener('DOMContentLoaded', function() { }); }; + // Click-to-copy on action cell and claude-suggestion code elements + document.querySelectorAll('.action-cell code, .claude-suggestion code, .claude-cmd-popup code').forEach(function(code) { + code.style.cursor = 'pointer'; + code.addEventListener('click', function(e) { + var text = code.textContent.trim(); + navigator.clipboard.writeText(text).then(function() { + var tooltip = document.createElement('span'); + tooltip.className = 'copy-tooltip'; + tooltip.textContent = 'Copied!'; + tooltip.style.left = e.clientX + 'px'; + tooltip.style.top = e.clientY + 'px'; + document.body.appendChild(tooltip); + setTimeout(function() { tooltip.remove(); }, 1500); + }); + }); + }); + + // Scroll to a job row and open it. Matches by job name. + window.scrollToJob = function(jobName) { + var rows = document.querySelectorAll('details.detail-row'); + var row = null; + for (var i = 0; i < rows.length; i++) { + if (rows[i].dataset.jobName === jobName) { row = rows[i]; break; } + } + if (row) { + if (row.classList.contains('collapsed-row')) { + sectionsCollapsed.details = false; + applyCollapse(); + updateExpandButtons(); + } + row.open = true; + row.scrollIntoView({ behavior: 'smooth', block: 'center' }); + row.querySelector('summary').style.background = 'rgba(210,153,34,0.15)'; + setTimeout(function() { row.querySelector('summary').style.background = ''; }, 2000); + } + }; + // Draggable column resizers document.querySelectorAll('table').forEach(table => { const headers = table.querySelectorAll('th'); @@ -290,6 +303,90 @@ document.addEventListener('DOMContentLoaded', function() { }); }); + // Detail grid sorting (failure analysis section) + const gridHeader = document.querySelector('.detail-grid-header'); + if (gridHeader) { + const cellClasses = ['detail-cell-job', 'detail-cell-version', 'detail-cell-payload', 'detail-cell-topology', 'detail-cell-type', 'detail-cell-recurrence', 'detail-cell-date']; + const sortableCols = gridHeader.querySelectorAll('[data-sort-col]'); + + sortableCols.forEach(col => { + col.addEventListener('click', function() { + const colIdx = parseInt(this.dataset.sortCol); + const isAsc = this.classList.contains('asc'); + sortableCols.forEach(c => c.classList.remove('asc', 'desc')); + this.classList.add(isAsc ? 'desc' : 'asc'); + const dir = isAsc ? -1 : 1; + + const rows = Array.from(document.querySelectorAll('details.detail-row')); + const cellClass = cellClasses[colIdx]; + const numPat = /^[-+]?\d+(\.\d+)?%?$/; + + rows.sort((a, b) => { + const aCell = a.querySelector('summary .' + cellClass); + const bCell = b.querySelector('summary .' + cellClass); + let aText = (aCell?.textContent || '').trim().replace(/\s+/g, ' ').toLowerCase(); + let bText = (bCell?.textContent || '').trim().replace(/\s+/g, ' ').toLowerCase(); + if (numPat.test(aText) && numPat.test(bText)) { + return (parseFloat(aText) - parseFloat(bText)) * dir; + } + return aText.localeCompare(bText) * dir; + }); + + let ref = gridHeader; + rows.forEach(row => { ref.after(row); ref = row; }); + applyCollapse(); + updateExpandButtons(); + }); + }); + + // Detail grid column resizing + const headerDivs = Array.from(gridHeader.querySelectorAll(':scope > div')); + let gridColWidths = null; + + function applyGridWidths() { + const tpl = gridColWidths.map(w => w + 'px').join(' '); + gridHeader.style.gridTemplateColumns = tpl; + document.querySelectorAll('details.detail-row > summary').forEach(s => { + s.style.gridTemplateColumns = tpl; + }); + } + + function freezeGridLayout() { + if (gridColWidths) return; + gridColWidths = headerDivs.map(d => d.offsetWidth); + applyGridWidths(); + } + + headerDivs.forEach((div, idx) => { + const resizer = document.createElement('div'); + resizer.className = 'col-resizer'; + div.appendChild(resizer); + resizer.addEventListener('click', e => e.stopPropagation()); + + let startX; + resizer.addEventListener('mousedown', function(e) { + e.preventDefault(); + e.stopPropagation(); + freezeGridLayout(); + startX = e.pageX; + const startWidth = gridColWidths[idx]; + resizer.classList.add('resizing'); + + function onMouseMove(e) { + gridColWidths[idx] = Math.max(40, startWidth + e.pageX - startX); + applyGridWidths(); + } + function onMouseUp() { + resizer.classList.remove('resizing'); + document.removeEventListener('mousemove', onMouseMove); + document.removeEventListener('mouseup', onMouseUp); + } + document.addEventListener('mousemove', onMouseMove); + document.addEventListener('mouseup', onMouseUp); + }); + }); + } + // Jump to failures table with filters pre-applied window.jumpToFailures = function(version, jobtype) { // Reset all filters first @@ -323,7 +420,7 @@ document.addEventListener('DOMContentLoaded', function() { applyFilters(); // Scroll to the failures heading - const heading = document.querySelector('#tab-payload-health h2:nth-of-type(2)'); + const heading = document.getElementById('section-edge-failures'); if (heading) heading.scrollIntoView({ behavior: 'smooth', block: 'start' }); }; @@ -354,10 +451,11 @@ document.addEventListener('DOMContentLoaded', function() { } }); + var jobNameEl = card.querySelector('.da-job-name'); + var jobName = jobNameEl ? jobNameEl.textContent.trim() : ''; var summaryText = summary.textContent.trim(); - var match = summaryText.match(/(?:AI Analyzed)?\s*(.+?)\s*\((\d+\.\d+)\)\s*$/); - var jobName = match ? match[1].trim() : ''; - var version = match ? match[2] : ''; + var match = summaryText.match(/\((\d+\.\d+)\)/); + var version = match ? match[1] : ''; var fields = card.querySelectorAll('.da-field'); var rootCause = '', failureType = '', recommendation = ''; @@ -409,6 +507,12 @@ document.addEventListener('DOMContentLoaded', function() { var header = document.createElement('div'); header.className = 'ai-highlight-header'; + if (item.jobName) { + var name = document.createElement('span'); + name.className = 'job-name'; + name.textContent = item.jobName; + header.appendChild(name); + } if (item.topology) { var b1 = document.createElement('span'); b1.className = 'badge ' + item.topology.toLowerCase(); @@ -433,19 +537,13 @@ document.addEventListener('DOMContentLoaded', function() { ver.textContent = item.version; header.appendChild(ver); } - if (item.jobName) { - var name = document.createElement('span'); - name.className = 'job-name'; - name.textContent = item.jobName; - header.appendChild(name); - } card.appendChild(header); if (item.rootCause) { var cause = document.createElement('div'); cause.className = 'ai-highlight-cause'; var lbl1 = document.createElement('span'); - lbl1.className = 'da-label'; + lbl1.className = 'da-label da-label-highlight'; lbl1.textContent = 'Root Cause:'; cause.appendChild(lbl1); cause.appendChild(document.createTextNode(' ' + item.rootCause)); @@ -455,7 +553,7 @@ document.addEventListener('DOMContentLoaded', function() { var rec = document.createElement('div'); rec.className = 'ai-highlight-rec'; var lbl2 = document.createElement('span'); - lbl2.className = 'da-label'; + lbl2.className = 'da-label da-label-highlight'; lbl2.textContent = 'Action:'; rec.appendChild(lbl2); rec.appendChild(document.createTextNode(' ' + item.recommendation)); diff --git a/payload-monitor/payload_monitor/report/templates/styles.css b/payload-monitor/payload_monitor/report/templates/styles.css index 8f18cff6..9428ae2b 100644 --- a/payload-monitor/payload_monitor/report/templates/styles.css +++ b/payload-monitor/payload_monitor/report/templates/styles.css @@ -467,7 +467,10 @@ a:hover { text-decoration: underline; } .fs-blocking-row { padding: 8px 18px 8px 52px; border-bottom: 1px solid rgba(248,81,73,0.08); + cursor: pointer; + transition: background 0.12s; } +.fs-blocking-row:hover { background: rgba(248,81,73,0.06); } .fs-blocking-row:last-child { border-bottom: none; } .fs-blocking-main { display: flex; @@ -538,6 +541,16 @@ a:hover { text-decoration: underline; } } .fs-escalation-item:last-child { border-bottom: none; } .fs-escalation-item:hover { background: rgba(210,153,34,0.06); text-decoration: none; } +.tooltip-run { + display: block; + padding: 3px 0; + border-top: 1px solid var(--border); +} +.tooltip-run a { + color: var(--accent); + font-size: 12px; + font-family: monospace; +} .fs-escalation-left { display: flex; align-items: center; @@ -559,6 +572,43 @@ a:hover { text-decoration: underline; } font-weight: 600; } +/* Escalation history inside job cards */ +.escalation-history { + border-left: 3px solid var(--orange); + background: rgba(210,153,34,0.06); + border-radius: 0 6px 6px 0; + padding: 10px 14px; + margin-bottom: 12px; +} +.escalation-history strong { + display: block; + font-size: 12px; + color: var(--orange); + margin-bottom: 6px; +} +.escalation-history-runs { + display: flex; + flex-wrap: wrap; + gap: 6px; +} +.escalation-history-run { + font-size: 12px; + font-family: monospace; + color: var(--accent); + background: var(--surface); + border: 1px solid var(--border); + border-radius: 4px; + padding: 2px 8px; + text-decoration: none; + transition: background 0.12s; +} +.escalation-history-run:hover { background: var(--surface-hover); } +.escalation-history-sippy { + font-family: inherit; + color: var(--orange); + border-color: rgba(210,153,34,0.3); +} + .fs-section-action { display: inline-flex; align-items: center; @@ -680,6 +730,13 @@ a:hover { text-decoration: underline; } gap: 8px; margin-bottom: 16px; flex-wrap: wrap; + align-items: center; +} +.filter-label { + color: var(--text-muted); + font-size: 11px; + font-weight: 600; + padding: 4px 0; } .filter-btn { padding: 4px 12px; @@ -707,22 +764,6 @@ table { th { position: relative; min-width: 60px; -} -th .col-resizer { - position: absolute; - right: 0; - top: 0; - bottom: 0; - width: 4px; - cursor: col-resize; - background: transparent; - z-index: 1; -} -th .col-resizer:hover, -th .col-resizer.resizing { - background: var(--accent); -} -th { text-align: left; padding: 8px 12px; border-bottom: 2px solid var(--border); @@ -746,21 +787,26 @@ table.layout-frozen td { } tr:hover { background: var(--surface-hover); } -.detail-link { - font-size: 11px; - padding: 2px 8px; - border: 1px solid var(--border); - border-radius: 4px; - white-space: nowrap; - color: var(--accent); +.col-resizer { + position: absolute; + right: 0; + top: 0; + bottom: 0; + width: 4px; + cursor: col-resize; + background: transparent; + z-index: 1; +} +.col-resizer:hover, +.col-resizer.resizing { + background: var(--accent); } -.detail-link:hover { background: var(--surface-hover); text-decoration: none; border-color: var(--accent); } -th.sortable { cursor: pointer; user-select: none; } -th.sortable:hover { color: var(--accent); } -th.sortable::after { content: ' ⇅'; font-size: 10px; opacity: 0.4; } -th.sortable.asc::after { content: ' ↑'; opacity: 1; } -th.sortable.desc::after { content: ' ↓'; opacity: 1; } +.sortable { cursor: pointer; user-select: none; } +.sortable:hover { color: var(--accent); } +.sortable::after { content: ' ⇅'; font-size: 10px; opacity: 0.4; } +.sortable.asc::after { content: ' ↑'; opacity: 1; } +.sortable.desc::after { content: ' ↓'; opacity: 1; } .badge { display: inline-block; @@ -777,7 +823,6 @@ th.sortable.desc::after { content: ' ↓'; opacity: 1; } .badge.recurring { background: rgba(210,153,34,0.15); color: #f0d070; } .badge.persistent { background: rgba(139,92,246,0.2); color: #b49cfa; } .badge.escalation-risk { background: rgba(219,109,40,0.25); color: #f0a050; } -.badge.retried { background: rgba(219,109,40,0.2); color: var(--orange); } .badge.green { background: rgba(63,185,80,0.2); color: var(--green); } .previous-attempts { @@ -785,11 +830,6 @@ th.sortable.desc::after { content: ' ↓'; opacity: 1; } padding-left: 12px; border-left: 3px solid var(--orange); } -.previous-attempts h4 { - font-size: 13px; - color: var(--orange); - margin-bottom: 8px; -} .previous-attempt-card { background: rgba(219,109,40,0.05); border: 1px solid rgba(219,109,40,0.2); @@ -843,10 +883,109 @@ details summary { padding: 12px 16px; cursor: pointer; font-weight: 500; - font-size: 13px; + font-size: 14px; } details summary:hover { background: var(--surface-hover); } -.detail-version { color: #f0f6fc; font-weight: 700; } + +/* Failure analysis grid layout */ +.detail-grid-header { + display: grid; + grid-template-columns: 24px minmax(0, 3fr) auto minmax(0, 2fr) auto auto auto auto; + gap: 0 8px; + align-items: center; + padding: 8px 16px; + max-width: 100%; + background: rgba(48,54,61,0.5); + border: 1px solid var(--border); + border-radius: 8px 8px 0 0; + margin-bottom: 0; + font-size: 13px; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.5px; + color: var(--text-muted); +} +details.detail-row { + border-radius: 0; + margin-bottom: 0; + border-top: none; +} +details.detail-row:last-of-type { + border-radius: 0 0 8px 8px; + margin-bottom: 8px; +} +details.detail-row > summary { + list-style: none; + display: grid; + grid-template-columns: 24px minmax(0, 3fr) auto minmax(0, 2fr) auto auto auto auto; + gap: 0 8px; + align-items: center; +} +details.detail-row > summary::-webkit-details-marker { display: none; } +details.detail-row > summary::before { + content: "\25B6"; + font-size: 10px; + color: var(--text-muted); + transition: transform 0.15s; + justify-self: center; +} +details.detail-row[open] > summary::before { transform: rotate(90deg); } +details.detail-row[open] { + border: 1px solid var(--accent); + border-left: 3px solid var(--accent); + background: rgba(88,166,255,0.04); + z-index: 1; + position: relative; +} +.detail-cell-topology { text-align: center; } +details.detail-row > summary:hover { border-radius: 0; } +details.detail-row:last-of-type > summary:hover { border-radius: 0 0 8px 8px; } +.detail-cell-job { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + font-size: 14px; + font-weight: 500; + min-width: 0; +} +.detail-cell-job .badge { font-size: 9px; padding: 1px 5px; margin-left: 4px; vertical-align: middle; } +.detail-cell-job .cross-topo-hint { font-size: 10px; margin-left: 4px; } +.detail-cell-job .cross-topo-hint .badge { font-size: 9px; padding: 1px 4px; margin-left: 1px; } +.detail-cell-version { + font-weight: 700; + color: #f0f6fc; + font-size: 14px; + text-align: center; +} +.detail-cell-payload { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + font-size: 13px; + min-width: 0; +} +.detail-cell-payload a { color: var(--accent); text-decoration: none; } +.detail-cell-payload a:hover { text-decoration: underline; } +.detail-cell-recurrence { + display: flex; + gap: 4px; + align-items: center; + flex-wrap: wrap; +} +.detail-cell-recurrence .badge { font-size: 11px; padding: 2px 7px; } +.detail-cell-date { + color: var(--text-muted); + font-size: 13px; + white-space: nowrap; +} +.detail-grid-header > div { position: relative; } +@media (max-width: 1024px) { + .detail-grid-header { display: none; } + details.detail-row { border-radius: 8px; margin-bottom: 8px; border-top: 1px solid var(--border); } + details.detail-row > summary { display: flex; flex-wrap: wrap; gap: 6px; } + .detail-cell-job { flex-basis: 100%; white-space: normal; } + .detail-cell-payload { display: none; } +} details .content { padding: 0 16px 16px; font-size: 13px; @@ -879,7 +1018,6 @@ details pre { } .jira-card .key { font-weight: 600; color: var(--accent); } .jira-card .summary { color: var(--text); margin-left: 12px; flex: 1; } -.jira-card .status { font-size: 12px; } /* Suggested bugs */ .suggest-card { @@ -921,6 +1059,13 @@ details pre { margin: 0 0 8px; font-size: 14px; } +.da-job-name { + color: var(--text); + font-weight: 600; + font-size: 13px; + margin: -4px 0 8px; + word-break: break-all; +} .da-field { margin: 6px 0; font-size: 13px; @@ -931,6 +1076,10 @@ details pre { margin-right: 6px; font-size: 14px; } +.da-label-highlight { + color: var(--accent); + font-weight: 700; +} .da-field span:not(.da-label, .badge) { color: rgba(230,237,243,0.7); font-size: 14px; @@ -1000,6 +1149,25 @@ details pre { .action-cell:focus-within .copy-btn { opacity: 1; } .copy-btn:hover, .copy-btn:focus { background: var(--surface-hover); } .copy-btn.copied { opacity: 1; color: var(--green); border-color: var(--green); } +.copy-tooltip { + position: fixed; + transform: translate(-50%, -100%); + margin-top: -8px; + padding: 4px 10px; + border-radius: 4px; + background: var(--green); + color: #000; + font-size: 12px; + font-weight: 600; + pointer-events: none; + z-index: 9999; + animation: copy-tooltip-fade 1.5s ease-out forwards; +} +@keyframes copy-tooltip-fade { + 0%, 60% { opacity: 1; } + 100% { opacity: 0; } +} +.action-cell code:hover, .claude-suggestion code:hover, .claude-cmd-popup code:hover { opacity: 0.8; cursor: pointer; } /* AI Analysis Highlights */ .ai-highlights { @@ -1045,11 +1213,17 @@ details pre { } .ai-highlight-cause, .ai-highlight-rec { - font-size: 12px; - color: var(--text-muted); + font-size: 13px; + color: var(--text); margin: 4px 0; line-height: 1.4; } +.ai-highlight-cause .da-label-highlight, +.ai-highlight-rec .da-label-highlight { + font-weight: 700; + font-size: 13px; + color: var(--accent); +} /* Regressions */ .regression-row { border-left: 3px solid var(--red); } diff --git a/payload-monitor/payload_monitor/report/timing_section.py b/payload-monitor/payload_monitor/report/timing_section.py index d7b62afd..daf5a446 100644 --- a/payload-monitor/payload_monitor/report/timing_section.py +++ b/payload-monitor/payload_monitor/report/timing_section.py @@ -4,7 +4,7 @@ import html from collections import defaultdict -from datetime import datetime, timedelta, timezone +from datetime import datetime from ..collectors.timing import compute_stats from ..models import TimingReport, TimingRun @@ -62,6 +62,17 @@ def _color_for(topology: str, run_type: str) -> str: return _COLORS.get((topology, run_type), "#8b949e") +def _stat_cells(stats: dict, keys: list[str]) -> str: + """Render ```` cells for the given stat keys.""" + parts = [] + for k in keys: + if k == "cv": + parts.append(f'{stats["cv"]}%') + else: + parts.append(f'{_fmt_duration(stats[k])}') + return "".join(parts) + + # --------------------------------------------------------------------------- # Sub-section renderers # --------------------------------------------------------------------------- @@ -98,13 +109,8 @@ def render_summary_table(report: TimingReport) -> str: f'{html.escape(rtype.capitalize())}' f'all' f'{stats["count"]}' - f'{_fmt_duration(stats["avg"])}' - f'{_fmt_duration(stats["median"])}' - f'{_fmt_duration(stats["p90"])}' - f'{_fmt_duration(stats["p95"])}' - f'{_fmt_duration(stats["p99"])}' - f'{stats["cv"]}%' - f'' + + _stat_cells(stats, ["avg", "median", "p90", "p95", "p99", "cv"]) + + '' ) else: rows.append( @@ -136,13 +142,8 @@ def render_summary_table(report: TimingReport) -> str: f'{html.escape(rtype.capitalize())}' f'{html.escape(ver)}' f'{ver_stats["count"]}' - f'{_fmt_duration(ver_stats["avg"])}' - f'{_fmt_duration(ver_stats["median"])}' - f'{_fmt_duration(ver_stats["p90"])}' - f'{_fmt_duration(ver_stats["p95"])}' - f'{_fmt_duration(ver_stats["p99"])}' - f'{ver_stats["cv"]}%' - f'' + + _stat_cells(ver_stats, ["avg", "median", "p90", "p95", "p99", "cv"]) + + '' ) else: rows.append( @@ -216,11 +217,8 @@ def render_variant_table(report: TimingReport) -> str: f'{html.escape(vk)}' f'all' f'{stats["count"]}' - f'{_fmt_duration(stats["avg"])}' - f'{_fmt_duration(stats["median"])}' - f'{_fmt_duration(stats["p95"])}' - f'{stats["cv"]}%' - f'' + + _stat_cells(stats, ["avg", "median", "p95", "cv"]) + + '' ) # Per-version rows — hidden by default, shown when version filter active for ver in sorted(set(r.release for r in group)): @@ -235,11 +233,8 @@ def render_variant_table(report: TimingReport) -> str: f'{html.escape(vk)}' f'{html.escape(ver)}' f'{ver_stats["count"]}' - f'{_fmt_duration(ver_stats["avg"])}' - f'{_fmt_duration(ver_stats["median"])}' - f'{_fmt_duration(ver_stats["p95"])}' - f'{ver_stats["cv"]}%' - f'' + + _stat_cells(ver_stats, ["avg", "median", "p95", "cv"]) + + '' ) all_versions = sorted(set(r.release for r in runs)) @@ -441,15 +436,17 @@ def ty(minutes: float) -> float: # Legend legend_x = pad_l + 8 legend_y = pad_t + 8 + n_series = len(series) + item_w = min(130, (w - pad_r - legend_x) / n_series) if n_series else 130 for i, (key, _) in enumerate(sorted(series.items())): topo, rtype = key color = _color_for(topo, rtype) - lx = legend_x + i * 130 + lx = legend_x + i * item_w parts.append( - f'' + f'' ) parts.append( - f'{topo} {rtype}' ) diff --git a/payload-monitor/tests/test_analyzer.py b/payload-monitor/tests/test_analyzer.py index 855dc416..c5c6f493 100644 --- a/payload-monitor/tests/test_analyzer.py +++ b/payload-monitor/tests/test_analyzer.py @@ -241,7 +241,14 @@ def j(r): assert risks[0].consecutive_failures == 3 assert risks[0].topology == "SNO" assert risks[0].version == "4.19" - assert "j1" in risks[0].sippy_url + assert risks[0].prow_url == "https://prow/j1" + assert "sippy.dptools.openshift.org" in risks[0].triage_url + assert "4.19" in risks[0].triage_url + assert len(risks[0].failing_runs) == 3 + assert risks[0].failing_runs[0]["payload_tag"] == "t5" + assert risks[0].failing_runs[1]["payload_tag"] == "t4" + assert risks[0].failing_runs[2]["payload_tag"] == "t3" + assert risks[0].failing_runs[0]["prow_url"] == "https://prow/j1" def test_non_consecutive(self): """Informing job fails in payloads 1, 3, 5 (gaps) -> no EscalationRisk.""" @@ -288,6 +295,24 @@ def test_empty_streams(self): """No streams -> empty list.""" assert _find_escalation_risks([], Config()) == [] + def test_latest_failure_empty_url_not_replaced_by_older(self): + """Newest failure has an empty prow_url -> it must not be replaced by + an older job's URL.""" + def informing_job(url): + return JobRun(name="j1", prow_url=url, result=JobResult.FAILURE, + job_type=JobType.INFORMING, topology="SNO") + + payloads = [ + _make_payload("t1", [informing_job("https://prow/old")]), + _make_payload("t2", [informing_job("https://prow/mid")]), + _make_payload("t3", [informing_job("")]), + ] + stream = StreamReport("s", "4.19", payloads=payloads) + + risks = _find_escalation_risks([stream], Config()) + assert len(risks) == 1 + assert risks[0].prow_url == "" + def test_all_passing(self): """All jobs pass -> empty list.""" payloads = [ diff --git a/payload-monitor/tests/test_collectors_release_controller.py b/payload-monitor/tests/test_collectors_release_controller.py index 68995672..36aa9929 100644 --- a/payload-monitor/tests/test_collectors_release_controller.py +++ b/payload-monitor/tests/test_collectors_release_controller.py @@ -174,10 +174,10 @@ def test_previous_attempts_filters_null_entries(self, config): assert result[0].previous_attempts[0].prow_url == "https://prow/1" -class TestDiscoverStreams: - def test_discover_streams(self, config): +class TestConfiguredStreamNames: + def test_configured_stream_names(self, config): config.versions = ["4.18", "4.19"] - streams = rc.discover_streams(config) + streams = rc.configured_stream_names(config) assert streams == ["4.18.0-0.nightly", "4.19.0-0.nightly"] diff --git a/payload-monitor/tests/test_collectors_sippy.py b/payload-monitor/tests/test_collectors_sippy.py index 3ff7488b..4748c7d4 100644 --- a/payload-monitor/tests/test_collectors_sippy.py +++ b/payload-monitor/tests/test_collectors_sippy.py @@ -58,12 +58,16 @@ def test_detects_regression(self): "jira_component": "", "id": 1, }] - regressions = sippy.identify_regressions(jobs) + regressions = sippy.identify_regressions(jobs, version="4.19") assert len(regressions) == 1 assert regressions[0].test_name == "periodic-sno-test" assert regressions[0].sample_pass_rate == 50.0 assert regressions[0].basis_pass_rate == 90.0 assert regressions[0].topology == "SNO" + assert regressions[0].triage_url.startswith( + "https://sippy.dptools.openshift.org/sippy-ng/jobs/4.19/analysis?filters=" + ) + assert "periodic-sno-test" in regressions[0].triage_url def test_skips_improvement(self): jobs = [{ diff --git a/payload-monitor/tests/test_models_timing.py b/payload-monitor/tests/test_models_timing.py index 169d31cc..bdb4215f 100644 --- a/payload-monitor/tests/test_models_timing.py +++ b/payload-monitor/tests/test_models_timing.py @@ -22,14 +22,6 @@ def test_is_not_success(self): ) assert run.is_success is False - def test_duration_minutes(self): - run = TimingRun( - job_name="test-job", topology="TNA", release="4.22", - start_time="2026-04-01T12:00:00Z", duration_seconds=5400, - result="S", run_type="install", - ) - assert run.duration_minutes == 90.0 - def test_default_variant(self): run = TimingRun( job_name="test-job", topology="TNA", release="4.22", @@ -38,32 +30,6 @@ def test_default_variant(self): ) assert run.variant == {} - def test_install_duration_from_setup(self): - run = TimingRun( - job_name="test-job", topology="TNA", release="4.22", - start_time="", duration_seconds=12000, - result="S", run_type="install", - step_durations={"install": 4752.0, "test": 7680.0}, - ) - assert run.install_duration_seconds == 4752.0 - - def test_install_duration_from_setup_key(self): - run = TimingRun( - job_name="test-job", topology="TNA", release="4.22", - start_time="", duration_seconds=12000, - result="S", run_type="install", - step_durations={"setup": 3000.0}, - ) - assert run.install_duration_seconds == 3000.0 - - def test_install_duration_missing(self): - run = TimingRun( - job_name="test-job", topology="TNA", release="4.22", - start_time="", duration_seconds=12000, - result="S", run_type="install", - ) - assert run.install_duration_seconds == 0.0 - class TestTimingReport: def test_empty_report(self): diff --git a/payload-monitor/tests/test_report_generator.py b/payload-monitor/tests/test_report_generator.py index 48bf56f6..0df22f0a 100644 --- a/payload-monitor/tests/test_report_generator.py +++ b/payload-monitor/tests/test_report_generator.py @@ -30,12 +30,14 @@ from payload_monitor.report.generator import ( _build_template_context, _extract_date, + _load_json, _render_analysis_card, _safe_dataclass_init, _safe_urls, generate_html, generate_json, load_json, + merge_analysis, patch_analysis_html, ) @@ -210,6 +212,19 @@ def test_no_per_attempt_breakdown_when_field_missing(self): html = _render_analysis_card(da) assert "Per-Attempt Breakdown" not in html + def test_job_name_rendered_when_provided(self): + da = {"root_cause": "x", "failure_type": "y", "impact": "z", + "suspect_prs": [], "recommendation": "r"} + html = _render_analysis_card(da, job_name="periodic-ci-openshift-tnf-e2e") + assert "da-job-name" in html + assert "periodic-ci-openshift-tnf-e2e" in html + + def test_no_job_name_div_when_not_provided(self): + da = {"root_cause": "x", "failure_type": "y", "impact": "z", + "suspect_prs": [], "recommendation": "r"} + html = _render_analysis_card(da) + assert "da-job-name" not in html + class TestGenerateHtml: def test_generates_html(self, sample_report): @@ -475,7 +490,7 @@ class TestEscalationRisksContext: def test_context_includes_escalation_risks(self): er = EscalationRisk( job_name="j1", topology="SNO", version="4.19", - consecutive_failures=3, sippy_url="https://sippy/j1", + consecutive_failures=3, prow_url="https://prow/j1", ) job = JobRun("j1", "url", JobResult.FAILURE, JobType.INFORMING, "SNO") payload = Payload("t", "s", "4.19", PayloadStatus.REJECTED, jobs=[job]) @@ -564,7 +579,13 @@ def test_jira_matches_round_trip(self, tmp_path): def test_escalation_risks_round_trip(self, tmp_path): er = EscalationRisk( job_name="j1", topology="SNO", version="4.19", - consecutive_failures=3, sippy_url="https://sippy/j1", + consecutive_failures=3, prow_url="https://prow/j1", + triage_url="https://sippy.dptools.openshift.org/sippy-ng/jobs/4.19/analysis?filters=x", + failing_runs=[ + {"payload_tag": "t5", "prow_url": "https://prow/j1"}, + {"payload_tag": "t4", "prow_url": "https://prow/j1"}, + {"payload_tag": "t3", "prow_url": "https://prow/j1"}, + ], ) report = MonitorReport(generated_at="now", escalation_risks=[er]) out = tmp_path / "report.json" @@ -573,6 +594,8 @@ def test_escalation_risks_round_trip(self, tmp_path): assert len(loaded.escalation_risks) == 1 assert loaded.escalation_risks[0].job_name == "j1" assert loaded.escalation_risks[0].consecutive_failures == 3 + assert loaded.escalation_risks[0].triage_url == er.triage_url + assert loaded.escalation_risks[0].failing_runs == er.failing_runs def test_cross_topology_round_trip(self, tmp_path): report = MonitorReport( @@ -788,17 +811,18 @@ class TestPatchAnalysisHtml: }, } - def _suggestion_div(self, attempt_count=1): + def _suggestion_div(self, attempt_count=1, prow_url=None): """Build a claude-suggestion div with data attributes matching the template.""" + url = prow_url or self.PROW_URL return ( - f'
\n' ' For deeper analysis, use Claude directly:
\n' '
\n' - f' /ci:prow-job-analyze-test-failure {self.PROW_URL}\n' + f' /ci:prow-job-analysis {url}\n' ' \n' '
\n' - '
' + '
' ) def _write_files(self, tmp_path, html_content, analysis=None): @@ -932,6 +956,57 @@ def test_badge_inserted_via_data_attribute(self, tmp_path): result = html_path.read_text() assert "ai-analyzed" in result + def test_badge_inserted_in_grid_layout(self, tmp_path): + html = ( + f'
\n' + ' \n' + ' job-name\n' + ' 4.19\n' + ' tag\n' + ' SNO\n' + ' blocking\n' + ' —\n' + ' 2026-03-25\n' + ' \n' + '
\n' + + self._suggestion_div(attempt_count=1) + '\n' + '
\n' + '
' + ) + html_path, analysis_path = self._write_files(tmp_path, html) + patch_analysis_html(html_path, analysis_path) + result = html_path.read_text() + assert "ai-analyzed" in result + assert 'job-name' in result + + def test_job_name_from_analysis_json(self, tmp_path): + analysis = {"by_prow_url": {self.PROW_URL: { + "job_name": "periodic-ci-openshift-tnf-e2e", + "root_cause": "test root cause", "failure_type": "Infrastructure flake", + "impact": "low impact", "suspect_prs": [], "recommendation": "retry", + }}} + html = ( + '

Prow Job: ' + f'View in Prow

\n' + + self._suggestion_div(attempt_count=1) + ) + html_path, analysis_path = self._write_files(tmp_path, html, analysis) + patch_analysis_html(html_path, analysis_path) + result = html_path.read_text() + assert 'class="da-job-name"' in result + assert "periodic-ci-openshift-tnf-e2e" in result + + def test_no_job_name_div_when_not_in_analysis_json(self, tmp_path): + html = ( + f'

Prow Job: ' + f'View in Prow

\n' + + self._suggestion_div(attempt_count=1) + ) + html_path, analysis_path = self._write_files(tmp_path, html) + patch_analysis_html(html_path, analysis_path) + result = html_path.read_text() + assert 'class="da-job-name"' not in result + def test_header_status_updated_via_data_attribute(self, tmp_path): html = ( '
\n' @@ -945,6 +1020,43 @@ def test_header_status_updated_via_data_attribute(self, tmp_path): assert "AI-enriched via Claude" in result assert "data-only" not in result + def test_url_with_ampersand(self, tmp_path): + """URLs containing & are HTML-escaped in data-prow-url but raw in analysis JSON.""" + raw_url = "https://prow.ci.openshift.org/view/gs/logs?a=1&b=2" + escaped_url = "https://prow.ci.openshift.org/view/gs/logs?a=1&b=2" + analysis = {"by_prow_url": {raw_url: { + "root_cause": "ampersand test", "failure_type": "test", + "impact": "none", "suspect_prs": [], "recommendation": "ok", + }}} + html = self._suggestion_div(attempt_count=1, prow_url=escaped_url) + html_path, analysis_path = self._write_files(tmp_path, html, analysis) + patch_analysis_html(html_path, analysis_path) + result = html_path.read_text() + assert "deep-analysis-card" in result + assert "ampersand test" in result + assert "claude-suggestion" not in result + + def test_extra_nested_div_inside_suggestion(self, tmp_path): + """Extra wrapper div inside .claude-suggestion doesn't break matching.""" + html = ( + f'
\n' + '
\n' + ' For deeper analysis, use Claude directly:
\n' + '
\n' + f' /ci:prow-job-analysis {self.PROW_URL}\n' + ' \n' + '
\n' + '
\n' + '
' + ) + html_path, analysis_path = self._write_files(tmp_path, html) + patch_analysis_html(html_path, analysis_path) + result = html_path.read_text() + assert "deep-analysis-card" in result + assert "test root cause" in result + assert "claude-suggestion" not in result + class TestTimingUnavailableContext: def test_false_when_timing_not_requested(self): @@ -1010,3 +1122,80 @@ def test_no_banner_in_html_when_timing_succeeded(self): ) html = generate_html(report) assert "Timing insights unavailable" not in html + + +class TestLoadJsonErrors: + def test_malformed_json_raises_value_error_with_path(self, tmp_path): + bad = tmp_path / "bad.json" + bad.write_text("{invalid json") + with pytest.raises(ValueError, match=str(bad)): + _load_json(bad, "Test JSON") + + def test_missing_file_raises_file_not_found_with_path(self, tmp_path): + missing = tmp_path / "nope.json" + with pytest.raises(FileNotFoundError, match=str(missing)): + _load_json(missing, "Test JSON") + + def test_patch_analysis_malformed_json(self, tmp_path): + html_path = tmp_path / "report.html" + html_path.write_text("") + bad_json = tmp_path / "analysis.json" + bad_json.write_text("{broken") + with pytest.raises(ValueError, match="Analysis JSON"): + patch_analysis_html(html_path, bad_json) + + def test_merge_analysis_missing_file(self, tmp_path): + report = MonitorReport(generated_at="now") + missing = tmp_path / "nope.json" + with pytest.raises(FileNotFoundError, match="Analysis JSON"): + merge_analysis(report, missing) + + def test_load_json_malformed(self, tmp_path): + bad = tmp_path / "report.json" + bad.write_text("not json") + with pytest.raises(ValueError, match="Report JSON"): + load_json(bad) + + +class TestStreamViewsContext: + def test_stream_views_populated(self): + b_job = JobRun("b1", "url", JobResult.FAILURE, JobType.BLOCKING, "SNO") + i_job = JobRun("i1", "url", JobResult.FAILURE, JobType.INFORMING, "TNA") + p1 = Payload("t1", "s", "4.19", PayloadStatus.REJECTED, jobs=[b_job]) + p2 = Payload("t2", "s", "4.19", PayloadStatus.ACCEPTED, jobs=[i_job]) + stream = StreamReport("s", "4.19", payloads=[p1, p2]) + report = MonitorReport(generated_at="now", streams=[stream]) + + ctx = _build_template_context(report) + assert len(ctx["stream_views"]) == 1 + sv = ctx["stream_views"][0] + assert sv["stream"] is stream + assert sv["affected_topologies"] == ["SNO", "TNA"] + assert isinstance(sv["recent_fails"], int) + assert isinstance(sv["older_fails"], int) + + def test_blocking_and_informing_counts(self): + b_job = JobRun("b1", "url", JobResult.FAILURE, JobType.BLOCKING, "SNO") + b_job2 = JobRun("b1", "url2", JobResult.FAILURE, JobType.BLOCKING, "SNO") + i_job = JobRun("i1", "url", JobResult.FAILURE, JobType.INFORMING, "TNA") + p = Payload("t", "s", "4.19", PayloadStatus.REJECTED, jobs=[b_job, b_job2, i_job]) + stream = StreamReport("s", "4.19", payloads=[p]) + report = MonitorReport(generated_at="now", streams=[stream]) + + ctx = _build_template_context(report) + assert ctx["blocking_failures"] == 2 + assert ctx["informing_failures"] == 1 + assert len(ctx["blocking_jobs_list"]) == 1 + assert ctx["blocking_jobs_list"][0]["job"].name == "b1" + + def test_total_and_rejected_payloads(self): + p1 = Payload("t1", "s", "4.19", PayloadStatus.REJECTED) + p2 = Payload("t2", "s", "4.19", PayloadStatus.ACCEPTED) + p3 = Payload("t3", "s2", "4.20", PayloadStatus.REJECTED) + s1 = StreamReport("s", "4.19", payloads=[p1, p2]) + s2 = StreamReport("s2", "4.20", payloads=[p3]) + report = MonitorReport(generated_at="now", streams=[s1, s2]) + + ctx = _build_template_context(report) + assert ctx["total_payloads"] == 3 + assert ctx["rejected_payloads"] == 2 diff --git a/plugins/edge-ocp-ci/.claude-plugin/plugin.json b/plugins/edge-ocp-ci/.claude-plugin/plugin.json index 6fe5bab3..02f15b20 100644 --- a/plugins/edge-ocp-ci/.claude-plugin/plugin.json +++ b/plugins/edge-ocp-ci/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "edge-ocp-ci", "description": "Edge OCP Payload Monitor — monitor OpenShift nightly payloads for edge topology (SNO/TNA/TNF) failures with AI-enriched analysis", - "version": "1.2.0", + "version": "1.2.1", "author": { "name": "vimauro" }, diff --git a/plugins/edge-ocp-ci/skills/generate-dashboard/SKILL.md b/plugins/edge-ocp-ci/skills/generate-dashboard/SKILL.md index 9c82cdb3..ebf907be 100644 --- a/plugins/edge-ocp-ci/skills/generate-dashboard/SKILL.md +++ b/plugins/edge-ocp-ci/skills/generate-dashboard/SKILL.md @@ -31,12 +31,8 @@ This skill composes with the following installed marketplace CI skills from the | Skill | When to Use | |-------|-------------| -| `ci:analyze-payload` | Full payload analysis with historical lookback and HTML report — use for rejected/failing payloads with edge blockers | -| `ci:prow-job-analyze-test-failure` | Analyze failed tests by inspecting test code, downloading artifacts, and optionally integrating must-gather — use for any failing edge test | -| `ci:prow-job-analyze-install-failure` | Analyze OpenShift install failures from installer logs, log bundles, and sosreports — use when edge jobs fail at install stage | -| `ci:prow-job-analyze-metal-install-failure` | Analyze bare metal install failures using dev-scripts artifacts — use for metal/baremetal SNO or TNF jobs with "metal" in name | -| `ci:prow-job-analyze-resource` | Analyze K8s resource lifecycle in Prow job artifacts (audit logs, pod logs) — use when failure involves resource state issues | -| `ci:prow-job-artifact-search` | Search, list, and fetch artifacts from Prow job runs in GCS — use when you need to find specific artifacts | +| `ci:prow-job-analysis` | General-purpose Prow job failure analysis — classifies job type and failure mode, then routes internally to the right investigation procedure (install, test, upgrade, resource exhaustion, networking, etc.) — use for any failing edge job | +| `ci:prow-job-analyze-resource` | Analyze K8s resource lifecycle in Prow job artifacts (audit logs, pod logs) — use for a deep resource-specific audit trail beyond what `ci:prow-job-analysis` covers | | `ci:analyze-regression` | Analyze Component Readiness regression details and suggest next steps — use for Sippy-detected regressions | | `ci:check-if-jira-regression-is-ongoing` | Check if a JIRA regression bug is still ongoing or resolved — use to validate whether known bugs still apply | @@ -110,7 +106,7 @@ The 7th field contains semicolon-separated Prow URLs for previous failed attempt If no `BLOCKING_JOBS_START` marker appears in the output, there are no blocking failures — skip to Step 6. -**For informing job failures:** Do NOT run deep analysis. The HTML report already includes a suggestion to use Claude directly with `/ci:prow-job-analyze-test-failure `. +**For informing job failures:** Do NOT run deep analysis. The HTML report already includes a suggestion to use Claude directly with `/ci:prow-job-analysis `. #### Analysis Prompt @@ -134,19 +130,15 @@ Steps: - Are different tests failing or different error messages? - Does the failure mode change between attempts? -Then based on failure type of the latest attempt: -- If install failure (error contains "install should succeed", "bootstrap", or failed in pre/setup phase): - Use `ci:prow-job-analyze-install-failure` (or `ci:prow-job-analyze-metal-install-failure` if job name contains "metal") -- If test failure (job passed install but failed during test phase): - Use `ci:prow-job-analyze-test-failure` -- If resource/state failure (etcd issues, operator degraded, node not ready): - Use `ci:prow-job-analyze-resource` +4. Use `ci:prow-job-analysis` on the latest attempt's Prow URL for root cause classification and analysis — it auto-detects job type and failure mode and routes internally to the right investigation procedure +5. If the failure is specifically about a Kubernetes resource's lifecycle (created/modified/deleted at an unexpected time) and needs an audit-log timeline beyond what `ci:prow-job-analysis` surfaced, use `ci:prow-job-analyze-resource` as a follow-up For JIRA context: use `ci:fetch-jira-issue` for any linked bugs. Return ONLY a JSON object with these fields: { "prow_url": "{prow_url}", + "job_name": "{job_name}", "root_cause": "Overall root cause (synthesize across all attempts if same cause, or describe the dominant pattern)", "failure_type": "Infrastructure flake | Test regression | Install failure | Platform issue", "impact": "How this affects payload acceptance and which topologies", @@ -179,11 +171,12 @@ When using subagents, launch all in parallel using multiple Agent tool calls in ```json { "prow_url": "{prow_url}", + "job_name": "{job_name}", "root_cause": "Analysis timed out or failed: {error_message}", "failure_type": "Analysis error", "impact": "Unable to determine — manual investigation needed", "suspect_prs": [], - "recommendation": "Run /ci:prow-job-analyze-test-failure {prow_url} manually for detailed analysis", + "recommendation": "Run /ci:prow-job-analysis {prow_url} manually for detailed analysis", "same_root_cause": true, "attempt_analyses": [] } @@ -200,6 +193,7 @@ Collect the deep analysis results (from subagents or inline analysis) and write { "by_prow_url": { "https://prow.ci.openshift.org/view/gs/.../123": { + "job_name": "periodic-ci-...", "root_cause": "Overall root cause synthesized across all attempts", "failure_type": "Infrastructure flake | Test regression | Install failure | Platform issue", "impact": "How this affects payload acceptance and which topologies", @@ -261,7 +255,7 @@ Offer follow-up actions the user can take from this session: - **Set release blocker** on a JIRA issue (`ci:set-release-blocker`) - **Triage a regression** in Component Readiness (`ci:triage-regression`) - **Trigger payload job** to test a fix (`ci:trigger-payload-job`) -- **Investigate an informing job** further (`ci:prow-job-analyze-test-failure`) +- **Investigate an informing job** further (`ci:prow-job-analysis`) ---