fix: preserve benchmark previews when publishing fails #2
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| "use strict"; | ||
|
|
||
| const assert = require("node:assert/strict"); | ||
| const fs = require("node:fs"); | ||
| const path = require("node:path"); | ||
| const test = require("node:test"); | ||
| const YAML = require("yaml"); | ||
|
|
||
| const workflow = YAML.parse( | ||
| fs.readFileSync( | ||
| path.join(__dirname, "..", ".github", "workflows", "publish.yml"), | ||
| "utf8", | ||
| ), | ||
| ); | ||
| const steps = workflow.jobs.publish.steps; | ||
| const step = (name) => steps.find((candidate) => candidate.name === name); | ||
|
|
||
| test("publisher exposes an optional data repository dispatch event", () => { | ||
| const input = workflow.on.workflow_call.inputs.data_dispatch_event; | ||
| assert.equal(input.required, undefined); | ||
| assert.equal(input.default, ""); | ||
| assert.equal(input.type, "string"); | ||
| }); | ||
|
|
||
| test("data publishing failures degrade to a commented preview", () => { | ||
| for (const name of [ | ||
| "Check out writable benchmark data", | ||
| "Commit benchmark data", | ||
| "Notify benchmark data repository", | ||
| ]) { | ||
| assert.equal(step(name)["continue-on-error"], true, name); | ||
| } | ||
| assert.equal(step("Require external data token"), undefined); | ||
| assert.match( | ||
| step("Check out public benchmark data for preview").if, | ||
| /data-checkout\.outcome != 'success'/u, | ||
| ); | ||
| assert.match( | ||
| step("Check out public benchmark data for preview").run, | ||
| /else\s+rm -rf "\$GITHUB_WORKSPACE\/benchmark-data"\s+mkdir benchmark-data/u, | ||
| ); | ||
| assert.match( | ||
| step("Add publishing status to report").run, | ||
| /Persistent publishing is unavailable/u, | ||
| ); | ||
| }); | ||
|
|
||
| test("benchmark validation and rendering remain hard failures", () => { | ||
| for (const name of [ | ||
| "Download platform artifacts", | ||
| "Check out trusted benchmark configuration for fork", | ||
| "Render benchmark history", | ||
| "Upload rendered preview", | ||
| "Create or update PR comment", | ||
| ]) { | ||
| assert.notEqual(step(name)["continue-on-error"], true, name); | ||
| } | ||
| }); |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Select data branchruns unconditionally with nocontinue-on-error, unlike every other data-side step in this PR.preview-dataguarantees abenchmark-datagit repo exists, but thegit fetch origin .../git checkout -B/git checkout --orphan/git rmcalls here run underset -euo pipefailand are unguarded. A transient failure (e.g. the remote becoming unreachable after the initial clone) will hard-fail the entire publish job, which defeats the PR's goal of degrading to a preview. Consider whether this step should also tolerate failure, or confirm the hard failure is intended.