From 5c0b6cb28df28d85b939a44075db644f9ee69190 Mon Sep 17 00:00:00 2001 From: Eric Blake Date: Tue, 22 Apr 2025 10:00:48 -0500 Subject: [PATCH 1/3] Link to build URL while waiting for status When watching a build in a gitlab job, seeing that the job is still executing in Cirrus is useful, but even more useful is having access to a clickable URL to jump from the gitlab job into the Cirrus job. --- cirrus_run/queries.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cirrus_run/queries.py b/cirrus_run/queries.py index e2978ff..64f1478 100644 --- a/cirrus_run/queries.py +++ b/cirrus_run/queries.py @@ -103,7 +103,7 @@ def wait_build(api, build_id: str, delay=3, abort=60*60): while time() < time_start + abort: response = api(query, params) status = response['build']['status'] - log.info('build {}: {}'.format(build_id, status)) + log.info('build https://cirrus-ci.com/build/{}: {}'.format(build_id, status)) if status in {'COMPLETED'}: return True if status in {'CREATED', 'TRIGGERED', 'EXECUTING'}: From c0701fc238ac0855e46798c7f6400a07e331f621 Mon Sep 17 00:00:00 2001 From: Peter Krempa Date: Mon, 21 Oct 2024 16:10:47 +0200 Subject: [PATCH 2/3] README: Document return values Add a section about return values from the script and document existing cases. Signed-off-by: Peter Krempa --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 0bc0e84..e32b7b4 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,11 @@ optional arguments: marker per line. If any marker is found in Cirrus CI output for a failed build, the build is retried once more. Default: $CIRRUS_FLAKY_MARKERS_FILE + +return values: + - 0: The cirrus job was successful + - 1: The job failed + - 2: Error in 'cirrus-run' ``` From fa345f1556cad28a0dcf246b138dd6db668e8959 Mon Sep 17 00:00:00 2001 From: Peter Krempa Date: Mon, 21 Oct 2024 17:10:28 +0200 Subject: [PATCH 3/3] Introduce check for "out of CI credits" situation When a Cirrus CI job started by 'cirrus-run' fails due lack of CI credits, it would look exactly like a failure in the job itself, thus the user would not be able to distinguish the problems. If a 'taks' failed due to CI minute lack, the 'notifications' field of given 'task' contains the following message: Failed to start an instance: FAILED_PRECONDITION: Monthly compute limit exceeded! To detect this, receive also task data and their notifications when waiting for a job and look for the message about CI credits. In order to see this status from scripts this will also return a new error code '3' if the out of CI credits situation is detected. Additionally in order to allow quick adaptation when the above string will be changed add '--cirrus-out-of-ci-credits-message' parameter which can configure the string. Signed-off-by: Peter Krempa --- README.md | 1 + cirrus_run/cli.py | 15 +++++++++++++-- cirrus_run/queries.py | 15 ++++++++++++++- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e32b7b4..3f3e75f 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,7 @@ return values: - 0: The cirrus job was successful - 1: The job failed - 2: Error in 'cirrus-run' + - 3: The Cirrus CI job ran out of CI minutes ``` diff --git a/cirrus_run/cli.py b/cirrus_run/cli.py index 60e5b11..1268d4c 100644 --- a/cirrus_run/cli.py +++ b/cirrus_run/cli.py @@ -14,7 +14,7 @@ from . import CirrusAPI from .throbber import ProgressBar -from .queries import build_log, get_repo, create_build, wait_build, CirrusBuildError +from .queries import build_log, get_repo, create_build, wait_build, CirrusBuildError, CirrusCreditsError log = logging.getLogger(__name__) @@ -47,10 +47,12 @@ def run(args, retry_index=0): print('Build created: {}'.format(build_url)) with ProgressBar('' if args.verbose else '.'): try: - wait_build(api, build_id, abort=args.timeout*60) + wait_build(api, build_id, abort=args.timeout*60, credits_error_message=args.cirrus_out_of_ci_credits_message) rc, status, message = 0, 'successful', '' except CirrusBuildError: rc, status, message = 1, 'failed', '' + except CirrusCreditsError: + rc, status, message = 3, 'error', 'Out of CI credits' except Exception as exc: rc, status, message = 2, 'error', '{exception}: {text}'.format( exception=exc.__class__.__name__, @@ -224,6 +226,15 @@ def parse_args(*a, **ka): 'the build is retried once more. Default: ${}' ).format(ENVIRONMENT['flaky_markers']), ) + parser.add_argument( + '--cirrus-out-of-ci-credits-message', + default='Monthly compute limit exceeded', + help=( + 'Error message passed via "notifications" from Cirrus CI reported' + 'when the CI job failed due to lack of CI credits.' + 'Default: "Monthly compute limit exceeded"' + ), + ) args = parser.parse_args(*a, **ka) if not args.token: diff --git a/cirrus_run/queries.py b/cirrus_run/queries.py index e2978ff..4fa1d9d 100644 --- a/cirrus_run/queries.py +++ b/cirrus_run/queries.py @@ -22,6 +22,8 @@ class CirrusQueryError(ValueError): class CirrusBuildError(RuntimeError): '''Raised on build failures''' +class CirrusCreditsError(RuntimeError): + '''Raised when build fails due to lack of CI credits''' class CirrusTimeoutError(RuntimeError): '''Raised when build takes too long''' @@ -85,7 +87,7 @@ def create_build(api: CirrusAPI, return answer['createBuild']['build']['id'] -def wait_build(api, build_id: str, delay=3, abort=60*60): +def wait_build(api, build_id: str, delay=3, abort=60*60, credits_error_message=None): '''Wait until build finishes''' ERROR_CONFIRM_TIMES = 3 @@ -93,6 +95,11 @@ def wait_build(api, build_id: str, delay=3, abort=60*60): query GetBuild($build: ID!) { build(id: $build) { status + tasks { + notifications { + message + } + } } } ''' @@ -116,6 +123,12 @@ def wait_build(api, build_id: str, delay=3, abort=60*60): sleep(2 * delay / (ERROR_CONFIRM_TIMES - 1)) continue else: + if credits_error_message is not None: + for task in response['build']['tasks']: + for notif in task['notifications']: + if credits_error_message in notif['message']: + raise CirrusCreditsError('build {} ran out of CI credits'.format(build_id)) + raise CirrusBuildError('build {} was terminated: {}'.format(build_id, status)) raise ValueError('build {} returned unknown status: {}'.format(build_id, status)) raise CirrusTimeoutError('build {} timed out'.format(build_id))