From 97cf4a58d8b7d1c846a5aae40d0ede4a1e087531 Mon Sep 17 00:00:00 2001 From: AkhileshNegi Date: Wed, 22 Apr 2026 09:48:04 +0530 Subject: [PATCH 1/5] setting up sentry cron observability --- backend/app/api/routes/cron.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/backend/app/api/routes/cron.py b/backend/app/api/routes/cron.py index ab072970b..5af996234 100644 --- a/backend/app/api/routes/cron.py +++ b/backend/app/api/routes/cron.py @@ -1,5 +1,8 @@ import logging +import sentry_sdk +from sentry_sdk.crons import MonitorConfig + from app.api.permissions import Permission, require_permission from fastapi import APIRouter, Depends @@ -10,12 +13,25 @@ router = APIRouter(tags=["Cron"]) +EVALUATION_CRON_MONITOR_CONFIG: MonitorConfig = { + "schedule": {"type": "interval", "value": 5, "unit": "minute"}, + "timezone": "UTC", + "checkin_margin": 2, + "max_runtime": 10, + "failure_issue_threshold": 2, + "recovery_threshold": 1, +} + @router.get( "/cron/evaluations", include_in_schema=False, dependencies=[Depends(require_permission(Permission.SUPERUSER))], ) +@sentry_sdk.monitor( + monitor_slug="evaluation-cron-job", + monitor_config=EVALUATION_CRON_MONITOR_CONFIG, +) def evaluation_cron_job( session: SessionDep, ) -> dict: @@ -34,7 +50,6 @@ def evaluation_cron_job( logger.info("[evaluation_cron_job] Cron job invoked") try: - # Process all pending evaluations across all organizations result = process_all_pending_evaluations_sync(session=session) logger.info( @@ -51,10 +66,5 @@ def evaluation_cron_job( f"[evaluation_cron_job] Error executing cron job: {e}", exc_info=True, ) - return { - "status": "error", - "error": str(e), - "total_processed": 0, - "total_failed": 0, - "total_still_processing": 0, - } + sentry_sdk.capture_exception(e) + raise From 68cc41815af5f360fe704caf4ca6f2abf4f394fe Mon Sep 17 00:00:00 2001 From: AkhileshNegi Date: Wed, 22 Apr 2026 10:50:44 +0530 Subject: [PATCH 2/5] using cron variable for consistency --- backend/app/api/routes/cron.py | 9 +++++++-- backend/app/core/config.py | 4 ++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/backend/app/api/routes/cron.py b/backend/app/api/routes/cron.py index 5af996234..1a4a8561c 100644 --- a/backend/app/api/routes/cron.py +++ b/backend/app/api/routes/cron.py @@ -7,6 +7,7 @@ from fastapi import APIRouter, Depends from app.api.deps import SessionDep +from app.core.config import settings from app.crud.evaluations import process_all_pending_evaluations_sync logger = logging.getLogger(__name__) @@ -14,10 +15,14 @@ router = APIRouter(tags=["Cron"]) EVALUATION_CRON_MONITOR_CONFIG: MonitorConfig = { - "schedule": {"type": "interval", "value": 5, "unit": "minute"}, + "schedule": { + "type": "interval", + "value": settings.CRON_INTERVAL_MINUTES, + "unit": "minute", + }, "timezone": "UTC", "checkin_margin": 2, - "max_runtime": 10, + "max_runtime": 2 * settings.CRON_INTERVAL_MINUTES, "failure_issue_threshold": 2, "recovery_threshold": 1, } diff --git a/backend/app/core/config.py b/backend/app/core/config.py index ee72442f8..478450b7f 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -156,6 +156,10 @@ def AWS_S3_BUCKET(self) -> str: CALLBACK_CONNECT_TIMEOUT: int = 3 CALLBACK_READ_TIMEOUT: int = 10 + # Evaluation cron invocation interval (minutes). Shared by the external + # invoker script and the Sentry cron monitor schedule so both stay aligned. + CRON_INTERVAL_MINUTES: int = 5 + @computed_field # type: ignore[prop-decorator] @property def COMPUTED_CELERY_WORKER_CONCURRENCY(self) -> int: From 011b404b57b2cee96b859f4c36f33dbc416034f8 Mon Sep 17 00:00:00 2001 From: AkhileshNegi Date: Wed, 22 Apr 2026 10:53:52 +0530 Subject: [PATCH 3/5] updated with comment --- backend/app/core/config.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 478450b7f..6e46112de 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -156,8 +156,10 @@ def AWS_S3_BUCKET(self) -> str: CALLBACK_CONNECT_TIMEOUT: int = 3 CALLBACK_READ_TIMEOUT: int = 10 - # Evaluation cron invocation interval (minutes). Shared by the external - # invoker script and the Sentry cron monitor schedule so both stay aligned. + # Evaluation cron invocation interval (minutes). In staging/production the + # endpoint is triggered by AWS EventBridge on this cadence; locally it can + # be driven by scripts/python/invoke-cron.py. The Sentry cron monitor reads + # this same value so its expected schedule stays aligned with the trigger. CRON_INTERVAL_MINUTES: int = 5 @computed_field # type: ignore[prop-decorator] From 1715ed07628f5fd3d29f8bd477910737519dff0a Mon Sep 17 00:00:00 2001 From: AkhileshNegi Date: Wed, 22 Apr 2026 10:55:09 +0530 Subject: [PATCH 4/5] add comments for readability and later reference --- backend/app/api/routes/cron.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/backend/app/api/routes/cron.py b/backend/app/api/routes/cron.py index 1a4a8561c..eb31e1322 100644 --- a/backend/app/api/routes/cron.py +++ b/backend/app/api/routes/cron.py @@ -15,15 +15,21 @@ router = APIRouter(tags=["Cron"]) EVALUATION_CRON_MONITOR_CONFIG: MonitorConfig = { + # Expected cadence: a check-in every CRON_INTERVAL_MINUTES minutes. "schedule": { "type": "interval", "value": settings.CRON_INTERVAL_MINUTES, "unit": "minute", }, + # Timezone for the schedule (only affects crontab-style schedules). "timezone": "UTC", + # Grace period (minutes) before a late check-in is marked as missed. "checkin_margin": 2, + # Max runtime (minutes) before an in-progress run is marked as timed out. "max_runtime": 2 * settings.CRON_INTERVAL_MINUTES, + # Consecutive failures/missed/timeouts required to open a Sentry issue. "failure_issue_threshold": 2, + # Consecutive successful check-ins required to auto-resolve the issue. "recovery_threshold": 1, } From 9b3b80ed202e0a88f7d95bd192203474e50f4e9e Mon Sep 17 00:00:00 2001 From: AkhileshNegi Date: Wed, 22 Apr 2026 12:11:27 +0530 Subject: [PATCH 5/5] CI fixes --- backend/app/api/routes/cron.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/app/api/routes/cron.py b/backend/app/api/routes/cron.py index eb31e1322..819185fce 100644 --- a/backend/app/api/routes/cron.py +++ b/backend/app/api/routes/cron.py @@ -1,7 +1,7 @@ import logging import sentry_sdk -from sentry_sdk.crons import MonitorConfig +from sentry_sdk.types import MonitorConfig from app.api.permissions import Permission, require_permission from fastapi import APIRouter, Depends