From c28808f676cd77acd397ba7697bfe88e7582e871 Mon Sep 17 00:00:00 2001 From: Tarun Tak Date: Tue, 21 Apr 2026 09:51:25 +0000 Subject: [PATCH 1/7] feat: improve database connection handling and update certificate deletion logic --- .../retired_user_cert_remover.py | 55 +++++++++++++++---- 1 file changed, 43 insertions(+), 12 deletions(-) diff --git a/util/jenkins/retired_user_cert_remover/retired_user_cert_remover.py b/util/jenkins/retired_user_cert_remover/retired_user_cert_remover.py index b3be873fe..a8a52d9b6 100644 --- a/util/jenkins/retired_user_cert_remover/retired_user_cert_remover.py +++ b/util/jenkins/retired_user_cert_remover/retired_user_cert_remover.py @@ -50,9 +50,16 @@ def delete_object(self, bucket, key): return self.client.delete_object(Bucket=bucket, Key=key) -def fetch_certificates_to_delete(db_host, db_user, db_password, db_name): +def get_db_connection(db_host, db_user, db_password, db_name): + try: + return pymysql.connect(host=db_host, user=db_user, password=db_password, database=db_name) + except Exception as ex: + logging.error(f"Database connection failed with error: {ex}") + sys.exit(1) + + +def fetch_certificates_to_delete(connection): try: - connection = pymysql.connect(host=db_host, user=db_user, password=db_password, database=db_name) cursor = connection.cursor() logging.info("Running query on database...") cursor.execute(""" @@ -79,32 +86,54 @@ def fetch_certificates_to_delete(db_host, db_user, db_password, db_name): """) result = cursor.fetchall() cursor.close() - connection.close() return result except Exception as ex: logging.error(f"Database query failed with error: {ex}") sys.exit(1) -def delete_certificates_from_s3(certificates, dry_run): +def mark_certificate_deleted(connection, certificate_id, user_id): + try: + cursor = connection.cursor() + cursor.execute( + """ + UPDATE certificates_generatedcertificate + SET status = 'deleted', download_url = '', download_uuid = '' + WHERE id = %s + """, + (certificate_id,) + ) + connection.commit() + cursor.close() + logging.info(f"Marked certificate {certificate_id} (user {user_id}) as deleted in database") + except Exception as ex: + logging.error(f"Failed to update certificate {certificate_id} (user {user_id}) in database: {ex}") + + +def delete_certificates_from_s3(certificates, connection, dry_run): s3_client = S3BotoWrapper() + logging.info(f"Found {len(certificates)} certificate(s) to process") for cert in certificates: - verify_uuid = cert[5] # VERIFY_UUID + user_id = cert[0] # LMS_USER_ID + certificate_id = cert[2] # CERTIFICATE_ID download_uuid = cert[4] # DOWNLOAD_UUID + verify_uuid = cert[5] # VERIFY_UUID verify_key = f"cert/{verify_uuid}" download_key = f"downloads/{download_uuid}/Certificate.pdf" try: if dry_run: - logging.info(f"[Dry Run] Would delete {verify_key} from S3") - logging.info(f"[Dry Run] Would delete {download_key} from S3") + logging.info(f"[Dry Run] Would delete {verify_key} from S3 (user {user_id})") + logging.info(f"[Dry Run] Would delete {download_key} from S3 (user {user_id})") + logging.info(f"[Dry Run] Would mark certificate {certificate_id} (user {user_id}) as deleted in database") else: - logging.info(f"Deleting {verify_key} from S3...") + logging.info(f"Deleting {verify_key} from S3 (user {user_id})...") s3_client.delete_object("verify.edx.org", verify_key) - logging.info(f"Deleting {download_key} from S3...") + logging.info(f"Deleting {download_key} from S3 (user {user_id})...") s3_client.delete_object("verify.edx.org", download_key) + mark_certificate_deleted(connection, certificate_id, user_id) except ClientError as e: - logging.error(f"Error deleting {verify_key} or {download_key}: {e}") + logging.error(f"Error deleting {verify_key} or {download_key} (user {user_id}): {e}") @click.command() @@ -114,8 +143,10 @@ def delete_certificates_from_s3(certificates, dry_run): @click.option('--db-name', '-db', required=True, help='Database name') @click.option('--dry-run', is_flag=True, help='Run the script in dry-run mode without making any changes') def controller(db_host, db_user, db_password, db_name, dry_run): - certificates = fetch_certificates_to_delete(db_host, db_user, db_password, db_name) - delete_certificates_from_s3(certificates, dry_run) + connection = get_db_connection(db_host, db_user, db_password, db_name) + certificates = fetch_certificates_to_delete(connection) + delete_certificates_from_s3(certificates, connection, dry_run) + connection.close() if __name__ == '__main__': From 43fa8b840f0b4aaa11540496c89d48cb7ecbec5e Mon Sep 17 00:00:00 2001 From: Tarun Tak Date: Tue, 21 Apr 2026 11:18:22 +0000 Subject: [PATCH 2/7] fix: username filter to search retired user --- .../retired_user_cert_remover/retired_user_cert_remover.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/jenkins/retired_user_cert_remover/retired_user_cert_remover.py b/util/jenkins/retired_user_cert_remover/retired_user_cert_remover.py index a8a52d9b6..1e5c61ffb 100644 --- a/util/jenkins/retired_user_cert_remover/retired_user_cert_remover.py +++ b/util/jenkins/retired_user_cert_remover/retired_user_cert_remover.py @@ -77,7 +77,7 @@ def fetch_certificates_to_delete(connection): ON gc.user_id = au.id WHERE - au.is_active = 0 + au.username LIKE 'retired__user_%' AND gc.download_url LIKE '%%https://%%' AND gc.status = 'downloadable' ORDER BY From 485889827dee06620d9e95f94c7e04ac8811314c Mon Sep 17 00:00:00 2001 From: Tarun Tak Date: Tue, 21 Apr 2026 12:51:38 +0000 Subject: [PATCH 3/7] fix: ensure proper cursor management and error handling in database operations --- .../retired_user_cert_remover.py | 31 ++++++++++++------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/util/jenkins/retired_user_cert_remover/retired_user_cert_remover.py b/util/jenkins/retired_user_cert_remover/retired_user_cert_remover.py index 1e5c61ffb..959142b9d 100644 --- a/util/jenkins/retired_user_cert_remover/retired_user_cert_remover.py +++ b/util/jenkins/retired_user_cert_remover/retired_user_cert_remover.py @@ -59,8 +59,8 @@ def get_db_connection(db_host, db_user, db_password, db_name): def fetch_certificates_to_delete(connection): + cursor = connection.cursor() try: - cursor = connection.cursor() logging.info("Running query on database...") cursor.execute(""" SELECT @@ -84,17 +84,17 @@ def fetch_certificates_to_delete(connection): LMS_USER_ID, COURSE_RUN_ID; """) - result = cursor.fetchall() - cursor.close() - return result + return cursor.fetchall() except Exception as ex: logging.error(f"Database query failed with error: {ex}") sys.exit(1) + finally: + cursor.close() def mark_certificate_deleted(connection, certificate_id, user_id): + cursor = connection.cursor() try: - cursor = connection.cursor() cursor.execute( """ UPDATE certificates_generatedcertificate @@ -104,15 +104,19 @@ def mark_certificate_deleted(connection, certificate_id, user_id): (certificate_id,) ) connection.commit() - cursor.close() logging.info(f"Marked certificate {certificate_id} (user {user_id}) as deleted in database") except Exception as ex: + connection.rollback() logging.error(f"Failed to update certificate {certificate_id} (user {user_id}) in database: {ex}") + raise + finally: + cursor.close() def delete_certificates_from_s3(certificates, connection, dry_run): s3_client = S3BotoWrapper() logging.info(f"Found {len(certificates)} certificate(s) to process") + failed = False for cert in certificates: user_id = cert[0] # LMS_USER_ID certificate_id = cert[2] # CERTIFICATE_ID @@ -132,8 +136,11 @@ def delete_certificates_from_s3(certificates, connection, dry_run): logging.info(f"Deleting {download_key} from S3 (user {user_id})...") s3_client.delete_object("verify.edx.org", download_key) mark_certificate_deleted(connection, certificate_id, user_id) - except ClientError as e: - logging.error(f"Error deleting {verify_key} or {download_key} (user {user_id}): {e}") + except (ClientError, Exception) as e: + logging.error(f"Error processing certificate {certificate_id} (user {user_id}): {e}") + failed = True + if failed: + sys.exit(1) @click.command() @@ -144,9 +151,11 @@ def delete_certificates_from_s3(certificates, connection, dry_run): @click.option('--dry-run', is_flag=True, help='Run the script in dry-run mode without making any changes') def controller(db_host, db_user, db_password, db_name, dry_run): connection = get_db_connection(db_host, db_user, db_password, db_name) - certificates = fetch_certificates_to_delete(connection) - delete_certificates_from_s3(certificates, connection, dry_run) - connection.close() + try: + certificates = fetch_certificates_to_delete(connection) + delete_certificates_from_s3(certificates, connection, dry_run) + finally: + connection.close() if __name__ == '__main__': From e8c5258dac3426a86348ed9406be49c0805b49cf Mon Sep 17 00:00:00 2001 From: Tarun Tak Date: Fri, 24 Apr 2026 06:18:33 +0000 Subject: [PATCH 4/7] fix: enhance logging for S3 certificate deletion with bucket information --- .../retired_user_cert_remover.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/util/jenkins/retired_user_cert_remover/retired_user_cert_remover.py b/util/jenkins/retired_user_cert_remover/retired_user_cert_remover.py index 959142b9d..b0307d2de 100644 --- a/util/jenkins/retired_user_cert_remover/retired_user_cert_remover.py +++ b/util/jenkins/retired_user_cert_remover/retired_user_cert_remover.py @@ -34,6 +34,7 @@ import click import sys import logging +from urllib.parse import urlparse MAX_TRIES = 5 # Configure logging @@ -120,21 +121,24 @@ def delete_certificates_from_s3(certificates, connection, dry_run): for cert in certificates: user_id = cert[0] # LMS_USER_ID certificate_id = cert[2] # CERTIFICATE_ID + certificate_url = cert[3] # CERTIFICATE_URL download_uuid = cert[4] # DOWNLOAD_UUID verify_uuid = cert[5] # VERIFY_UUID + parsed_url = urlparse(certificate_url) + s3_bucket = parsed_url.path.lstrip("/").split("/")[0] verify_key = f"cert/{verify_uuid}" download_key = f"downloads/{download_uuid}/Certificate.pdf" try: if dry_run: - logging.info(f"[Dry Run] Would delete {verify_key} from S3 (user {user_id})") - logging.info(f"[Dry Run] Would delete {download_key} from S3 (user {user_id})") + logging.info(f"[Dry Run] Would delete {verify_key} from S3 bucket {s3_bucket} (user {user_id})") + logging.info(f"[Dry Run] Would delete {download_key} from S3 bucket {s3_bucket} (user {user_id})") logging.info(f"[Dry Run] Would mark certificate {certificate_id} (user {user_id}) as deleted in database") else: - logging.info(f"Deleting {verify_key} from S3 (user {user_id})...") - s3_client.delete_object("verify.edx.org", verify_key) - logging.info(f"Deleting {download_key} from S3 (user {user_id})...") - s3_client.delete_object("verify.edx.org", download_key) + logging.info(f"Deleting {verify_key} from S3 bucket {s3_bucket} (user {user_id})...") + s3_client.delete_object(s3_bucket, verify_key) + logging.info(f"Deleting {download_key} from S3 bucket {s3_bucket} (user {user_id})...") + s3_client.delete_object(s3_bucket, download_key) mark_certificate_deleted(connection, certificate_id, user_id) except (ClientError, Exception) as e: logging.error(f"Error processing certificate {certificate_id} (user {user_id}): {e}") From fc8749c9b7c3b82cc0d0f7619824da0da14e1ff8 Mon Sep 17 00:00:00 2001 From: Tarun Tak Date: Fri, 24 Apr 2026 06:31:47 +0000 Subject: [PATCH 5/7] fix: enhance logging for S3 certificate deletion with bucket information --- .../retired_user_cert_remover/retired_user_cert_remover.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/jenkins/retired_user_cert_remover/retired_user_cert_remover.py b/util/jenkins/retired_user_cert_remover/retired_user_cert_remover.py index b0307d2de..d668039a8 100644 --- a/util/jenkins/retired_user_cert_remover/retired_user_cert_remover.py +++ b/util/jenkins/retired_user_cert_remover/retired_user_cert_remover.py @@ -157,7 +157,7 @@ def controller(db_host, db_user, db_password, db_name, dry_run): connection = get_db_connection(db_host, db_user, db_password, db_name) try: certificates = fetch_certificates_to_delete(connection) - delete_certificates_from_s3(certificates, connection, dry_run) + delete_certificates_from_s3(certificates, connection, False) finally: connection.close() From 5643f92ec3ea5c03c17f593af9fdc1214812d36e Mon Sep 17 00:00:00 2001 From: Tarun Tak Date: Tue, 28 Apr 2026 11:36:23 +0000 Subject: [PATCH 6/7] fix: update retired user certificate removal to use LMS API --- .../retired_user_cert_remover.py | 232 ++++++++---------- 1 file changed, 96 insertions(+), 136 deletions(-) diff --git a/util/jenkins/retired_user_cert_remover/retired_user_cert_remover.py b/util/jenkins/retired_user_cert_remover/retired_user_cert_remover.py index d668039a8..4754cb8c9 100644 --- a/util/jenkins/retired_user_cert_remover/retired_user_cert_remover.py +++ b/util/jenkins/retired_user_cert_remover/retired_user_cert_remover.py @@ -1,165 +1,125 @@ """ -Script to delete downloadable certificates of inactive users from S3, based on RDS MySQL database entries. +Script to delete downloadable certificates of inactive users from S3 by calling +the LMS retire_certs_s3 API endpoint. -Usage: - python retired_user_cert_remover.py --db-host=my-db-host --db-name=my-db --dry-run +This script no longer connects directly to RDS. All certificate discovery, S3 +deletion, and database updates are handled by the LMS API endpoint: + POST /api/certificates/v1/retire_certs_s3 + +The LMS endpoint requires an OAuth token obtained by exchanging client_id / +client_secret (stored in AWS Secrets Manager) for a bearer token. -Arguments: - --db-host The RDS database host. - --db-name The database name. - --dry-run Run the script in dry-run mode (logs actions without deleting). - --db-user The RDS database user (also settable via DB_USER env var). - --db-password The RDS database password (also settable via DB_PASSWORD env var). +Usage: + python retired_user_cert_remover.py \ + --lms-host=https://lms.example.com \ + --client-id= \ + --client-secret= \ + [--dry-run] Environment Variables: - DB_USER Database username (alternative to --db-user). - DB_PASSWORD Database password (alternative to --db-password). - -Functionality: - - Connects to an RDS MySQL database and fetches certificates for inactive users. - - Targets only certificates with a valid download URL and status 'downloadable'. - - Deletes corresponding certificate files from S3 (verify and download locations). - - Supports dry-run mode to simulate deletions for review. - -Example: - export DB_USER=admin - export DB_PASSWORD=securepass - python retired_user_cert_remover.py --db-host=mydb.amazonaws.com --db-name=edxapp --dry-run + LMS_CLIENT_ID OAuth client id (alternative to --client-id). + LMS_CLIENT_SECRET OAuth client secret (alternative to --client-secret). + +Dry-run: + Passes ?dry_run=true to the API. The LMS logs what would be deleted without + making any changes to S3 or the database. """ -import boto3 -from botocore.exceptions import ClientError -import pymysql -import backoff -import click -import sys import logging -from urllib.parse import urlparse +import sys -MAX_TRIES = 5 -# Configure logging -LOGGER = logging.getLogger(__name__) -logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +import backoff +import click +import requests +MAX_TOKEN_ATTEMPTS = 3 -class S3BotoWrapper: - def __init__(self): - self.client = boto3.client("s3") +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +LOGGER = logging.getLogger(__name__) - @backoff.on_exception(backoff.expo, ClientError, max_tries=MAX_TRIES) - def delete_object(self, bucket, key): - return self.client.delete_object(Bucket=bucket, Key=key) +def get_oauth_token(lms_host, client_id, client_secret): + """ + Exchange client credentials for a bearer token via LMS DOT. + + Returns the access token string, or exits on failure. + """ + token_url = f'{lms_host.rstrip("/")}/oauth2/access_token/' + + @backoff.on_exception(backoff.expo, requests.RequestException, max_tries=MAX_TOKEN_ATTEMPTS) + def _request(): + response = requests.post( + token_url, + data={ + 'grant_type': 'client_credentials', + 'client_id': client_id, + 'client_secret': client_secret, + }, + timeout=30, + ) + response.raise_for_status() + return response.json()['access_token'] -def get_db_connection(db_host, db_user, db_password, db_name): try: - return pymysql.connect(host=db_host, user=db_user, password=db_password, database=db_name) - except Exception as ex: - logging.error(f"Database connection failed with error: {ex}") + token = _request() + LOGGER.info('Successfully obtained OAuth token from %s', token_url) + return token + except Exception as exc: + LOGGER.error('Failed to obtain OAuth token: %s', exc) sys.exit(1) -def fetch_certificates_to_delete(connection): - cursor = connection.cursor() +def call_retire_certs_api(lms_host, token, dry_run): + """ + Call POST /api/certificates/v1/retire_certs_s3 on the LMS. + + Returns the parsed JSON response body. + Exits with code 1 if the call fails entirely (non-2xx after retries). + Exits with code 2 if the call returns 207 (partial failure). + """ + url = f'{lms_host.rstrip("/")}/api/certificates/v1/retire_certs_s3' + params = {'dry_run': 'true'} if dry_run else {} + headers = {'Authorization': f'Bearer {token}', 'Content-Type': 'application/json'} + + LOGGER.info('Calling %s (dry_run=%s)', url, dry_run) try: - logging.info("Running query on database...") - cursor.execute(""" - SELECT - au.id as "LMS_USER_ID", - gc.course_id as "COURSE_RUN_ID", - gc.id as "CERTIFICATE_ID", - gc.download_url as "CERTIFICATE_URL", - gc.download_uuid as "DOWNLOAD_UUID", - gc.verify_uuid as "VERIFY_UUID" - FROM - auth_user as au - JOIN - certificates_generatedcertificate as gc - ON - gc.user_id = au.id - WHERE - au.username LIKE 'retired__user_%' - AND gc.download_url LIKE '%%https://%%' - AND gc.status = 'downloadable' - ORDER BY - LMS_USER_ID, - COURSE_RUN_ID; - """) - return cursor.fetchall() - except Exception as ex: - logging.error(f"Database query failed with error: {ex}") + response = requests.post(url, params=params, headers=headers, timeout=600) + except requests.RequestException as exc: + LOGGER.error('HTTP request to retire_certs_s3 failed: %s', exc) sys.exit(1) - finally: - cursor.close() - -def mark_certificate_deleted(connection, certificate_id, user_id): - cursor = connection.cursor() + body = {} try: - cursor.execute( - """ - UPDATE certificates_generatedcertificate - SET status = 'deleted', download_url = '', download_uuid = '' - WHERE id = %s - """, - (certificate_id,) + body = response.json() + except ValueError: + pass + + if response.status_code == 200: + LOGGER.info('retire_certs_s3 completed successfully: %s', body) + return body + + if response.status_code == 207: + LOGGER.warning( + 'retire_certs_s3 completed with partial failures: processed=%s failed=%s', + body.get('processed'), body.get('failed'), ) - connection.commit() - logging.info(f"Marked certificate {certificate_id} (user {user_id}) as deleted in database") - except Exception as ex: - connection.rollback() - logging.error(f"Failed to update certificate {certificate_id} (user {user_id}) in database: {ex}") - raise - finally: - cursor.close() - - -def delete_certificates_from_s3(certificates, connection, dry_run): - s3_client = S3BotoWrapper() - logging.info(f"Found {len(certificates)} certificate(s) to process") - failed = False - for cert in certificates: - user_id = cert[0] # LMS_USER_ID - certificate_id = cert[2] # CERTIFICATE_ID - certificate_url = cert[3] # CERTIFICATE_URL - download_uuid = cert[4] # DOWNLOAD_UUID - verify_uuid = cert[5] # VERIFY_UUID - - parsed_url = urlparse(certificate_url) - s3_bucket = parsed_url.path.lstrip("/").split("/")[0] - verify_key = f"cert/{verify_uuid}" - download_key = f"downloads/{download_uuid}/Certificate.pdf" - try: - if dry_run: - logging.info(f"[Dry Run] Would delete {verify_key} from S3 bucket {s3_bucket} (user {user_id})") - logging.info(f"[Dry Run] Would delete {download_key} from S3 bucket {s3_bucket} (user {user_id})") - logging.info(f"[Dry Run] Would mark certificate {certificate_id} (user {user_id}) as deleted in database") - else: - logging.info(f"Deleting {verify_key} from S3 bucket {s3_bucket} (user {user_id})...") - s3_client.delete_object(s3_bucket, verify_key) - logging.info(f"Deleting {download_key} from S3 bucket {s3_bucket} (user {user_id})...") - s3_client.delete_object(s3_bucket, download_key) - mark_certificate_deleted(connection, certificate_id, user_id) - except (ClientError, Exception) as e: - logging.error(f"Error processing certificate {certificate_id} (user {user_id}): {e}") - failed = True - if failed: - sys.exit(1) + sys.exit(2) + + LOGGER.error( + 'retire_certs_s3 returned unexpected status %s: %s', + response.status_code, body, + ) + sys.exit(1) @click.command() -@click.option('--db-host', '-h', required=True, help='Database host') -@click.option('--db-user', envvar='DB_USER', required=True, help='Database user') -@click.option('--db-password', envvar='DB_PASSWORD', required=True, help='Database password') -@click.option('--db-name', '-db', required=True, help='Database name') -@click.option('--dry-run', is_flag=True, help='Run the script in dry-run mode without making any changes') -def controller(db_host, db_user, db_password, db_name, dry_run): - connection = get_db_connection(db_host, db_user, db_password, db_name) - try: - certificates = fetch_certificates_to_delete(connection) - delete_certificates_from_s3(certificates, connection, False) - finally: - connection.close() +@click.option('--lms-host', required=True, help='Base URL of the LMS (e.g. https://lms.edx.org)') +@click.option('--client-id', envvar='LMS_CLIENT_ID', required=True, help='OAuth DOT client id') +@click.option('--client-secret', envvar='LMS_CLIENT_SECRET', required=True, help='OAuth DOT client secret') +@click.option('--dry-run', is_flag=True, help='Run in dry-run mode without making any changes') +def controller(lms_host, client_id, client_secret, dry_run): + token = get_oauth_token(lms_host, client_id, client_secret) + call_retire_certs_api(lms_host, token, dry_run) if __name__ == '__main__': From b5f94eed3e6e96b244738be774065db520084772 Mon Sep 17 00:00:00 2001 From: Tarun Tak Date: Wed, 29 Apr 2026 11:09:52 +0000 Subject: [PATCH 7/7] fix: implement retry logic for API calls to handle transient network errors --- .../retired_user_cert_remover.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/util/jenkins/retired_user_cert_remover/retired_user_cert_remover.py b/util/jenkins/retired_user_cert_remover/retired_user_cert_remover.py index 4754cb8c9..f92d929ab 100644 --- a/util/jenkins/retired_user_cert_remover/retired_user_cert_remover.py +++ b/util/jenkins/retired_user_cert_remover/retired_user_cert_remover.py @@ -33,6 +33,7 @@ import requests MAX_TOKEN_ATTEMPTS = 3 +MAX_API_ATTEMPTS = 3 logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') LOGGER = logging.getLogger(__name__) @@ -74,18 +75,27 @@ def call_retire_certs_api(lms_host, token, dry_run): Call POST /api/certificates/v1/retire_certs_s3 on the LMS. Returns the parsed JSON response body. - Exits with code 1 if the call fails entirely (non-2xx after retries). + Retries up to MAX_API_ATTEMPTS times on transient network errors. + Exits with code 1 if the call fails entirely after retries. Exits with code 2 if the call returns 207 (partial failure). """ url = f'{lms_host.rstrip("/")}/api/certificates/v1/retire_certs_s3' params = {'dry_run': 'true'} if dry_run else {} headers = {'Authorization': f'Bearer {token}', 'Content-Type': 'application/json'} + @backoff.on_exception(backoff.expo, requests.RequestException, max_tries=MAX_API_ATTEMPTS) + def _request(): + response = requests.post(url, params=params, headers=headers, timeout=600) + # Retry on 5xx server errors; 2xx/207/4xx are handled below. + if response.status_code >= 500: + response.raise_for_status() + return response + LOGGER.info('Calling %s (dry_run=%s)', url, dry_run) try: - response = requests.post(url, params=params, headers=headers, timeout=600) + response = _request() except requests.RequestException as exc: - LOGGER.error('HTTP request to retire_certs_s3 failed: %s', exc) + LOGGER.error('HTTP request to retire_certs_s3 failed after retries: %s', exc) sys.exit(1) body = {}