From 71874c8158a1247c04d128b28ae3a80ea5bf94a2 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Wed, 4 Mar 2026 13:39:48 -0600 Subject: [PATCH 1/5] Tracking tasks per each new collection --- specifyweb/backend/accounts/views.py | 17 +--- specifyweb/backend/context/views.py | 10 +-- specifyweb/backend/setup_tool/api.py | 16 +++- specifyweb/backend/setup_tool/redis.py | 2 + specifyweb/backend/setup_tool/setup_tasks.py | 84 ++++++++++++++++++-- 5 files changed, 101 insertions(+), 28 deletions(-) diff --git a/specifyweb/backend/accounts/views.py b/specifyweb/backend/accounts/views.py index 6dc56aab016..0ef586ad614 100644 --- a/specifyweb/backend/accounts/views.py +++ b/specifyweb/backend/accounts/views.py @@ -1,5 +1,4 @@ import base64 -from datetime import timedelta import hashlib import hmac import json @@ -17,7 +16,7 @@ from django.db.models import Max from django.shortcuts import render from django.template.response import TemplateResponse -from django.utils import crypto, timezone +from django.utils import crypto from django.utils.http import url_has_allowed_host_and_scheme, urlencode from django.views.decorators.cache import never_cache from typing import cast @@ -295,8 +294,7 @@ def choose_collection(request) -> http.HttpResponse: id to the user if one is provided. """ from specifyweb.backend.context.views import set_collection_cookie, users_collections_for_sp7 - from specifyweb.backend.setup_tool.api import is_config_task_running - + from specifyweb.backend.setup_tool.api import get_collections_with_busy_config from specifyweb.specify.api.serializers import obj_to_data, toJson @@ -320,14 +318,8 @@ def choose_collection(request) -> http.HttpResponse: ) available_collections = users_collections_for_sp7(request.specify_user.id) - if is_config_task_running(): - # If config tasks are running, filter out newly created collections - fifteen_minutes_ago = timezone.now() - timedelta(minutes=15) - available_collections = [ - c - for c in available_collections - if c.timestampcreated is None or c.timestampcreated <= fifteen_minutes_ago - ] + busy_collection_ids = get_collections_with_busy_config(c.id for c in available_collections) + available_collections = [c for c in available_collections if c.id not in busy_collection_ids] if len(available_collections) == 1: set_collection_cookie(redirect_resp, available_collections[0].id) @@ -588,4 +580,3 @@ def set_admin_status(request, userid): else: user.clear_admin() return http.HttpResponse('false', content_type='text/plain') - diff --git a/specifyweb/backend/context/views.py b/specifyweb/backend/context/views.py index 373d599f3f6..77f29b19c8e 100644 --- a/specifyweb/backend/context/views.py +++ b/specifyweb/backend/context/views.py @@ -5,7 +5,6 @@ import json import os import re -from datetime import timedelta from typing import List from django.conf import settings @@ -38,7 +37,7 @@ from .remote_prefs import get_remote_prefs from .schema_localization import get_schema_languages, get_schema_localization from .viewsets import get_views -from specifyweb.backend.setup_tool.api import get_config_progress, is_config_task_running +from specifyweb.backend.setup_tool.api import get_collections_with_busy_config, get_config_progress def set_collection_cookie(response, collection_id): # pragma: no cover @@ -647,15 +646,14 @@ def get_server_time(request): def _filter_collections_not_ready_for_config_task(collections): - if not is_config_task_running(): + busy_collection_ids = get_collections_with_busy_config(c.id for c in collections) + if not busy_collection_ids: return collections - # If config tasks are running, filter out newly created collections. - fifteen_minutes_ago = timezone.now() - timedelta(minutes=15) return [ c for c in collections - if c.timestampcreated is None or c.timestampcreated <= fifteen_minutes_ago + if c.id not in busy_collection_ids ] diff --git a/specifyweb/backend/setup_tool/api.py b/specifyweb/backend/setup_tool/api.py index 05ad45c3b17..da61ea93500 100644 --- a/specifyweb/backend/setup_tool/api.py +++ b/specifyweb/backend/setup_tool/api.py @@ -22,6 +22,8 @@ from specifyweb.backend.setup_tool.prep_type_defaults import create_default_prep_types from specifyweb.backend.setup_tool.app_resource_defaults import ensure_discipline_resource_dir from specifyweb.backend.setup_tool.setup_tasks import ( + get_collections_with_busy_config as _get_collections_with_busy_config, + is_collection_config_busy, setup_database_background, get_active_setup_task, get_last_setup_error, @@ -343,7 +345,7 @@ def create_collection(data, run_fix_schema_config_async: bool = True): # Create picklists create_default_picklists(new_collection, discipline.type) if run_fix_schema_config_async: - queue_fix_schema_config_background() + queue_fix_schema_config_background(new_collection.id) else: fix_schema_config() @@ -518,6 +520,12 @@ def get_config_resource_progress(running_task_names: Optional[list[str]] = None) active_task_names = set(running_task_names or []) return _get_config_resource_progress_from_active_names(active_task_names) + +def get_collections_with_busy_config(collection_ids) -> set[int]: + """Returns collection ids that still have active tracked config tasks.""" + return _get_collections_with_busy_config(collection_ids) + + def get_config_progress(collection_id: Optional[int] = None) -> dict: """Returns a dict of the status of config/setup related background tasks""" try: @@ -525,7 +533,11 @@ def get_config_progress(collection_id: Optional[int] = None) -> dict: except MissingWorkerError: running_task_names = [] - busy = is_config_task_running(running_task_names) + busy = ( + is_collection_config_busy(collection_id) + if collection_id is not None + else is_config_task_running(running_task_names) + ) last_error = None completed_resources = get_config_resource_progress(running_task_names) diff --git a/specifyweb/backend/setup_tool/redis.py b/specifyweb/backend/setup_tool/redis.py index 1a70c241a9c..c53cdcaf4b5 100644 --- a/specifyweb/backend/setup_tool/redis.py +++ b/specifyweb/backend/setup_tool/redis.py @@ -2,5 +2,7 @@ # Also defined separately in setup_tool/apps.py ACTIVE_TASK_REDIS_KEY = "specify:{database}:setup:active_task_id" ACTIVE_TASK_TTL = 60*60*2 # setup should be less than 2 hours +# Keep track of config/setup celery task ids by collection. +COLLECTION_TASK_IDS_REDIS_KEY = "specify:{database}:setup:collection:{collection_id}:task_ids" # Keep track of last error. LAST_ERROR_REDIS_KEY = "specify:{database}:setup:last_error" \ No newline at end of file diff --git a/specifyweb/backend/setup_tool/setup_tasks.py b/specifyweb/backend/setup_tool/setup_tasks.py index bc6bf0dd1d9..3fdba62c87e 100644 --- a/specifyweb/backend/setup_tool/setup_tasks.py +++ b/specifyweb/backend/setup_tool/setup_tasks.py @@ -4,7 +4,7 @@ from django.db import transaction from specifyweb.celery_tasks import app -from typing import Tuple, Optional +from typing import Iterable, Optional, Tuple from celery.result import AsyncResult from specifyweb.backend.setup_tool import api from specifyweb.backend.setup_tool.app_resource_defaults import create_app_resource_defaults @@ -12,13 +12,73 @@ from specifyweb.specify.management.commands.run_key_migration_functions import fix_schema_config from specifyweb.specify.models_utils.model_extras import PALEO_DISCIPLINES, GEOLOGY_DISCIPLINES from specifyweb.celery_tasks import is_worker_alive, MissingWorkerError -from specifyweb.backend.redis_cache.store import set_string, get_string -from specifyweb.backend.setup_tool.redis import ACTIVE_TASK_REDIS_KEY, ACTIVE_TASK_TTL, LAST_ERROR_REDIS_KEY +from specifyweb.backend.redis_cache.store import ( + add_to_set, + delete_key, + get_string, + remove_from_set, + set_members, + set_string, +) +from specifyweb.backend.setup_tool.redis import ( + ACTIVE_TASK_REDIS_KEY, + ACTIVE_TASK_TTL, + COLLECTION_TASK_IDS_REDIS_KEY, + LAST_ERROR_REDIS_KEY, +) from uuid import uuid4 import logging logger = logging.getLogger(__name__) +ACTIVE_CELERY_STATES = frozenset(("PENDING", "RECEIVED", "STARTED", "RETRY", "PROGRESS")) + +def _collection_task_ids_key(collection_id: int) -> str: + return COLLECTION_TASK_IDS_REDIS_KEY.replace("{collection_id}", str(collection_id)) + +def _track_collection_task(collection_id: Optional[int], task_id: str) -> None: + if collection_id is None: + return + add_to_set(_collection_task_ids_key(collection_id), task_id) + +def _untrack_collection_task(collection_id: Optional[int], task_id: Optional[str]) -> None: + if collection_id is None or task_id is None: + return + key = _collection_task_ids_key(collection_id) + remove_from_set(key, task_id) + if not set_members(key): + delete_key(key) + +def _is_task_active(task_id: str) -> bool: + return app.AsyncResult(task_id).state in ACTIVE_CELERY_STATES + +def is_collection_config_busy(collection_id: Optional[int]) -> bool: + if collection_id is None: + return False + + key = _collection_task_ids_key(collection_id) + task_ids = set_members(key) + if not task_ids: + return False + + inactive_task_ids = [] + for task_id in task_ids: + if _is_task_active(task_id): + return True + inactive_task_ids.append(task_id) + + remove_from_set(key, *inactive_task_ids) + if not set_members(key): + delete_key(key) + return False + +def get_collections_with_busy_config(collection_ids: Iterable[int]) -> set[int]: + return { + collection_id + for collection_id in collection_ids + if is_collection_config_busy(collection_id) + } + def setup_database_background(data: dict) -> str: # Clear any previous error logs. set_last_setup_error(None) @@ -38,9 +98,15 @@ def setup_database_background(data: dict) -> str: return task.id -def queue_fix_schema_config_background() -> str: +def queue_fix_schema_config_background(collection_id: Optional[int] = None) -> str: """Queue fix_schema_config to run asynchronously and return the task id""" - task = fix_schema_config_task.apply_async() + task_id = str(uuid4()) + _track_collection_task(collection_id, task_id) + try: + task = fix_schema_config_task.apply_async(args=[collection_id], task_id=task_id) + except Exception: + _untrack_collection_task(collection_id, task_id) + raise return task.id def get_active_setup_task() -> Tuple[Optional[AsyncResult], bool]: @@ -171,9 +237,13 @@ def update_progress(): raise @app.task(bind=True) -def fix_schema_config_task(self): +def fix_schema_config_task(self, collection_id: Optional[int] = None): """Run schema config migration fixups in a background worker""" - fix_schema_config() + task_id = getattr(self.request, "id", None) + try: + fix_schema_config() + finally: + _untrack_collection_task(collection_id, task_id) def get_last_setup_error() -> Optional[str]: err = get_string(LAST_ERROR_REDIS_KEY) From 5d0e488fbf1cc6e0695d7d19c836bca7c746148c Mon Sep 17 00:00:00 2001 From: alec_dev Date: Thu, 5 Mar 2026 00:14:22 -0600 Subject: [PATCH 2/5] check discipline status for new collections --- specifyweb/backend/context/views.py | 3 + specifyweb/backend/setup_tool/api.py | 32 +++++++++- specifyweb/backend/setup_tool/redis.py | 4 +- .../backend/setup_tool/schema_defaults.py | 60 +++++++++++++++++-- specifyweb/backend/setup_tool/setup_tasks.py | 3 +- 5 files changed, 93 insertions(+), 9 deletions(-) diff --git a/specifyweb/backend/context/views.py b/specifyweb/backend/context/views.py index 77f29b19c8e..f16e8bbb937 100644 --- a/specifyweb/backend/context/views.py +++ b/specifyweb/backend/context/views.py @@ -305,6 +305,9 @@ def collection(request): set_collection_cookie(response, collection.id) return response else: + available_collections = _filter_collections_not_ready_for_config_task( + available_collections + ) response = dict( available=[obj_to_data(c) for c in available_collections], current=(current and int(current)) diff --git a/specifyweb/backend/setup_tool/api.py b/specifyweb/backend/setup_tool/api.py index da61ea93500..808a9343336 100644 --- a/specifyweb/backend/setup_tool/api.py +++ b/specifyweb/backend/setup_tool/api.py @@ -16,6 +16,7 @@ from specifyweb.backend.setup_tool.utils import normalize_keys, resolve_uri_or_fallback from specifyweb.backend.setup_tool.schema_defaults import ( apply_schema_defaults, + is_discipline_setup_busy, queue_apply_schema_defaults_background, ) from specifyweb.backend.setup_tool.picklist_defaults import create_default_picklists @@ -50,7 +51,6 @@ def get_setup_progress() -> dict: """Returns a dictionary of the status of the database setup.""" # Check if setup is currently in progress active_setup_task, busy = get_active_setup_task() - busy = busy or is_config_task_running() completed_resources = None last_error = None @@ -67,6 +67,11 @@ def get_setup_progress() -> dict: completed_resources = get_setup_resource_progress() last_error = get_last_setup_error() + has_incomplete_resources = not all(completed_resources.values()) + if has_incomplete_resources: + # During initial setup, include tracked config/setup tasks in busy state. + busy = busy or _tracked_setup_tasks_busy() or is_config_task_running() + return { "resources": completed_resources, "last_error": last_error, @@ -302,7 +307,11 @@ def create_discipline(data, run_apply_schema_defaults_async: bool = True): except Exception as e: raise SetupError(e) -def create_collection(data, run_fix_schema_config_async: bool = True): +def create_collection( + data, + run_fix_schema_config_async: bool = True, + require_discipline_ready: bool = True, +): from specifyweb.specify.models import Collection, Discipline # If collection_id is provided and exists, return success @@ -326,6 +335,11 @@ def create_collection(data, run_fix_schema_config_async: bool = True): data['discipline'] = discipline else: raise SetupError("No discipline available") + + if require_discipline_ready and is_discipline_setup_busy(discipline.id): + raise SetupError( + "This discipline is still being created. Please wait for background setup tasks to finish before creating a collection." + ) # The discipline needs a Taxon Tree in order for the Collection Object Type to be created. if not discipline.taxontreedef_id: @@ -491,6 +505,18 @@ def create_tree(name: str, data: dict) -> dict: "specifyweb.backend.setup_tool.schema_defaults.apply_schema_defaults_task", }) + +def _tracked_setup_tasks_busy() -> bool: + """True if any tracked collection/discipline setup task is active.""" + if any( + is_discipline_setup_busy(discipline_id) + for discipline_id in Discipline.objects.values_list('id', flat=True) + ): + return True + collection_ids = models.Collection.objects.values_list('id', flat=True) + return bool(_get_collections_with_busy_config(collection_ids)) + + def _task_name_to_progress_key(task_name: str) -> str: """Convert a task function name into a camelCase progress key.""" task_function_name = task_name.rsplit(".", 1)[-1] @@ -536,7 +562,7 @@ def get_config_progress(collection_id: Optional[int] = None) -> dict: busy = ( is_collection_config_busy(collection_id) if collection_id is not None - else is_config_task_running(running_task_names) + else is_config_task_running(running_task_names) or _tracked_setup_tasks_busy() ) last_error = None completed_resources = get_config_resource_progress(running_task_names) diff --git a/specifyweb/backend/setup_tool/redis.py b/specifyweb/backend/setup_tool/redis.py index c53cdcaf4b5..eb852080ead 100644 --- a/specifyweb/backend/setup_tool/redis.py +++ b/specifyweb/backend/setup_tool/redis.py @@ -4,5 +4,7 @@ ACTIVE_TASK_TTL = 60*60*2 # setup should be less than 2 hours # Keep track of config/setup celery task ids by collection. COLLECTION_TASK_IDS_REDIS_KEY = "specify:{database}:setup:collection:{collection_id}:task_ids" +# Keep track of discipline setup celery task ids by discipline. +DISCIPLINE_TASK_IDS_REDIS_KEY = "specify:{database}:setup:discipline:{discipline_id}:task_ids" # Keep track of last error. -LAST_ERROR_REDIS_KEY = "specify:{database}:setup:last_error" \ No newline at end of file +LAST_ERROR_REDIS_KEY = "specify:{database}:setup:last_error" diff --git a/specifyweb/backend/setup_tool/schema_defaults.py b/specifyweb/backend/setup_tool/schema_defaults.py index 6c9e7f9a8d3..e9d11b85f61 100644 --- a/specifyweb/backend/setup_tool/schema_defaults.py +++ b/specifyweb/backend/setup_tool/schema_defaults.py @@ -3,12 +3,55 @@ from specifyweb.celery_tasks import app from .utils import load_json_from_file from specifyweb.specify.models import Discipline +from specifyweb.backend.redis_cache.store import add_to_set, delete_key, remove_from_set, set_members +from specifyweb.backend.setup_tool.redis import DISCIPLINE_TASK_IDS_REDIS_KEY from pathlib import Path +from typing import Optional +from uuid import uuid4 import logging logger = logging.getLogger(__name__) +ACTIVE_CELERY_STATES = frozenset(("PENDING", "RECEIVED", "STARTED", "RETRY", "PROGRESS")) + +def _discipline_task_ids_key(discipline_id: int) -> str: + return DISCIPLINE_TASK_IDS_REDIS_KEY.replace("{discipline_id}", str(discipline_id)) + +def _track_discipline_task(discipline_id: int, task_id: str) -> None: + add_to_set(_discipline_task_ids_key(discipline_id), task_id) + +def _untrack_discipline_task(discipline_id: Optional[int], task_id: Optional[str]) -> None: + if discipline_id is None or task_id is None: + return + key = _discipline_task_ids_key(discipline_id) + remove_from_set(key, task_id) + if not set_members(key): + delete_key(key) + +def _is_task_active(task_id: str) -> bool: + return app.AsyncResult(task_id).state in ACTIVE_CELERY_STATES + +def is_discipline_setup_busy(discipline_id: Optional[int]) -> bool: + if discipline_id is None: + return False + + key = _discipline_task_ids_key(discipline_id) + task_ids = set_members(key) + if not task_ids: + return False + + inactive_task_ids = [] + for task_id in task_ids: + if _is_task_active(task_id): + return True + inactive_task_ids.append(task_id) + + remove_from_set(key, *inactive_task_ids) + if not set_members(key): + delete_key(key) + return False + def apply_schema_defaults(discipline: Discipline): """ Apply schema config localization defaults for this discipline. @@ -56,14 +99,23 @@ def apply_schema_defaults(discipline: Discipline): description=table_description, defaults=table_defaults, ) - def queue_apply_schema_defaults_background(discipline_id: int) -> str: """Queue apply_schema_defaults to run asynchronously and return the task id.""" - task = apply_schema_defaults_task.apply_async(args=[discipline_id]) + task_id = str(uuid4()) + _track_discipline_task(discipline_id, task_id) + try: + task = apply_schema_defaults_task.apply_async(args=[discipline_id], task_id=task_id) + except Exception: + _untrack_discipline_task(discipline_id, task_id) + raise return task.id @app.task(bind=True) def apply_schema_defaults_task(self, discipline_id: int): """Run schema localization defaults for one discipline in a background worker.""" - discipline = Discipline.objects.get(id=discipline_id) - apply_schema_defaults(discipline) + task_id = getattr(self.request, "id", None) + try: + discipline = Discipline.objects.get(id=discipline_id) + apply_schema_defaults(discipline) + finally: + _untrack_discipline_task(discipline_id, task_id) diff --git a/specifyweb/backend/setup_tool/setup_tasks.py b/specifyweb/backend/setup_tool/setup_tasks.py index 3fdba62c87e..83efcf4c25e 100644 --- a/specifyweb/backend/setup_tool/setup_tasks.py +++ b/specifyweb/backend/setup_tool/setup_tasks.py @@ -203,7 +203,8 @@ def update_progress(): logger.info('Creating collection') collection_result = api.create_collection( data['collection'], - run_fix_schema_config_async=False + run_fix_schema_config_async=False, + require_discipline_ready=False, ) collection_id = collection_result.get('collection_id') update_progress() From e3ae1db687396dc082af76e1aa1d95c186858019 Mon Sep 17 00:00:00 2001 From: alec_dev Date: Thu, 5 Mar 2026 12:38:16 -0600 Subject: [PATCH 3/5] Turn redis task key identifiers into uniquer tuples with database_name --- specifyweb/backend/setup_tool/schema_defaults.py | 4 +++- specifyweb/backend/setup_tool/setup_tasks.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/specifyweb/backend/setup_tool/schema_defaults.py b/specifyweb/backend/setup_tool/schema_defaults.py index e9d11b85f61..b4f6d450eb4 100644 --- a/specifyweb/backend/setup_tool/schema_defaults.py +++ b/specifyweb/backend/setup_tool/schema_defaults.py @@ -16,7 +16,9 @@ ACTIVE_CELERY_STATES = frozenset(("PENDING", "RECEIVED", "STARTED", "RETRY", "PROGRESS")) def _discipline_task_ids_key(discipline_id: int) -> str: - return DISCIPLINE_TASK_IDS_REDIS_KEY.replace("{discipline_id}", str(discipline_id)) + return DISCIPLINE_TASK_IDS_REDIS_KEY.replace( + "{discipline_id}", f"({{database}},{discipline_id})" + ) def _track_discipline_task(discipline_id: int, task_id: str) -> None: add_to_set(_discipline_task_ids_key(discipline_id), task_id) diff --git a/specifyweb/backend/setup_tool/setup_tasks.py b/specifyweb/backend/setup_tool/setup_tasks.py index 83efcf4c25e..426d22df76f 100644 --- a/specifyweb/backend/setup_tool/setup_tasks.py +++ b/specifyweb/backend/setup_tool/setup_tasks.py @@ -34,7 +34,9 @@ ACTIVE_CELERY_STATES = frozenset(("PENDING", "RECEIVED", "STARTED", "RETRY", "PROGRESS")) def _collection_task_ids_key(collection_id: int) -> str: - return COLLECTION_TASK_IDS_REDIS_KEY.replace("{collection_id}", str(collection_id)) + return COLLECTION_TASK_IDS_REDIS_KEY.replace( + "{collection_id}", f"({{database}},{collection_id})" + ) def _track_collection_task(collection_id: Optional[int], task_id: str) -> None: if collection_id is None: From b7955342e861e93c397a66551b460ebd7b2dcc0b Mon Sep 17 00:00:00 2001 From: alec_dev Date: Thu, 5 Mar 2026 23:54:35 -0600 Subject: [PATCH 4/5] task tracking refactoring --- specifyweb/backend/accounts/views.py | 6 +- specifyweb/backend/context/views.py | 22 ++- specifyweb/backend/setup_tool/api.py | 88 ++++++------ specifyweb/backend/setup_tool/redis.py | 8 +- .../backend/setup_tool/schema_defaults.py | 16 ++- specifyweb/backend/setup_tool/setup_tasks.py | 90 ++---------- .../backend/setup_tool/task_tracking.py | 133 ++++++++++++++++++ 7 files changed, 218 insertions(+), 145 deletions(-) create mode 100644 specifyweb/backend/setup_tool/task_tracking.py diff --git a/specifyweb/backend/accounts/views.py b/specifyweb/backend/accounts/views.py index 0ef586ad614..22f1e477401 100644 --- a/specifyweb/backend/accounts/views.py +++ b/specifyweb/backend/accounts/views.py @@ -294,7 +294,8 @@ def choose_collection(request) -> http.HttpResponse: id to the user if one is provided. """ from specifyweb.backend.context.views import set_collection_cookie, users_collections_for_sp7 - from specifyweb.backend.setup_tool.api import get_collections_with_busy_config + from specifyweb.backend.setup_tool.api import filter_ready_collections_for_config_tasks + from specifyweb.specify.api.serializers import obj_to_data, toJson @@ -318,8 +319,7 @@ def choose_collection(request) -> http.HttpResponse: ) available_collections = users_collections_for_sp7(request.specify_user.id) - busy_collection_ids = get_collections_with_busy_config(c.id for c in available_collections) - available_collections = [c for c in available_collections if c.id not in busy_collection_ids] + available_collections = filter_ready_collections_for_config_tasks(available_collections) if len(available_collections) == 1: set_collection_cookie(redirect_resp, available_collections[0].id) diff --git a/specifyweb/backend/context/views.py b/specifyweb/backend/context/views.py index f16e8bbb937..977b980e512 100644 --- a/specifyweb/backend/context/views.py +++ b/specifyweb/backend/context/views.py @@ -37,7 +37,11 @@ from .remote_prefs import get_remote_prefs from .schema_localization import get_schema_languages, get_schema_localization from .viewsets import get_views -from specifyweb.backend.setup_tool.api import get_collections_with_busy_config, get_config_progress +from specifyweb.backend.setup_tool.api import ( + get_config_progress, + filter_ready_collections_for_config_tasks, + filter_ready_disciplines_for_config_tasks, +) def set_collection_cookie(response, collection_id): # pragma: no cover @@ -305,9 +309,6 @@ def collection(request): set_collection_cookie(response, collection.id) return response else: - available_collections = _filter_collections_not_ready_for_config_task( - available_collections - ) response = dict( available=[obj_to_data(c) for c in available_collections], current=(current and int(current)) @@ -649,15 +650,11 @@ def get_server_time(request): def _filter_collections_not_ready_for_config_task(collections): - busy_collection_ids = get_collections_with_busy_config(c.id for c in collections) - if not busy_collection_ids: - return collections + return filter_ready_collections_for_config_tasks(collections) - return [ - c - for c in collections - if c.id not in busy_collection_ids - ] + +def _filter_disciplines_not_ready_for_config_task(disciplines): + return filter_ready_disciplines_for_config_tasks(disciplines) def _build_system_data(*, filter_not_ready_collections: bool): @@ -667,6 +664,7 @@ def _build_system_data(*, filter_not_ready_collections: bool): collections = list(Collection.objects.all()) if filter_not_ready_collections: + disciplines = _filter_disciplines_not_ready_for_config_task(disciplines) collections = _filter_collections_not_ready_for_config_task(collections) discipline_map = {} diff --git a/specifyweb/backend/setup_tool/api.py b/specifyweb/backend/setup_tool/api.py index fcd906c480d..5f7f804d77a 100644 --- a/specifyweb/backend/setup_tool/api.py +++ b/specifyweb/backend/setup_tool/api.py @@ -5,11 +5,9 @@ import json from typing import Optional -from datetime import timedelta from django.http import (JsonResponse) from django.db.models import Max from django.db import transaction -from django.utils import timezone from specifyweb.backend.permissions.models import UserPolicy @@ -18,15 +16,12 @@ from specifyweb.backend.setup_tool.utils import normalize_keys, resolve_uri_or_fallback from specifyweb.backend.setup_tool.schema_defaults import ( apply_schema_defaults, - is_discipline_setup_busy, queue_apply_schema_defaults_background, ) from specifyweb.backend.setup_tool.picklist_defaults import create_default_picklists from specifyweb.backend.setup_tool.prep_type_defaults import create_default_prep_types from specifyweb.backend.setup_tool.app_resource_defaults import ensure_discipline_resource_dir from specifyweb.backend.setup_tool.setup_tasks import ( - get_collections_with_busy_config as _get_collections_with_busy_config, - is_collection_config_busy, setup_database_background, get_active_setup_task, get_last_setup_error, @@ -35,6 +30,10 @@ ) from specifyweb.celery_tasks import MissingWorkerError, get_running_worker_task_names from specifyweb.backend.setup_tool.tree_defaults import start_default_tree_from_configuration, update_tree_scoping +from specifyweb.backend.setup_tool.task_tracking import ( + is_collection_ready_for_config_tasks, + is_discipline_ready_for_config_tasks, +) from specifyweb.specify.models import Institution, Discipline from specifyweb.backend.businessrules.uniqueness_rules import apply_default_uniqueness_rules from specifyweb.specify.management.commands.run_key_migration_functions import fix_cots, fix_schema_config @@ -44,7 +43,6 @@ APP_VERSION = "7" SCHEMA_VERSION = "2.10" -CONFIG_TASK_COLLECTION_BLOCK_WINDOW = timedelta(minutes=15) class SetupError(Exception): """Raised by any setup tasks.""" @@ -312,11 +310,7 @@ def create_discipline(data, run_apply_schema_defaults_async: bool = True): except Exception as e: raise SetupError(e) -def create_collection( - data, - run_fix_schema_config_async: bool = True, - require_discipline_ready: bool = True, -): +def create_collection(data, run_fix_schema_config_async: bool = True): from specifyweb.specify.models import Collection, Discipline # If collection_id is provided and exists, return success @@ -340,11 +334,6 @@ def create_collection( data['discipline'] = discipline else: raise SetupError("No discipline available") - - if require_discipline_ready and is_discipline_setup_busy(discipline.id): - raise SetupError( - "This discipline is still being created. Please wait for background setup tasks to finish before creating a collection." - ) # The discipline needs a Taxon Tree in order for the Collection Object Type to be created. if not discipline.taxontreedef_id: @@ -364,7 +353,7 @@ def create_collection( # Create picklists create_default_picklists(new_collection, discipline.type) if run_fix_schema_config_async: - queue_fix_schema_config_background(new_collection.id) + queue_fix_schema_config_background(collection_id=new_collection.id) else: fix_schema_config() @@ -510,18 +499,6 @@ def create_tree(name: str, data: dict) -> dict: "specifyweb.backend.setup_tool.schema_defaults.apply_schema_defaults_task", }) - -def _tracked_setup_tasks_busy() -> bool: - """True if any tracked collection/discipline setup task is active.""" - if any( - is_discipline_setup_busy(discipline_id) - for discipline_id in Discipline.objects.values_list('id', flat=True) - ): - return True - collection_ids = models.Collection.objects.values_list('id', flat=True) - return bool(_get_collections_with_busy_config(collection_ids)) - - def _task_name_to_progress_key(task_name: str) -> str: """Convert a task function name into a camelCase progress key.""" task_function_name = task_name.rsplit(".", 1)[-1] @@ -552,9 +529,36 @@ def get_config_resource_progress(running_task_names: Optional[list[str]] = None) return _get_config_resource_progress_from_active_names(active_task_names) -def get_collections_with_busy_config(collection_ids) -> set[int]: - """Returns collection ids that still have active tracked config tasks.""" - return _get_collections_with_busy_config(collection_ids) +def is_collection_busy_for_config_tasks( + collection_id: int, + discipline_id: Optional[int] = None, +) -> bool: + if discipline_id is None: + collection = models.Collection.objects.filter(id=collection_id).only("discipline_id").first() + if collection is None: + return False + discipline_id = collection.discipline_id + return not is_collection_ready_for_config_tasks(collection_id, discipline_id) + + +def is_discipline_busy_for_config_tasks(discipline_id: int) -> bool: + return not is_discipline_ready_for_config_tasks(discipline_id) + + +def filter_ready_collections_for_config_tasks(collections: list) -> list: + return [ + collection + for collection in collections + if not is_collection_busy_for_config_tasks(collection.id, collection.discipline_id) + ] + + +def filter_ready_disciplines_for_config_tasks(disciplines: list) -> list: + return [ + discipline + for discipline in disciplines + if not is_discipline_busy_for_config_tasks(discipline.id) + ] def get_config_progress(collection_id: Optional[int] = None) -> dict: @@ -564,9 +568,11 @@ def get_config_progress(collection_id: Optional[int] = None) -> dict: except MissingWorkerError: running_task_names = [] - busy = is_config_task_running(running_task_names) - if busy and collection_id is not None: - busy = _is_new_collection_in_time_window(collection_id) + busy = ( + is_config_task_running(running_task_names) + if collection_id is None + else is_collection_busy_for_config_tasks(collection_id) + ) last_error = None completed_resources = get_config_resource_progress(running_task_names) @@ -575,15 +581,3 @@ def get_config_progress(collection_id: Optional[int] = None) -> dict: "last_error": last_error, "busy": busy, } - -def _is_new_collection_in_time_window(collection_id: int) -> bool: - """Return True when a newly created collection is still in the new collection time window.""" - from specifyweb.specify.models import Collection - - collection = Collection.objects.filter(id=collection_id).only("timestampcreated").first() - if collection is None: - return False - if collection.timestampcreated is None: - return False - - return collection.timestampcreated > timezone.now() - CONFIG_TASK_COLLECTION_BLOCK_WINDOW diff --git a/specifyweb/backend/setup_tool/redis.py b/specifyweb/backend/setup_tool/redis.py index eb852080ead..ff2f77efa9f 100644 --- a/specifyweb/backend/setup_tool/redis.py +++ b/specifyweb/backend/setup_tool/redis.py @@ -2,9 +2,9 @@ # Also defined separately in setup_tool/apps.py ACTIVE_TASK_REDIS_KEY = "specify:{database}:setup:active_task_id" ACTIVE_TASK_TTL = 60*60*2 # setup should be less than 2 hours -# Keep track of config/setup celery task ids by collection. -COLLECTION_TASK_IDS_REDIS_KEY = "specify:{database}:setup:collection:{collection_id}:task_ids" -# Keep track of discipline setup celery task ids by discipline. -DISCIPLINE_TASK_IDS_REDIS_KEY = "specify:{database}:setup:discipline:{discipline_id}:task_ids" # Keep track of last error. LAST_ERROR_REDIS_KEY = "specify:{database}:setup:last_error" + +# Track async setup/config tasks by resource scope. +COLLECTION_TASKS_REDIS_KEY = "specify:{database}:setup:collection:{collection_id}:tasks" +DISCIPLINE_TASKS_REDIS_KEY = "specify:{database}:setup:discipline:{discipline_id}:tasks" diff --git a/specifyweb/backend/setup_tool/schema_defaults.py b/specifyweb/backend/setup_tool/schema_defaults.py index 1426ccec0ed..791ac736ac5 100644 --- a/specifyweb/backend/setup_tool/schema_defaults.py +++ b/specifyweb/backend/setup_tool/schema_defaults.py @@ -2,6 +2,7 @@ from specifyweb.specify.migration_utils.update_schema_config import update_table_schema_config_with_defaults from specifyweb.celery_tasks import app from .utils import load_json_from_file +from .task_tracking import queue_discipline_background_task, finish_discipline_background_task from specifyweb.specify.models import Discipline from django.db import transaction from celery.exceptions import MaxRetriesExceededError @@ -62,16 +63,21 @@ def apply_schema_defaults(discipline: Discipline): description=table_description, defaults=table_defaults, ) + def queue_apply_schema_defaults_background(discipline_id: int) -> str: """Queue apply_schema_defaults to run asynchronously and return the task id.""" task_id = str(uuid4()) # Dispatch only after the discipline row is committed so workers can read it. - transaction.on_commit( - lambda: apply_schema_defaults_task.apply_async( + def enqueue() -> None: + async_result = apply_schema_defaults_task.apply_async( args=[discipline_id], task_id=task_id, ) + queue_discipline_background_task(discipline_id, async_result.id) + + transaction.on_commit( + enqueue ) return task_id @@ -92,5 +98,9 @@ def apply_schema_defaults_task(self, discipline_id: int): discipline_id, SCHEMA_DEFAULTS_MISSING_DISCIPLINE_MAX_RETRIES, ) + finish_discipline_background_task(discipline_id, self.request.id) return - apply_schema_defaults(discipline) + try: + apply_schema_defaults(discipline) + finally: + finish_discipline_background_task(discipline_id, self.request.id) diff --git a/specifyweb/backend/setup_tool/setup_tasks.py b/specifyweb/backend/setup_tool/setup_tasks.py index 426d22df76f..3618acbdc55 100644 --- a/specifyweb/backend/setup_tool/setup_tasks.py +++ b/specifyweb/backend/setup_tool/setup_tasks.py @@ -4,7 +4,7 @@ from django.db import transaction from specifyweb.celery_tasks import app -from typing import Iterable, Optional, Tuple +from typing import Tuple, Optional from celery.result import AsyncResult from specifyweb.backend.setup_tool import api from specifyweb.backend.setup_tool.app_resource_defaults import create_app_resource_defaults @@ -12,75 +12,17 @@ from specifyweb.specify.management.commands.run_key_migration_functions import fix_schema_config from specifyweb.specify.models_utils.model_extras import PALEO_DISCIPLINES, GEOLOGY_DISCIPLINES from specifyweb.celery_tasks import is_worker_alive, MissingWorkerError -from specifyweb.backend.redis_cache.store import ( - add_to_set, - delete_key, - get_string, - remove_from_set, - set_members, - set_string, -) -from specifyweb.backend.setup_tool.redis import ( - ACTIVE_TASK_REDIS_KEY, - ACTIVE_TASK_TTL, - COLLECTION_TASK_IDS_REDIS_KEY, - LAST_ERROR_REDIS_KEY, +from specifyweb.backend.redis_cache.store import set_string, get_string +from specifyweb.backend.setup_tool.redis import ACTIVE_TASK_REDIS_KEY, ACTIVE_TASK_TTL, LAST_ERROR_REDIS_KEY +from specifyweb.backend.setup_tool.task_tracking import ( + queue_collection_background_task, + finish_collection_background_task, ) from uuid import uuid4 import logging logger = logging.getLogger(__name__) -ACTIVE_CELERY_STATES = frozenset(("PENDING", "RECEIVED", "STARTED", "RETRY", "PROGRESS")) - -def _collection_task_ids_key(collection_id: int) -> str: - return COLLECTION_TASK_IDS_REDIS_KEY.replace( - "{collection_id}", f"({{database}},{collection_id})" - ) - -def _track_collection_task(collection_id: Optional[int], task_id: str) -> None: - if collection_id is None: - return - add_to_set(_collection_task_ids_key(collection_id), task_id) - -def _untrack_collection_task(collection_id: Optional[int], task_id: Optional[str]) -> None: - if collection_id is None or task_id is None: - return - key = _collection_task_ids_key(collection_id) - remove_from_set(key, task_id) - if not set_members(key): - delete_key(key) - -def _is_task_active(task_id: str) -> bool: - return app.AsyncResult(task_id).state in ACTIVE_CELERY_STATES - -def is_collection_config_busy(collection_id: Optional[int]) -> bool: - if collection_id is None: - return False - - key = _collection_task_ids_key(collection_id) - task_ids = set_members(key) - if not task_ids: - return False - - inactive_task_ids = [] - for task_id in task_ids: - if _is_task_active(task_id): - return True - inactive_task_ids.append(task_id) - - remove_from_set(key, *inactive_task_ids) - if not set_members(key): - delete_key(key) - return False - -def get_collections_with_busy_config(collection_ids: Iterable[int]) -> set[int]: - return { - collection_id - for collection_id in collection_ids - if is_collection_config_busy(collection_id) - } - def setup_database_background(data: dict) -> str: # Clear any previous error logs. set_last_setup_error(None) @@ -102,13 +44,10 @@ def setup_database_background(data: dict) -> str: def queue_fix_schema_config_background(collection_id: Optional[int] = None) -> str: """Queue fix_schema_config to run asynchronously and return the task id""" - task_id = str(uuid4()) - _track_collection_task(collection_id, task_id) - try: - task = fix_schema_config_task.apply_async(args=[collection_id], task_id=task_id) - except Exception: - _untrack_collection_task(collection_id, task_id) - raise + args = [collection_id] if collection_id is not None else [] + task = fix_schema_config_task.apply_async(args=args) + if collection_id is not None: + queue_collection_background_task(collection_id, task.id) return task.id def get_active_setup_task() -> Tuple[Optional[AsyncResult], bool]: @@ -205,8 +144,7 @@ def update_progress(): logger.info('Creating collection') collection_result = api.create_collection( data['collection'], - run_fix_schema_config_async=False, - require_discipline_ready=False, + run_fix_schema_config_async=False ) collection_id = collection_result.get('collection_id') update_progress() @@ -242,11 +180,11 @@ def update_progress(): @app.task(bind=True) def fix_schema_config_task(self, collection_id: Optional[int] = None): """Run schema config migration fixups in a background worker""" - task_id = getattr(self.request, "id", None) try: fix_schema_config() finally: - _untrack_collection_task(collection_id, task_id) + if collection_id is not None: + finish_collection_background_task(collection_id, self.request.id) def get_last_setup_error() -> Optional[str]: err = get_string(LAST_ERROR_REDIS_KEY) @@ -331,4 +269,4 @@ def create_discipline_and_trees_task(data: dict): )) except Exception as e: logger.exception(f'Error creating discipline: {e}') - raise \ No newline at end of file + raise diff --git a/specifyweb/backend/setup_tool/task_tracking.py b/specifyweb/backend/setup_tool/task_tracking.py new file mode 100644 index 00000000000..8cbcf32cf44 --- /dev/null +++ b/specifyweb/backend/setup_tool/task_tracking.py @@ -0,0 +1,133 @@ +from typing import Optional +import logging + +from specifyweb.backend.redis_cache.store import add_to_set, delete_key, remove_from_set, set_members +from specifyweb.celery_tasks import CELERY_TASK_STATE, app +from specifyweb.backend.setup_tool.redis import ( + COLLECTION_TASKS_REDIS_KEY, + DISCIPLINE_TASKS_REDIS_KEY, +) + +logger = logging.getLogger(__name__) + +ACTIVE_TASK_STATES = frozenset( + { + CELERY_TASK_STATE.PENDING, + CELERY_TASK_STATE.RECEIVED, + CELERY_TASK_STATE.STARTED, + CELERY_TASK_STATE.RETRY, + "PROGRESS", + "RUNNING", + } +) + +TERMINAL_TASK_STATES = frozenset( + { + CELERY_TASK_STATE.SUCCESS, + CELERY_TASK_STATE.FAILURE, + CELERY_TASK_STATE.REVOKED, + } +) + +def _collection_tasks_key(collection_id: int) -> str: + return COLLECTION_TASKS_REDIS_KEY.replace("{collection_id}", str(collection_id)) + +def _discipline_tasks_key(discipline_id: int) -> str: + return DISCIPLINE_TASKS_REDIS_KEY.replace("{discipline_id}", str(discipline_id)) + +def queue_collection_background_task(collection_id: int, task_id: str) -> None: + try: + add_to_set(_collection_tasks_key(collection_id), task_id) + except Exception: + logger.warning( + "Failed to track collection task %s for collection %s.", + task_id, + collection_id, + ) + +def finish_collection_background_task(collection_id: int, task_id: str) -> None: + try: + key = _collection_tasks_key(collection_id) + remove_from_set(key, task_id) + if len(set_members(key)) == 0: + delete_key(key) + except Exception: + logger.warning( + "Failed to clear tracked collection task %s for collection %s.", + task_id, + collection_id, + ) + +def queue_discipline_background_task(discipline_id: int, task_id: str) -> None: + try: + add_to_set(_discipline_tasks_key(discipline_id), task_id) + except Exception: + logger.warning( + "Failed to track discipline task %s for discipline %s.", + task_id, + discipline_id, + ) + +def finish_discipline_background_task(discipline_id: int, task_id: str) -> None: + try: + key = _discipline_tasks_key(discipline_id) + remove_from_set(key, task_id) + if len(set_members(key)) == 0: + delete_key(key) + except Exception: + logger.warning( + "Failed to clear tracked discipline task %s for discipline %s.", + task_id, + discipline_id, + ) + +def _active_task_ids_from_redis_key(key: str) -> set[str]: + try: + task_ids = set_members(key) + if not task_ids: + return set() + + active_task_ids: set[str] = set() + finished_task_ids: list[str] = [] + for task_id in task_ids: + task_state = app.AsyncResult(task_id).state + if task_state in ACTIVE_TASK_STATES: + active_task_ids.add(task_id) + continue + if task_state in TERMINAL_TASK_STATES: + finished_task_ids.append(task_id) + continue + # Unknown states should block readiness until they transition. + active_task_ids.add(task_id) + + if finished_task_ids: + remove_from_set(key, *finished_task_ids) + if len(set_members(key)) == 0: + delete_key(key) + + return active_task_ids + except Exception: + logger.warning("Failed to read task tracking key %s.", key) + return set() + +def get_active_collection_background_tasks(collection_id: int) -> set[str]: + return _active_task_ids_from_redis_key(_collection_tasks_key(collection_id)) + +def get_active_discipline_background_tasks(discipline_id: int) -> set[str]: + return _active_task_ids_from_redis_key(_discipline_tasks_key(discipline_id)) + +def has_collection_background_tasks(collection_id: int) -> bool: + return len(get_active_collection_background_tasks(collection_id)) > 0 + +def has_discipline_background_tasks(discipline_id: int) -> bool: + return len(get_active_discipline_background_tasks(discipline_id)) > 0 + +def is_collection_ready_for_config_tasks(collection_id: int, discipline_id: Optional[int] = None) -> bool: + if has_collection_background_tasks(collection_id): + return False + if discipline_id is not None and has_discipline_background_tasks(discipline_id): + return False + return True + +def is_discipline_ready_for_config_tasks(discipline_id: int) -> bool: + return not has_discipline_background_tasks(discipline_id) From c85bed786bf266a0452c0a36e0f7c2c34adc6acd Mon Sep 17 00:00:00 2001 From: alec_dev Date: Fri, 6 Mar 2026 10:39:51 -0600 Subject: [PATCH 5/5] ensure clean redis after tasks --- specifyweb/backend/context/views.py | 21 +++++++------------ specifyweb/backend/setup_tool/api.py | 5 ----- specifyweb/backend/setup_tool/redis.py | 1 + .../backend/setup_tool/task_tracking.py | 19 +++++++---------- 4 files changed, 16 insertions(+), 30 deletions(-) diff --git a/specifyweb/backend/context/views.py b/specifyweb/backend/context/views.py index 977b980e512..87861acc126 100644 --- a/specifyweb/backend/context/views.py +++ b/specifyweb/backend/context/views.py @@ -42,8 +42,7 @@ filter_ready_collections_for_config_tasks, filter_ready_disciplines_for_config_tasks, ) - - + def set_collection_cookie(response, collection_id): # pragma: no cover response.set_cookie('collection', str(collection_id), max_age=365*24*60*60) @@ -619,8 +618,7 @@ def viewsets(request): file not in FORM_RESOURCE_EXCLUDED_LST, all_files)) return HttpResponse(json.dumps(viewsets), content_type="application/json") - - + def view_helper(request, limit): if 'collectionid' in request.GET: # Allow a URL parameter to override the logged in collection. @@ -647,16 +645,13 @@ def remote_prefs(request): @require_http_methods(['GET', 'HEAD']) def get_server_time(request): return JsonResponse({"server_time": timezone.now().isoformat()}) - - + def _filter_collections_not_ready_for_config_task(collections): return filter_ready_collections_for_config_tasks(collections) - - + def _filter_disciplines_not_ready_for_config_task(disciplines): return filter_ready_disciplines_for_config_tasks(disciplines) - - + def _build_system_data(*, filter_not_ready_collections: bool): institution = Institution.objects.get() divisions = list(Division.objects.all()) @@ -805,8 +800,7 @@ def get_endpoint_tags(endpoint): list = [endpoint[method]['tags'] for method in methods] return [item for sublist in list for item in sublist] # flatten the list - - + def get_tags(endpoints): tag_names = [get_endpoint_tags(endpoint) for endpoint in endpoints.values()] @@ -933,8 +927,7 @@ def get_endpoints( prefix + path, preparams + params ) - - + def generate_openapi_for_endpoints(all_endpoints=False): # pragma: no cover """Returns a JSON description of endpoints. diff --git a/specifyweb/backend/setup_tool/api.py b/specifyweb/backend/setup_tool/api.py index 5f7f804d77a..e06ba640f65 100644 --- a/specifyweb/backend/setup_tool/api.py +++ b/specifyweb/backend/setup_tool/api.py @@ -528,7 +528,6 @@ def get_config_resource_progress(running_task_names: Optional[list[str]] = None) active_task_names = set(running_task_names or []) return _get_config_resource_progress_from_active_names(active_task_names) - def is_collection_busy_for_config_tasks( collection_id: int, discipline_id: Optional[int] = None, @@ -540,11 +539,9 @@ def is_collection_busy_for_config_tasks( discipline_id = collection.discipline_id return not is_collection_ready_for_config_tasks(collection_id, discipline_id) - def is_discipline_busy_for_config_tasks(discipline_id: int) -> bool: return not is_discipline_ready_for_config_tasks(discipline_id) - def filter_ready_collections_for_config_tasks(collections: list) -> list: return [ collection @@ -552,7 +549,6 @@ def filter_ready_collections_for_config_tasks(collections: list) -> list: if not is_collection_busy_for_config_tasks(collection.id, collection.discipline_id) ] - def filter_ready_disciplines_for_config_tasks(disciplines: list) -> list: return [ discipline @@ -560,7 +556,6 @@ def filter_ready_disciplines_for_config_tasks(disciplines: list) -> list: if not is_discipline_busy_for_config_tasks(discipline.id) ] - def get_config_progress(collection_id: Optional[int] = None) -> dict: """Returns a dict of the status of config/setup related background tasks""" try: diff --git a/specifyweb/backend/setup_tool/redis.py b/specifyweb/backend/setup_tool/redis.py index ff2f77efa9f..259effa029e 100644 --- a/specifyweb/backend/setup_tool/redis.py +++ b/specifyweb/backend/setup_tool/redis.py @@ -2,6 +2,7 @@ # Also defined separately in setup_tool/apps.py ACTIVE_TASK_REDIS_KEY = "specify:{database}:setup:active_task_id" ACTIVE_TASK_TTL = 60*60*2 # setup should be less than 2 hours + # Keep track of last error. LAST_ERROR_REDIS_KEY = "specify:{database}:setup:last_error" diff --git a/specifyweb/backend/setup_tool/task_tracking.py b/specifyweb/backend/setup_tool/task_tracking.py index 8cbcf32cf44..a7190612b87 100644 --- a/specifyweb/backend/setup_tool/task_tracking.py +++ b/specifyweb/backend/setup_tool/task_tracking.py @@ -35,6 +35,11 @@ def _collection_tasks_key(collection_id: int) -> str: def _discipline_tasks_key(discipline_id: int) -> str: return DISCIPLINE_TASKS_REDIS_KEY.replace("{discipline_id}", str(discipline_id)) +def _remove_task_ids_and_delete_empty_key(key: str, *task_ids: str) -> None: + remove_from_set(key, *task_ids) + if len(set_members(key)) == 0: + delete_key(key) + def queue_collection_background_task(collection_id: int, task_id: str) -> None: try: add_to_set(_collection_tasks_key(collection_id), task_id) @@ -47,10 +52,7 @@ def queue_collection_background_task(collection_id: int, task_id: str) -> None: def finish_collection_background_task(collection_id: int, task_id: str) -> None: try: - key = _collection_tasks_key(collection_id) - remove_from_set(key, task_id) - if len(set_members(key)) == 0: - delete_key(key) + _remove_task_ids_and_delete_empty_key(_collection_tasks_key(collection_id), task_id) except Exception: logger.warning( "Failed to clear tracked collection task %s for collection %s.", @@ -70,10 +72,7 @@ def queue_discipline_background_task(discipline_id: int, task_id: str) -> None: def finish_discipline_background_task(discipline_id: int, task_id: str) -> None: try: - key = _discipline_tasks_key(discipline_id) - remove_from_set(key, task_id) - if len(set_members(key)) == 0: - delete_key(key) + _remove_task_ids_and_delete_empty_key(_discipline_tasks_key(discipline_id), task_id) except Exception: logger.warning( "Failed to clear tracked discipline task %s for discipline %s.", @@ -101,9 +100,7 @@ def _active_task_ids_from_redis_key(key: str) -> set[str]: active_task_ids.add(task_id) if finished_task_ids: - remove_from_set(key, *finished_task_ids) - if len(set_members(key)) == 0: - delete_key(key) + _remove_task_ids_and_delete_empty_key(key, *finished_task_ids) return active_task_ids except Exception: