From d329517d9af202c3353afcc5e7b5912d0911b3f5 Mon Sep 17 00:00:00 2001 From: hoangtrann Date: Sat, 22 Nov 2025 06:05:08 +0700 Subject: [PATCH 1/3] [IMP] queue_job: requeue orphaned jobs --- queue_job/jobrunner/runner.py | 37 ++++++++++++++++++++++++ queue_job/tests/test_requeue_dead_job.py | 31 ++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/queue_job/jobrunner/runner.py b/queue_job/jobrunner/runner.py index a0db6751db..ccdbeaec11 100644 --- a/queue_job/jobrunner/runner.py +++ b/queue_job/jobrunner/runner.py @@ -386,6 +386,35 @@ def _query_requeue_dead_jobs(self): RETURNING uuid """ + def _query_requeue_orphaned_jobs(self): + """Query to requeue jobs stuck in 'enqueued' state without a lock. + + This handles the edge case where the runner marks a job as 'enqueued' + but the HTTP request to start the job never reaches the Odoo server + (e.g., due to server shutdown/crash between setting enqueued and + the controller receiving the request). These jobs have no lock record + because set_started() was never called, so they are invisible to + _query_requeue_dead_jobs(). + """ + return """ + UPDATE + queue_job + SET + state='pending' + WHERE + state = 'enqueued' + AND date_enqueued < (now() AT TIME ZONE 'utc' - INTERVAL '10 sec') + AND NOT EXISTS ( + SELECT + 1 + FROM + queue_job_lock + WHERE + queue_job_id = queue_job.id + ) + RETURNING uuid + """ + def requeue_dead_jobs(self): """ Set started and enqueued jobs but not locked to pending @@ -414,6 +443,14 @@ def requeue_dead_jobs(self): for (uuid,) in cr.fetchall(): _logger.warning("Re-queued dead job with uuid: %s", uuid) + # Requeue orphaned jobs (enqueued but never started, no lock) + query = self._query_requeue_orphaned_jobs() + cr.execute(query) + for (uuid,) in cr.fetchall(): + _logger.warning( + "Re-queued orphaned job (enqueued without lock) with uuid: %s", uuid + ) + class QueueJobRunner: def __init__( diff --git a/queue_job/tests/test_requeue_dead_job.py b/queue_job/tests/test_requeue_dead_job.py index c6c82a2f4d..5c6ca47e52 100644 --- a/queue_job/tests/test_requeue_dead_job.py +++ b/queue_job/tests/test_requeue_dead_job.py @@ -131,3 +131,34 @@ def test_requeue_dead_jobs(self): # because we committed the cursor, the savepoint of the test method is # gone, and this would break TransactionCase cleanups self.cr.execute("SAVEPOINT test_%d" % self._savepoint_id) + + def test_requeue_orphaned_jobs(self): + uuid = "test_enqueued_job" + queue_job = self.create_dummy_job(uuid) + job_obj = Job.load(self.env, queue_job.uuid) + + # Only enqueued job, don't set it to started to simulate the scenario + # that system shutdown before job is starting + job_obj.set_enqueued() + job_obj.date_enqueued = datetime.now() - timedelta(minutes=1) + job_obj.store() + + # job ins't actually picked up by the first requeue attempt + query = Database(self.env.cr.dbname)._query_requeue_dead_jobs() + self.env.cr.execute(query) + uuids_requeued = self.env.cr.fetchall() + self.assertFalse(uuids_requeued) + + # job is picked up by the 2nd requeue attempt + query = Database(self.env.cr.dbname)._query_requeue_orphaned_jobs() + self.env.cr.execute(query) + uuids_requeued = self.env.cr.fetchall() + self.assertTrue(queue_job.uuid in j[0] for j in uuids_requeued) + + # clean up + queue_job.unlink() + self.env.cr.commit() # pylint: disable=E8102 + + # because we committed the cursor, the savepoint of the test method is + # gone, and this would break TransactionCase cleanups + self.cr.execute("SAVEPOINT test_%d" % self._savepoint_id) From c7324dbf91311d84f4636b91e0917211c152531c Mon Sep 17 00:00:00 2001 From: hoangtrann Date: Wed, 31 Dec 2025 19:27:16 +0700 Subject: [PATCH 2/3] [IMP] queue_job: query orphaned dead job not exist in lock table --- queue_job/jobrunner/runner.py | 78 ++++++++---------------- queue_job/tests/test_requeue_dead_job.py | 8 +-- 2 files changed, 26 insertions(+), 60 deletions(-) diff --git a/queue_job/jobrunner/runner.py b/queue_job/jobrunner/runner.py index ccdbeaec11..4cf063e818 100644 --- a/queue_job/jobrunner/runner.py +++ b/queue_job/jobrunner/runner.py @@ -365,52 +365,26 @@ def _query_requeue_dead_jobs(self): ELSE exc_info END) WHERE - id in ( - SELECT - queue_job_id - FROM - queue_job_lock - WHERE - queue_job_id in ( - SELECT - id - FROM - queue_job - WHERE - state IN ('enqueued','started') - AND date_enqueued < - (now() AT TIME ZONE 'utc' - INTERVAL '10 sec') - ) - FOR UPDATE SKIP LOCKED - ) - RETURNING uuid - """ - - def _query_requeue_orphaned_jobs(self): - """Query to requeue jobs stuck in 'enqueued' state without a lock. - - This handles the edge case where the runner marks a job as 'enqueued' - but the HTTP request to start the job never reaches the Odoo server - (e.g., due to server shutdown/crash between setting enqueued and - the controller receiving the request). These jobs have no lock record - because set_started() was never called, so they are invisible to - _query_requeue_dead_jobs(). - """ - return """ - UPDATE - queue_job - SET - state='pending' - WHERE - state = 'enqueued' + state IN ('enqueued','started') AND date_enqueued < (now() AT TIME ZONE 'utc' - INTERVAL '10 sec') - AND NOT EXISTS ( - SELECT - 1 - FROM - queue_job_lock - WHERE - queue_job_id = queue_job.id + AND ( + id in ( + SELECT + queue_job_id + FROM + queue_job_lock + WHERE + queue_job_lock.queue_job_id = queue_job.id + FOR UPDATE SKIP LOCKED + ) + OR NOT EXISTS ( + SELECT + 1 + FROM + queue_job_lock + WHERE + queue_job_lock.queue_job_id = queue_job.id + ) ) RETURNING uuid """ @@ -433,6 +407,12 @@ def requeue_dead_jobs(self): However, when the Odoo server crashes or is otherwise force-stopped, running jobs are interrupted while the runner has no chance to know they have been aborted. + + This also handles orphaned jobs (enqueued but never started, no lock). + This edge case occurs when the runner marks a job as 'enqueued' + but the HTTP request to start the job never reaches the Odoo server + (e.g., due to server shutdown/crash between setting enqueued and + the controller receiving the request). """ with closing(self.conn.cursor()) as cr: @@ -443,14 +423,6 @@ def requeue_dead_jobs(self): for (uuid,) in cr.fetchall(): _logger.warning("Re-queued dead job with uuid: %s", uuid) - # Requeue orphaned jobs (enqueued but never started, no lock) - query = self._query_requeue_orphaned_jobs() - cr.execute(query) - for (uuid,) in cr.fetchall(): - _logger.warning( - "Re-queued orphaned job (enqueued without lock) with uuid: %s", uuid - ) - class QueueJobRunner: def __init__( diff --git a/queue_job/tests/test_requeue_dead_job.py b/queue_job/tests/test_requeue_dead_job.py index 5c6ca47e52..180e1294eb 100644 --- a/queue_job/tests/test_requeue_dead_job.py +++ b/queue_job/tests/test_requeue_dead_job.py @@ -143,16 +143,10 @@ def test_requeue_orphaned_jobs(self): job_obj.date_enqueued = datetime.now() - timedelta(minutes=1) job_obj.store() - # job ins't actually picked up by the first requeue attempt + # job is now picked up by the requeue query (which includes orphaned jobs) query = Database(self.env.cr.dbname)._query_requeue_dead_jobs() self.env.cr.execute(query) uuids_requeued = self.env.cr.fetchall() - self.assertFalse(uuids_requeued) - - # job is picked up by the 2nd requeue attempt - query = Database(self.env.cr.dbname)._query_requeue_orphaned_jobs() - self.env.cr.execute(query) - uuids_requeued = self.env.cr.fetchall() self.assertTrue(queue_job.uuid in j[0] for j in uuids_requeued) # clean up From 7f5e5bec2fb9eb0d231a144290a37992ee07a426 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Wed, 7 Jan 2026 11:31:38 +0000 Subject: [PATCH 3/3] [BOT] post-merge updates --- README.md | 2 +- queue_job/README.rst | 2 +- queue_job/__manifest__.py | 2 +- queue_job/static/description/index.html | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4fa5be4e2b..53b61dc039 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Available addons addon | version | maintainers | summary --- | --- | --- | --- [base_import_async](base_import_async/) | 17.0.1.0.0 | | Import CSV files in the background -[queue_job](queue_job/) | 17.0.1.5.0 | guewen | Job Queue +[queue_job](queue_job/) | 17.0.1.5.1 | guewen | Job Queue [queue_job_cron](queue_job_cron/) | 17.0.1.1.0 | | Scheduled Actions as Queue Jobs [queue_job_cron_jobrunner](queue_job_cron_jobrunner/) | 17.0.1.1.0 | ivantodorovich | Run jobs without a dedicated JobRunner [queue_job_subscribe](queue_job_subscribe/) | 17.0.1.0.0 | | Control which users are subscribed to queue job notifications diff --git a/queue_job/README.rst b/queue_job/README.rst index c6950adcc9..9b659a7457 100644 --- a/queue_job/README.rst +++ b/queue_job/README.rst @@ -11,7 +11,7 @@ Job Queue !! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - !! source digest: sha256:10e03ffe452b93247cdca483f5d4597ae8d6f572bc00de63bcc7f7238d2ce33d + !! source digest: sha256:20857af17bb6802106b5203b0d4d7daca00ab1510dd6beb2131aa17e0657df05 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! .. |badge1| image:: https://img.shields.io/badge/maturity-Mature-brightgreen.png diff --git a/queue_job/__manifest__.py b/queue_job/__manifest__.py index b552e7d52a..1cd367c571 100644 --- a/queue_job/__manifest__.py +++ b/queue_job/__manifest__.py @@ -2,7 +2,7 @@ { "name": "Job Queue", - "version": "17.0.1.5.0", + "version": "17.0.1.5.1", "author": "Camptocamp,ACSONE SA/NV,Odoo Community Association (OCA)", "website": "https://github.com/OCA/queue", "license": "LGPL-3", diff --git a/queue_job/static/description/index.html b/queue_job/static/description/index.html index 627abec65a..e80c9ffb9d 100644 --- a/queue_job/static/description/index.html +++ b/queue_job/static/description/index.html @@ -372,7 +372,7 @@

Job Queue

!! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -!! source digest: sha256:10e03ffe452b93247cdca483f5d4597ae8d6f572bc00de63bcc7f7238d2ce33d +!! source digest: sha256:20857af17bb6802106b5203b0d4d7daca00ab1510dd6beb2131aa17e0657df05 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -->

Mature License: LGPL-3 OCA/queue Translate me on Weblate Try me on Runboat

This addon adds an integrated Job Queue to Odoo.