chore: skip steps

app/Jobs/ProcessPolydockAppInstanceJobs/BaseJob.php

@@ -3,6 +3,7 @@
 namespace App\Jobs\ProcessPolydockAppInstanceJobs;
 
 use App\Models\PolydockAppInstance;
+use FreedomtechHosting\PolydockApp\Enums\PolydockAppInstanceStatus;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
@@ -18,6 +19,8 @@ abstract class BaseJob implements ShouldQueue
     use Queueable;
     use SerializesModels;
 
+    protected const OVERLAP_LOCK_SECONDS = 120;
+
     protected PolydockAppInstance $appInstance;
 
     /**
@@ -81,12 +84,142 @@ public function middleware()
 
         return [
             (new WithoutOverlapping($uniqueId))
-                ->expireAfter(5) // 5 seconds
+                ->expireAfter(self::OVERLAP_LOCK_SECONDS)
                 ->shared() // Use shared lock across different queues
                 ->dontRelease(),
         ];
     }
 
+    protected function shouldSkipBecauseStatusAdvanced(PolydockAppInstanceStatus $expectedStatus): bool
+    {
+        if (! isset($this->appInstance)) {
+            return false;
+        }
+
+        $currentStatus = $this->appInstance->status;
+
+        if ($currentStatus === $expectedStatus) {
+            return false;
+        }
+
+        if (! $this->isKnownStatusProgression($expectedStatus, $currentStatus)) {
+            return false;
+        }
+
+        Log::info('Skipping stale lifecycle job because app instance already advanced', [
+            'job_type' => class_basename(static::class),
+            'app_instance_id' => $this->appInstance->id,
+            'expected_status' => $expectedStatus->value,
+            'current_status' => $currentStatus->value,
+        ]);
+
+        return true;
+    }
+
+    /**
+     * Lifecycle stage ordinals, used to detect when a queued job is stale
+     * because the instance has already advanced past the *stage* the job was
+     * scheduled for.
+     *
+     * Each logical stage of the lifecycle gets a single ordinal. All statuses
+     * within the same stage (e.g. `PENDING_CREATE`, `CREATE_RUNNING`,
+     * `CREATE_COMPLETED`) share that ordinal, so a stale job is only
+     * considered "advanced past" when the instance has moved into a strictly
+     * later stage. In-stage progression is left alone — `WithoutOverlapping`
+     * handles dedup of jobs targeting the same stage.
+     *
+     * The four `RUNNING_*` statuses share a single ordinal because they are
+     * alternative running states rather than sequential ones.
+     *
+     * Upgrade stages sit after the running stage because an in-place upgrade
+     * is initiated against an already-claimed, running instance and returns
+     * to a running/claimed state when complete.
+     *
+     * Keep this in sync with the status flow handled by
+     * {@see \App\PolydockEngine\Engine}, the dispatch table in
+     * {@see \App\Listeners\ProcessPolydockAppInstanceStatusChange}, and the
+     * stage groupings on {@see \App\Models\PolydockAppInstance}.
+     */
+    private static function lifecycleStageOrdinal(PolydockAppInstanceStatus $status): ?int
+    {
+        return match ($status) {
+            PolydockAppInstanceStatus::NEW => 0,
+
+            PolydockAppInstanceStatus::PENDING_PRE_CREATE,
+            PolydockAppInstanceStatus::PRE_CREATE_RUNNING,
+            PolydockAppInstanceStatus::PRE_CREATE_COMPLETED => 10,
+
+            PolydockAppInstanceStatus::PENDING_CREATE,
+            PolydockAppInstanceStatus::CREATE_RUNNING,
+            PolydockAppInstanceStatus::CREATE_COMPLETED => 20,
+
+            PolydockAppInstanceStatus::PENDING_POST_CREATE,
+            PolydockAppInstanceStatus::POST_CREATE_RUNNING,
+            PolydockAppInstanceStatus::POST_CREATE_COMPLETED => 30,
+
+            PolydockAppInstanceStatus::PENDING_PRE_DEPLOY,
+            PolydockAppInstanceStatus::PRE_DEPLOY_RUNNING,
+            PolydockAppInstanceStatus::PRE_DEPLOY_COMPLETED => 40,
+
+            PolydockAppInstanceStatus::PENDING_DEPLOY,
+            PolydockAppInstanceStatus::DEPLOY_RUNNING,
+            PolydockAppInstanceStatus::DEPLOY_COMPLETED => 50,
+
+            PolydockAppInstanceStatus::PENDING_POST_DEPLOY,
+            PolydockAppInstanceStatus::POST_DEPLOY_RUNNING,
+            PolydockAppInstanceStatus::POST_DEPLOY_COMPLETED => 60,
+
+            PolydockAppInstanceStatus::PENDING_POLYDOCK_CLAIM,
+            PolydockAppInstanceStatus::POLYDOCK_CLAIM_RUNNING,
+            PolydockAppInstanceStatus::POLYDOCK_CLAIM_COMPLETED => 70,
+
+            PolydockAppInstanceStatus::RUNNING_HEALTHY_UNCLAIMED,
+            PolydockAppInstanceStatus::RUNNING_HEALTHY_CLAIMED,
+            PolydockAppInstanceStatus::RUNNING_UNHEALTHY,
+            PolydockAppInstanceStatus::RUNNING_UNRESPONSIVE => 80,
+
+            PolydockAppInstanceStatus::PENDING_PRE_UPGRADE,
+            PolydockAppInstanceStatus::PRE_UPGRADE_RUNNING,
+            PolydockAppInstanceStatus::PRE_UPGRADE_COMPLETED => 90,
+
+            PolydockAppInstanceStatus::PENDING_UPGRADE,
+            PolydockAppInstanceStatus::UPGRADE_RUNNING,
+            PolydockAppInstanceStatus::UPGRADE_COMPLETED => 100,
+
+            PolydockAppInstanceStatus::PENDING_POST_UPGRADE,
+            PolydockAppInstanceStatus::POST_UPGRADE_RUNNING,
+            PolydockAppInstanceStatus::POST_UPGRADE_COMPLETED => 110,
+
+            PolydockAppInstanceStatus::PENDING_PRE_REMOVE,
+            PolydockAppInstanceStatus::PRE_REMOVE_RUNNING,
+            PolydockAppInstanceStatus::PRE_REMOVE_COMPLETED => 120,
+
+            PolydockAppInstanceStatus::PENDING_REMOVE,
+            PolydockAppInstanceStatus::REMOVE_RUNNING,
+            PolydockAppInstanceStatus::REMOVE_COMPLETED => 130,
+
+            PolydockAppInstanceStatus::PENDING_POST_REMOVE,
+            PolydockAppInstanceStatus::POST_REMOVE_RUNNING,
+            PolydockAppInstanceStatus::POST_REMOVE_COMPLETED => 140,
+
+            PolydockAppInstanceStatus::REMOVED => 150,
+
+            default => null,
+        };
+    }
+
+    private function isKnownStatusProgression(PolydockAppInstanceStatus $expectedStatus, PolydockAppInstanceStatus $currentStatus): bool
+    {
+        $expectedOrdinal = self::lifecycleStageOrdinal($expectedStatus);
+        $currentOrdinal = self::lifecycleStageOrdinal($currentStatus);
+
+        if ($expectedOrdinal === null || $currentOrdinal === null) {
+            return false;
+        }
+
+        return $currentOrdinal > $expectedOrdinal;
+    }
+
     public function polydockJobStart()
     {
         $this->appInstance = PolydockAppInstance::find($this->appInstanceId)->refresh();

app/Jobs/ProcessPolydockAppInstanceJobs/Claim/ClaimJob.php

@@ -25,6 +25,12 @@ public function handle(): void
         }
 
         if ($appInstance->status != PolydockAppInstanceStatus::PENDING_POLYDOCK_CLAIM) {
+            if ($this->shouldSkipBecauseStatusAdvanced(PolydockAppInstanceStatus::PENDING_POLYDOCK_CLAIM)) {
+                $this->polydockJobDone();
+
+                return;
+            }
+
             throw new PolydockAppInstanceStatusFlowException(
                 'ClaimJob must be in status PENDING_POLYDOCK_CLAIM',
                 $appInstance->status,

app/Jobs/ProcessPolydockAppInstanceJobs/Create/CreateJob.php

@@ -28,6 +28,12 @@ public function handle(): void
         }
 
         if ($appInstance->status != PolydockAppInstanceStatus::PENDING_CREATE) {
+            if ($this->shouldSkipBecauseStatusAdvanced(PolydockAppInstanceStatus::PENDING_CREATE)) {
+                $this->polydockJobDone();
+
+                return;
+            }
+
             throw new PolydockAppInstanceStatusFlowException(
                 'CreateJob must be in status PENDING_CREATE',
             );

app/Jobs/ProcessPolydockAppInstanceJobs/Create/PostCreateJob.php

@@ -27,6 +27,12 @@ public function handle(): void
         }
 
         if ($appInstance->status != PolydockAppInstanceStatus::PENDING_POST_CREATE) {
+            if ($this->shouldSkipBecauseStatusAdvanced(PolydockAppInstanceStatus::PENDING_POST_CREATE)) {
+                $this->polydockJobDone();
+
+                return;
+            }
+
             throw new PolydockAppInstanceStatusFlowException(
                 'PostCreateJob must be in status PENDING_POST_CREATE',
             );

app/Jobs/ProcessPolydockAppInstanceJobs/Create/PreCreateJob.php

@@ -28,6 +28,12 @@ public function handle(): void
         }
 
         if ($appInstance->status != PolydockAppInstanceStatus::PENDING_PRE_CREATE) {
+            if ($this->shouldSkipBecauseStatusAdvanced(PolydockAppInstanceStatus::PENDING_PRE_CREATE)) {
+                $this->polydockJobDone();
+
+                return;
+            }
+
             throw new PolydockAppInstanceStatusFlowException(
                 'PreCreateJob must be in status PENDING_PRE_CREATE',
             );

app/Jobs/ProcessPolydockAppInstanceJobs/Deploy/DeployJob.php

@@ -27,6 +27,12 @@ public function handle(): void
         }
 
         if ($appInstance->status != PolydockAppInstanceStatus::PENDING_DEPLOY) {
+            if ($this->shouldSkipBecauseStatusAdvanced(PolydockAppInstanceStatus::PENDING_DEPLOY)) {
+                $this->polydockJobDone();
+
+                return;
+            }
+
             throw new PolydockAppInstanceStatusFlowException(
                 'DeployJob must be in status PENDING_DEPLOY',
             );

app/Jobs/ProcessPolydockAppInstanceJobs/Deploy/PostDeployJob.php

@@ -28,6 +28,12 @@ public function handle(): void
         }
 
         if ($appInstance->status != PolydockAppInstanceStatus::PENDING_POST_DEPLOY) {
+            if ($this->shouldSkipBecauseStatusAdvanced(PolydockAppInstanceStatus::PENDING_POST_DEPLOY)) {
+                $this->polydockJobDone();
+
+                return;
+            }
+
             throw new PolydockAppInstanceStatusFlowException(
                 'PostDeployJob must be in status PENDING_POST_DEPLOY',
             );

app/Jobs/ProcessPolydockAppInstanceJobs/Deploy/PreDeployJob.php

@@ -27,6 +27,12 @@ public function handle(): void
         }
 
         if ($appInstance->status != PolydockAppInstanceStatus::PENDING_PRE_DEPLOY) {
+            if ($this->shouldSkipBecauseStatusAdvanced(PolydockAppInstanceStatus::PENDING_PRE_DEPLOY)) {
+                $this->polydockJobDone();
+
+                return;
+            }
+
             throw new PolydockAppInstanceStatusFlowException(
                 'PreDeployJob must be in status PENDING_PRE_DEPLOY',
             );

app/Jobs/ProcessPolydockAppInstanceJobs/New/ProcessNewJob.php

@@ -25,6 +25,12 @@ public function handle(): void
         }
 
         if ($appInstance->status != PolydockAppInstanceStatus::NEW) {
+            if ($this->shouldSkipBecauseStatusAdvanced(PolydockAppInstanceStatus::NEW)) {
+                $this->polydockJobDone();
+
+                return;
+            }
+
             throw new PolydockAppInstanceStatusFlowException(
                 'New PolydockAppInstance must be in status NEW',
             );

app/Jobs/ProcessPolydockAppInstanceJobs/Remove/PostRemoveJob.php

@@ -27,6 +27,12 @@ public function handle(): void
         }
 
         if ($appInstance->status != PolydockAppInstanceStatus::PENDING_POST_REMOVE) {
+            if ($this->shouldSkipBecauseStatusAdvanced(PolydockAppInstanceStatus::PENDING_POST_REMOVE)) {
+                $this->polydockJobDone();
+
+                return;
+            }
+
             throw new PolydockAppInstanceStatusFlowException(
                 'PostRemoveJob must be in status PENDING_POST_REMOVE',
             );

app/Jobs/ProcessPolydockAppInstanceJobs/Remove/PreRemoveJob.php

@@ -27,6 +27,12 @@ public function handle(): void
         }
 
         if ($appInstance->status != PolydockAppInstanceStatus::PENDING_PRE_REMOVE) {
+            if ($this->shouldSkipBecauseStatusAdvanced(PolydockAppInstanceStatus::PENDING_PRE_REMOVE)) {
+                $this->polydockJobDone();
+
+                return;
+            }
+
             throw new PolydockAppInstanceStatusFlowException(
                 'PreRemoveJob must be in status PENDING_PRE_REMOVE',
             );

app/Jobs/ProcessPolydockAppInstanceJobs/Remove/RemoveJob.php

@@ -27,6 +27,12 @@ public function handle(): void
         }
 
         if ($appInstance->status != PolydockAppInstanceStatus::PENDING_REMOVE) {
+            if ($this->shouldSkipBecauseStatusAdvanced(PolydockAppInstanceStatus::PENDING_REMOVE)) {
+                $this->polydockJobDone();
+
+                return;
+            }
+
             throw new PolydockAppInstanceStatusFlowException(
                 'RemoveJob must be in status PENDING_REMOVE',
             );