[Issue #1352] harden daemon lifecycle and pixels-install reliability

AGENTS.md

@@ -43,4 +43,7 @@
 - For serverless changes, ensure planner/invoker/worker settings remain consistent (input/intermediate/output storage schemes in `pixels-turbo/README.md`).
 
 ## Optional skills
-- For guided Pixels deployment, install and use the `pixels-install` skill from `skills/pixels-install`.
+- For guided Pixels deployment, install and use the `pixels-install` skill from `skills/pixels-install`:
+  - Cursor: `./skills/install.sh --skill pixels-install --tool cursor --scope project`
+  - Codex: `./skills/install.sh --skill pixels-install --tool codex --scope project`
+  - Claude: `./skills/install.sh --skill pixels-install --tool claude --scope project`

pixels-common/src/main/resources/pixels.properties

@@ -279,8 +279,6 @@ retina.enable=false
 retina.tile.visibility.capacity=10240
 # number of rows recorded in memTable, must be a multiple of 64
 retina.buffer.memTable.size=10240
-# the scheme of the storage for retina buffer object storage, e.g., s3, minio
-retina.buffer.object.storage.scheme=s3
 # the folder path for retina buffer object storage
 retina.buffer.object.storage.folder=s3://object-storage
 # number of threads to flush shared storage

pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/Daemon.java

@@ -67,7 +67,7 @@ public void setup (String lockFilePath)
             if (myLock == null)
             {
                 // this process has been started
-                this.clean();
+                this.releaseLock();
                 log.info("another daemon process is holding lock on " + lockFilePath + ", exiting...");
                 this.running = false;
                 System.exit(0);
@@ -82,54 +82,74 @@ public void setup (String lockFilePath)
              * Issue #181:
              * We should not bind the SIGTERM handler.
              * If it is bind, the shutdown hooks will not be called.
-             * Daemon.shutdown() will be called in the shutdown hook in DaemonMain,
-             * instead of in the signal handler.
+             * Daemon shutdown is coordinated by the shutdown hook in DaemonMain,
+             * instead of by the signal handler.
              */
             // bind handler for SIGTERM(15) signal.
             //this.shutdownHandler = new ShutdownHandler(this);
             //Signal.handle(new Signal("TERM"), shutdownHandler);
         } catch (IOException e)
         {
             log.error("I/O exception when creating lock file channels.", e);
-            this.clean();
+            this.releaseLock();
         }
     }
 
-    public void clean ()
+    /**
+     * Stop the daemon main loop without releasing the process lock.
+     */
+    public void requestShutdown()
     {
+        this.running = false;
+    }
+
+    /**
+     * Release the process lock after all managed servers have been stopped.
+     */
+    public synchronized void releaseLock()
+    {
+        if (this.cleaned)
+        {
+            return;
+        }
+        log.info("releasing daemon lock and cleaning the file channel...");
         try
         {
-            if (myLock != null)
+            if (this.myLock != null && this.myLock.isValid())
             {
-                myLock.release();
+                this.myLock.release();
             }
         } catch (IOException e1)
         {
-            log.error("error when releasing daemon lock");
+            log.error("error when releasing daemon lock", e1);
         }
 
-        try
+        if (this.myChannel != null)
         {
-            if (this.myChannel != null)
+            try
             {
                 this.myChannel.truncate(0);
+            }
+            catch (IOException e)
+            {
+                log.error("error when truncating my own channel", e);
+            }
+            try
+            {
                 this.myChannel.close();
             }
-        } catch (IOException e)
-        {
-            log.error("error when closing my own channel.", e);
+            catch (IOException e)
+            {
+                log.error("error when closing my own channel", e);
+            }
         }
         this.cleaned = true;
     }
 
     public void shutdown()
     {
-        this.running = false;
-        while (!this.cleaned)
-        {
-            log.info("the daemon thread is stopped, cleaning the file channels...");
-            this.clean();
-        }
+        this.requestShutdown();
+        this.releaseLock();
     }
 
     public boolean isRunning()
@@ -153,8 +173,15 @@ public void handle(Signal signal)
         {
             if (signal.getNumber() == 15)
             {
-                this.daemon.shutdown();
-                this.executor.run();
+                this.daemon.requestShutdown();
+                try
+                {
+                    this.executor.run();
+                }
+                finally
+                {
+                    this.daemon.releaseLock();
+                }
             }
         }
     }

pixels-daemon/src/main/java/io/pixelsdb/pixels/daemon/DaemonMain.java

@@ -6,10 +6,11 @@
 import io.pixelsdb.pixels.daemon.cache.CacheWorker;
 import io.pixelsdb.pixels.daemon.index.IndexServer;
 import io.pixelsdb.pixels.daemon.node.NodeServer;
+import io.pixelsdb.pixels.daemon.retina.RetinaReadyCheck;
 import io.pixelsdb.pixels.daemon.retina.RetinaServer;
-import io.pixelsdb.pixels.daemon.exception.NoSuchServerException;
 import io.pixelsdb.pixels.daemon.heartbeat.HeartbeatCoordinator;
 import io.pixelsdb.pixels.daemon.heartbeat.HeartbeatWorker;
+import io.pixelsdb.pixels.daemon.metadata.MetadataReadyCheck;
 import io.pixelsdb.pixels.daemon.metadata.MetadataServer;
 import io.pixelsdb.pixels.daemon.metrics.MetricsServer;
 import io.pixelsdb.pixels.daemon.scaling.ScalingMetricsServer;
@@ -108,7 +109,7 @@ public static void main(String[] args)
                     container.addServer("metadata", metadataServer);
                     // start transaction server
                     TransServer transServer = new TransServer(transServerPort);
-                    container.addServer("transaction", transServer);
+                    container.addServer("transaction", transServer, new RetinaReadyCheck());
                     // start query schedule server
                     QueryScheduleServer queryScheduleServer = new QueryScheduleServer(queryScheduleServerPort);
                     container.addServer("query_schedule", queryScheduleServer);
@@ -174,7 +175,8 @@ else if(role.equals(NodeProto.NodeRole.RETINA))
                 {
                     // start retina server on worker node
                     RetinaServer retinaServer = new RetinaServer(retinaServerPort);
-                    container.addServer("retina", retinaServer);
+                    container.addServer("retina", retinaServer,
+                            new MetadataReadyCheck());
 
                     if (indexServerEnabled)
                     {
@@ -202,47 +204,27 @@ else if(role.equals(NodeProto.NodeRole.RETINA))
             // The shutdown hook ensures the servers are shutdown graceful
             // if this main daemon is terminated by SIGTERM(15) signal.
             ShutdownHookManager.Instance().registerShutdownHook(DaemonMain.class, false, () -> {
-                for (String serverName : container.getServerNames())
+                // Stop the watchdog before shutting down servers, but retain the
+                // process lock until all server shutdown attempts have completed.
+                mainDaemon.requestShutdown();
+                try
                 {
-                    // shutdown the server threads.
-                    try
-                    {
-                        container.shutdownServer(serverName);
-                    } catch (NoSuchServerException e)
+                    container.shutdownAll();
+                    if (!container.awaitTermination(30, TimeUnit.SECONDS))
                     {
-                        log.error("error when stopping server threads", e);
+                        log.warn("not all servers stopped before the shutdown timeout");
                     }
                 }
-                for (int i = 60; i > 0; --i)
+                catch (InterruptedException e)
                 {
-                    try
-                    {
-                        boolean done = true;
-                        for (String serverName : container.getServerNames())
-                        {
-                            if (container.checkServer(serverName, 0))
-                            {
-                                done = false;
-                                break;
-                            }
-                        }
-                        if (done)
-                        {
-                            break;
-                        }
-                        TimeUnit.SECONDS.sleep(1);
-                    }
-                    catch (Throwable e)
-                    {
-                        log.error("error when waiting server threads shutdown", e);
-                    }
+                    Thread.currentThread().interrupt();
+                    log.warn("interrupted while waiting for servers to stop", e);
                 }
-                /**
-                 * Issue #181:
-                 * Shutdown the daemon thread here instead of using the SIGTERM handler.
-                 */
-                mainDaemon.shutdown();
-                log.info("all the servers are shutdown, bye...");
+                finally
+                {
+                    mainDaemon.releaseLock();
+                }
+                log.info("daemon shutdown completed, bye...");
             });
 
             // continue the main thread, start and check the server threads.
@@ -253,10 +235,7 @@ else if(role.equals(NodeProto.NodeRole.RETINA))
                 {
                     for (String serverName : container.getServerNames())
                     {
-                        if (!container.checkServer(serverName))
-                        {
-                            container.startServer(serverName);
-                        }
+                        container.startServer(serverName);
                     }
                     TimeUnit.SECONDS.sleep(1);
                 }