Improve Docker Container Signal Handling & Graceful Shutdown

.github/workflows/build-node-container.yml

@@ -1,42 +1,98 @@
-name: Build and Publish Node Container
+name: Build Node Container
 
 on:
   push:
-    branches:
-      - main
-      - k-mistele/add-compose-configurations
     paths:
-      - 'Dockerfile-node'
-      - 'node/**'
+      - 'Dockerfile*'
+      - '!node/**'
       - '.github/**'
-      - '!docker-compose*'
+      - 'docker-compose*'
       - 'poetry.lock'
       - 'pyproject.toml'
 
 jobs:
-  build-and-push:
+  build-only:
     runs-on: ubuntu-latest
-
+    env:
+      HUB_URL: ws://localhost:3001/rpc
+      NODE_URL: http://localhost:7001
+      NAPTHA_HUB_USERNAME: test_user
+      NAPTHA_HUB_PVT_KEY: a5141246773dc97480b376369da8215fd04d654a46bd3b1815c9913b1c2572d9
+      NAPTHA_HUB_PASSWORD: test_password
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-
-      - name: Set up Docker Buildx
+      
+      - name: Set up Docker
         uses: docker/setup-buildx-action@v3
 
-      - name: Login to DockerHub
-        uses: docker/login-action@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
         with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
+          python-version: '3.12'
+
+      - name: Set up env variables
+        run: |
+          cp .env.example .env
+          sed -i 's/^LAUNCH_DOCKER=.*/LAUNCH_DOCKER=true/' .env
+          sed -i 's/^NODE_IP=.*/NODE_IP=127.0.0.1/' .env
+          sed -i 's/^LOCAL_HUB=.*/LOCAL_HUB=true/' .env
+          sed -i 's/^REGISTER_NODE_WITH_HUB=.*/REGISTER_NODE_WITH_HUB=true/' .env
+          sed -i 's/^OLLAMA_MODELS=.*/OLLAMA_MODELS=""/' .env
+          sed -i 's/^VLLM_MODELS=.*/VLLM_MODELS=""/' .env
+          sed -i 's/^OPENAI_MODELS=.*/OPENAI_MODELS=""/' .env
 
-      - name: Build and push
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          file: ./Dockerfile-node
-          push: true
-          platforms: linux/amd64,linux/arm64
-          tags: |
-            napthaai/node:latest
-            napthaai/node:${{ github.sha }}
+          echo "Creating PEM file for user: $NAPTHA_HUB_USERNAME"
+          PEM_FILE="$NAPTHA_HUB_USERNAME.pem"
+          echo "$NAPTHA_HUB_PVT_KEY" > "$PEM_FILE"
+          chmod 600 "$PEM_FILE"
+
+          # Replace existing placeholders in .env instead of appending duplicates
+          sed -i "s/^PRIVATE_KEY=.*/PRIVATE_KEY=$PEM_FILE/" .env
+          sed -i "s/^HUB_USERNAME=.*/HUB_USERNAME=$NAPTHA_HUB_USERNAME/" .env
+          sed -i "s/^HUB_PASSWORD=.*/HUB_PASSWORD=$NAPTHA_HUB_PASSWORD/" .env
+
+          sed -i 's|/home/<youruser>/.cache/huggingface|/home/runner/.cache/huggingface|g' .env
+          echo "=== .env contents (sanitized) ==="
+          grep -v "PASSWORD\|KEY" .env
+
+      - name: Install Naptha SDK
+        run: pip install naptha-sdk
+
+      - name: Launch node with Docker
+        run: bash launch.sh
+
+      - name: Check Docker containers
+        run: |
+          sleep 30  # Give containers time to start
+          docker ps
+          docker compose logs
+
+      - name: Test Naptha SDK
+        env:
+          HUB_USERNAME: test_user
+          HUB_PASSWORD: test_password
+          NODE_URL: http://localhost:7001
+          HUB_URL: ws://localhost:3001/rpc
+        run: |
+          # Use the generated PEM file from the previous step
+          export PRIVATE_KEY="${HUB_USERNAME}.pem"
+          echo "Using PRIVATE_KEY file: $PRIVATE_KEY"
+
+          echo "Signing up with Naptha..."
+          naptha signup
+
+          echo "Checking nodes (should show 1 node):"
+          naptha nodes
+
+          docker stop node-app
+          sleep 30
+
+          echo "Checking nodes (should be empty):"
+          naptha nodes
+
+      - name: Cleanup
+        if: always()
+        run: |
+          ./docker-ctl.sh down || true
+          docker system prune -f

.github/workflows/build-push-node-container.yml

@@ -0,0 +1,42 @@
+name: Build and Publish Node Container
+
+on:
+  push:
+    branches:
+      - main
+      - k-mistele/add-compose-configurations
+    paths:
+      - 'Dockerfile-node'
+      - 'node/**'
+      - '.github/**'
+      - '!docker-compose*'
+      - 'poetry.lock'
+      - 'pyproject.toml'
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./Dockerfile-node
+          push: true
+          platforms: linux/amd64,linux/arm64
+          tags: |
+            napthaai/node:latest
+            napthaai/node:${{ github.sha }}

Dockerfile-node-dev

@@ -35,11 +35,6 @@ RUN curl -sSf https://install.surrealdb.com | sh
 
 EXPOSE 7001 7002
 
-# run db migrations / config & server
-CMD (poetry run python -m node.storage.db.init_db | tee /dev/stdout) && \
-    (poetry run python -m node.storage.hub.init_hub | tee /dev/stdout) && \
-    (poetry run python -m node.storage.hub.init_hub --user | tee /dev/stdout) && \
-    ((poetry run celery -A node.worker.main:app worker --loglevel=info | tee /dev/stdout) & ) && \
-    (poetry run python -m node.server.server --communication-protocol http --port 7001 | tee /dev/stdout) & \
-    poetry run python -m node.server.server --communication-protocol ws --port 7002 | tee /dev/stdout
+RUN chmod +x /app/entrypoint.sh
+CMD ["/app/entrypoint.sh"]
 

docker-compose.yml

@@ -51,6 +51,8 @@ services:
     container_name: node-app
     build:
       dockerfile: Dockerfile-node-dev
+    init: true
+    stop_grace_period: 30s
     env_file:
       - ./.env
     volumes:

entrypoint.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -e
+
+# Run DB migrations and hub initialization
+poetry run python -m node.storage.db.init_db | tee /dev/stdout
+poetry run python -m node.storage.hub.init_hub | tee /dev/stdout
+poetry run python -m node.storage.hub.init_hub --user | tee /dev/stdout
+
+# Start celery worker detached
+nohup poetry run celery -A node.worker.main:app worker --loglevel=info > /dev/stdout 2>&1 &
+
+# Start WS server detached
+nohup poetry run python -m node.server.server --communication-protocol ws --port 7002 > /dev/stdout 2>&1 &
+
+# Run HTTP server as the main foreground process so it receives SIGTERM
+exec poetry run python -m node.server.server --communication-protocol http --port 7001