From 27d759ff46562d3e2b2f8bc3fd7cef9bfc9177b2 Mon Sep 17 00:00:00 2001 From: Mohamed Arshath Date: Thu, 13 Feb 2025 08:19:48 +0000 Subject: [PATCH 1/8] refactor: Improve Docker container startup with entrypoint script --- Dockerfile-node-dev | 9 ++------- docker-compose.yml | 2 ++ entrypoint.sh | 16 ++++++++++++++++ 3 files changed, 20 insertions(+), 7 deletions(-) create mode 100644 entrypoint.sh diff --git a/Dockerfile-node-dev b/Dockerfile-node-dev index d7743644..9d9d295a 100644 --- a/Dockerfile-node-dev +++ b/Dockerfile-node-dev @@ -31,11 +31,6 @@ RUN curl -sSf https://install.surrealdb.com | sh EXPOSE 7001 7002 -# run db migrations / config & server -CMD (poetry run python -m node.storage.db.init_db | tee /dev/stdout) && \ - (poetry run python -m node.storage.hub.init_hub | tee /dev/stdout) && \ - (poetry run python -m node.storage.hub.init_hub --user | tee /dev/stdout) && \ - ((poetry run celery -A node.worker.main:app worker --loglevel=info | tee /dev/stdout) & ) && \ - (poetry run python -m node.server.server --communication-protocol http --port 7001 | tee /dev/stdout) & \ - poetry run python -m node.server.server --communication-protocol ws --port 7002 | tee /dev/stdout +RUN chmod +x /app/entrypoint.sh +CMD ["/app/entrypoint.sh"] diff --git a/docker-compose.yml b/docker-compose.yml index 1764beca..1ffcace8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -51,6 +51,8 @@ services: container_name: node-app build: dockerfile: Dockerfile-node-dev + init: true + stop_grace_period: 30s env_file: - ./.env volumes: diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 00000000..e88c0a7e --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -e + +# Run DB migrations and hub initialization +poetry run python -m node.storage.db.init_db | tee /dev/stdout +poetry run python -m node.storage.hub.init_hub | tee /dev/stdout +poetry run python -m node.storage.hub.init_hub --user | tee /dev/stdout + +# Start celery worker detached +nohup poetry run celery -A node.worker.main:app worker --loglevel=info > /dev/stdout 2>&1 & + +# Start WS server detached +nohup poetry run python -m node.server.server --communication-protocol ws --port 7002 > /dev/stdout 2>&1 & + +# Run HTTP server as the main foreground process so it receives SIGTERM +exec poetry run python -m node.server.server --communication-protocol http --port 7001 \ No newline at end of file From 4349a8bfd89a36018c772773b248ada4eb030018 Mon Sep 17 00:00:00 2001 From: Enrico Rotundo Date: Mon, 17 Feb 2025 11:20:25 +0100 Subject: [PATCH 2/8] ci: run docker build --- .github/workflows/build-node-contaier.yml | 44 +++++++++++++++++++ ...iner.yml => build-push-node-container.yml} | 0 2 files changed, 44 insertions(+) create mode 100644 .github/workflows/build-node-contaier.yml rename .github/workflows/{build-node-container.yml => build-push-node-container.yml} (100%) diff --git a/.github/workflows/build-node-contaier.yml b/.github/workflows/build-node-contaier.yml new file mode 100644 index 00000000..1a523b72 --- /dev/null +++ b/.github/workflows/build-node-contaier.yml @@ -0,0 +1,44 @@ +name: Build Node Container + +on: + push: + paths: + - 'Dockerfile*' + - '!node/**' + - '.github/**' + - 'docker-compose*' + - 'poetry.lock' + - 'pyproject.toml' + +jobs: + build-only: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build Node Container + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile-node + push: false + platforms: linux/amd64,linux/arm64 + tags: | + napthaai/node:latest + napthaai/node:${{ github.sha }} + + - name: Build Node Dev Container + uses: docker/build-push-action@v5 + with: + context: . + file: ./Dockerfile-node-dev + push: false + platforms: linux/amd64,linux/arm64,darwin/arm64 + tags: | + napthaai/node-dev:latest + napthaai/node-dev:${{ github.sha }} \ No newline at end of file diff --git a/.github/workflows/build-node-container.yml b/.github/workflows/build-push-node-container.yml similarity index 100% rename from .github/workflows/build-node-container.yml rename to .github/workflows/build-push-node-container.yml From 69091edd204ccb564d99c4b793bf5dafe14d4acf Mon Sep 17 00:00:00 2001 From: moarshy Date: Mon, 17 Feb 2025 20:33:16 +0800 Subject: [PATCH 3/8] ci: Enhance GitHub Actions workflow for Naptha node deployment and testing --- .github/workflows/build-node-contaier.yml | 95 +++++++++++++++++------ 1 file changed, 73 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build-node-contaier.yml b/.github/workflows/build-node-contaier.yml index 1a523b72..dcf5a63e 100644 --- a/.github/workflows/build-node-contaier.yml +++ b/.github/workflows/build-node-contaier.yml @@ -13,32 +13,83 @@ on: jobs: build-only: runs-on: ubuntu-latest - + env: + HUB_URL: wss://hub.naptha.ai/rpc + NODE_URL: http://localhost:7001 steps: - name: Checkout code uses: actions/checkout@v4 - - - name: Set up Docker Buildx + + - name: Set up Docker uses: docker/setup-buildx-action@v3 - - name: Build Node Container - uses: docker/build-push-action@v5 + - name: Set up Python + uses: actions/setup-python@v4 with: - context: . - file: ./Dockerfile-node - push: false - platforms: linux/amd64,linux/arm64 - tags: | - napthaai/node:latest - napthaai/node:${{ github.sha }} + python-version: '3.12' - - name: Build Node Dev Container - uses: docker/build-push-action@v5 - with: - context: . - file: ./Dockerfile-node-dev - push: false - platforms: linux/amd64,linux/arm64,darwin/arm64 - tags: | - napthaai/node-dev:latest - napthaai/node-dev:${{ github.sha }} \ No newline at end of file + - name: Set up env variables + env: + NAPTHA_HUB_USERNAME: ${{ vars.NAPTHA_HUB_USERNAME }} + NAPTHA_HUB_PVT_KEY: ${{ secrets.NAPTHA_HUB_PVT_KEY }} + NAPTHA_HUB_PASSWORD: ${{ secrets.NAPTHA_HUB_PASSWORD }} + run: | + cp .env.example .env + + # override defaults for docker launch + echo -e "\n" >> .env + echo "LAUNCH_DOCKER=true" >> .env + echo "LOCAL_HUB=true" >> .env + + # avoid downloading large models + echo "OLLAMA_MODELS=" >> .env + echo "VLLM_MODELS=" >> .env + echo "OPENAI_MODELS=" >> .env + + # set credentials + echo "${NAPTHA_HUB_PVT_KEY}" > "${NAPTHA_HUB_USERNAME}.pem" + echo "PRIVATE_KEY=${NAPTHA_HUB_USERNAME}.pem" >> .env + echo "HUB_USERNAME=${NAPTHA_HUB_USERNAME}" >> .env + echo "HUB_PASSWORD=${NAPTHA_HUB_PASSWORD}" >> .env + + - name: Install Naptha SDK + run: pip install naptha-sdk + + - name: Launch node with Docker + run: bash launch.sh + + - name: Check Docker containers + run: | + sleep 30 # Give containers time to start + docker ps + docker compose logs + + - name: Test Naptha SDK + env: + HUB_USERNAME: ${{ vars.NAPTHA_HUB_USERNAME }} + HUB_PASSWORD: ${{ secrets.NAPTHA_HUB_PASSWORD }} + HUB_PRIVATE_KEY: ${{ secrets.NAPTHA_HUB_PVT_KEY }} + run: | + # Sign up with naptha + echo "Naptha signup" + naptha signup + + # Check nodes - should show 1 node + echo "Checking nodes (should show 1 node):" + naptha nodes + + # Stop node container + docker stop node-app + + # Wait a moment for node to be unregistered + sleep 30 + + # Check nodes again - should be empty + echo "Checking nodes (should be empty):" + naptha nodes + + - name: Cleanup + if: always() + run: | + ./docker-ctl.sh down || true + docker system prune -f \ No newline at end of file From b737ac714840f7ac0b8712abd3d07842832454ca Mon Sep 17 00:00:00 2001 From: moarshy Date: Mon, 17 Feb 2025 20:48:37 +0800 Subject: [PATCH 4/8] ci: Add GitHub Actions workflow for Node container build and testing --- ...ild-node-contaier.yml => build-node-container.yml} | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) rename .github/workflows/{build-node-contaier.yml => build-node-container.yml} (86%) diff --git a/.github/workflows/build-node-contaier.yml b/.github/workflows/build-node-container.yml similarity index 86% rename from .github/workflows/build-node-contaier.yml rename to .github/workflows/build-node-container.yml index dcf5a63e..cfaea145 100644 --- a/.github/workflows/build-node-contaier.yml +++ b/.github/workflows/build-node-container.yml @@ -40,17 +40,22 @@ jobs: echo -e "\n" >> .env echo "LAUNCH_DOCKER=true" >> .env echo "LOCAL_HUB=true" >> .env + echo "REGISTER_NODE_WITH_HUB=true" >> .env # avoid downloading large models echo "OLLAMA_MODELS=" >> .env echo "VLLM_MODELS=" >> .env echo "OPENAI_MODELS=" >> .env - # set credentials + # Write private key to file and set environment variables echo "${NAPTHA_HUB_PVT_KEY}" > "${NAPTHA_HUB_USERNAME}.pem" + chmod 600 "${NAPTHA_HUB_USERNAME}.pem" echo "PRIVATE_KEY=${NAPTHA_HUB_USERNAME}.pem" >> .env echo "HUB_USERNAME=${NAPTHA_HUB_USERNAME}" >> .env echo "HUB_PASSWORD=${NAPTHA_HUB_PASSWORD}" >> .env + + # Replace HF_HOME placeholder + sed -i 's|/home//.cache/huggingface|/home/runner/.cache/huggingface|g' .env - name: Install Naptha SDK run: pip install naptha-sdk @@ -71,8 +76,8 @@ jobs: HUB_PRIVATE_KEY: ${{ secrets.NAPTHA_HUB_PVT_KEY }} run: | # Sign up with naptha - echo "Naptha signup" - naptha signup + echo "Signing up with Naptha..." + echo "$HUB_PASSWORD" | naptha signup "$HUB_USERNAME" # Check nodes - should show 1 node echo "Checking nodes (should show 1 node):" From 84a9dfe9ab5295711c40a62d0fdc30c9975def35 Mon Sep 17 00:00:00 2001 From: moarshy Date: Mon, 17 Feb 2025 21:33:26 +0800 Subject: [PATCH 5/8] ci: Refactor GitHub Actions workflow for Naptha node testing --- .github/workflows/build-node-container.yml | 55 ++++++++++------------ 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build-node-container.yml b/.github/workflows/build-node-container.yml index cfaea145..15d852db 100644 --- a/.github/workflows/build-node-container.yml +++ b/.github/workflows/build-node-container.yml @@ -16,6 +16,9 @@ jobs: env: HUB_URL: wss://hub.naptha.ai/rpc NODE_URL: http://localhost:7001 + NAPTHA_HUB_USERNAME: ${{ secrets.NAPTHA_HUB_USERNAME }} + NAPTHA_HUB_PVT_KEY: ${{ secrets.NAPTHA_HUB_PVT_KEY }} + NAPTHA_HUB_PASSWORD: ${{ secrets.NAPTHA_HUB_PASSWORD }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -29,33 +32,28 @@ jobs: python-version: '3.12' - name: Set up env variables - env: - NAPTHA_HUB_USERNAME: ${{ vars.NAPTHA_HUB_USERNAME }} - NAPTHA_HUB_PVT_KEY: ${{ secrets.NAPTHA_HUB_PVT_KEY }} - NAPTHA_HUB_PASSWORD: ${{ secrets.NAPTHA_HUB_PASSWORD }} run: | cp .env.example .env - - # override defaults for docker launch - echo -e "\n" >> .env - echo "LAUNCH_DOCKER=true" >> .env + sed -i 's/^LAUNCH_DOCKER=.*/LAUNCH_DOCKER=true/' .env echo "LOCAL_HUB=true" >> .env - echo "REGISTER_NODE_WITH_HUB=true" >> .env - - # avoid downloading large models + echo "REGISTER_NODE_WITH_HUB=false" >> .env echo "OLLAMA_MODELS=" >> .env echo "VLLM_MODELS=" >> .env echo "OPENAI_MODELS=" >> .env - - # Write private key to file and set environment variables - echo "${NAPTHA_HUB_PVT_KEY}" > "${NAPTHA_HUB_USERNAME}.pem" - chmod 600 "${NAPTHA_HUB_USERNAME}.pem" - echo "PRIVATE_KEY=${NAPTHA_HUB_USERNAME}.pem" >> .env - echo "HUB_USERNAME=${NAPTHA_HUB_USERNAME}" >> .env - echo "HUB_PASSWORD=${NAPTHA_HUB_PASSWORD}" >> .env - - # Replace HF_HOME placeholder + + echo "Creating PEM file for user: $NAPTHA_HUB_USERNAME" + PEM_FILE="$NAPTHA_HUB_USERNAME.pem" + echo "$NAPTHA_HUB_PVT_KEY" > "$PEM_FILE" + chmod 600 "$PEM_FILE" + + # Replace existing placeholders in .env instead of appending duplicates + sed -i "s/^PRIVATE_KEY=.*/PRIVATE_KEY=$PEM_FILE/" .env + sed -i "s/^HUB_USERNAME=.*/HUB_USERNAME=$NAPTHA_HUB_USERNAME/" .env + sed -i "s/^HUB_PASSWORD=.*/HUB_PASSWORD=$NAPTHA_HUB_PASSWORD/" .env + sed -i 's|/home//.cache/huggingface|/home/runner/.cache/huggingface|g' .env + echo "=== .env contents (sanitized) ===" + grep -v "PASSWORD\|KEY" .env - name: Install Naptha SDK run: pip install naptha-sdk @@ -71,25 +69,24 @@ jobs: - name: Test Naptha SDK env: - HUB_USERNAME: ${{ vars.NAPTHA_HUB_USERNAME }} + HUB_USERNAME: ${{ secrets.NAPTHA_HUB_USERNAME }} HUB_PASSWORD: ${{ secrets.NAPTHA_HUB_PASSWORD }} - HUB_PRIVATE_KEY: ${{ secrets.NAPTHA_HUB_PVT_KEY }} + NODE_URL: http://localhost:7001 + HUB_URL: ws://localhost:3001/rpc run: | - # Sign up with naptha + # Use the generated PEM file from the previous step + export PRIVATE_KEY="${HUB_USERNAME}.pem" + echo "Using PRIVATE_KEY file: $PRIVATE_KEY" + echo "Signing up with Naptha..." - echo "$HUB_PASSWORD" | naptha signup "$HUB_USERNAME" + naptha signup - # Check nodes - should show 1 node echo "Checking nodes (should show 1 node):" naptha nodes - # Stop node container docker stop node-app - - # Wait a moment for node to be unregistered sleep 30 - # Check nodes again - should be empty echo "Checking nodes (should be empty):" naptha nodes From 32ab2ff2d82662c79cf3d37d8ce1f21427698642 Mon Sep 17 00:00:00 2001 From: moarshy Date: Tue, 18 Feb 2025 08:51:27 +0800 Subject: [PATCH 6/8] ci: Update GitHub Actions workflow configuration for local testing --- .github/workflows/build-node-container.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-node-container.yml b/.github/workflows/build-node-container.yml index 15d852db..9b2e9a53 100644 --- a/.github/workflows/build-node-container.yml +++ b/.github/workflows/build-node-container.yml @@ -14,9 +14,9 @@ jobs: build-only: runs-on: ubuntu-latest env: - HUB_URL: wss://hub.naptha.ai/rpc + HUB_URL: ws://localhost:3001/rpc NODE_URL: http://localhost:7001 - NAPTHA_HUB_USERNAME: ${{ secrets.NAPTHA_HUB_USERNAME }} + NAPTHA_HUB_USERNAME: test_user NAPTHA_HUB_PVT_KEY: ${{ secrets.NAPTHA_HUB_PVT_KEY }} NAPTHA_HUB_PASSWORD: ${{ secrets.NAPTHA_HUB_PASSWORD }} steps: @@ -35,11 +35,12 @@ jobs: run: | cp .env.example .env sed -i 's/^LAUNCH_DOCKER=.*/LAUNCH_DOCKER=true/' .env - echo "LOCAL_HUB=true" >> .env - echo "REGISTER_NODE_WITH_HUB=false" >> .env - echo "OLLAMA_MODELS=" >> .env - echo "VLLM_MODELS=" >> .env - echo "OPENAI_MODELS=" >> .env + sed -i 's/^NODE_IP=.*/NODE_IP=127.0.0.1/' .env + sed -i 's/^LOCAL_HUB=.*/LOCAL_HUB=true/' .env + sed -i 's/^REGISTER_NODE_WITH_HUB=.*/REGISTER_NODE_WITH_HUB=true/' .env + sed -i 's/^OLLAMA_MODELS=.*/OLLAMA_MODELS=""/' .env + sed -i 's/^VLLM_MODELS=.*/VLLM_MODELS=""/' .env + sed -i 's/^OPENAI_MODELS=.*/OPENAI_MODELS=""/' .env echo "Creating PEM file for user: $NAPTHA_HUB_USERNAME" PEM_FILE="$NAPTHA_HUB_USERNAME.pem" From aae4da971571d46fcf937c47e293aaf6ebcc5a62 Mon Sep 17 00:00:00 2001 From: moarshy Date: Tue, 18 Feb 2025 08:54:31 +0800 Subject: [PATCH 7/8] ci: Update GitHub Actions workflow with hardcoded credentials for testing --- .github/workflows/build-node-container.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-node-container.yml b/.github/workflows/build-node-container.yml index 9b2e9a53..53348c33 100644 --- a/.github/workflows/build-node-container.yml +++ b/.github/workflows/build-node-container.yml @@ -17,8 +17,8 @@ jobs: HUB_URL: ws://localhost:3001/rpc NODE_URL: http://localhost:7001 NAPTHA_HUB_USERNAME: test_user - NAPTHA_HUB_PVT_KEY: ${{ secrets.NAPTHA_HUB_PVT_KEY }} - NAPTHA_HUB_PASSWORD: ${{ secrets.NAPTHA_HUB_PASSWORD }} + NAPTHA_HUB_PVT_KEY: a5141246773dc97480b376369da8215fd04d654a46bd3b1815c9913b1c2572d9 + NAPTHA_HUB_PASSWORD: test_password steps: - name: Checkout code uses: actions/checkout@v4 From 6cf0817ea899d2420e2685a3b0488257e37168ac Mon Sep 17 00:00:00 2001 From: moarshy Date: Tue, 18 Feb 2025 09:00:43 +0800 Subject: [PATCH 8/8] ci: Use test credentials in GitHub Actions workflow --- .github/workflows/build-node-container.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-node-container.yml b/.github/workflows/build-node-container.yml index 53348c33..0ea900d3 100644 --- a/.github/workflows/build-node-container.yml +++ b/.github/workflows/build-node-container.yml @@ -70,8 +70,8 @@ jobs: - name: Test Naptha SDK env: - HUB_USERNAME: ${{ secrets.NAPTHA_HUB_USERNAME }} - HUB_PASSWORD: ${{ secrets.NAPTHA_HUB_PASSWORD }} + HUB_USERNAME: test_user + HUB_PASSWORD: test_password NODE_URL: http://localhost:7001 HUB_URL: ws://localhost:3001/rpc run: |