feat: Dockerfile (#1473)

Co-authored-by: Hien To <tominhhien97@gmail.com>

Author: hiento09

docker/Dockerfile

@@ -0,0 +1,82 @@
+FROM ubuntu:22.04 as base
+
+FROM base as build
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    curl \
+    wget \
+    jq \
+    tar \
+    openmpi-bin \
+    libopenmpi-dev && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git \
+    uuid-dev \
+    lsb-release \
+    software-properties-common \
+    gpg \
+    zip \
+    unzip \
+    gcc \
+    g++ \
+    ninja-build \
+    pkg-config \
+    openssl && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null && \
+    apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" && \
+    apt-get update && \
+    apt-get install -y cmake && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY . /app/
+
+RUN git submodule update --init && cd engine && make configure-vcpkg && make build CMAKE_EXTRA_FLAGS="-DCORTEX_CPP_VERSION=$(git rev-parse HEAD) -DCMAKE_BUILD_TEST=OFF -DCMAKE_TOOLCHAIN_FILE=vcpkg/scripts/buildsystems/vcpkg.cmake"
+
+FROM base as runtime
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates \
+    curl \
+    wget \
+    jq \
+    tar \
+    openmpi-bin \
+    libopenmpi-dev && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+ARG CORTEX_LLAMACPP_VERSION=latest
+
+COPY --from=build /app/engine/build/cortex /usr/local/bin/cortex
+COPY --from=build /app/engine/build/cortex-server /usr/local/bin/cortex-server
+
+COPY ./docker/download-cortex.llamacpp.sh /tmp/download-cortex.llamacpp.sh
+
+# Get the latest version of the Cortex Llama
+RUN chmod +x /tmp/download-cortex.llamacpp.sh && /bin/bash /tmp/download-cortex.llamacpp.sh ${CORTEX_LLAMACPP_VERSION}
+
+# Copy the entrypoint script
+COPY ./docker/entrypoint.sh /usr/local/bin/entrypoint.sh
+
+EXPOSE 39281
+
+HEALTHCHECK --interval=300s --timeout=30s --start-period=10s --retries=3 \
+    CMD curl -f http://127.0.0.1:39281/healthz || exit 1
+  
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

docker/README.md

@@ -0,0 +1,48 @@
+## Cortex Docker Setup
+
+This guide will help you set up and run Cortex using Docker.
+
+### Prerequisites
+- Docker / Docker Desktop
+- nvidia-container-toolkit (for GPU support)
+
+### Instructions
+1. Clone the Cortex repository
+    ```bash
+    git clone https://github.com/janhq/cortex.cpp.git
+
+    cd cortex.cpp/docker
+    ```
+2. Build the Docker image
+    ```bash
+    # Default always uses the latest cortex.cpp and cortex.llamacpp
+    docker build -t cortex .
+
+    # Use specific version of cortex.cpp and cortex.llamacpp
+    docker build --build-arg CORTEX_LLAMACPP_VERSION=0.1.34 -t cortex .
+    ```
+
+3. Run the Docker container
+    ```bash
+    # Create Volume to store models and data
+    docker volume create cortex_data
+
+    # CPU mode
+    docker run -it -d --name cortex -v cortex_data:/root/cortexcpp -p 39281:39281 cortex
+
+    # GPU mode - nvidia-docker required, it will automatically use all available GPUs
+    docker run --gpus all -it -d --name cortex -v cortex_data:/root/cortexcpp -p 39281:39281 cortex
+    ```
+
+4. Check logs (Optional)
+    ```bash
+    docker logs cortex
+    ```
+
+5. Access to http://localhost:39281 to check the cortex docs API.
+
+6. Execute to container and try out cortex cli
+    ```bash
+    docker exec -it cortex bash
+    cortex --help
+    ```

docker/download-cortex.llamacpp.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+VERSION=${1:-latest}
+
+# Get the latest version of the cortex.llamacpp
+if [ "$VERSION" = "latest" ]; then
+    VERSION=$(curl -s https://api.github.com/repos/janhq/cortex.llamacpp/releases/latest | jq -r '.tag_name' | sed 's/^v//');
+fi
+
+# Create the directory to store the cortex.llamacpp
+mkdir -p /opt/cortex.llamacpp
+cd /opt/cortex.llamacpp
+
+# Download the cortex.llamacpp engines
+echo -e "Downloading Cortex Llama version $VERSION"
+wget https://github.com/janhq/cortex.llamacpp/releases/download/v$VERSION/cortex.llamacpp-$VERSION-linux-amd64-avx-cuda-11-7.tar.gz
+wget https://github.com/janhq/cortex.llamacpp/releases/download/v$VERSION/cortex.llamacpp-$VERSION-linux-amd64-avx-cuda-12-0.tar.gz
+wget https://github.com/janhq/cortex.llamacpp/releases/download/v$VERSION/cortex.llamacpp-$VERSION-linux-amd64-avx.tar.gz
+wget https://github.com/janhq/cortex.llamacpp/releases/download/v$VERSION/cortex.llamacpp-$VERSION-linux-amd64-avx2-cuda-11-7.tar.gz
+wget https://github.com/janhq/cortex.llamacpp/releases/download/v$VERSION/cortex.llamacpp-$VERSION-linux-amd64-avx2-cuda-12-0.tar.gz
+wget https://github.com/janhq/cortex.llamacpp/releases/download/v$VERSION/cortex.llamacpp-$VERSION-linux-amd64-avx2.tar.gz
+wget https://github.com/janhq/cortex.llamacpp/releases/download/v$VERSION/cortex.llamacpp-$VERSION-linux-amd64-avx512-cuda-11-7.tar.gz
+wget https://github.com/janhq/cortex.llamacpp/releases/download/v$VERSION/cortex.llamacpp-$VERSION-linux-amd64-avx512-cuda-12-0.tar.gz
+wget https://github.com/janhq/cortex.llamacpp/releases/download/v$VERSION/cortex.llamacpp-$VERSION-linux-amd64-avx512.tar.gz
+wget https://github.com/janhq/cortex.llamacpp/releases/download/v$VERSION/cortex.llamacpp-$VERSION-linux-amd64-noavx-cuda-11-7.tar.gz
+wget https://github.com/janhq/cortex.llamacpp/releases/download/v$VERSION/cortex.llamacpp-$VERSION-linux-amd64-noavx-cuda-12-0.tar.gz
+wget https://github.com/janhq/cortex.llamacpp/releases/download/v$VERSION/cortex.llamacpp-$VERSION-linux-amd64-noavx.tar.gz
+wget https://github.com/janhq/cortex.llamacpp/releases/download/v$VERSION/cortex.llamacpp-$VERSION-linux-amd64-vulkan.tar.gz
+wget https://github.com/janhq/cortex.llamacpp/releases/download/v$VERSION/cuda-11-7-linux-amd64.tar.gz
+wget https://github.com/janhq/cortex.llamacpp/releases/download/v$VERSION/cuda-12-0-linux-amd64.tar.gz

docker/entrypoint.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+# Install cortex.llamacpp engine
+
+cortex engines install llama-cpp -s /opt/cortex.llamacpp
+cortex -v
+
+# Start the cortex server
+
+sed -i 's/apiServerHost: 127.0.0.1/apiServerHost: 0.0.0.0/' /root/.cortexrc
+
+cortex start
+
+# Keep the container running by tailing the log files
+tail -f /root/cortexcpp/logs/cortex.log &
+tail -f /root/cortexcpp/logs/cortex-cli.log &
+wait