From 0b56f5205ce12828e35a2e209ffef817f7780acc Mon Sep 17 00:00:00 2001 From: Sindre Nistad Date: Fri, 14 Feb 2025 09:39:50 +0100 Subject: [PATCH] refactor: Utilize multistage builds to improve caching and compatibility The main changes here are; * using multi-stage builds for better caching and smaller images * explicitly listing the files that should be included / "COPY"able into the containers For the API, I moved the tests outside the `src` folder. That way, we do not include the tests in the finished image. Since we use multi-stage builds, we can install the necessary compilers for running on ARM. The compilers are not part of the "production" image, and we don't need to worry about them taking up more space in the image layers. I also created a user (similar to what's done in `nginx`) so that we can run the tests without running the container as root. When only installing the necessary packages, I noticed that `click` is not explicitly defined, so it wasn't included in `site-packages`. By explicitly listing it in `pyproject.toml`, that issue is resolved. --- .pre-commit-config.yaml | 4 +- api/.dockerignore | 14 +- api/Dockerfile | 76 +++++++-- api/poetry.lock | 4 +- api/pyproject.toml | 12 +- api/src/init.sh | 7 +- api/{src => }/tests/__init__.py | 0 api/{src => }/tests/conftest.py | 0 api/{src => }/tests/integration/__init__.py | 0 .../common/test_exception_handler.py | 0 .../health_check/test_health_check_feature.py | 0 .../features/todo/test_todo_feature.py | 0 .../features/whoami/test_whoami_feature.py | 0 .../tests/integration/mock_authentication.py | 0 api/{src => }/tests/unit/__init__.py | 0 .../test_exception_handler_integration.py | 0 .../mongodb/test_mongo_database_client.py | 0 api/{src => }/tests/unit/features/__init__.py | 0 .../tests/unit/features/todo/__init__.py | 0 .../tests/unit/features/todo/conftest.py | 0 .../unit/features/todo/entities/__init__.py | 0 .../features/todo/entities/test_todo_item.py | 0 .../unit/features/todo/repository/__init__.py | 0 .../todo/repository/test_todo_repository.py | 0 .../features/todo/use_cases/test_add_todo.py | 34 ++-- .../todo/use_cases/test_delete_todo_by_id.py | 0 .../todo/use_cases/test_get_todo_all.py | 0 .../todo/use_cases/test_get_todo_by_id.py | 0 .../todo/use_cases/test_update_todo.py | 0 docker-compose.override.yml | 1 + .../development-guide/03-testing.md | 4 +- .../extending-the-api/02-adding-entities.md | 2 +- .../adding-data-providers/01-clients.md | 2 +- .../adding-data-providers/03-repositories.md | 2 +- .../adding-features/01-controllers.md | 2 +- .../adding-features/02-use-cases.md | 2 +- web/.dockerignore | 25 ++- web/Dockerfile | 151 ++++++++++-------- 38 files changed, 227 insertions(+), 115 deletions(-) rename api/{src => }/tests/__init__.py (100%) rename api/{src => }/tests/conftest.py (100%) rename api/{src => }/tests/integration/__init__.py (100%) rename api/{src => }/tests/integration/common/test_exception_handler.py (100%) rename api/{src => }/tests/integration/features/health_check/test_health_check_feature.py (100%) rename api/{src => }/tests/integration/features/todo/test_todo_feature.py (100%) rename api/{src => }/tests/integration/features/whoami/test_whoami_feature.py (100%) rename api/{src => }/tests/integration/mock_authentication.py (100%) rename api/{src => }/tests/unit/__init__.py (100%) rename api/{src => }/tests/unit/common/test_exception_handler_integration.py (100%) rename api/{src => }/tests/unit/data_providers/clients/mongodb/test_mongo_database_client.py (100%) rename api/{src => }/tests/unit/features/__init__.py (100%) rename api/{src => }/tests/unit/features/todo/__init__.py (100%) rename api/{src => }/tests/unit/features/todo/conftest.py (100%) rename api/{src => }/tests/unit/features/todo/entities/__init__.py (100%) rename api/{src => }/tests/unit/features/todo/entities/test_todo_item.py (100%) rename api/{src => }/tests/unit/features/todo/repository/__init__.py (100%) rename api/{src => }/tests/unit/features/todo/repository/test_todo_repository.py (100%) rename api/{src => }/tests/unit/features/todo/use_cases/test_add_todo.py (97%) rename api/{src => }/tests/unit/features/todo/use_cases/test_delete_todo_by_id.py (100%) rename api/{src => }/tests/unit/features/todo/use_cases/test_get_todo_all.py (100%) rename api/{src => }/tests/unit/features/todo/use_cases/test_get_todo_by_id.py (100%) rename api/{src => }/tests/unit/features/todo/use_cases/test_update_todo.py (100%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 33975a53..095b93e8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,7 +26,7 @@ repos: - id: mixed-line-ending exclude: ^.*\.(lock)$ - id: detect-private-key - exclude: api/src/tests/integration/mock_authentication.py + exclude: api/tests/integration/mock_authentication.py - id: no-commit-to-branch args: [--branch, main, --branch, master] stages: [commit-msg] @@ -80,7 +80,7 @@ repos: hooks: - id: pytest name: pytest-check - entry: sh -c "cd ./api/src/ && poetry run pytest ./tests/" + entry: sh -c "cd ./api/ && poetry run pytest ./tests/" language: system pass_filenames: false always_run: true diff --git a/api/.dockerignore b/api/.dockerignore index 88789aa7..771c2628 100644 --- a/api/.dockerignore +++ b/api/.dockerignore @@ -1,3 +1,11 @@ -.venv -.pytest_cache -.mypy_cache +# Ignore all by default +* +!pyproject.toml +!poetry.lock + +!src/**/*.py +!src/*.py +!src/init.sh + +!tests/**/*.py +!tests/test_data/ diff --git a/api/Dockerfile b/api/Dockerfile index 33780b7c..3712ca65 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -1,26 +1,76 @@ -FROM --platform=linux/amd64 python:3.12-slim AS base +# syntax=docker/dockerfile:1 +FROM python:3.13.2-alpine3.21 AS base +ENV XDG_CACHE_HOME=/var/lib/ +ENV SITE_PACKAGES=/usr/local/lib/python3.13/site-packages + +ENV USER="api-user" + +RUN adduser \ + --disabled-password \ + --home /code \ + --gecos "" \ + --uid 1000 \ + "$USER" + WORKDIR /code CMD ["/code/src/init.sh", "api"] EXPOSE 5000 ENV PYTHONUNBUFFERED=1 -ENV PYTHONPATH=/code +ENV PYTHONPATH=/code/src + +FROM base AS dependencies +ENV POETRY_VERSION=1.8.5 + +ENV TO_BE_REMOVED="/usr/local/share/to-be-removed.txt" +# Enumerate files that should not be present in dependencies-slim +# that way, the relevant folders can be COPY-ed without incurring the size cost of already existing items +RUN < "$TO_BE_REMOVED" +find "$SITE_PACKAGES" -maxdepth 1 -mindepth 1 >> "$TO_BE_REMOVED" + +EOF + +ENV PATH="/root/.local/bin:$PATH" +RUN \ + --mount=type=cache,target="$XDG_CACHE_HOME/pip" \ + pip install --upgrade pip && \ + pip install --user "poetry==$POETRY_VERSION" && \ poetry config virtualenvs.create false -COPY pyproject.toml pyproject.toml -COPY poetry.lock poetry.lock +RUN --mount=type=cache,target=/var/cache/apk \ + apk add \ + linux-headers \ + musl-dev \ + gcc + +COPY --chown="$USER:$USER" \ + pyproject.toml poetry.lock ./ +RUN --mount=type=cache,target="$XDG_CACHE_HOME/pip" \ + poetry install --no-dev + +FROM dependencies AS dependencies-slim +RUN < @@ -36,7 +36,7 @@ As a general rule, unit tests should not have any external dependencies - especi ### Integration tests -The integrations tests can be found under `src/tests/integration`. +The integrations tests can be found under `tests/integration`. To run integration tests add `--integration` as argument for pytest. diff --git a/documentation/docs/contribute/development-guide/coding/extending-the-api/02-adding-entities.md b/documentation/docs/contribute/development-guide/coding/extending-the-api/02-adding-entities.md index 31730fe8..50e6b1de 100644 --- a/documentation/docs/contribute/development-guide/coding/extending-the-api/02-adding-entities.md +++ b/documentation/docs/contribute/development-guide/coding/extending-the-api/02-adding-entities.md @@ -24,7 +24,7 @@ Entities should not be affected by any change external to them. ## Testing entities ```mdx-code-block -import Test from '!!raw-loader!@site/../api/src/tests/unit/features/todo/entities/test_todo_item.py'; +import Test from '!!raw-loader!@site/../api/tests/unit/features/todo/entities/test_todo_item.py'; {Test} ``` diff --git a/documentation/docs/contribute/development-guide/coding/extending-the-api/adding-data-providers/01-clients.md b/documentation/docs/contribute/development-guide/coding/extending-the-api/adding-data-providers/01-clients.md index d383198f..45b12492 100644 --- a/documentation/docs/contribute/development-guide/coding/extending-the-api/adding-data-providers/01-clients.md +++ b/documentation/docs/contribute/development-guide/coding/extending-the-api/adding-data-providers/01-clients.md @@ -15,7 +15,7 @@ import MongoClient from '!!raw-loader!@site/../api/src/data_providers/clients/mo The `test_client` fixture are using the mongomock instead of real database. ```mdx-code-block -import Test from '!!raw-loader!@site/../api/src/tests/unit/data_providers/clients/mongodb/test_mongo_database_client.py'; +import Test from '!!raw-loader!@site/../api/tests/unit/data_providers/clients/mongodb/test_mongo_database_client.py'; {Test} ``` diff --git a/documentation/docs/contribute/development-guide/coding/extending-the-api/adding-data-providers/03-repositories.md b/documentation/docs/contribute/development-guide/coding/extending-the-api/adding-data-providers/03-repositories.md index fd3519b2..9939cf62 100644 --- a/documentation/docs/contribute/development-guide/coding/extending-the-api/adding-data-providers/03-repositories.md +++ b/documentation/docs/contribute/development-guide/coding/extending-the-api/adding-data-providers/03-repositories.md @@ -17,7 +17,7 @@ Use the `test_client` fixture as input to TodoRepository. The `test_client` fixt real database. ```mdx-code-block -import Test from '!!raw-loader!@site/../api/src/tests/unit/features/todo/repository/test_todo_repository.py'; +import Test from '!!raw-loader!@site/../api/tests/unit/features/todo/repository/test_todo_repository.py'; {Test} ``` diff --git a/documentation/docs/contribute/development-guide/coding/extending-the-api/adding-features/01-controllers.md b/documentation/docs/contribute/development-guide/coding/extending-the-api/adding-features/01-controllers.md index ec156619..6bf0f5d0 100644 --- a/documentation/docs/contribute/development-guide/coding/extending-the-api/adding-features/01-controllers.md +++ b/documentation/docs/contribute/development-guide/coding/extending-the-api/adding-features/01-controllers.md @@ -29,7 +29,7 @@ FastAPI is built around the [OpenAPI Specification](https://github.com/OAI/OpenA Use the `test_client` fixture to populate the database with test data and `test_app` fixture to perform REST API calls. ```mdx-code-block -import Test from '!!raw-loader!@site/../api/src/tests/integration/features/todo/test_todo_feature.py'; +import Test from '!!raw-loader!@site/../api/tests/integration/features/todo/test_todo_feature.py'; {Test} ``` diff --git a/documentation/docs/contribute/development-guide/coding/extending-the-api/adding-features/02-use-cases.md b/documentation/docs/contribute/development-guide/coding/extending-the-api/adding-features/02-use-cases.md index 7a8be020..84608b9e 100644 --- a/documentation/docs/contribute/development-guide/coding/extending-the-api/adding-features/02-use-cases.md +++ b/documentation/docs/contribute/development-guide/coding/extending-the-api/adding-features/02-use-cases.md @@ -30,7 +30,7 @@ The use-case should only know of the repository interface (abstract class) befor Use the `todo_repository` fixture as input to use_cases. ```mdx-code-block -import Test from '!!raw-loader!@site/../api/src/tests/unit/features/todo/use_cases/test_add_todo.py'; +import Test from '!!raw-loader!@site/../api/tests/unit/features/todo/use_cases/test_add_todo.py'; {Test} ``` diff --git a/web/.dockerignore b/web/.dockerignore index 50eb0783..4c6aef2b 100644 --- a/web/.dockerignore +++ b/web/.dockerignore @@ -1,2 +1,23 @@ -node_modules -src/build +# ignore all by default +* + +# Nginx +!nginx/*.conf +!nginx/*/*.conf + +# Single Page Application +# Dependencies +!package.json +!yarn.lock + +# Application +!public/ +!index.html +!src/**/*.tsx +!src/**/*.ts +!src/*.tsx +!src/*.ts + +# Configuration / build files +!vite.config.mts +!tsconfig.json diff --git a/web/Dockerfile b/web/Dockerfile index b620604c..331f7284 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -1,63 +1,88 @@ -FROM nginx:1.22.0-alpine AS server - -RUN apk upgrade --update-cache - -# Run as non-root -RUN deluser nginx -RUN adduser --disabled-password --no-create-home --gecos "" --uid 1000 nginx - -# Copy configs -COPY nginx/nginx.conf /etc/nginx/nginx.conf -COPY nginx/config/ /etc/nginx/config - -# Remove default nginx config -RUN rm /etc/nginx/conf.d/default.conf - -# Copy sites-available into sites-enabled -COPY nginx/sites-available/default.conf /etc/nginx/sites-enabled/default.conf - -# Create log directory if not present, set permissions -RUN mkdir -p /var/log/nginx && \ - chown -R nginx:nginx /var/log/nginx - -# Create tmp directory if not present, set permissions -RUN mkdir -p /tmp/nginx && \ - chown -R nginx:nginx /tmp/nginx - -# Create pidfile, set permissions -RUN touch /var/run/nginx.pid && \ - chown -R nginx:nginx /var/run/nginx.pid - -# Run master process as non-root user -USER 1000 - -FROM node:22 AS base -ARG AUTH_ENABLED=0 -# Azure AD requires a scope. -ARG AUTH_SCOPE="" -ARG CLIENT_ID="" -ARG TENANT_ID="" -ENV VITE_AUTH_SCOPE=$AUTH_SCOPE -ENV VITE_AUTH=$AUTH_ENABLED -ENV VITE_AUTH_CLIENT_ID=$CLIENT_ID -ENV VITE_AUTH_TENANT=$TENANT_ID -ENV VITE_TOKEN_ENDPOINT=https://login.microsoftonline.com/${VITE_AUTH_TENANT}/oauth2/v2.0/token -ENV VITE_AUTH_ENDPOINT=https://login.microsoftonline.com/${VITE_AUTH_TENANT}/oauth2/v2.0/authorize -ENV VITE_LOGOUT_ENDPOINT=https://login.microsoftonline.com/${VITE_AUTH_TENANT}/oauth2/logout - -WORKDIR /code -COPY ./ ./ -RUN yarn install - -FROM base AS development -CMD ["yarn", "start"] - -FROM server AS nginx-dev -COPY nginx/environments/web.dev.conf /etc/nginx/environments/ - -FROM base AS build -RUN yarn build - -FROM server AS nginx-prod -COPY nginx/environments/web.prod.conf /etc/nginx/environments/ -COPY --from=build /code/build /data/www +# syntax=docker/dockerfile:1 +FROM nginx:1.27.4-alpine3.21-slim AS server + +# Create a non-root user, which will be running the server + +RUN <= 1000 +# that way, it is not a privileged user, and radix will be happy +deluser "nginx" +adduser \ + --disabled-password \ + --no-create-home \ + --gecos "" \ + --uid 1000 \ + "nginx" + +# Remove default nginx config +rm /etc/nginx/conf.d/default.conf + +# Create log directory if not present, set permissions +mkdir -p /var/log/nginx +chown -R "nginx:nginx" /var/log/nginx + +# Create tmp directory if not present, set permissions +mkdir -p /tmp/nginx +chown -R "nginx:nginx" /tmp/nginx + +# Create pidfile, set permissions +touch /var/run/nginx.pid +chown -R "nginx:nginx" /var/run/nginx.pid +EOF + +# Copy configs +COPY nginx/nginx.conf /etc/nginx/nginx.conf +COPY nginx/config/ /etc/nginx/config + +# Copy sites-available into sites-enabled +COPY nginx/sites-available/default.conf /etc/nginx/sites-enabled/default.conf + +# Run master process as non-root user +USER 1000 + +FROM node:22.13.1-alpine3.21 AS base +ARG AUTH_ENABLED=0 +# Azure AD requires a scope. +ARG AUTH_SCOPE="" +ARG CLIENT_ID="" +ARG TENANT_ID="" +ENV VITE_AUTH_SCOPE=$AUTH_SCOPE +ENV VITE_AUTH=$AUTH_ENABLED +ENV VITE_AUTH_CLIENT_ID=$CLIENT_ID +ENV VITE_AUTH_TENANT=$TENANT_ID +ENV VITE_TOKEN_ENDPOINT=https://login.microsoftonline.com/${VITE_AUTH_TENANT}/oauth2/v2.0/token +ENV VITE_AUTH_ENDPOINT=https://login.microsoftonline.com/${VITE_AUTH_TENANT}/oauth2/v2.0/authorize +ENV VITE_LOGOUT_ENDPOINT=https://login.microsoftonline.com/${VITE_AUTH_TENANT}/oauth2/logout + +ENV YARN_CACHE_FOLDER=/var/cache/yarn + +RUN mkdir /code && \ + chown -R "node:node" /code + +WORKDIR /code +USER "node" +COPY --chown="node:node" package.json yarn.lock ./ +RUN --mount=type=cache,target=$YARN_CACHE_FOLDER,uid=1000 \ + yarn install + +COPY --chown="node:node" public public +COPY --chown="node:node" tsconfig.json vite.config.mts index.html ./ +COPY --chown="node:node" src src + +FROM base AS development +ENV YARN_CACHE_FOLDER="" +CMD ["yarn", "start"] + +FROM server AS nginx-dev +COPY nginx/environments/web.dev.conf /etc/nginx/environments/ + +FROM base AS build +RUN --mount=type=cache,target=$YARN_CACHE_FOLDER,uid=1000 \ + yarn build + +FROM server AS nginx-prod +COPY nginx/environments/web.prod.conf /etc/nginx/environments/ +COPY --from=build /code/build /data/www