From 0dbf6cd791d74680b3b3e35e693a45cb24bd3ad2 Mon Sep 17 00:00:00 2001 From: Joshua Creek <8179928+jcreek@users.noreply.github.com> Date: Tue, 12 May 2026 13:55:41 +0100 Subject: [PATCH 01/10] feat(*): add Dockerfile and docker-compose for self-hosting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a multi-stage Dockerfile (web/worker/frontend targets), a docker-compose.yml that brings up the full stack (web, worker, redis, postgres, c5tools, nginx frontend), an nginx config that serves the Vue SPA and proxies the Django API, an extended .env.example, and a rewritten "Using Docker" section in docs/own-instance.rst. Resolves the "currently in preparation" note in docs/own-instance.rst. c5tools is pinned to v0.8.3 via build.context git URL — no second submodule needed. Verified locally end to end: compose up -> validator-web entrypoint runs migrate + collectstatic + optional superuser creation from DJANGO_SUPERUSER_* env -> gunicorn. counter-api-validation POST -> worker -> c5tools -> messages all reachable via http://localhost (nginx -> validator-web) and direct on :8000. --- .env.example | 45 +++++++++++++- Dockerfile | 67 +++++++++++++++++++++ docker-compose.yml | 124 +++++++++++++++++++++++++++++++++++++++ docker/nginx.conf | 54 +++++++++++++++++ docker/web-entrypoint.sh | 65 ++++++++++++++++++++ docs/own-instance.rst | 105 ++++++++++++++++++++++++++++++++- 6 files changed, 454 insertions(+), 6 deletions(-) create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100644 docker/nginx.conf create mode 100644 docker/web-entrypoint.sh diff --git a/.env.example b/.env.example index d9d2b55..db2fa76 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,45 @@ -SECRET_KEY=some-random-text +# Copy this file to `.env` and edit the values before `docker compose up`. + +# ---------- Django ---------- +SECRET_KEY=replace-me-with-a-long-random-string DEBUG=False -ALLOWED_HOSTS='foo.bar, bar.baz' +ALLOWED_HOSTS=localhost,127.0.0.1 +CSRF_TRUSTED_ORIGINS=http://localhost + +# ---------- Database ---------- DB_NAME=coval -DB_PASSWORD=coval +DB_USER=coval +DB_PASSWORD=replace-me-too DB_PORT=5432 +# DB_HOST is set to the in-compose hostname (validator-db) automatically; +# override only when running validator-web outside of compose. + +# ---------- Redis (in-compose defaults work as-is) ---------- +# REDIS_HOST=validator-redis +# REDIS_PORT=6379 + +# ---------- Validation engine (in-compose c5tools by default) ---------- +# Override to point at a different c5tools instance, or a comma-separated +# list to fan out across multiple instances. +# VALIDATION_MODULES_URLS=http://c5tools:8080/ + +# ---------- User account behaviour ---------- +ALLOW_USER_REGISTRATION=True + +# ---------- Initial superuser (optional) ---------- +# If both are set, validator-web idempotently creates a superuser with a +# verified email on first boot. Re-running `docker compose up` with the same +# values is a no-op. Leave commented out to use `manage.py createsuperuser` +# manually instead. +# DJANGO_SUPERUSER_EMAIL=admin@example.com +# DJANGO_SUPERUSER_PASSWORD=replace-me-with-a-real-password + +# ---------- Email (uncomment if you want validator emails to send) ---------- +# EMAIL_HOST=smtp.example.com +# DEFAULT_FROM_EMAIL=noreply@example.com +# SERVER_EMAIL=noreply@example.com +# +# For Mailgun specifically: +# MAILGUN_API_KEY= +# MAILGUN_SENDER_DOMAIN= +# MAILGUN_API_URL=https://api.eu.mailgun.net/v3 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8fa6828 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,67 @@ +# Multi-stage Dockerfile for the COUNTER Validator self-hosting stack. +# +# Build targets: +# web Django + gunicorn (REST API + admin), exposes :8000 +# worker Celery worker consuming `celery,validation` queues +# frontend nginx serving the Vue SPA and reverse-proxying the Django app +# +# Built and orchestrated by docker-compose.yml at the repo root. To build a single +# target manually: +# docker build --target web -t counter-validator-web . +# docker build --target worker -t counter-validator-worker . +# docker build --target frontend -t counter-validator-frontend . + +# ---------- frontend-build: produce /frontend/dist ---------- +FROM node:20-alpine AS frontend-build +WORKDIR /frontend +# .yarnrc.yml sets `nodeLinker: node-modules` for this project; it must be +# present before `yarn install` so the right linker mode is picked. +COPY frontend/.yarnrc.yml frontend/package.json frontend/yarn.lock ./ +RUN corepack enable && yarn install --immutable +COPY frontend/ ./ +RUN yarn build + +# ---------- python-base: shared layer for web and worker ---------- +FROM python:3.12 AS python-base +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + POETRY_VIRTUALENVS_CREATE=false \ + POETRY_NO_INTERACTION=1 +RUN apt-get update && apt-get install -y --no-install-recommends \ + libmagic1 \ + && rm -rf /var/lib/apt/lists/* +RUN pip install --no-cache-dir "poetry==1.8.*" +WORKDIR /app + +# Install Python deps first for cache friendliness. +COPY pyproject.toml poetry.lock ./ +RUN poetry install --no-root --without dev + +# Copy the application source. +COPY manage.py ./ +COPY apps/ ./apps/ +COPY config/ ./config/ +COPY tools/ ./tools/ + +# ---------- web: Django + gunicorn ---------- +FROM python-base AS web +RUN pip install --no-cache-dir gunicorn +COPY docker/web-entrypoint.sh /usr/local/bin/web-entrypoint.sh +RUN chmod +x /usr/local/bin/web-entrypoint.sh +ENV DJANGO_SETTINGS_MODULE=config.settings.production \ + STATIC_ROOT=/app/static +EXPOSE 8000 +# Migrate + collectstatic + (optional) auto-create superuser from +# DJANGO_SUPERUSER_* env, then gunicorn. See docker/web-entrypoint.sh. +CMD ["/usr/local/bin/web-entrypoint.sh"] + +# ---------- worker: Celery ---------- +FROM python-base AS worker +ENV DJANGO_SETTINGS_MODULE=config.settings.production +CMD ["celery", "-A", "config", "worker", "-c", "2", "-Q", "celery,validation", "-l", "INFO"] + +# ---------- frontend: nginx + Vue SPA ---------- +FROM nginx:alpine AS frontend +COPY --from=frontend-build /frontend/dist /usr/share/nginx/html +COPY docker/nginx.conf /etc/nginx/conf.d/default.conf +EXPOSE 80 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..dd60a29 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,124 @@ +# Self-hosting compose stack for the COUNTER Validator. +# +# Six services on a single bridge network: +# validator-db Postgres for the validator's persistent state +# validator-redis Celery broker +# c5tools PHP validation engine (built from ubfr/c5tools git URL) +# validator-web Django REST API + admin (gunicorn) +# validator-worker Celery worker +# validator-frontend nginx serving the Vue SPA + reverse-proxying the API +# +# Quick start: +# cp .env.example .env # then edit SECRET_KEY and DB_PASSWORD +# docker compose up -d # validator-web auto-runs migrate + collectstatic on boot +# docker compose exec validator-web python manage.py createsuperuser +# # open http://localhost + +services: + + validator-db: + image: postgres:16-alpine + volumes: + - postgres_data:/var/lib/postgresql/data + environment: + POSTGRES_DB: ${DB_NAME:-coval} + POSTGRES_USER: ${DB_USER:-coval} + POSTGRES_PASSWORD: ${DB_PASSWORD:?DB_PASSWORD is required. Copy .env.example to .env and edit it before running docker compose up.} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-coval} -d ${DB_NAME:-coval}"] + interval: 5s + timeout: 5s + retries: 20 + + validator-redis: + image: redis:7-alpine + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 5s + retries: 20 + + # c5tools is the PHP validation engine (ubfr/c5tools). The validator's + # Celery worker hard-requires it (see config/settings/base.py: + # VALIDATION_MODULES_URLS). Built from a pinned upstream tag — bump + # the fragment below to update. + c5tools: + build: + context: https://github.com/ubfr/c5tools.git#v0.8.3 + environment: + SSL_MODE: "off" + PHP_OPCACHE_ENABLE: "1" + + validator-web: + build: + context: . + target: web + depends_on: + validator-db: + condition: service_healthy + validator-redis: + condition: service_healthy + c5tools: + condition: service_started + environment: + SECRET_KEY: ${SECRET_KEY:?SECRET_KEY is required. Copy .env.example to .env and edit it before running docker compose up.} + DEBUG: "${DEBUG:-False}" + ALLOWED_HOSTS: "${ALLOWED_HOSTS:-localhost,127.0.0.1}" + CSRF_TRUSTED_ORIGINS: "${CSRF_TRUSTED_ORIGINS:-http://localhost}" + DB_NAME: ${DB_NAME:-coval} + DB_USER: ${DB_USER:-coval} + DB_PASSWORD: ${DB_PASSWORD:?DB_PASSWORD is required. Copy .env.example to .env and edit it before running docker compose up.} + DB_HOST: validator-db + DB_PORT: ${DB_PORT:-5432} + REDIS_HOST: ${REDIS_HOST:-validator-redis} + REDIS_PORT: ${REDIS_PORT:-6379} + VALIDATION_MODULES_URLS: ${VALIDATION_MODULES_URLS:-http://c5tools:8080/} + ALLOW_USER_REGISTRATION: "${ALLOW_USER_REGISTRATION:-True}" + EMAIL_HOST: "${EMAIL_HOST:-localhost}" + DEFAULT_FROM_EMAIL: "${DEFAULT_FROM_EMAIL:-root@localhost}" + # Optional: if both are set, web-entrypoint.sh idempotently creates a + # superuser with a verified email on boot. Empty defaults disable it. + DJANGO_SUPERUSER_EMAIL: "${DJANGO_SUPERUSER_EMAIL:-}" + DJANGO_SUPERUSER_PASSWORD: "${DJANGO_SUPERUSER_PASSWORD:-}" + volumes: + - static_data:/app/static + ports: + - "8000:8000" + + validator-worker: + build: + context: . + target: worker + depends_on: + validator-db: + condition: service_healthy + validator-redis: + condition: service_healthy + c5tools: + condition: service_started + environment: + SECRET_KEY: ${SECRET_KEY:?SECRET_KEY is required. Copy .env.example to .env and edit it before running docker compose up.} + DEBUG: "${DEBUG:-False}" + DB_NAME: ${DB_NAME:-coval} + DB_USER: ${DB_USER:-coval} + DB_PASSWORD: ${DB_PASSWORD:?DB_PASSWORD is required. Copy .env.example to .env and edit it before running docker compose up.} + DB_HOST: validator-db + DB_PORT: ${DB_PORT:-5432} + REDIS_HOST: ${REDIS_HOST:-validator-redis} + REDIS_PORT: ${REDIS_PORT:-6379} + VALIDATION_MODULES_URLS: ${VALIDATION_MODULES_URLS:-http://c5tools:8080/} + + validator-frontend: + build: + context: . + target: frontend + depends_on: + - validator-web + volumes: + - static_data:/var/www/static:ro + ports: + - "80:80" + +volumes: + postgres_data: + static_data: diff --git a/docker/nginx.conf b/docker/nginx.conf new file mode 100644 index 0000000..bba7f4e --- /dev/null +++ b/docker/nginx.conf @@ -0,0 +1,54 @@ +server { + listen 80; + server_name _; + + # Vue SPA assets and entry point + root /usr/share/nginx/html; + index index.html; + + # Some validator endpoints accept file uploads (TR/PR/IR JSON, Excel, etc.) + client_max_body_size 100M; + + # Django/DRF static assets (admin, browseable API). Populated by + # validator-web's `manage.py collectstatic` into the shared volume. + location /static/ { + alias /var/www/static/; + expires 30d; + access_log off; + } + + # User-uploaded media (validation files awaiting cleanup) + location /media/ { + proxy_pass http://validator-web:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + # Django REST API + location /api/ { + proxy_pass http://validator-web:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 300s; + proxy_send_timeout 300s; + } + + # Django admin + location /admin/ { + proxy_pass http://validator-web:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + # Vue SPA fallback — try the requested file, then directory, then index.html + # so client-side routing works on deep links. + location / { + try_files $uri $uri/ /index.html; + } +} diff --git a/docker/web-entrypoint.sh b/docker/web-entrypoint.sh new file mode 100644 index 0000000..acea943 --- /dev/null +++ b/docker/web-entrypoint.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Entrypoint for the validator-web container. +# +# 1. Apply database migrations. +# 2. Collect Django/DRF static assets into the shared /app/static volume +# so the nginx frontend can serve /static/*. +# 3. If DJANGO_SUPERUSER_EMAIL and DJANGO_SUPERUSER_PASSWORD are both set, +# idempotently create (or find) a superuser with a verified email so the +# operator can sign in without running `manage.py createsuperuser` by +# hand. Re-running with the same values is a no-op. +# 4. Hand off to gunicorn. + +set -euo pipefail + +python manage.py migrate --noinput +python manage.py collectstatic --noinput + +if [ -n "${DJANGO_SUPERUSER_EMAIL:-}" ] && [ -n "${DJANGO_SUPERUSER_PASSWORD:-}" ]; then + python manage.py shell <<'PYEOF' +import os +from django.contrib.auth import get_user_model +from allauth.account.models import EmailAddress + +email = os.environ["DJANGO_SUPERUSER_EMAIL"] +password = os.environ["DJANGO_SUPERUSER_PASSWORD"] + +User = get_user_model() +user, created = User.objects.get_or_create( + email=email, + defaults={"is_active": True, "is_superuser": True, "is_staff": True}, +) +if created: + user.set_password(password) + user.save() + EmailAddress.objects.get_or_create( + user=user, + email=email, + defaults={"verified": True, "primary": True}, + ) + print(f"[web-entrypoint] Created superuser: {email}") +else: + # Make sure an existing user is still flagged as superuser+staff with + # a verified email, but don't reset their password. + dirty = False + if not user.is_superuser: + user.is_superuser = True + dirty = True + if not user.is_staff: + user.is_staff = True + dirty = True + if dirty: + user.save() + EmailAddress.objects.get_or_create( + user=user, + email=email, + defaults={"verified": True, "primary": True}, + ) + print(f"[web-entrypoint] Superuser already exists: {email}") +PYEOF +fi + +exec gunicorn config.wsgi:application \ + --bind 0.0.0.0:8000 \ + --workers 2 \ + --access-logfile - diff --git a/docs/own-instance.rst b/docs/own-instance.rst index e347a05..563bd11 100644 --- a/docs/own-instance.rst +++ b/docs/own-instance.rst @@ -20,7 +20,106 @@ applications, you should be able to figure it out. Using Docker ------------ -The easiest way to run the COUNTER Validator is by using Docker. Because the application comprises of several components, -it uses Docker Compose to make it easy to spin up all the necessary services. +The repository ships a `Dockerfile` and `docker-compose.yml` that bring up the +validator and all of its dependencies in a single command. -This method is currently in preparation and will be documented here in the future. +Prerequisites +~~~~~~~~~~~~~ + +* Docker 20.10+ with Docker Compose v2 (`docker compose`). +* About 2 GB of free disk space for the built images. +* Outbound network access on the first build (`compose` clones the + `ubfr/c5tools` repository on the fly). + +Quick start +~~~~~~~~~~~ + +1. Copy the env example and edit at least `SECRET_KEY` and `DB_PASSWORD`: + + .. code-block:: bash + + cp .env.example .env + $EDITOR .env + +2. Build and start the stack: + + .. code-block:: bash + + docker compose up -d + + The ``validator-web`` container runs ``manage.py migrate`` and + ``manage.py collectstatic`` automatically before starting gunicorn, so the + database schema is ready as soon as the container is up. Tail the logs to + watch progress: + + .. code-block:: bash + + docker compose logs -f validator-web + +3. Create an admin user. The easiest path is to set + ``DJANGO_SUPERUSER_EMAIL`` and ``DJANGO_SUPERUSER_PASSWORD`` in ``.env`` + before step 2 — ``validator-web`` auto-creates a superuser (with a + verified email so it can use the API immediately) on first boot, and + is a no-op on subsequent boots. + + If you'd rather create it interactively, do that now: + + .. code-block:: bash + + docker compose exec validator-web python manage.py createsuperuser + +4. (Optional) Load the COUNTER registry data so the platform picker is + populated: + + .. code-block:: bash + + docker compose exec validator-web python manage.py download_registry + +5. Open http://localhost/ in a browser. + +Services +~~~~~~~~ + +The compose file defines six services on a single bridge network: + +* `validator-web` — Django REST API and admin (gunicorn on :8000). +* `validator-worker` — Celery worker consuming `celery,validation` queues. +* `validator-redis` — Redis 7 as the Celery broker. +* `validator-db` — PostgreSQL 16, persisted via the `postgres_data` + named volume so validation history survives `docker compose down`. +* `c5tools` — the PHP validation engine (`ubfr/c5tools + `_). Built from a pinned upstream tag + via the `build.context` git URL in `docker-compose.yml`; bump that tag to + update. +* `validator-frontend` — nginx serving the built Vue SPA on :80 and + reverse-proxying `/api/*`, `/admin/*`, `/media/*` and `/static/*` + to `validator-web`. + +Production deployment +~~~~~~~~~~~~~~~~~~~~~ + +For a production instance: + +* Generate a strong `SECRET_KEY` and `DB_PASSWORD`; do not commit `.env`. +* Set `ALLOWED_HOSTS` and `CSRF_TRUSTED_ORIGINS` to your real hostname(s). +* Put a TLS-terminating reverse proxy (Caddy, nginx, Cloudflare, etc.) in + front of `validator-frontend` on port 80. The COUNTER Code of Practice + requires that COUNTER APIs be served over HTTPS. +* Configure email settings (`EMAIL_HOST` plus `DEFAULT_FROM_EMAIL`, or + the `MAILGUN_*` variables for Mailgun) so the registration flow can + send verification emails. +* Schedule a periodic `docker compose exec validator-web python manage.py + download_registry` (e.g. via cron on the host) to keep the registry data + fresh. + +Updating to a new version +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + git pull + docker compose build + docker compose up -d + +The ``validator-web`` container applies any new migrations automatically on +startup before gunicorn binds. From 69f5cf5a7f8936858845ea3a3d68eb960e9de6ae Mon Sep 17 00:00:00 2001 From: Josh Creek <8179928+jcreek@users.noreply.github.com> Date: Sun, 24 May 2026 15:20:16 +0100 Subject: [PATCH 02/10] build(*): Update node version Co-authored-by: Beda Kosata --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8fa6828..fa031d6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # docker build --target frontend -t counter-validator-frontend . # ---------- frontend-build: produce /frontend/dist ---------- -FROM node:20-alpine AS frontend-build +FROM node:24-alpine AS frontend-build WORKDIR /frontend # .yarnrc.yml sets `nodeLinker: node-modules` for this project; it must be # present before `yarn install` so the right linker mode is picked. From 7b66f1d081e8b0fda0103d04764c33dab04cce6f Mon Sep 17 00:00:00 2001 From: Josh Creek <8179928+jcreek@users.noreply.github.com> Date: Sun, 24 May 2026 15:20:49 +0100 Subject: [PATCH 03/10] build(*): Update docker-compose.yml to use port from env var Co-authored-by: Beda Kosata --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index dd60a29..ad1f06a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -117,7 +117,7 @@ services: volumes: - static_data:/var/www/static:ro ports: - - "80:80" + - "${DOCKER_FRONTEND_PORT:-8888}:80" volumes: postgres_data: From 431aad1111ae425684e7a0631f88ffb2d1ae5618 Mon Sep 17 00:00:00 2001 From: Josh Creek <8179928+jcreek@users.noreply.github.com> Date: Sun, 24 May 2026 15:21:16 +0100 Subject: [PATCH 04/10] build(*): Remove backend port from docker-compose.yml Co-authored-by: Beda Kosata --- docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index ad1f06a..8390af5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -82,8 +82,6 @@ services: DJANGO_SUPERUSER_PASSWORD: "${DJANGO_SUPERUSER_PASSWORD:-}" volumes: - static_data:/app/static - ports: - - "8000:8000" validator-worker: build: From f6f1d1bae4c85026f5ad31f770d83bebadcb95a7 Mon Sep 17 00:00:00 2001 From: Josh Creek <8179928+jcreek@users.noreply.github.com> Date: Sun, 24 May 2026 15:22:54 +0100 Subject: [PATCH 05/10] docs(*): Update .env.example to add frontend port env var Co-authored-by: Beda Kosata --- .env.example | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index db2fa76..dd5b446 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,7 @@ # Copy this file to `.env` and edit the values before `docker compose up`. - +# ---------- Compose / frontend ---------- +# When running local instance using Docker compose, expose the frontend on this port. +DOCKER_FRONTEND_PORT=8888 # ---------- Django ---------- SECRET_KEY=replace-me-with-a-long-random-string DEBUG=False From 52d8cfaacb55c98a07db48263b39d94f4b083c5b Mon Sep 17 00:00:00 2001 From: Josh Creek <8179928+jcreek@users.noreply.github.com> Date: Sun, 24 May 2026 15:23:41 +0100 Subject: [PATCH 06/10] docs(*): Remove inaccurate reference to the COUNTER Code of Practice Co-authored-by: Beda Kosata --- docs/own-instance.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/own-instance.rst b/docs/own-instance.rst index 563bd11..d4f46b6 100644 --- a/docs/own-instance.rst +++ b/docs/own-instance.rst @@ -103,8 +103,7 @@ For a production instance: * Generate a strong `SECRET_KEY` and `DB_PASSWORD`; do not commit `.env`. * Set `ALLOWED_HOSTS` and `CSRF_TRUSTED_ORIGINS` to your real hostname(s). * Put a TLS-terminating reverse proxy (Caddy, nginx, Cloudflare, etc.) in - front of `validator-frontend` on port 80. The COUNTER Code of Practice - requires that COUNTER APIs be served over HTTPS. + front of `validator-frontend`. * Configure email settings (`EMAIL_HOST` plus `DEFAULT_FROM_EMAIL`, or the `MAILGUN_*` variables for Mailgun) so the registration flow can send verification emails. From 2a34b49000c4715749d97046853ad344e27de62d Mon Sep 17 00:00:00 2001 From: Josh Creek <8179928+jcreek@users.noreply.github.com> Date: Sun, 24 May 2026 15:24:08 +0100 Subject: [PATCH 07/10] docs(*): Update .env.example with new default port Co-authored-by: Beda Kosata --- .env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.example b/.env.example index dd5b446..c181fa3 100644 --- a/.env.example +++ b/.env.example @@ -6,7 +6,7 @@ DOCKER_FRONTEND_PORT=8888 SECRET_KEY=replace-me-with-a-long-random-string DEBUG=False ALLOWED_HOSTS=localhost,127.0.0.1 -CSRF_TRUSTED_ORIGINS=http://localhost +CSRF_TRUSTED_ORIGINS=http://localhost:8888 # ---------- Database ---------- DB_NAME=coval From f0fba7f3c40864afccf84c59d48aad4fd9a64ef5 Mon Sep 17 00:00:00 2001 From: Josh Creek <8179928+jcreek@users.noreply.github.com> Date: Sun, 24 May 2026 15:25:11 +0100 Subject: [PATCH 08/10] docs(*): Add clarification for local vs production deployment --- docs/own-instance.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/own-instance.rst b/docs/own-instance.rst index d4f46b6..252f2ce 100644 --- a/docs/own-instance.rst +++ b/docs/own-instance.rst @@ -23,6 +23,8 @@ Using Docker The repository ships a `Dockerfile` and `docker-compose.yml` that bring up the validator and all of its dependencies in a single command. +N.B. The default setup is intended mostly for local experimentation. For a Production deployment, please read the `Production deployment` section. + Prerequisites ~~~~~~~~~~~~~ From 583f4fddd1462fd62411b42dd168f1cc2157d482 Mon Sep 17 00:00:00 2001 From: Josh Creek <8179928+jcreek@users.noreply.github.com> Date: Sun, 24 May 2026 15:25:51 +0100 Subject: [PATCH 09/10] docs(*): Change default ALLOW_USER_REGISTRATION to false in .env.example --- .env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.example b/.env.example index c181fa3..8baee8b 100644 --- a/.env.example +++ b/.env.example @@ -26,7 +26,7 @@ DB_PORT=5432 # VALIDATION_MODULES_URLS=http://c5tools:8080/ # ---------- User account behaviour ---------- -ALLOW_USER_REGISTRATION=True +ALLOW_USER_REGISTRATION=False # ---------- Initial superuser (optional) ---------- # If both are set, validator-web idempotently creates a superuser with a From 7b0e7e256a662004642e5769c77c83732fe6cd19 Mon Sep 17 00:00:00 2001 From: Joshua Creek <8179928+jcreek@users.noreply.github.com> Date: Tue, 30 Jun 2026 18:13:50 +0100 Subject: [PATCH 10/10] fix(*): Fix changelog, email, admin routing, CSRF and line endings This commit makes the self-hosting stack work out of the box by copying CHANGELOG.md into the image so the changelog endpoint no longer 500s, defaulting both web and worker to the console email backend so registration and notification emails work without SMTP, serving the Django admin under /django-admin/ so it no longer shadows the SPA's /admin/* routes, re-resolving validator-web via Docker DNS while forwarding the original Host:port so CSRF checks pass on any published port, and normalising line endings so the container entrypoint runs from Windows checkouts --- .env.example | 26 ++++++++++++++++++++++++-- .gitattributes | 6 ++++++ Dockerfile | 7 ++++++- config/settings/base.py | 4 ++++ docker-compose.yml | 37 +++++++++++++++++++++++++++++++++---- docker/nginx.conf | 41 +++++++++++++++++++++++++++++------------ 6 files changed, 102 insertions(+), 19 deletions(-) create mode 100644 .gitattributes diff --git a/.env.example b/.env.example index 8baee8b..4904aeb 100644 --- a/.env.example +++ b/.env.example @@ -6,8 +6,19 @@ DOCKER_FRONTEND_PORT=8888 SECRET_KEY=replace-me-with-a-long-random-string DEBUG=False ALLOWED_HOSTS=localhost,127.0.0.1 +# Scheme + host:port the frontend is published on. The default tracks +# DOCKER_FRONTEND_PORT automatically; set explicitly if you front the stack +# with a domain or TLS. CSRF_TRUSTED_ORIGINS=http://localhost:8888 +# ---------- Django admin ---------- +# The Django admin is served under /django-admin/ rather than /admin/ so it +# doesn't shadow the SPA's own /admin/* routes (e.g. /admin/users). +# IMPORTANT: docker/nginx.conf hardcodes this exact prefix in its location +# block, so if you change this you MUST update docker/nginx.conf to match, +# or the admin site becomes unreachable. +# DJANGO_ADMIN_PATH=django-admin/ + # ---------- Database ---------- DB_NAME=coval DB_USER=coval @@ -36,12 +47,23 @@ ALLOW_USER_REGISTRATION=False # DJANGO_SUPERUSER_EMAIL=admin@example.com # DJANGO_SUPERUSER_PASSWORD=replace-me-with-a-real-password -# ---------- Email (uncomment if you want validator emails to send) ---------- +# ---------- Email ---------- +# Used by validator-web (registration / verification / password reset) and by +# validator-worker (operator notifications + the daily validation report). +# By default the stack uses the console backend, which prints emails to the +# container logs so registration works without an SMTP server: +# docker compose logs validator-web # registration/verification/reset +# docker compose logs validator-worker # operator + daily-report emails +# +# For REAL delivery, either switch to SMTP (uncomment the lines below and point +# EMAIL_HOST at a reachable server) or configure Mailgun further down. +# EMAIL_BACKEND=django.core.mail.backends.smtp.EmailBackend # EMAIL_HOST=smtp.example.com # DEFAULT_FROM_EMAIL=noreply@example.com # SERVER_EMAIL=noreply@example.com # -# For Mailgun specifically: +# For Mailgun specifically (if MAILGUN_API_KEY is set it takes precedence over +# EMAIL_BACKEND above): # MAILGUN_API_KEY= # MAILGUN_SENDER_DOMAIN= # MAILGUN_API_URL=https://api.eu.mailgun.net/v3 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a3eee32 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# Normalise line endings. Shell scripts MUST stay LF or they break when run in +# Linux containers (a CRLF shebang makes the kernel look for an interpreter that +# does not exist — e.g. `env: 'bash\r': No such file or directory`). This guards +# every checkout regardless of the developer's core.autocrlf setting. +* text=auto eol=lf +*.sh text eol=lf diff --git a/Dockerfile b/Dockerfile index fa031d6..415887a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,6 +39,8 @@ RUN poetry install --no-root --without dev # Copy the application source. COPY manage.py ./ +# Served at runtime by apps/core/changelog.py (reads BASE_DIR / "CHANGELOG.md"). +COPY CHANGELOG.md ./ COPY apps/ ./apps/ COPY config/ ./config/ COPY tools/ ./tools/ @@ -47,7 +49,10 @@ COPY tools/ ./tools/ FROM python-base AS web RUN pip install --no-cache-dir gunicorn COPY docker/web-entrypoint.sh /usr/local/bin/web-entrypoint.sh -RUN chmod +x /usr/local/bin/web-entrypoint.sh +# Strip any CR so the shebang works even when the host checked the script out +# with CRLF line endings (Windows + core.autocrlf), then make it executable. +RUN sed -i 's/\r$//' /usr/local/bin/web-entrypoint.sh \ + && chmod +x /usr/local/bin/web-entrypoint.sh ENV DJANGO_SETTINGS_MODULE=config.settings.production \ STATIC_ROOT=/app/static EXPOSE 8000 diff --git a/config/settings/base.py b/config/settings/base.py index 5946ecd..03af174 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -240,6 +240,10 @@ SERVER_EMAIL = config("SERVER_EMAIL", default="root@localhost") DEFAULT_FROM_EMAIL = config("DEFAULT_FROM_EMAIL", default="root@localhost") EMAIL_HOST = config("EMAIL_HOST", default="localhost") +# Overridable so the self-hosting Docker stack can default to the console +# backend (see docker-compose.yml) and work without an SMTP server. A real +# deployment sets MAILGUN_API_KEY (below) or EMAIL_BACKEND + EMAIL_HOST. +EMAIL_BACKEND = config("EMAIL_BACKEND", default="django.core.mail.backends.smtp.EmailBackend") if MAILGUN_API_KEY := config("MAILGUN_API_KEY", default=""): # if we have the mailgun api key, we activate mailgun diff --git a/docker-compose.yml b/docker-compose.yml index 8390af5..2ae3b7d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,18 @@ # cp .env.example .env # then edit SECRET_KEY and DB_PASSWORD # docker compose up -d # validator-web auto-runs migrate + collectstatic on boot # docker compose exec validator-web python manage.py createsuperuser -# # open http://localhost +# # open http://localhost:8888 (or whatever DOCKER_FRONTEND_PORT you set) + +# Shared email config for the services that send mail: validator-web (registration, +# verification, password reset) and validator-worker (operator notifications + the +# daily report). Defaults to the console backend so everything works out of the box +# without an SMTP server — emails print to the container logs. For real delivery set +# MAILGUN_API_KEY, or EMAIL_BACKEND + EMAIL_HOST. Kept in one anchor so web and +# worker can never drift apart. +x-email-env: &email-env + EMAIL_BACKEND: "${EMAIL_BACKEND:-django.core.mail.backends.console.EmailBackend}" + EMAIL_HOST: "${EMAIL_HOST:-localhost}" + DEFAULT_FROM_EMAIL: "${DEFAULT_FROM_EMAIL:-root@localhost}" services: @@ -64,7 +75,10 @@ services: SECRET_KEY: ${SECRET_KEY:?SECRET_KEY is required. Copy .env.example to .env and edit it before running docker compose up.} DEBUG: "${DEBUG:-False}" ALLOWED_HOSTS: "${ALLOWED_HOSTS:-localhost,127.0.0.1}" - CSRF_TRUSTED_ORIGINS: "${CSRF_TRUSTED_ORIGINS:-http://localhost}" + # Defaults to localhost on DOCKER_FRONTEND_PORT (the nginx config forwards + # the original Host:port so Django's CSRF origin check matches). If you + # front this with a domain or TLS, set CSRF_TRUSTED_ORIGINS explicitly. + CSRF_TRUSTED_ORIGINS: "${CSRF_TRUSTED_ORIGINS:-http://localhost:${DOCKER_FRONTEND_PORT:-8888}}" DB_NAME: ${DB_NAME:-coval} DB_USER: ${DB_USER:-coval} DB_PASSWORD: ${DB_PASSWORD:?DB_PASSWORD is required. Copy .env.example to .env and edit it before running docker compose up.} @@ -74,14 +88,20 @@ services: REDIS_PORT: ${REDIS_PORT:-6379} VALIDATION_MODULES_URLS: ${VALIDATION_MODULES_URLS:-http://c5tools:8080/} ALLOW_USER_REGISTRATION: "${ALLOW_USER_REGISTRATION:-True}" - EMAIL_HOST: "${EMAIL_HOST:-localhost}" - DEFAULT_FROM_EMAIL: "${DEFAULT_FROM_EMAIL:-root@localhost}" + # Served under /django-admin/ so it doesn't shadow the SPA's own /admin/* + # routes (e.g. /admin/users). The nginx config hardcodes this exact prefix + # in its location block, so if you change DJANGO_ADMIN_PATH you MUST update + # docker/nginx.conf to match — otherwise the admin site becomes unreachable. + DJANGO_ADMIN_PATH: "${DJANGO_ADMIN_PATH:-django-admin/}" + # Email backend etc., shared with validator-worker — see x-email-env above. + <<: *email-env # Optional: if both are set, web-entrypoint.sh idempotently creates a # superuser with a verified email on boot. Empty defaults disable it. DJANGO_SUPERUSER_EMAIL: "${DJANGO_SUPERUSER_EMAIL:-}" DJANGO_SUPERUSER_PASSWORD: "${DJANGO_SUPERUSER_PASSWORD:-}" volumes: - static_data:/app/static + - media_data:/app/media validator-worker: build: @@ -105,6 +125,13 @@ services: REDIS_HOST: ${REDIS_HOST:-validator-redis} REDIS_PORT: ${REDIS_PORT:-6379} VALIDATION_MODULES_URLS: ${VALIDATION_MODULES_URLS:-http://c5tools:8080/} + # The worker sends operator-notification and daily-report emails, so it + # needs the same mail config as web (see x-email-env above). Without this + # it would fall back to the SMTP default and fail once operators/admins + # are configured to receive mail. + <<: *email-env + volumes: + - media_data:/app/media validator-frontend: build: @@ -114,9 +141,11 @@ services: - validator-web volumes: - static_data:/var/www/static:ro + - media_data:/var/www/media:ro ports: - "${DOCKER_FRONTEND_PORT:-8888}:80" volumes: postgres_data: static_data: + media_data: diff --git a/docker/nginx.conf b/docker/nginx.conf index bba7f4e..f74caa1 100644 --- a/docker/nginx.conf +++ b/docker/nginx.conf @@ -9,6 +9,13 @@ server { # Some validator endpoints accept file uploads (TR/PR/IR JSON, Excel, etc.) client_max_body_size 100M; + # Docker's embedded DNS. Combined with a variabilised proxy_pass below, this + # makes nginx re-resolve the validator-web service name at request time + # instead of caching its IP at startup — so recreating the web container + # (a routine redeploy) no longer leaves the frontend serving 502s, and the + # frontend can start even if web isn't up yet. + resolver 127.0.0.11 valid=10s ipv6=off; + # Django/DRF static assets (admin, browseable API). Populated by # validator-web's `manage.py collectstatic` into the shared volume. location /static/ { @@ -17,19 +24,26 @@ server { access_log off; } - # User-uploaded media (validation files awaiting cleanup) + # User-uploaded media (validation files awaiting cleanup), served straight + # from the shared media_data volume that validator-web/worker write to. + # Django only serves /media/ when DEBUG=True, so it must be served here. + # NOTE: no per-user auth — any file is downloadable by anyone with its URL + # (unguessable random filename + expiry are the only protection). See the + # security note in docker-compose.yml before exposing this publicly. location /media/ { - proxy_pass http://validator-web:8000; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; + alias /var/www/media/; + access_log off; } # Django REST API location /api/ { - proxy_pass http://validator-web:8000; - proxy_set_header Host $host; + # `set` + variable proxy_pass forces runtime DNS resolution (see resolver + # above). Host carries the original host:port ($http_host, not $host which + # strips the port) so Django's CSRF origin check matches the browser's + # Origin on any published port. + set $upstream_web validator-web; + proxy_pass http://$upstream_web:8000; + proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; @@ -37,10 +51,13 @@ server { proxy_send_timeout 300s; } - # Django admin - location /admin/ { - proxy_pass http://validator-web:8000; - proxy_set_header Host $host; + # Django admin. Served under /django-admin/ (DJANGO_ADMIN_PATH) rather than + # /admin/ so it doesn't shadow the Vue SPA's own /admin/* routes (e.g. + # /admin/users), which must fall through to the SPA index.html below. + location /django-admin/ { + set $upstream_web validator-web; + proxy_pass http://$upstream_web:8000; + proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme;