diff --git a/.env.example b/.env.example index d9d2b55..4904aeb 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,69 @@ -SECRET_KEY=some-random-text +# Copy this file to `.env` and edit the values before `docker compose up`. +# ---------- Compose / frontend ---------- +# When running local instance using Docker compose, expose the frontend on this port. +DOCKER_FRONTEND_PORT=8888 +# ---------- Django ---------- +SECRET_KEY=replace-me-with-a-long-random-string DEBUG=False -ALLOWED_HOSTS='foo.bar, bar.baz' +ALLOWED_HOSTS=localhost,127.0.0.1 +# Scheme + host:port the frontend is published on. The default tracks +# DOCKER_FRONTEND_PORT automatically; set explicitly if you front the stack +# with a domain or TLS. +CSRF_TRUSTED_ORIGINS=http://localhost:8888 + +# ---------- Django admin ---------- +# The Django admin is served under /django-admin/ rather than /admin/ so it +# doesn't shadow the SPA's own /admin/* routes (e.g. /admin/users). +# IMPORTANT: docker/nginx.conf hardcodes this exact prefix in its location +# block, so if you change this you MUST update docker/nginx.conf to match, +# or the admin site becomes unreachable. +# DJANGO_ADMIN_PATH=django-admin/ + +# ---------- Database ---------- DB_NAME=coval -DB_PASSWORD=coval +DB_USER=coval +DB_PASSWORD=replace-me-too DB_PORT=5432 +# DB_HOST is set to the in-compose hostname (validator-db) automatically; +# override only when running validator-web outside of compose. + +# ---------- Redis (in-compose defaults work as-is) ---------- +# REDIS_HOST=validator-redis +# REDIS_PORT=6379 + +# ---------- Validation engine (in-compose c5tools by default) ---------- +# Override to point at a different c5tools instance, or a comma-separated +# list to fan out across multiple instances. +# VALIDATION_MODULES_URLS=http://c5tools:8080/ + +# ---------- User account behaviour ---------- +ALLOW_USER_REGISTRATION=False + +# ---------- Initial superuser (optional) ---------- +# If both are set, validator-web idempotently creates a superuser with a +# verified email on first boot. Re-running `docker compose up` with the same +# values is a no-op. Leave commented out to use `manage.py createsuperuser` +# manually instead. +# DJANGO_SUPERUSER_EMAIL=admin@example.com +# DJANGO_SUPERUSER_PASSWORD=replace-me-with-a-real-password + +# ---------- Email ---------- +# Used by validator-web (registration / verification / password reset) and by +# validator-worker (operator notifications + the daily validation report). +# By default the stack uses the console backend, which prints emails to the +# container logs so registration works without an SMTP server: +# docker compose logs validator-web # registration/verification/reset +# docker compose logs validator-worker # operator + daily-report emails +# +# For REAL delivery, either switch to SMTP (uncomment the lines below and point +# EMAIL_HOST at a reachable server) or configure Mailgun further down. +# EMAIL_BACKEND=django.core.mail.backends.smtp.EmailBackend +# EMAIL_HOST=smtp.example.com +# DEFAULT_FROM_EMAIL=noreply@example.com +# SERVER_EMAIL=noreply@example.com +# +# For Mailgun specifically (if MAILGUN_API_KEY is set it takes precedence over +# EMAIL_BACKEND above): +# MAILGUN_API_KEY= +# MAILGUN_SENDER_DOMAIN= +# MAILGUN_API_URL=https://api.eu.mailgun.net/v3 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..a3eee32 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# Normalise line endings. Shell scripts MUST stay LF or they break when run in +# Linux containers (a CRLF shebang makes the kernel look for an interpreter that +# does not exist — e.g. `env: 'bash\r': No such file or directory`). This guards +# every checkout regardless of the developer's core.autocrlf setting. +* text=auto eol=lf +*.sh text eol=lf diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..415887a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,72 @@ +# Multi-stage Dockerfile for the COUNTER Validator self-hosting stack. +# +# Build targets: +# web Django + gunicorn (REST API + admin), exposes :8000 +# worker Celery worker consuming `celery,validation` queues +# frontend nginx serving the Vue SPA and reverse-proxying the Django app +# +# Built and orchestrated by docker-compose.yml at the repo root. To build a single +# target manually: +# docker build --target web -t counter-validator-web . +# docker build --target worker -t counter-validator-worker . +# docker build --target frontend -t counter-validator-frontend . + +# ---------- frontend-build: produce /frontend/dist ---------- +FROM node:24-alpine AS frontend-build +WORKDIR /frontend +# .yarnrc.yml sets `nodeLinker: node-modules` for this project; it must be +# present before `yarn install` so the right linker mode is picked. +COPY frontend/.yarnrc.yml frontend/package.json frontend/yarn.lock ./ +RUN corepack enable && yarn install --immutable +COPY frontend/ ./ +RUN yarn build + +# ---------- python-base: shared layer for web and worker ---------- +FROM python:3.12 AS python-base +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + POETRY_VIRTUALENVS_CREATE=false \ + POETRY_NO_INTERACTION=1 +RUN apt-get update && apt-get install -y --no-install-recommends \ + libmagic1 \ + && rm -rf /var/lib/apt/lists/* +RUN pip install --no-cache-dir "poetry==1.8.*" +WORKDIR /app + +# Install Python deps first for cache friendliness. +COPY pyproject.toml poetry.lock ./ +RUN poetry install --no-root --without dev + +# Copy the application source. +COPY manage.py ./ +# Served at runtime by apps/core/changelog.py (reads BASE_DIR / "CHANGELOG.md"). +COPY CHANGELOG.md ./ +COPY apps/ ./apps/ +COPY config/ ./config/ +COPY tools/ ./tools/ + +# ---------- web: Django + gunicorn ---------- +FROM python-base AS web +RUN pip install --no-cache-dir gunicorn +COPY docker/web-entrypoint.sh /usr/local/bin/web-entrypoint.sh +# Strip any CR so the shebang works even when the host checked the script out +# with CRLF line endings (Windows + core.autocrlf), then make it executable. +RUN sed -i 's/\r$//' /usr/local/bin/web-entrypoint.sh \ + && chmod +x /usr/local/bin/web-entrypoint.sh +ENV DJANGO_SETTINGS_MODULE=config.settings.production \ + STATIC_ROOT=/app/static +EXPOSE 8000 +# Migrate + collectstatic + (optional) auto-create superuser from +# DJANGO_SUPERUSER_* env, then gunicorn. See docker/web-entrypoint.sh. +CMD ["/usr/local/bin/web-entrypoint.sh"] + +# ---------- worker: Celery ---------- +FROM python-base AS worker +ENV DJANGO_SETTINGS_MODULE=config.settings.production +CMD ["celery", "-A", "config", "worker", "-c", "2", "-Q", "celery,validation", "-l", "INFO"] + +# ---------- frontend: nginx + Vue SPA ---------- +FROM nginx:alpine AS frontend +COPY --from=frontend-build /frontend/dist /usr/share/nginx/html +COPY docker/nginx.conf /etc/nginx/conf.d/default.conf +EXPOSE 80 diff --git a/config/settings/base.py b/config/settings/base.py index 5946ecd..03af174 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -240,6 +240,10 @@ SERVER_EMAIL = config("SERVER_EMAIL", default="root@localhost") DEFAULT_FROM_EMAIL = config("DEFAULT_FROM_EMAIL", default="root@localhost") EMAIL_HOST = config("EMAIL_HOST", default="localhost") +# Overridable so the self-hosting Docker stack can default to the console +# backend (see docker-compose.yml) and work without an SMTP server. A real +# deployment sets MAILGUN_API_KEY (below) or EMAIL_BACKEND + EMAIL_HOST. +EMAIL_BACKEND = config("EMAIL_BACKEND", default="django.core.mail.backends.smtp.EmailBackend") if MAILGUN_API_KEY := config("MAILGUN_API_KEY", default=""): # if we have the mailgun api key, we activate mailgun diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..2ae3b7d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,151 @@ +# Self-hosting compose stack for the COUNTER Validator. +# +# Six services on a single bridge network: +# validator-db Postgres for the validator's persistent state +# validator-redis Celery broker +# c5tools PHP validation engine (built from ubfr/c5tools git URL) +# validator-web Django REST API + admin (gunicorn) +# validator-worker Celery worker +# validator-frontend nginx serving the Vue SPA + reverse-proxying the API +# +# Quick start: +# cp .env.example .env # then edit SECRET_KEY and DB_PASSWORD +# docker compose up -d # validator-web auto-runs migrate + collectstatic on boot +# docker compose exec validator-web python manage.py createsuperuser +# # open http://localhost:8888 (or whatever DOCKER_FRONTEND_PORT you set) + +# Shared email config for the services that send mail: validator-web (registration, +# verification, password reset) and validator-worker (operator notifications + the +# daily report). Defaults to the console backend so everything works out of the box +# without an SMTP server — emails print to the container logs. For real delivery set +# MAILGUN_API_KEY, or EMAIL_BACKEND + EMAIL_HOST. Kept in one anchor so web and +# worker can never drift apart. +x-email-env: &email-env + EMAIL_BACKEND: "${EMAIL_BACKEND:-django.core.mail.backends.console.EmailBackend}" + EMAIL_HOST: "${EMAIL_HOST:-localhost}" + DEFAULT_FROM_EMAIL: "${DEFAULT_FROM_EMAIL:-root@localhost}" + +services: + + validator-db: + image: postgres:16-alpine + volumes: + - postgres_data:/var/lib/postgresql/data + environment: + POSTGRES_DB: ${DB_NAME:-coval} + POSTGRES_USER: ${DB_USER:-coval} + POSTGRES_PASSWORD: ${DB_PASSWORD:?DB_PASSWORD is required. Copy .env.example to .env and edit it before running docker compose up.} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-coval} -d ${DB_NAME:-coval}"] + interval: 5s + timeout: 5s + retries: 20 + + validator-redis: + image: redis:7-alpine + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 5s + retries: 20 + + # c5tools is the PHP validation engine (ubfr/c5tools). The validator's + # Celery worker hard-requires it (see config/settings/base.py: + # VALIDATION_MODULES_URLS). Built from a pinned upstream tag — bump + # the fragment below to update. + c5tools: + build: + context: https://github.com/ubfr/c5tools.git#v0.8.3 + environment: + SSL_MODE: "off" + PHP_OPCACHE_ENABLE: "1" + + validator-web: + build: + context: . + target: web + depends_on: + validator-db: + condition: service_healthy + validator-redis: + condition: service_healthy + c5tools: + condition: service_started + environment: + SECRET_KEY: ${SECRET_KEY:?SECRET_KEY is required. Copy .env.example to .env and edit it before running docker compose up.} + DEBUG: "${DEBUG:-False}" + ALLOWED_HOSTS: "${ALLOWED_HOSTS:-localhost,127.0.0.1}" + # Defaults to localhost on DOCKER_FRONTEND_PORT (the nginx config forwards + # the original Host:port so Django's CSRF origin check matches). If you + # front this with a domain or TLS, set CSRF_TRUSTED_ORIGINS explicitly. + CSRF_TRUSTED_ORIGINS: "${CSRF_TRUSTED_ORIGINS:-http://localhost:${DOCKER_FRONTEND_PORT:-8888}}" + DB_NAME: ${DB_NAME:-coval} + DB_USER: ${DB_USER:-coval} + DB_PASSWORD: ${DB_PASSWORD:?DB_PASSWORD is required. Copy .env.example to .env and edit it before running docker compose up.} + DB_HOST: validator-db + DB_PORT: ${DB_PORT:-5432} + REDIS_HOST: ${REDIS_HOST:-validator-redis} + REDIS_PORT: ${REDIS_PORT:-6379} + VALIDATION_MODULES_URLS: ${VALIDATION_MODULES_URLS:-http://c5tools:8080/} + ALLOW_USER_REGISTRATION: "${ALLOW_USER_REGISTRATION:-True}" + # Served under /django-admin/ so it doesn't shadow the SPA's own /admin/* + # routes (e.g. /admin/users). The nginx config hardcodes this exact prefix + # in its location block, so if you change DJANGO_ADMIN_PATH you MUST update + # docker/nginx.conf to match — otherwise the admin site becomes unreachable. + DJANGO_ADMIN_PATH: "${DJANGO_ADMIN_PATH:-django-admin/}" + # Email backend etc., shared with validator-worker — see x-email-env above. + <<: *email-env + # Optional: if both are set, web-entrypoint.sh idempotently creates a + # superuser with a verified email on boot. Empty defaults disable it. + DJANGO_SUPERUSER_EMAIL: "${DJANGO_SUPERUSER_EMAIL:-}" + DJANGO_SUPERUSER_PASSWORD: "${DJANGO_SUPERUSER_PASSWORD:-}" + volumes: + - static_data:/app/static + - media_data:/app/media + + validator-worker: + build: + context: . + target: worker + depends_on: + validator-db: + condition: service_healthy + validator-redis: + condition: service_healthy + c5tools: + condition: service_started + environment: + SECRET_KEY: ${SECRET_KEY:?SECRET_KEY is required. Copy .env.example to .env and edit it before running docker compose up.} + DEBUG: "${DEBUG:-False}" + DB_NAME: ${DB_NAME:-coval} + DB_USER: ${DB_USER:-coval} + DB_PASSWORD: ${DB_PASSWORD:?DB_PASSWORD is required. Copy .env.example to .env and edit it before running docker compose up.} + DB_HOST: validator-db + DB_PORT: ${DB_PORT:-5432} + REDIS_HOST: ${REDIS_HOST:-validator-redis} + REDIS_PORT: ${REDIS_PORT:-6379} + VALIDATION_MODULES_URLS: ${VALIDATION_MODULES_URLS:-http://c5tools:8080/} + # The worker sends operator-notification and daily-report emails, so it + # needs the same mail config as web (see x-email-env above). Without this + # it would fall back to the SMTP default and fail once operators/admins + # are configured to receive mail. + <<: *email-env + volumes: + - media_data:/app/media + + validator-frontend: + build: + context: . + target: frontend + depends_on: + - validator-web + volumes: + - static_data:/var/www/static:ro + - media_data:/var/www/media:ro + ports: + - "${DOCKER_FRONTEND_PORT:-8888}:80" + +volumes: + postgres_data: + static_data: + media_data: diff --git a/docker/nginx.conf b/docker/nginx.conf new file mode 100644 index 0000000..f74caa1 --- /dev/null +++ b/docker/nginx.conf @@ -0,0 +1,71 @@ +server { + listen 80; + server_name _; + + # Vue SPA assets and entry point + root /usr/share/nginx/html; + index index.html; + + # Some validator endpoints accept file uploads (TR/PR/IR JSON, Excel, etc.) + client_max_body_size 100M; + + # Docker's embedded DNS. Combined with a variabilised proxy_pass below, this + # makes nginx re-resolve the validator-web service name at request time + # instead of caching its IP at startup — so recreating the web container + # (a routine redeploy) no longer leaves the frontend serving 502s, and the + # frontend can start even if web isn't up yet. + resolver 127.0.0.11 valid=10s ipv6=off; + + # Django/DRF static assets (admin, browseable API). Populated by + # validator-web's `manage.py collectstatic` into the shared volume. + location /static/ { + alias /var/www/static/; + expires 30d; + access_log off; + } + + # User-uploaded media (validation files awaiting cleanup), served straight + # from the shared media_data volume that validator-web/worker write to. + # Django only serves /media/ when DEBUG=True, so it must be served here. + # NOTE: no per-user auth — any file is downloadable by anyone with its URL + # (unguessable random filename + expiry are the only protection). See the + # security note in docker-compose.yml before exposing this publicly. + location /media/ { + alias /var/www/media/; + access_log off; + } + + # Django REST API + location /api/ { + # `set` + variable proxy_pass forces runtime DNS resolution (see resolver + # above). Host carries the original host:port ($http_host, not $host which + # strips the port) so Django's CSRF origin check matches the browser's + # Origin on any published port. + set $upstream_web validator-web; + proxy_pass http://$upstream_web:8000; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 300s; + proxy_send_timeout 300s; + } + + # Django admin. Served under /django-admin/ (DJANGO_ADMIN_PATH) rather than + # /admin/ so it doesn't shadow the Vue SPA's own /admin/* routes (e.g. + # /admin/users), which must fall through to the SPA index.html below. + location /django-admin/ { + set $upstream_web validator-web; + proxy_pass http://$upstream_web:8000; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + # Vue SPA fallback — try the requested file, then directory, then index.html + # so client-side routing works on deep links. + location / { + try_files $uri $uri/ /index.html; + } +} diff --git a/docker/web-entrypoint.sh b/docker/web-entrypoint.sh new file mode 100644 index 0000000..acea943 --- /dev/null +++ b/docker/web-entrypoint.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Entrypoint for the validator-web container. +# +# 1. Apply database migrations. +# 2. Collect Django/DRF static assets into the shared /app/static volume +# so the nginx frontend can serve /static/*. +# 3. If DJANGO_SUPERUSER_EMAIL and DJANGO_SUPERUSER_PASSWORD are both set, +# idempotently create (or find) a superuser with a verified email so the +# operator can sign in without running `manage.py createsuperuser` by +# hand. Re-running with the same values is a no-op. +# 4. Hand off to gunicorn. + +set -euo pipefail + +python manage.py migrate --noinput +python manage.py collectstatic --noinput + +if [ -n "${DJANGO_SUPERUSER_EMAIL:-}" ] && [ -n "${DJANGO_SUPERUSER_PASSWORD:-}" ]; then + python manage.py shell <<'PYEOF' +import os +from django.contrib.auth import get_user_model +from allauth.account.models import EmailAddress + +email = os.environ["DJANGO_SUPERUSER_EMAIL"] +password = os.environ["DJANGO_SUPERUSER_PASSWORD"] + +User = get_user_model() +user, created = User.objects.get_or_create( + email=email, + defaults={"is_active": True, "is_superuser": True, "is_staff": True}, +) +if created: + user.set_password(password) + user.save() + EmailAddress.objects.get_or_create( + user=user, + email=email, + defaults={"verified": True, "primary": True}, + ) + print(f"[web-entrypoint] Created superuser: {email}") +else: + # Make sure an existing user is still flagged as superuser+staff with + # a verified email, but don't reset their password. + dirty = False + if not user.is_superuser: + user.is_superuser = True + dirty = True + if not user.is_staff: + user.is_staff = True + dirty = True + if dirty: + user.save() + EmailAddress.objects.get_or_create( + user=user, + email=email, + defaults={"verified": True, "primary": True}, + ) + print(f"[web-entrypoint] Superuser already exists: {email}") +PYEOF +fi + +exec gunicorn config.wsgi:application \ + --bind 0.0.0.0:8000 \ + --workers 2 \ + --access-logfile - diff --git a/docs/own-instance.rst b/docs/own-instance.rst index e347a05..252f2ce 100644 --- a/docs/own-instance.rst +++ b/docs/own-instance.rst @@ -20,7 +20,107 @@ applications, you should be able to figure it out. Using Docker ------------ -The easiest way to run the COUNTER Validator is by using Docker. Because the application comprises of several components, -it uses Docker Compose to make it easy to spin up all the necessary services. +The repository ships a `Dockerfile` and `docker-compose.yml` that bring up the +validator and all of its dependencies in a single command. -This method is currently in preparation and will be documented here in the future. +N.B. The default setup is intended mostly for local experimentation. For a Production deployment, please read the `Production deployment` section. + +Prerequisites +~~~~~~~~~~~~~ + +* Docker 20.10+ with Docker Compose v2 (`docker compose`). +* About 2 GB of free disk space for the built images. +* Outbound network access on the first build (`compose` clones the + `ubfr/c5tools` repository on the fly). + +Quick start +~~~~~~~~~~~ + +1. Copy the env example and edit at least `SECRET_KEY` and `DB_PASSWORD`: + + .. code-block:: bash + + cp .env.example .env + $EDITOR .env + +2. Build and start the stack: + + .. code-block:: bash + + docker compose up -d + + The ``validator-web`` container runs ``manage.py migrate`` and + ``manage.py collectstatic`` automatically before starting gunicorn, so the + database schema is ready as soon as the container is up. Tail the logs to + watch progress: + + .. code-block:: bash + + docker compose logs -f validator-web + +3. Create an admin user. The easiest path is to set + ``DJANGO_SUPERUSER_EMAIL`` and ``DJANGO_SUPERUSER_PASSWORD`` in ``.env`` + before step 2 — ``validator-web`` auto-creates a superuser (with a + verified email so it can use the API immediately) on first boot, and + is a no-op on subsequent boots. + + If you'd rather create it interactively, do that now: + + .. code-block:: bash + + docker compose exec validator-web python manage.py createsuperuser + +4. (Optional) Load the COUNTER registry data so the platform picker is + populated: + + .. code-block:: bash + + docker compose exec validator-web python manage.py download_registry + +5. Open http://localhost/ in a browser. + +Services +~~~~~~~~ + +The compose file defines six services on a single bridge network: + +* `validator-web` — Django REST API and admin (gunicorn on :8000). +* `validator-worker` — Celery worker consuming `celery,validation` queues. +* `validator-redis` — Redis 7 as the Celery broker. +* `validator-db` — PostgreSQL 16, persisted via the `postgres_data` + named volume so validation history survives `docker compose down`. +* `c5tools` — the PHP validation engine (`ubfr/c5tools + `_). Built from a pinned upstream tag + via the `build.context` git URL in `docker-compose.yml`; bump that tag to + update. +* `validator-frontend` — nginx serving the built Vue SPA on :80 and + reverse-proxying `/api/*`, `/admin/*`, `/media/*` and `/static/*` + to `validator-web`. + +Production deployment +~~~~~~~~~~~~~~~~~~~~~ + +For a production instance: + +* Generate a strong `SECRET_KEY` and `DB_PASSWORD`; do not commit `.env`. +* Set `ALLOWED_HOSTS` and `CSRF_TRUSTED_ORIGINS` to your real hostname(s). +* Put a TLS-terminating reverse proxy (Caddy, nginx, Cloudflare, etc.) in + front of `validator-frontend`. +* Configure email settings (`EMAIL_HOST` plus `DEFAULT_FROM_EMAIL`, or + the `MAILGUN_*` variables for Mailgun) so the registration flow can + send verification emails. +* Schedule a periodic `docker compose exec validator-web python manage.py + download_registry` (e.g. via cron on the host) to keep the registry data + fresh. + +Updating to a new version +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: bash + + git pull + docker compose build + docker compose up -d + +The ``validator-web`` container applies any new migrations automatically on +startup before gunicorn binds.