From af2b9c255142ce4e1887ac3550a5990af4ead652 Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Wed, 18 Feb 2026 11:38:51 +0100 Subject: [PATCH 1/9] fix(ci): run publish only on tag or push on main, run lint/tests/build on any push Co-authored-by: Cursor # Conflicts: # .circleci/config.yml --- .circleci/config.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index b35c907..029f0e5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,6 +8,12 @@ parameters: publish-branch: type: string default: "main" + python-module: + type: string + default: "api_tabular" + api-port: + type: string + default: "8005" jobs: lint: From e024a2234efd78a829b1443d9e7149bb8c14318f Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Tue, 17 Feb 2026 17:46:24 +0100 Subject: [PATCH 2/9] build: add Dockerfile --- Dockerfile | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..5689450 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +FROM astral/uv:python3.11-trixie-slim + +# install needed apt packages +RUN apt-get update -y && \ + apt-get install -y --no-install-recommends git && \ + rm -rf /var/lib/apt/lists/* + +# create user & group +RUN groupadd --system datagouv && \ + useradd --system --gid datagouv --create-home datagouv + +# install +WORKDIR /home/datagouv +ADD . /home/datagouv/ +RUN uv sync --frozen +RUN chown -R datagouv:datagouv /home/datagouv + +# run +USER datagouv +# Use `python -m gunicorn` instead of `gunicorn` due to uv issue #15246: https://github.com/astral-sh/uv/issues/15246 +ENTRYPOINT ["uv", "run", "python", "-m", "gunicorn"] +# Gunicorn config: 2 workers for ~1 vCPU allocation, aiohttp.GunicornWebWorker for async support, default timeouts suitable for async workers +CMD ["api_tabular.tabular.app:app_factory", "--bind", "0.0.0.0:8005", "--worker-class", "aiohttp.GunicornWebWorker", "--workers", "2", "--access-logfile", "-"] From b8167aa3564e36b8aa6be9ce1cf092e602f64314 Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Tue, 17 Feb 2026 17:51:22 +0100 Subject: [PATCH 3/9] ci: refactor CI to build on main, send to docker registry, and publish on pypo on tag # Conflicts: # .circleci/config.yml --- .circleci/config.yml | 51 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 029f0e5..4fe2314 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,6 +14,12 @@ parameters: api-port: type: string default: "8005" + docker-registry-url: + type: string + default: "registry.gitlab.com/etalab/data.gouv.fr/infra" + docker-image-name: + type: string + default: "tabular-api" jobs: lint: @@ -70,13 +76,21 @@ jobs: path: reports/python build: - docker: - - image: ghcr.io/astral-sh/uv:python<< pipeline.parameters.python-version >>-trixie + machine: + image: ubuntu-2404:current steps: - checkout - run: - name: Compute RELEASE_VERSION via setuptools_scm + name: Install uv and Python << pipeline.parameters.python-version >> + command: | + curl -LsSf https://astral.sh/uv/install.sh | sh + export PATH="$HOME/.local/bin:$PATH" + uv python install << pipeline.parameters.python-version >> + uv sync --frozen + - run: + name: Compute RELEASE_VERSION via setuptools_scm and build wheel command: | + export PATH="$HOME/.local/bin:$PATH" uv pip install --system setuptools-scm # derive from setuptools_scm or base version + build number RELEASE_VERSION=$(python -m setuptools_scm) @@ -84,19 +98,38 @@ jobs: echo "Build number: $CIRCLE_BUILD_NUM" echo "Commit hash: ${CIRCLE_SHA1:0:7}" echo "Git tag: $CIRCLE_TAG" - - run: - name: Build a distributable package as a wheel release - command: | uv build --wheel - # Build already executed above; artifacts are in dist/ - store_artifacts: path: dist - persist_to_workspace: root: . paths: - . + - run: + name: Set Docker image tag + command: | + if [ -n "$CIRCLE_TAG" ]; then + export DOCKER_IMAGE_TAG="$CIRCLE_TAG" + elif [ -n "$CIRCLE_BRANCH" ]; then + export DOCKER_IMAGE_TAG="$CIRCLE_BRANCH" + else + export DOCKER_IMAGE_TAG="${CIRCLE_SHA1:0:7}" + fi + echo "DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG" >> "$BASH_ENV" + - run: + name: Log in to Docker registry + command: | + echo "${DOCKER_REGISTRY_PASSWORD}" | docker login -u "${DOCKER_REGISTRY_USER}" --password-stdin registry.gitlab.com + - run: + name: Build and push Docker image + command: | + source "$BASH_ENV" + export PATH="$HOME/.local/bin:$PATH" + IMAGE="<< pipeline.parameters.docker-registry-url >>/<< pipeline.parameters.docker-image-name >>:${DOCKER_IMAGE_TAG}" + docker build -t "$IMAGE" . + docker push "$IMAGE" - publish: + publish-pypi: docker: - image: ghcr.io/astral-sh/uv:python<< pipeline.parameters.python-version >>-trixie-slim steps: @@ -116,7 +149,7 @@ workflows: requires: - lint - tests - - publish: + - publish-pypi: requires: - build filters: From 144e52f358be32b9a5754fcec0ee450e6808f1e2 Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Tue, 17 Feb 2026 18:36:06 +0100 Subject: [PATCH 4/9] ci: tag docker image the same way we tag release --- .circleci/config.yml | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4fe2314..125b6ca 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -88,7 +88,7 @@ jobs: uv python install << pipeline.parameters.python-version >> uv sync --frozen - run: - name: Compute RELEASE_VERSION via setuptools_scm and build wheel + name: Compute RELEASE_VERSION via setuptools_scm, and build wheel command: | export PATH="$HOME/.local/bin:$PATH" uv pip install --system setuptools-scm @@ -98,6 +98,7 @@ jobs: echo "Build number: $CIRCLE_BUILD_NUM" echo "Commit hash: ${CIRCLE_SHA1:0:7}" echo "Git tag: $CIRCLE_TAG" + echo "RELEASE_VERSION=$RELEASE_VERSION" >> "$BASH_ENV" uv build --wheel - store_artifacts: path: dist @@ -105,17 +106,6 @@ jobs: root: . paths: - . - - run: - name: Set Docker image tag - command: | - if [ -n "$CIRCLE_TAG" ]; then - export DOCKER_IMAGE_TAG="$CIRCLE_TAG" - elif [ -n "$CIRCLE_BRANCH" ]; then - export DOCKER_IMAGE_TAG="$CIRCLE_BRANCH" - else - export DOCKER_IMAGE_TAG="${CIRCLE_SHA1:0:7}" - fi - echo "DOCKER_IMAGE_TAG=$DOCKER_IMAGE_TAG" >> "$BASH_ENV" - run: name: Log in to Docker registry command: | @@ -125,7 +115,9 @@ jobs: command: | source "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" - IMAGE="<< pipeline.parameters.docker-registry-url >>/<< pipeline.parameters.docker-image-name >>:${DOCKER_IMAGE_TAG}" + # Docker tags allow only [a-zA-Z0-9_.-]; setuptools_scm can output e.g. 0.4.0.dev5+gabc1234 + DOCKER_TAG=$(echo "$RELEASE_VERSION" | tr '+' '-') + IMAGE="<< pipeline.parameters.docker-registry-url >>/<< pipeline.parameters.docker-image-name >>:${DOCKER_TAG}" docker build -t "$IMAGE" . docker push "$IMAGE" From d6b51ae12ce353e2e99b71b373a9b03056dfbe8d Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Tue, 17 Feb 2026 19:21:11 +0100 Subject: [PATCH 5/9] build: also build Metrics API --- .circleci/config.yml | 17 ++++++++++++----- Dockerfile | 12 ++++++++---- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 125b6ca..7f57fa3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,9 +17,12 @@ parameters: docker-registry-url: type: string default: "registry.gitlab.com/etalab/data.gouv.fr/infra" - docker-image-name: + docker-image-name-tabular: type: string default: "tabular-api" + docker-image-name-metrics: + type: string + default: "metrics-api" jobs: lint: @@ -111,15 +114,19 @@ jobs: command: | echo "${DOCKER_REGISTRY_PASSWORD}" | docker login -u "${DOCKER_REGISTRY_USER}" --password-stdin registry.gitlab.com - run: - name: Build and push Docker image + name: Build and push Docker images (Tabular API and Metrics API) command: | source "$BASH_ENV" export PATH="$HOME/.local/bin:$PATH" + REGISTRY="<< pipeline.parameters.docker-registry-url >>" # Docker tags allow only [a-zA-Z0-9_.-]; setuptools_scm can output e.g. 0.4.0.dev5+gabc1234 DOCKER_TAG=$(echo "$RELEASE_VERSION" | tr '+' '-') - IMAGE="<< pipeline.parameters.docker-registry-url >>/<< pipeline.parameters.docker-image-name >>:${DOCKER_TAG}" - docker build -t "$IMAGE" . - docker push "$IMAGE" + # Tabular API: tabular endpoints only + docker build --build-arg APP_MODULE=api_tabular.tabular.app:app_factory -t "${REGISTRY}/<< pipeline.parameters.docker-image-name-tabular >>:${DOCKER_TAG}" . + docker push "${REGISTRY}/<< pipeline.parameters.docker-image-name-tabular >>:${DOCKER_TAG}" + # Metrics API: metrics endpoints only + docker build --build-arg APP_MODULE=api_tabular.metrics.app:app_factory -t "${REGISTRY}/<< pipeline.parameters.docker-image-name-metrics >>:${DOCKER_TAG}" . + docker push "${REGISTRY}/<< pipeline.parameters.docker-image-name-metrics >>:${DOCKER_TAG}" publish-pypi: docker: diff --git a/Dockerfile b/Dockerfile index 5689450..1daaedb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,8 @@ FROM astral/uv:python3.11-trixie-slim +# Which app to run (e.g. api_tabular.tabular.app:app_factory or api_tabular.metrics.app:app_factory) +ARG APP_MODULE=api_tabular.tabular.app:app_factory + # install needed apt packages RUN apt-get update -y && \ apt-get install -y --no-install-recommends git && \ @@ -15,9 +18,10 @@ ADD . /home/datagouv/ RUN uv sync --frozen RUN chown -R datagouv:datagouv /home/datagouv -# run +# run (ENV from ARG so shell can expand APP_MODULE at runtime) USER datagouv +ENV APP_MODULE=${APP_MODULE} # Use `python -m gunicorn` instead of `gunicorn` due to uv issue #15246: https://github.com/astral-sh/uv/issues/15246 -ENTRYPOINT ["uv", "run", "python", "-m", "gunicorn"] -# Gunicorn config: 2 workers for ~1 vCPU allocation, aiohttp.GunicornWebWorker for async support, default timeouts suitable for async workers -CMD ["api_tabular.tabular.app:app_factory", "--bind", "0.0.0.0:8005", "--worker-class", "aiohttp.GunicornWebWorker", "--workers", "2", "--access-logfile", "-"] +# Shell so APP_MODULE is expanded; bind to 8005 (map to different host ports when running both containers) +ENTRYPOINT ["/bin/sh", "-c"] +CMD ["uv run python -m gunicorn $APP_MODULE --bind 0.0.0.0:8005 --worker-class aiohttp.GunicornWebWorker --workers 2 --access-logfile -"] From c72157e7918258ecf2d12930d2b6467b0378b502 Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Tue, 17 Feb 2026 20:00:51 +0100 Subject: [PATCH 6/9] build: update docker compose --- docker-compose.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 57ffa5c..469a9b8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,3 +47,35 @@ services: - POSTGRES_PASSWORD=csvapi profiles: - test + + # Tabular API (run locally via Docker, uses same Dockerfile as CI) + tabular-api: + build: + context: . + dockerfile: Dockerfile + args: + APP_MODULE: api_tabular.tabular.app:app_factory + ports: + - "8005:8005" + environment: + - PGREST_ENDPOINT=http://postgrest-test:8080 + depends_on: + - postgrest-test + profiles: + - test + + # Metrics API (run locally via Docker, uses same Dockerfile as CI) + metrics-api: + build: + context: . + dockerfile: Dockerfile + args: + APP_MODULE: api_tabular.metrics.app:app_factory + ports: + - "8006:8005" + environment: + - PGREST_ENDPOINT=http://postgrest-test:8080 + depends_on: + - postgrest-test + profiles: + - test From 61aefa11772b6144ddb2ca56abe64e2bb82a98c8 Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Tue, 17 Feb 2026 20:00:58 +0100 Subject: [PATCH 7/9] docs: update README --- README.md | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index dac3375..57a6fc5 100644 --- a/README.md +++ b/README.md @@ -18,26 +18,39 @@ The production API is deployed on data.gouv.fr infrastructure at [`https://tabul - **[uv](https://docs.astral.sh/uv/)** for dependency management - **Docker & Docker Compose** +### Docker Compose profiles + +| Profile | Use when | Services started | +|----------|----------|------------------| +| **test** | Run locally with the test database (PostgreSQL + PostgREST + optional Tabular API + Metrics API in Docker) | `postgres-test`, `postgrest-test`, `tabular-api`, `metrics-api` | +| **hydra** | Use a real Hydra CSV database; PostgREST only in Docker, run the API with uv on the host | `postgrest` | + +**Run everything locally in Docker:** `docker compose --profile test up -d` โ†’ Tabular API at http://localhost:8005, Metrics API at http://localhost:8006, PostgREST at http://localhost:8080. + ### ๐Ÿงช Run with a test database -1. **Start the Infrastructure** +1. **Start the stack** + + With the **test** profile, you can either run the full stack in Docker (recommended for a quick local run) or only the infrastructure and run the API with uv (for development with hot reload). - Start the test CSV database and test PostgREST container: + **Option A โ€“ All in Docker (easiest for running locally):** ```shell docker compose --profile test up -d ``` - The `--profile test` flag tells Docker Compose to start the PostgREST and PostgreSQL services for the test CSV database. This starts PostgREST on port 8080, connecting to the test CSV database. You can access the raw PostgREST API on http://localhost:8080. + This starts the test PostgreSQL, PostgREST, Tabular API (port 8005), and Metrics API (port 8006). You can use the APIs at http://localhost:8005 and http://localhost:8006. -2. **Launch the main API proxy** - - Install dependencies and start the proxy services: + **Option B โ€“ Only infrastructure in Docker, API with uv (for development):** + ```shell + docker compose --profile test up -d postgres-test postgrest-test + ``` + PostgREST is available at http://localhost:8080. Then start the API proxy on the host: ```shell uv sync - uv run adev runserver -p8005 api_tabular/tabular/app.py # Api related to apified CSV files by udata-hydra (dev server) - uv run adev runserver -p8006 api_tabular/metrics/app.py # Api related to udata's metrics (dev server) + uv run adev runserver -p8005 api_tabular/tabular/app.py # Tabular API (dev server) + uv run adev runserver -p8006 api_tabular/metrics/app.py # Metrics API (dev server) ``` - **Note:** For production, use gunicorn with aiohttp worker: + **Note:** For production (on the host), use gunicorn with aiohttp worker: ```shell # Tabular API (port 8005) uv run gunicorn api_tabular.tabular.app:app_factory \ @@ -54,9 +67,9 @@ The production API is deployed on data.gouv.fr infrastructure at [`https://tabul --access-logfile - ``` - The main API provides a controlled layer over PostgREST - exposing PostgREST directly would be too permissive, so this adds a security and access control layer. + The API provides a controlled layer over PostgREST (security and access control); exposing PostgREST directly would be too permissive. -3. **Test the API** +2. **Test the API** Query the API using a `resource_id`. Several test resources are available in the fake database: From b0e8d758d695bc0c4becbc9410f9e61a63fbdc50 Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Wed, 18 Feb 2026 12:19:10 +0100 Subject: [PATCH 8/9] ci: build/publish to docker registry only when publishing to PyPi --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7f57fa3..49ca220 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -148,6 +148,9 @@ workflows: requires: - lint - tests + filters: + branches: + only: << pipeline.parameters.publish-branch >> - publish-pypi: requires: - build From e98ebdf81a80a38775a18490f86361846b89188c Mon Sep 17 00:00:00 2001 From: Adrien Carpentier Date: Tue, 24 Feb 2026 16:09:22 +0100 Subject: [PATCH 9/9] ci: fix docker login --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 49ca220..0e62015 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -112,7 +112,7 @@ jobs: - run: name: Log in to Docker registry command: | - echo "${DOCKER_REGISTRY_PASSWORD}" | docker login -u "${DOCKER_REGISTRY_USER}" --password-stdin registry.gitlab.com + echo "${DOCKER_REGISTRY_PASSWORD}" | docker login -u "oauth2" --password-stdin registry.gitlab.com - run: name: Build and push Docker images (Tabular API and Metrics API) command: | @@ -151,6 +151,7 @@ workflows: filters: branches: only: << pipeline.parameters.publish-branch >> + context: org-global - publish-pypi: requires: - build