Draft: Sync development into main for next release

.github/workflows/lint.yml

@@ -26,4 +26,4 @@ jobs:
 
       - name: Run linter
         run: |
-          ruff check src/ --output-format=github
+          ruff check app/ --output-format=github

.gitignore

@@ -26,4 +26,7 @@ src/inputs/*.pdf
 frontend/release/
 
 # Local Claude Code instructions
-CLAUDE.md
+CLAUDE.md
+
+*temp/
+*.env*

CONTRIBUTING.md

@@ -39,25 +39,14 @@ If you have a great idea for FireForm, we'd love to hear it! Please open an issu
 4. Make sure your code lints.
 5. Issue that pull request!
 
-## 🛠️ Local Development Setup
-
-FireForm uses Docker and Docker Compose for the backend to ensure a consistent environment.
+**Issues are not formally assigned.** You are free to pick any open issue, work on it, and raise a PR directly. If multiple PRs address the same issue, the first one that actually fixes it generally gets preference. To avoid duplicating someone else's work, coordinate with other contributors on our [Discord](https://discord.gg/nBv5b6kF68) before starting.
 
-### Prerequisites
+## 💬 Community
 
-- [Docker](https://docs.docker.com/get-docker/)
-- [Docker Compose](https://docs.docker.com/compose/install/)
-- `make` (optional, but recommended)
-- [Node.js](https://nodejs.org/) 20+ (only needed for the desktop app)
+Join our Discord server to ask questions, discuss issues, and coordinate work with other contributors: https://discord.gg/nBv5b6kF68
 
-### Desktop App Development
-
-The frontend is a vanilla HTML/CSS/JS app wrapped in Electron. To run it locally:
+## 🛠️ Local Development Setup
 
-```bash
-cd frontend
-npm install    # one-time setup
-npm start      # launches the Electron desktop window
-```
+See the [Setup Guide](docs/1.%20SETUP.md) for the full walkthrough: prerequisites, running the backend with Docker, testing endpoints via Swagger UI, day-to-day commands, and troubleshooting.
 
-The backend (API + Ollama) must be running separately via Docker — see `make fireform`.
+Before writing code, read the [Project Structure](docs/2.%20PROJECT_STRUCTURE.md) guide — it explains how the codebase is organized and where new code should go.

Makefile

@@ -1,8 +1,10 @@
-.PHONY: help build up down logs shell exec pull-model test clean fireform logs-app logs-ollama logs-frontend super-clean
+.PHONY: help init fireform build up down logs logs-app logs-ollama shell pull-model test clean super-clean
 
-# The extraction model pulled into Ollama and used by src/llm.py. Override with
-# `make pull-model OLLAMA_MODEL=...`. A 1.5B model keeps per-field fills fast.
-OLLAMA_MODEL ?= qwen2.5:1.5b
+COMPOSE     = docker compose -f docker/dev/compose.yml --env-file docker/.env.dev
+ENV_DEV     = docker/.env.dev
+
+# Read OLLAMA_MODEL from .env.dev at runtime; fall back to default if file absent.
+OLLAMA_MODEL = $(shell grep -E '^OLLAMA_MODEL=' $(ENV_DEV) 2>/dev/null | cut -d= -f2 | tr -d '[:space:]' || echo qwen2.5:1.5b)
 
 help:
 	@printf '%s\n' \
@@ -13,74 +15,88 @@ help:
 	'/_/    /_//_/   \___/ /_/    \____/_/  /_/ /_/ /_/ ' \
 	''
 	@echo ""
-	@echo "Fireform Development Commands"
+	@echo "FireForm Development Commands"
 	@echo "=============================="
-	@echo "make fireform     - Build and start containers, then open a shell"
+	@echo "make init         - First-time setup: check deps, create .env.dev, pick model"
+	@echo "make fireform     - Build images, start containers, pull Ollama model"
 	@echo "make build        - Build Docker images"
-	@echo "make up           - Start all containers"
+	@echo "make up           - Start all containers (detached)"
 	@echo "make down         - Stop all containers"
-	@echo "make logs         - View container logs"
-	@echo "make logs-app     - View API container logs"
-	@echo "make logs-frontend - View frontend container logs"
-	@echo "make logs-ollama  - View Ollama container logs"
-	@echo "make shell        - Open Python shell in app container"
-	@echo "make exec         - Execute Python script in container"
-	@echo "make pull-model   - Pull the extraction model ($(OLLAMA_MODEL)) into Ollama"
-	@echo "make test         - Run tests"
-	@echo "make clean        - Remove containers"
-	@echo "make super-clean  - [CAUTION] Use carefully. Cleans up ALL stopped  containers, networks, build cache..."
-
-# Fix #382 — pull-model is now part of the main setup flow
-# The extraction model is pulled automatically before you need it
-fireform: build up pull-model
+	@echo "make logs         - Stream all container logs"
+	@echo "make logs-app     - Stream app container logs"
+	@echo "make logs-ollama  - Stream Ollama container logs"
+	@echo "make logs-worker  - Stream Celery worker logs"
+	@echo "make shell        - Open shell in running app container"
+	@echo "make pull-model   - Pull Ollama model from .env.dev ($(OLLAMA_MODEL))"
+	@echo "make test         - Run test suite"
+	@echo "make clean        - Stop containers (preserves volumes)"
+	@echo "make super-clean  - [CAUTION] Stop containers, delete volumes, prune Docker"
+
+init:
+	@chmod +x scripts/check-deps.sh scripts/init-env.sh scripts/select-model.sh
+	@sh scripts/check-deps.sh
+	@sh scripts/init-env.sh
+	@sh scripts/select-model.sh
+	@printf "Build containers and pull model now? [y/N] "; \
+	read answer; \
+	case "$$answer" in \
+		[yY]*) $(MAKE) fireform ;; \
+		*) echo "Run 'make fireform' when ready." ;; \
+	esac
+
+fireform: build up
+	@printf "Waiting for Ollama to be ready..."
+	@until $(COMPOSE) exec -T ollama ollama list > /dev/null 2>&1; do \
+		printf '.'; sleep 2; \
+	done
+	@echo " ready."
+	@if $(COMPOSE) exec -T ollama ollama list 2>/dev/null | grep -q "^$(OLLAMA_MODEL)"; then \
+		echo "  Model $(OLLAMA_MODEL) already pulled."; \
+	else \
+		echo "  Pulling $(OLLAMA_MODEL)..."; \
+		$(COMPOSE) exec -T ollama ollama pull $(OLLAMA_MODEL); \
+	fi
 	@echo ""
-	@echo "✅ FireForm is ready!"
-	@echo "   Frontend: http://localhost:5173"
+	@echo "FireForm is ready!"
 	@echo "   API:      http://localhost:8000"
 	@echo "   API Docs: http://localhost:8000/docs"
 	@echo ""
 	@echo "Run 'make logs' to view live logs, 'make down' to stop."
 
 build:
-	docker compose build
+	@$(COMPOSE) build
 
 up:
-	docker compose up -d
+	@$(COMPOSE) up -d
 
 down:
-	docker compose down
+	@$(COMPOSE) down --remove-orphans
 
 logs:
-	docker compose logs -f
+	@$(COMPOSE) logs -f
 
 logs-app:
-	docker compose logs -f app
+	@$(COMPOSE) logs -f app
 
 logs-ollama:
-	docker compose logs -f ollama
+	@$(COMPOSE) logs -f ollama
 
-logs-frontend:
-	docker compose logs -f frontend
+logs-worker:
+	@$(COMPOSE) logs -f celery-worker
 
 shell:
-	docker compose exec app /bin/bash
-
-# Start the FastAPI server inside the running container
-run:
-	docker compose exec app uvicorn api.main:app --host 0.0.0.0 --port 8000 --reload
-
-exec:
-	docker compose exec app python3 src/main.py
+	@$(COMPOSE) exec app /bin/sh
 
 pull-model:
-	docker compose exec ollama ollama pull $(OLLAMA_MODEL)
+	@$(COMPOSE) exec -T ollama ollama pull $(OLLAMA_MODEL)
 
-# Fix — correct test directory (was src/test/ which doesn't exist)
 test:
-	docker compose exec app python3 -m pytest tests/ -v
+	@$(COMPOSE) exec -T app python3 -m pytest tests/ -v
 
 clean:
-	docker compose down -v
+	@$(COMPOSE) down
+
 super-clean:
-	docker compose down -v
-	docker system prune 
+	@echo "WARNING: this will delete all volumes (database, uploads, model weights)."
+	@$(COMPOSE) down -v
+	@docker system prune -f