diff --git a/CLAUDE.md b/CLAUDE.md index 6fb4ef6e..e5919167 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -56,9 +56,9 @@ submitqueue/ # repo root (Go module github.com/uber/submi ├── runway/ # Runway domain (single service — the domain *is* the service) │ └── controller/ # Runway service controllers (consumes the merge queues; no gateway/orchestrator split) ├── tool/ # Development and CI tooling -├── example/ +├── service/ # Runnable server/client wiring (entry points + Docker Compose) │ ├── submitqueue/ # Runnable SubmitQueue servers/clients + Docker Compose -│ ├── stovepipe/ # Runnable Stovepipe servers/clients +│ ├── stovepipe/ # Runnable Stovepipe server/client + Docker Compose │ └── runway/ # Runnable Runway server/client + Docker Compose ├── test/ │ ├── e2e/submitqueue/ # End-to-end tests (full stack) @@ -132,7 +132,7 @@ Vendor-agnostic, pluggable interfaces with implementations in subdirectories: A `{domain}/extension/{ext}` or `platform/extension/{ext}` package contains the behavioral interface, its `Config`, the `Factory` *interface*, and impl constructors `New(...)` that return the interface. It must **not** contain `Factory` *implementations* (`NewFactory()` constructors or factory structs) or any queue-selection logic. -Why: an impl package (e.g. `scorer/heuristic`) can't know the queue topology or the other impls, so a "which impl for which queue" decision doesn't belong there. Per-queue routing — and the small adapters that wrap a `New(...)` impl in the `Factory` interface — live in the wiring layer (e.g. `example/{domain}/{service}/server/main.go`), the one place that knows the full queue set. That's where you route on `Config.QueueName`. +Why: an impl package (e.g. `scorer/heuristic`) can't know the queue topology or the other impls, so a "which impl for which queue" decision doesn't belong there. Per-queue routing — and the small adapters that wrap a `New(...)` impl in the `Factory` interface — live in the wiring layer (e.g. `service/{domain}/{service}/server/main.go`), the one place that knows the full queue set. That's where you route on `Config.QueueName`. Rule of thumb: if you're about to add a `NewFactory()` or a `map[queue]impl` under `{domain}/extension/` or `platform/extension/`, it belongs in the wiring layer instead. @@ -239,11 +239,11 @@ make clean # Clean Bazel cache **Add new RPC method:** 1. Edit `api/{domain}/{service}/proto/*.proto` → `make proto` 2. Add controller in `{domain}/{service}/controller/` -3. Wire up in `example/{domain}/{service}/server/main.go` +3. Wire up in `service/{domain}/{service}/server/main.go` **Add new queue message controller:** 1. Create `{domain}/{service}/controller/{step}/` implementing `consumer.Controller` -2. Wire up in `example/{domain}/{service}/server/main.go` +2. Wire up in `service/{domain}/{service}/server/main.go` **Add new extension:** 1. Create the extension under `{domain}/extension/{ext}/{impl}/` (domain-specific, e.g. `submitqueue/extension/...`) or `platform/extension/{ext}/{impl}/` (shared across domains) with factory and interfaces diff --git a/Makefile b/Makefile index 92546046..4adf3464 100644 --- a/Makefile +++ b/Makefile @@ -5,21 +5,21 @@ BAZEL = ./tool/bazel COMPOSE = docker-compose # SubmitQueue compose files -COMPOSE_FILE = example/submitqueue/docker-compose.yml -GATEWAY_COMPOSE_FILE = example/submitqueue/gateway/server/docker-compose.yml -ORCHESTRATOR_COMPOSE_FILE = example/submitqueue/orchestrator/server/docker-compose.yml +COMPOSE_FILE = service/submitqueue/docker-compose.yml +GATEWAY_COMPOSE_FILE = service/submitqueue/gateway/server/docker-compose.yml +ORCHESTRATOR_COMPOSE_FILE = service/submitqueue/orchestrator/server/docker-compose.yml # Fixed project name for local manual testing (tests use unique random names) SUBMITQUEUE_LOCAL_PROJECT = submitqueue # Stovepipe compose file (single Ping-only service) -STOVEPIPE_COMPOSE_FILE = example/stovepipe/docker-compose.yml +STOVEPIPE_COMPOSE_FILE = service/stovepipe/docker-compose.yml # Fixed project name for local manual testing (tests use unique random names) STOVEPIPE_LOCAL_PROJECT = stovepipe # Runway compose files -RUNWAY_COMPOSE_FILE = example/runway/server/docker-compose.yml +RUNWAY_COMPOSE_FILE = service/runway/server/docker-compose.yml # Fixed project name for local manual testing (tests use unique random names) RUNWAY_LOCAL_PROJECT = runway @@ -65,34 +65,34 @@ build-all-linux: build-submitqueue-gateway-linux build-submitqueue-orchestrator- build-runway-linux: ## Build Runway Linux binary for Docker @echo "Building Runway Linux binary for Docker..." - @$(BAZEL) build --platforms=@rules_go//go/toolchain:linux_amd64 //example/runway/server:runway + @$(BAZEL) build --platforms=@rules_go//go/toolchain:linux_amd64 //service/runway/server:runway @mkdir -p .docker-bin - @cp -f bazel-bin/example/runway/server/runway_/runway .docker-bin/runway 2>/dev/null || \ - cp -f bazel-bin/example/runway/server/runway .docker-bin/runway + @cp -f bazel-bin/service/runway/server/runway_/runway .docker-bin/runway 2>/dev/null || \ + cp -f bazel-bin/service/runway/server/runway .docker-bin/runway @echo "Runway Linux binary ready at .docker-bin/runway" build-submitqueue-gateway-linux: ## Build Gateway Linux binary for Docker @echo "Building Gateway Linux binary for Docker..." - @$(BAZEL) build --platforms=@rules_go//go/toolchain:linux_amd64 //example/submitqueue/gateway/server:gateway + @$(BAZEL) build --platforms=@rules_go//go/toolchain:linux_amd64 //service/submitqueue/gateway/server:gateway @mkdir -p .docker-bin - @cp -f bazel-bin/example/submitqueue/gateway/server/gateway_/gateway .docker-bin/gateway 2>/dev/null || \ - cp -f bazel-bin/example/submitqueue/gateway/server/gateway .docker-bin/gateway + @cp -f bazel-bin/service/submitqueue/gateway/server/gateway_/gateway .docker-bin/gateway 2>/dev/null || \ + cp -f bazel-bin/service/submitqueue/gateway/server/gateway .docker-bin/gateway @echo "Gateway Linux binary ready at .docker-bin/gateway" build-submitqueue-orchestrator-linux: ## Build Orchestrator Linux binary for Docker @echo "Building Orchestrator Linux binary for Docker..." - @$(BAZEL) build --platforms=@rules_go//go/toolchain:linux_amd64 //example/submitqueue/orchestrator/server:orchestrator + @$(BAZEL) build --platforms=@rules_go//go/toolchain:linux_amd64 //service/submitqueue/orchestrator/server:orchestrator @mkdir -p .docker-bin - @cp -f bazel-bin/example/submitqueue/orchestrator/server/orchestrator_/orchestrator .docker-bin/orchestrator 2>/dev/null || \ - cp -f bazel-bin/example/submitqueue/orchestrator/server/orchestrator .docker-bin/orchestrator + @cp -f bazel-bin/service/submitqueue/orchestrator/server/orchestrator_/orchestrator .docker-bin/orchestrator 2>/dev/null || \ + cp -f bazel-bin/service/submitqueue/orchestrator/server/orchestrator .docker-bin/orchestrator @echo "Orchestrator Linux binary ready at .docker-bin/orchestrator" build-stovepipe-linux: ## Build Stovepipe Linux binary for Docker @echo "Building Stovepipe Linux binary for Docker..." - @$(BAZEL) build --platforms=@rules_go//go/toolchain:linux_amd64 //example/stovepipe/server:stovepipe + @$(BAZEL) build --platforms=@rules_go//go/toolchain:linux_amd64 //service/stovepipe/server:stovepipe @mkdir -p .docker-bin - @cp -f bazel-bin/example/stovepipe/server/stovepipe_/stovepipe .docker-bin/stovepipe 2>/dev/null || \ - cp -f bazel-bin/example/stovepipe/server/stovepipe .docker-bin/stovepipe + @cp -f bazel-bin/service/stovepipe/server/stovepipe_/stovepipe .docker-bin/stovepipe 2>/dev/null || \ + cp -f bazel-bin/service/stovepipe/server/stovepipe .docker-bin/stovepipe @echo "Stovepipe Linux binary ready at .docker-bin/stovepipe" check-gazelle: ## Check BUILD.bazel files are up to date @@ -361,26 +361,26 @@ proto: ## Generate protobuf files from .proto definitions # Bazel query helpers query-deps: - @$(BAZEL) query 'deps(//example/submitqueue/gateway/server:gateway)' + @$(BAZEL) query 'deps(//service/submitqueue/gateway/server:gateway)' query-targets: @$(BAZEL) query //... # Run gateway client (connects to any running gateway service) run-client-submitqueue-gateway: - @$(BAZEL) run //example/submitqueue/gateway/client:gateway -- -addr $(or $(SERVER_ADDR),localhost:8081) -message "$(or $(MESSAGE),ping)" + @$(BAZEL) run //service/submitqueue/gateway/client:gateway -- -addr $(or $(SERVER_ADDR),localhost:8081) -message "$(or $(MESSAGE),ping)" # Run orchestrator client (connects to any running orchestrator service) run-client-submitqueue-orchestrator: - @$(BAZEL) run //example/submitqueue/orchestrator/client:orchestrator -- -addr $(or $(SERVER_ADDR),localhost:8082) -message "$(or $(MESSAGE),ping)" + @$(BAZEL) run //service/submitqueue/orchestrator/client:orchestrator -- -addr $(or $(SERVER_ADDR),localhost:8082) -message "$(or $(MESSAGE),ping)" # Run stovepipe client (connects to any running stovepipe service) run-client-stovepipe: - @$(BAZEL) run //example/stovepipe/client:stovepipe -- -addr $(or $(SERVER_ADDR),localhost:8083) -message "$(or $(MESSAGE),ping)" + @$(BAZEL) run //service/stovepipe/client:stovepipe -- -addr $(or $(SERVER_ADDR),localhost:8083) -message "$(or $(MESSAGE),ping)" # Run runway client (connects to any running runway service) run-client-runway: - @$(BAZEL) run //example/runway/client:runway -- -addr $(or $(SERVER_ADDR),localhost:8086) -message "$(or $(MESSAGE),ping)" + @$(BAZEL) run //service/runway/client:runway -- -addr $(or $(SERVER_ADDR),localhost:8086) -message "$(or $(MESSAGE),ping)" run-queue-admin: ## Run queue-admin CLI (use ARGS to pass arguments, e.g. make run-queue-admin ARGS="list-topics") @$(BAZEL) run //platform/extension/messagequeue/mysql/ctl -- $(ARGS) diff --git a/README.md b/README.md index dbc81ab6..bbae9246 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ grpcurl -plaintext -d '{"message": "hello"}' localhost:8081 uber.submitqueue.gat make local-stop ``` -See [example/README.md](example/README.md) for more examples including running individual services and clients. +See [service/README.md](service/README.md) for more examples including running individual services and clients. ## Documentation @@ -43,7 +43,7 @@ See [example/README.md](example/README.md) for more examples including running i | [Contributing](CONTRIBUTING.md) | How to contribute, workflow, guidelines | | [Testing Guide](doc/howto/TESTING.md) | Unit, integration, and E2E testing patterns | | [Architecture Guide](CLAUDE.md) | Project layout, patterns, conventions | -| [Examples](example/README.md) | Running services, clients, API reference | +| [Examples](service/README.md) | Running services, clients, API reference | | [RFCs](doc/rfc/index.md) | Design documents and proposals | ## Project Status diff --git a/doc/howto/TESTING.md b/doc/howto/TESTING.md index 0d051197..d2839740 100644 --- a/doc/howto/TESTING.md +++ b/doc/howto/TESTING.md @@ -388,6 +388,6 @@ assert.Equal(s.T(), "expected", resp.Value) ## See Also - [CLAUDE.md](../../CLAUDE.md) - Development guidelines and project structure -- [example/submitqueue/docker-compose.yml](../../example/submitqueue/docker-compose.yml) - Full stack service definitions -- [example/submitqueue/gateway/server/docker-compose.yml](../../example/submitqueue/gateway/server/docker-compose.yml) - Gateway isolation -- [example/submitqueue/orchestrator/server/docker-compose.yml](../../example/submitqueue/orchestrator/server/docker-compose.yml) - Orchestrator isolation +- [service/submitqueue/docker-compose.yml](../../service/submitqueue/docker-compose.yml) - Full stack service definitions +- [service/submitqueue/gateway/server/docker-compose.yml](../../service/submitqueue/gateway/server/docker-compose.yml) - Gateway isolation +- [service/submitqueue/orchestrator/server/docker-compose.yml](../../service/submitqueue/orchestrator/server/docker-compose.yml) - Orchestrator isolation diff --git a/doc/rfc/submitqueue/extension-contract.md b/doc/rfc/submitqueue/extension-contract.md index 98245322..15d006cb 100644 --- a/doc/rfc/submitqueue/extension-contract.md +++ b/doc/rfc/submitqueue/extension-contract.md @@ -52,7 +52,7 @@ Non-obvious points: ## Mechanism -Dependencies are injected per-extension at the existing `Factory.For` (wiring: `example/submitqueue/orchestrator/server/main.go`) — only the handles a contract justifies, never the whole storage aggregator. The repeated batch→changes walk becomes one shared resolver (today's duplicated `collectChanges`, consolidated, and preserving the batch boundaries build's copy flattens). Controllers shrink to passing the identity entity they already load. +Dependencies are injected per-extension at the existing `Factory.For` (wiring: `service/submitqueue/orchestrator/server/main.go`) — only the handles a contract justifies, never the whole storage aggregator. The repeated batch→changes walk becomes one shared resolver (today's duplicated `collectChanges`, consolidated, and preserving the batch boundaries build's copy flattens). Controllers shrink to passing the identity entity they already load. `entity.BatchChanges` is kept, not removed — it becomes the shared resolver's *detailed output* (URIs + provider details for a batch, what the scorer consumes) rather than a value the score controller assembles and passes in. Its line/file helpers move with it; only its producer changes. diff --git a/example/README.md b/example/README.md deleted file mode 100644 index 106a6386..00000000 --- a/example/README.md +++ /dev/null @@ -1,153 +0,0 @@ -# Examples - -Example gRPC servers and clients for running the submitqueue services locally. - -## Services - -- **SubmitQueue Gateway** (port 8081) — entry point for land requests. Exposes `Ping` and `Land` RPCs. -- **SubmitQueue Orchestrator** (port 8082) — coordinates the pipeline. Exposes `Ping` RPC and consumes queue messages across 9 pipeline topics. -- **Stovepipe** (port 8083) - single Ping-only service. Exposes `Ping` RPC. - -Services require MySQL (app database + queue database). Docker Compose handles this automatically. - -## Directory Structure - -``` -example/ -├── submitqueue/ -│ ├── docker-compose.yml # Full stack (Gateway + Orchestrator + 2x MySQL) -│ ├── gateway/ -│ │ ├── server/ -│ │ │ ├── main.go # Gateway server entry point -│ │ │ ├── Dockerfile -│ │ │ └── docker-compose.yml # Gateway-only stack -│ │ └── client/main.go # Gateway ping client -│ └── orchestrator/ -│ ├── server/ -│ │ ├── main.go # Orchestrator server entry point -│ │ ├── Dockerfile -│ │ └── docker-compose.yml # Orchestrator-only stack -│ └── client/main.go # Orchestrator ping client -└── stovepipe/ - ├── docker-compose.yml # Single Ping-only service stack - ├── server/ - │ ├── main.go # Stovepipe gRPC server (Docker: :8080; go run default :8083) - │ └── Dockerfile - └── client/main.go # Stovepipe ping client -``` - -## Running - -### Docker Compose (recommended) - - -```bash -# Start full SubmitQueue stack (Gateway + Orchestrator + MySQL) -make local-submitqueue-start -make local-submitqueue-gateway-start -make local-submitqueue-orchestrator-start - -# Start Stovepipe service (single Ping-only gRPC service) -make local-stovepipe-start - -# View logs and status -make local-submitqueue-logs -make local-submitqueue-ps - -# Stop (SubmitQueue + Stovepipe default projects) -make local-stop -``` - -For Docker, `make build-stovepipe-linux` copies a Linux binary to `.docker-bin/stovepipe` so it does not overwrite SubmitQueue’s `.docker-bin/gateway`. Stovepipe is currently a single Ping-only service with no storage or queue dependencies, so the compose stack has no MySQL. The compose service key is **`stovepipe-service`**, so with default project **`stovepipe`** you should see **`stovepipe-stovepipe-service-1`**. `make local-stop` stops the SubmitQueue, Stovepipe, and Runway stacks; `make local-stovepipe-stop` stops only Stovepipe. - -### Bazel - -```bash -# Build servers -bazel build //example/submitqueue/gateway/server:gateway -bazel build //example/submitqueue/orchestrator/server:orchestrator -bazel build //example/stovepipe/server:stovepipe - -# Build clients -bazel build //example/submitqueue/gateway/client:gateway -bazel build //example/submitqueue/orchestrator/client:orchestrator -bazel build //example/stovepipe/client:stovepipe -``` - -### Go - -```bash -go run example/submitqueue/gateway/server/main.go -go run example/submitqueue/orchestrator/server/main.go -go run example/stovepipe/server/main.go -``` - -## Testing with Clients - -```bash -# Go clients -go run example/submitqueue/gateway/client/main.go -addr localhost:8081 -message "hello" -go run example/submitqueue/orchestrator/client/main.go -addr localhost:8082 -message "hello" -go run example/stovepipe/client/main.go -addr localhost:8083 -message "hello" -``` - -Client flags: -- `-addr`: Server address (default: service-specific port) -- `-message`: Message to send in the ping request -- `-timeout`: Request timeout (default: 5s) - -### grpcurl - -Install grpcurl if you don't have it: -```bash -brew install grpcurl # macOS -# or -go install github.com/fullstorydev/grpcurl/cmd/grpcurl@latest -``` - -```bash -# Ping -grpcurl -plaintext -d '{"message": "hello"}' localhost:8081 uber.submitqueue.gateway.SubmitQueueGateway/Ping -grpcurl -plaintext -d '{"message": "hello"}' localhost:8082 uber.submitqueue.orchestrator.SubmitQueueOrchestrator/Ping -grpcurl -plaintext -d '{"message": "hello"}' localhost:8083 uber.submitqueue.stovepipe.Stovepipe/Ping - -# List services -grpcurl -plaintext localhost:8081 list -grpcurl -plaintext localhost:8082 list -grpcurl -plaintext localhost:8083 list - -# Describe a service -grpcurl -plaintext localhost:8081 describe uber.submitqueue.gateway.SubmitQueueGateway -grpcurl -plaintext localhost:8082 describe uber.submitqueue.orchestrator.SubmitQueueOrchestrator -grpcurl -plaintext localhost:8083 describe uber.submitqueue.stovepipe.Stovepipe -``` - -## API Reference - -### Gateway Service - -**Service**: `uber.submitqueue.gateway.SubmitQueueGateway` -**Proto**: `api/submitqueue/gateway/proto/gateway.proto` - -| Method | Description | -|--------|-------------| -| `Ping` | Health check, returns service name and timestamp | -| `Land` | Submit a land request for code changes | - -### Orchestrator Service - -**Service**: `uber.submitqueue.orchestrator.SubmitQueueOrchestrator` -**Proto**: `api/submitqueue/orchestrator/proto/orchestrator.proto` - -| Method | Description | -|--------|-------------| -| `Ping` | Health check, returns service name and timestamp | - -### Stovepipe - -**Service**: `uber.submitqueue.stovepipe.Stovepipe` -**Proto**: `api/stovepipe/proto/stovepipe.proto` - -| Method | Description | -|--------|-------------| -| `Ping` | Health check | diff --git a/service/README.md b/service/README.md new file mode 100644 index 00000000..35e9fe08 --- /dev/null +++ b/service/README.md @@ -0,0 +1,176 @@ +# Service Wiring + +Runnable gRPC servers and clients that wire each domain's controllers and extensions together and run them locally. This is the composition root — the one layer that knows the full queue topology, picks concrete extension implementations, and owns per-queue routing. Controllers, entities, and extensions stay transport- and wiring-agnostic; everything here turns them into a process you can start. + +Each domain has its own subdirectory with a dedicated README: + +- [`submitqueue/`](submitqueue/README.md) — the multi-service SubmitQueue domain (Gateway + Orchestrator). +- [`stovepipe/`](stovepipe/README.md) — the single-service Stovepipe domain (ingest → process pipeline). +- [`runway/`](runway/README.md) — the single-service Runway landing service (consumes the merge queues). + +## Services + +| Service | Port | Domain | RPCs | Backing stores | +|---------|------|--------|------|----------------| +| **SubmitQueue Gateway** | 8081 | `submitqueue` | `Ping`, `Land` | MySQL app + queue | +| **SubmitQueue Orchestrator** | 8082 | `submitqueue` | `Ping` (+ consumes 9 pipeline topics) | MySQL app + queue | +| **Stovepipe** | 8083 | `stovepipe` | `Ping`, `Ingest` (+ consumes the process topic) | MySQL storage + queue | +| **Runway** | 8086 | `runway` | `Ping` (+ consumes merge-conflict-check & merge topics) | MySQL queue | + +Ports above are the `go run` defaults; under Docker Compose each server listens on `:8080` inside its container and is published on a random ephemeral host port (use `make local-*-ps` / `docker port` to discover it). + +## Directory Structure + +``` +service/ +├── submitqueue/ +│ ├── docker-compose.yml # Full stack (Gateway + Orchestrator + 2x MySQL) +│ ├── gateway/ +│ │ ├── server/ # Gateway server entry point + Dockerfile + compose +│ │ │ └── queues.yaml # Per-queue extension profiles +│ │ └── client/ # Gateway ping client +│ └── orchestrator/ +│ ├── server/ # Orchestrator server entry point + Dockerfile + compose +│ └── client/ # Orchestrator ping client +├── stovepipe/ +│ ├── docker-compose.yml # Stovepipe service + storage MySQL + queue MySQL +│ ├── server/ # Stovepipe gRPC server + Dockerfile +│ └── client/ # Stovepipe ping client +└── runway/ + ├── server/ # Runway gRPC server + Dockerfile + compose + └── client/ # Runway ping client +``` + +## Running + +### Docker Compose (recommended) + +```bash +# Full SubmitQueue stack (Gateway + Orchestrator + MySQL) +make local-submitqueue-start +make local-submitqueue-gateway-start # gateway-only stack +make local-submitqueue-orchestrator-start # orchestrator-only stack + +# Stovepipe service (gRPC service + storage MySQL + queue MySQL) +make local-stovepipe-start + +# Runway service (consumer + queue MySQL) +make local-runway-start + +# Logs and status (SubmitQueue) +make local-submitqueue-logs +make local-submitqueue-ps + +# Stop everything (SubmitQueue + Stovepipe + Runway) +make local-stop +``` + +`make local-stop` stops the SubmitQueue, Stovepipe, and Runway stacks; the per-domain `make local-stovepipe-stop` / `make local-runway-stop` targets stop just one. Each `build-*-linux` target copies a distinct Linux binary into `.docker-bin/` so the compose stacks don't clobber each other's artifacts. + +### Bazel + +```bash +# Servers +bazel build //service/submitqueue/gateway/server:gateway +bazel build //service/submitqueue/orchestrator/server:orchestrator +bazel build //service/stovepipe/server:stovepipe +bazel build //service/runway/server:runway + +# Clients +bazel build //service/submitqueue/gateway/client:gateway +bazel build //service/submitqueue/orchestrator/client:orchestrator +bazel build //service/stovepipe/client:stovepipe +bazel build //service/runway/client:runway +``` + +### Go + +```bash +go run ./service/submitqueue/gateway/server +go run ./service/submitqueue/orchestrator/server +go run ./service/stovepipe/server +go run ./service/runway/server +``` + +## Testing with Clients + +```bash +# Go clients +go run ./service/submitqueue/gateway/client -addr localhost:8081 -message "hello" +go run ./service/submitqueue/orchestrator/client -addr localhost:8082 -message "hello" +go run ./service/stovepipe/client -addr localhost:8083 -message "hello" +go run ./service/runway/client -addr localhost:8086 -message "hello" + +# Bazel-run clients (honor SERVER_ADDR / MESSAGE) +make run-client-submitqueue-gateway +make run-client-submitqueue-orchestrator +make run-client-stovepipe +make run-client-runway +``` + +Client flags: +- `-addr`: Server address (default: service-specific port) +- `-message`: Message to send in the ping request +- `-timeout`: Request timeout (default: 5s) + +### grpcurl + +Install grpcurl if you don't have it: +```bash +brew install grpcurl # macOS +# or +go install github.com/fullstorydev/grpcurl/cmd/grpcurl@latest +``` + +```bash +# Ping +grpcurl -plaintext -d '{"message": "hello"}' localhost:8081 uber.submitqueue.gateway.SubmitQueueGateway/Ping +grpcurl -plaintext -d '{"message": "hello"}' localhost:8082 uber.submitqueue.orchestrator.SubmitQueueOrchestrator/Ping +grpcurl -plaintext -d '{"message": "hello"}' localhost:8083 uber.submitqueue.stovepipe.Stovepipe/Ping +grpcurl -plaintext -d '{"message": "hello"}' localhost:8086 uber.runway.Runway/Ping + +# List / describe services (reflection is registered on every server) +grpcurl -plaintext localhost:8081 list +grpcurl -plaintext localhost:8081 describe uber.submitqueue.gateway.SubmitQueueGateway +``` + +## API Reference + +### Gateway Service + +**Service**: `uber.submitqueue.gateway.SubmitQueueGateway` +**Proto**: `api/submitqueue/gateway/proto/gateway.proto` + +| Method | Description | +|--------|-------------| +| `Ping` | Health check, returns service name and timestamp | +| `Land` | Submit a land request for code changes | + +### Orchestrator Service + +**Service**: `uber.submitqueue.orchestrator.SubmitQueueOrchestrator` +**Proto**: `api/submitqueue/orchestrator/proto/orchestrator.proto` + +| Method | Description | +|--------|-------------| +| `Ping` | Health check, returns service name and timestamp | + +### Stovepipe + +**Service**: `uber.submitqueue.stovepipe.Stovepipe` +**Proto**: `api/stovepipe/proto/stovepipe.proto` + +| Method | Description | +|--------|-------------| +| `Ping` | Health check | +| `Ingest` | Ingest a queue's head commit, persist the Request, and publish it to the process stage | + +### Runway + +**Service**: `uber.runway.Runway` +**Proto**: `api/runway/proto/runway.proto` + +| Method | Description | +|--------|-------------| +| `Ping` | Health check | + diff --git a/service/runway/README.md b/service/runway/README.md new file mode 100644 index 00000000..a60348e5 --- /dev/null +++ b/service/runway/README.md @@ -0,0 +1,72 @@ +# Runway Service + +Runnable wiring for the **Runway** domain — a single-service, consumer-only landing service. Runway has no gateway/orchestrator split: the domain *is* the service. It exposes a thin `Ping` RPC for health checks and does its real work as a queue consumer, draining the merge pipeline queues that the SubmitQueue orchestrator publishes to. + +## What it does + +Runway registers two consuming subscriptions against the shared MySQL-backed message queue: + +- **merge-conflict-check** (`TopicKeyMergeConflictCheck`) — handled by `runway/controller/mergeconflictcheck`. +- **merge** (`TopicKeyMerge`) — handled by `runway/controller/merge`. + +These topic keys and their wire contracts are owned by the queue's producer side and published under `api/runway/messagequeue/` (the external, cross-domain contract). The corresponding signal queues where Runway will publish results are not wired yet. + +Because Runway only consumes queues and serves `Ping`, it needs a **queue** database but no application/storage database. + +## Layout + +``` +runway/ +├── server/ +│ ├── main.go # gRPC server (Ping) + primary consumer wiring +│ ├── Dockerfile +│ └── docker-compose.yml # Runway service + queue MySQL +└── client/ + └── main.go # Ping client (default :8086) +``` + +The Runway controllers themselves live under [`runway/controller/`](../../runway/controller); this directory only contains the runnable wiring and a Docker Compose stack for manual testing. + +## Configuration + +| Variable | Required | Description | Default | +|-------------------|----------|-----------------------------------------------|--------------------------| +| `QUEUE_MYSQL_DSN` | yes | Queue database DSN | — | +| `PORT` | no | gRPC listen address | `:8086` | +| `HOSTNAME` | no | Subscriber name for the queue consumer | `runway-` | + +## Running + +### Docker Compose (recommended) + +```bash +make local-runway-start # builds the Linux binary, starts runway + queue MySQL, applies the queue schema +make local-runway-stop # tears the stack down +``` + +`local-runway-start` prints the ephemeral host ports for the gRPC server and the queue MySQL. Only the queue schema is applied — there is no Runway app schema. + +### Bazel / Go + +```bash +bazel build //service/runway/server:runway +bazel build //service/runway/client:runway + +go run ./service/runway/server +``` + +## Testing the Ping RPC + +```bash +go run ./service/runway/client -addr localhost:8086 -message "hello" +# or +make run-client-runway SERVER_ADDR=localhost:8086 MESSAGE=hello + +# grpcurl +grpcurl -plaintext -d '{"message": "hello"}' localhost:8086 uber.runway.Runway/Ping +``` + +## Shutdown + +The server handles `SIGINT` / `SIGTERM` gracefully: it drains in-flight RPCs, then stops the queue consumer (30s timeout). It exits `0` on clean shutdown, `143` (128 + SIGTERM) when stopped by signal, and `1` on startup/runtime errors (details on stderr). Shutdown errors override the signal exit code. + diff --git a/example/runway/client/BUILD.bazel b/service/runway/client/BUILD.bazel similarity index 86% rename from example/runway/client/BUILD.bazel rename to service/runway/client/BUILD.bazel index 12dad7a9..e16263d2 100644 --- a/example/runway/client/BUILD.bazel +++ b/service/runway/client/BUILD.bazel @@ -3,7 +3,7 @@ load("@rules_go//go:def.bzl", "go_binary", "go_library") go_library( name = "client_lib", srcs = ["main.go"], - importpath = "github.com/uber/submitqueue/example/runway/client", + importpath = "github.com/uber/submitqueue/service/runway/client", visibility = ["//visibility:private"], deps = [ "//api/runway/protopb", diff --git a/example/runway/client/main.go b/service/runway/client/main.go similarity index 100% rename from example/runway/client/main.go rename to service/runway/client/main.go diff --git a/example/runway/server/BUILD.bazel b/service/runway/server/BUILD.bazel similarity index 93% rename from example/runway/server/BUILD.bazel rename to service/runway/server/BUILD.bazel index e8bb1200..240c9d8a 100644 --- a/example/runway/server/BUILD.bazel +++ b/service/runway/server/BUILD.bazel @@ -3,7 +3,7 @@ load("@rules_go//go:def.bzl", "go_binary", "go_library") go_library( name = "server_lib", srcs = ["main.go"], - importpath = "github.com/uber/submitqueue/example/runway/server", + importpath = "github.com/uber/submitqueue/service/runway/server", visibility = ["//visibility:private"], deps = [ "//api/runway/messagequeue", diff --git a/example/runway/server/Dockerfile b/service/runway/server/Dockerfile similarity index 100% rename from example/runway/server/Dockerfile rename to service/runway/server/Dockerfile diff --git a/example/runway/server/docker-compose.yml b/service/runway/server/docker-compose.yml similarity index 94% rename from example/runway/server/docker-compose.yml rename to service/runway/server/docker-compose.yml index e900cb06..d39026ab 100644 --- a/example/runway/server/docker-compose.yml +++ b/service/runway/server/docker-compose.yml @@ -4,7 +4,7 @@ # IMPORTANT: Before running compose, build the Linux binary: # make build-runway-linux # OR -# bazel build --platforms=@rules_go//go/toolchain:linux_amd64 //example/runway/server:runway +# bazel build --platforms=@rules_go//go/toolchain:linux_amd64 //service/runway/server:runway # # Quick start: # make local-runway-start @@ -33,7 +33,7 @@ services: runway-service: build: context: ${REPO_ROOT} - dockerfile: example/runway/server/Dockerfile + dockerfile: service/runway/server/Dockerfile ports: - "8080" # Random ephemeral port to avoid conflicts environment: diff --git a/example/runway/server/main.go b/service/runway/server/main.go similarity index 100% rename from example/runway/server/main.go rename to service/runway/server/main.go diff --git a/example/stovepipe/BUILD.bazel b/service/stovepipe/BUILD.bazel similarity index 100% rename from example/stovepipe/BUILD.bazel rename to service/stovepipe/BUILD.bazel diff --git a/service/stovepipe/README.md b/service/stovepipe/README.md new file mode 100644 index 00000000..a6554f3e --- /dev/null +++ b/service/stovepipe/README.md @@ -0,0 +1,78 @@ +# Stovepipe Service + +Runnable wiring for the **Stovepipe** domain — a single-service domain (the domain *is* the service). The server exposes two RPCs and runs one internal pipeline stage as a queue consumer: + +- **`Ping`** — health check. +- **`Ingest`** — resolves a queue's head commit, persists a `Request` (and its head URI) to storage, and publishes the request to the **process** stage. +- **process consumer** (`TopicKeyProcess`) — reloads the persisted `Request` from storage and runs the process stage (`stovepipe/controller/process`). + +The ingest → process hop stays inside one service and one store, so only the request **ID** travels on the queue; the consumer reloads from storage (the source of truth), which keeps messages small and redelivery idempotent. The process topic key and its internal wire contract are owned by the domain under `stovepipe/core/messagequeue/`. + +Stovepipe therefore needs two MySQL databases: a **storage** database (the `request` and `request_uri` tables) and a **queue** database (messaging infrastructure). + +## Wiring notes + +`server/main.go` is the composition root and supplies the concrete extension implementations. Two are deliberately demo-only and must be replaced for any real deployment: + +- **`inMemoryCounter`** — a process-local `counter.Counter` for sequence numbers; not durable. A real deployment uses a persistent implementation (e.g. `platform/extension/counter/mysql`). +- **`fakeSourceControlFactory`** — seeds each queue with a deterministic single-commit history so ingest resolves a stable head URI (and re-ingesting the same queue exercises the dedup path). A real deployment supplies a VCS-backed `sourcecontrol.Factory`. + +## Layout + +``` +stovepipe/ +├── docker-compose.yml # Stovepipe service + storage MySQL + queue MySQL +├── server/ +│ ├── main.go # gRPC server (Ping, Ingest) + process-stage consumer wiring +│ └── Dockerfile +└── client/ + └── main.go # Ping client (default :8083) +``` + +The Stovepipe controllers live under [`stovepipe/controller/`](../../stovepipe/controller) and its extensions under [`stovepipe/extension/`](../../stovepipe/extension); this directory only contains the runnable wiring and a Docker Compose stack for manual testing. + +## Configuration + +| Variable | Required | Description | Default | +|---------------------|----------|------------------------------------------|----------------------| +| `STORAGE_MYSQL_DSN` | yes | Storage database DSN (`request`, `request_uri`) | — | +| `QUEUE_MYSQL_DSN` | yes | Queue database DSN | — | +| `PORT` | no | gRPC listen address | `:8083` | +| `HOSTNAME` | no | Subscriber name for the process consumer | `stovepipe-` | + +## Running + +### Docker Compose (recommended) + +```bash +make local-stovepipe-start # builds the Linux binary, starts the service + both MySQL DBs, applies storage + queue schemas +make local-stovepipe-stop # tears the stack down +make local-stovepipe-logs # follow logs +``` + +The compose service key is **`stovepipe-service`**, so under the default project **`stovepipe`** the container is **`stovepipe-stovepipe-service-1`**. Inside the container the server listens on `:8080`, published on a random ephemeral host port. + +### Bazel / Go + +```bash +bazel build //service/stovepipe/server:stovepipe +bazel build //service/stovepipe/client:stovepipe + +go run ./service/stovepipe/server +``` + +## Testing the Ping RPC + +```bash +go run ./service/stovepipe/client -addr localhost:8083 -message "hello" +# or +make run-client-stovepipe SERVER_ADDR=localhost:8083 MESSAGE=hello + +# grpcurl +grpcurl -plaintext -d '{"message": "hello"}' localhost:8083 uber.submitqueue.stovepipe.Stovepipe/Ping +``` + +## Shutdown + +The server handles `SIGINT` / `SIGTERM` gracefully: it drains in-flight RPCs, then stops the process consumer (30s timeout). It exits `0` on clean shutdown, `143` (128 + SIGTERM) when stopped by signal, and `1` on startup/runtime errors (details on stderr). Shutdown errors override the signal exit code. + diff --git a/example/stovepipe/client/BUILD.bazel b/service/stovepipe/client/BUILD.bazel similarity index 87% rename from example/stovepipe/client/BUILD.bazel rename to service/stovepipe/client/BUILD.bazel index 2adf44c4..a3e81225 100644 --- a/example/stovepipe/client/BUILD.bazel +++ b/service/stovepipe/client/BUILD.bazel @@ -3,7 +3,7 @@ load("@rules_go//go:def.bzl", "go_binary", "go_library") go_library( name = "client_lib", srcs = ["main.go"], - importpath = "github.com/uber/submitqueue/example/stovepipe/client", + importpath = "github.com/uber/submitqueue/service/stovepipe/client", visibility = ["//visibility:private"], deps = [ "//api/stovepipe/protopb", diff --git a/example/stovepipe/client/main.go b/service/stovepipe/client/main.go similarity index 100% rename from example/stovepipe/client/main.go rename to service/stovepipe/client/main.go diff --git a/example/stovepipe/docker-compose.yml b/service/stovepipe/docker-compose.yml similarity index 95% rename from example/stovepipe/docker-compose.yml rename to service/stovepipe/docker-compose.yml index 7bc64cc9..2dadd8d4 100644 --- a/example/stovepipe/docker-compose.yml +++ b/service/stovepipe/docker-compose.yml @@ -9,7 +9,7 @@ # make build-stovepipe-linux # OR # bazel build --platforms=@rules_go//go/toolchain:linux_amd64 \ -# //example/stovepipe/server:stovepipe +# //service/stovepipe/server:stovepipe # # Quick start: # make local-stovepipe-start @@ -51,7 +51,7 @@ services: stovepipe-service: build: context: ${REPO_ROOT} - dockerfile: example/stovepipe/server/Dockerfile + dockerfile: service/stovepipe/server/Dockerfile ports: - "8080" # Random ephemeral port to avoid conflicts environment: diff --git a/example/stovepipe/server/BUILD.bazel b/service/stovepipe/server/BUILD.bazel similarity index 94% rename from example/stovepipe/server/BUILD.bazel rename to service/stovepipe/server/BUILD.bazel index 68572b79..c38fd059 100644 --- a/example/stovepipe/server/BUILD.bazel +++ b/service/stovepipe/server/BUILD.bazel @@ -3,7 +3,7 @@ load("@rules_go//go:def.bzl", "go_binary", "go_library") go_library( name = "server_lib", srcs = ["main.go"], - importpath = "github.com/uber/submitqueue/example/stovepipe/server", + importpath = "github.com/uber/submitqueue/service/stovepipe/server", visibility = ["//visibility:private"], deps = [ "//api/stovepipe/protopb", diff --git a/example/stovepipe/server/Dockerfile b/service/stovepipe/server/Dockerfile similarity index 100% rename from example/stovepipe/server/Dockerfile rename to service/stovepipe/server/Dockerfile diff --git a/example/stovepipe/server/main.go b/service/stovepipe/server/main.go similarity index 100% rename from example/stovepipe/server/main.go rename to service/stovepipe/server/main.go diff --git a/example/submitqueue/BUILD.bazel b/service/submitqueue/BUILD.bazel similarity index 100% rename from example/submitqueue/BUILD.bazel rename to service/submitqueue/BUILD.bazel diff --git a/example/submitqueue/README.md b/service/submitqueue/README.md similarity index 88% rename from example/submitqueue/README.md rename to service/submitqueue/README.md index a395790e..1cdb2bea 100644 --- a/example/submitqueue/README.md +++ b/service/submitqueue/README.md @@ -1,6 +1,6 @@ -# Example Servers +# SubmitQueue Services -Reference implementations of the Gateway and Orchestrator services, wired with MySQL-backed extensions and runnable via Docker Compose. +Runnable wiring for the **SubmitQueue** domain's two services — the Gateway (entry point for land requests) and the Orchestrator (coordinates the pipeline) — wired with MySQL-backed extensions and runnable via Docker Compose. ## Starting @@ -30,10 +30,10 @@ export MYSQL_DSN='root:root@tcp(127.0.0.1:3306)/submitqueue?parseTime=true' export QUEUE_MYSQL_DSN='root:root@tcp(127.0.0.1:3307)/submitqueue?parseTime=true' # Start gateway (default :8081) -go run ./example/submitqueue/gateway/server +go run ./service/submitqueue/gateway/server # Start orchestrator (default :8082) -go run ./example/submitqueue/orchestrator/server +go run ./service/submitqueue/orchestrator/server ``` ## Stopping diff --git a/example/submitqueue/docker-compose.yml b/service/submitqueue/docker-compose.yml similarity index 93% rename from example/submitqueue/docker-compose.yml rename to service/submitqueue/docker-compose.yml index 1b3ed04b..c8c49d53 100644 --- a/example/submitqueue/docker-compose.yml +++ b/service/submitqueue/docker-compose.yml @@ -3,7 +3,7 @@ # IMPORTANT: Before running docker-compose, build the Linux binaries: # make build-gateway-linux build-orchestrator-linux # OR -# bazel build --platforms=@rules_go//go/toolchain:linux_amd64 //example/submitqueue/gateway/server //example/submitqueue/orchestrator/server +# bazel build --platforms=@rules_go//go/toolchain:linux_amd64 //service/submitqueue/gateway/server //service/submitqueue/orchestrator/server # # Quick start: # make e2e-test @@ -44,7 +44,7 @@ services: gateway-service: build: context: ${REPO_ROOT} - dockerfile: example/submitqueue/gateway/server/Dockerfile + dockerfile: service/submitqueue/gateway/server/Dockerfile ports: - "8080" # Random ephemeral port to avoid conflicts environment: @@ -66,7 +66,7 @@ services: orchestrator-service: build: context: ${REPO_ROOT} - dockerfile: example/submitqueue/orchestrator/server/Dockerfile + dockerfile: service/submitqueue/orchestrator/server/Dockerfile ports: - "8080" # Random ephemeral port to avoid conflicts environment: diff --git a/example/submitqueue/gateway/client/BUILD.bazel b/service/submitqueue/gateway/client/BUILD.bazel similarity index 87% rename from example/submitqueue/gateway/client/BUILD.bazel rename to service/submitqueue/gateway/client/BUILD.bazel index 7af3bc36..e759f17b 100644 --- a/example/submitqueue/gateway/client/BUILD.bazel +++ b/service/submitqueue/gateway/client/BUILD.bazel @@ -3,7 +3,7 @@ load("@rules_go//go:def.bzl", "go_binary", "go_library") go_library( name = "gateway_lib", srcs = ["main.go"], - importpath = "github.com/uber/submitqueue/example/submitqueue/gateway/client", + importpath = "github.com/uber/submitqueue/service/submitqueue/gateway/client", visibility = ["//visibility:private"], deps = [ "//api/submitqueue/gateway/protopb", diff --git a/example/submitqueue/gateway/client/main.go b/service/submitqueue/gateway/client/main.go similarity index 100% rename from example/submitqueue/gateway/client/main.go rename to service/submitqueue/gateway/client/main.go diff --git a/example/submitqueue/gateway/server/BUILD.bazel b/service/submitqueue/gateway/server/BUILD.bazel similarity index 95% rename from example/submitqueue/gateway/server/BUILD.bazel rename to service/submitqueue/gateway/server/BUILD.bazel index 4cc925d3..dad699c8 100644 --- a/example/submitqueue/gateway/server/BUILD.bazel +++ b/service/submitqueue/gateway/server/BUILD.bazel @@ -8,7 +8,7 @@ exports_files( go_library( name = "gateway_lib", srcs = ["main.go"], - importpath = "github.com/uber/submitqueue/example/submitqueue/gateway/server", + importpath = "github.com/uber/submitqueue/service/submitqueue/gateway/server", visibility = ["//visibility:private"], deps = [ "//api/submitqueue/gateway/protopb", diff --git a/example/submitqueue/gateway/server/Dockerfile b/service/submitqueue/gateway/server/Dockerfile similarity index 85% rename from example/submitqueue/gateway/server/Dockerfile rename to service/submitqueue/gateway/server/Dockerfile index 992bcab4..73840ae0 100644 --- a/example/submitqueue/gateway/server/Dockerfile +++ b/service/submitqueue/gateway/server/Dockerfile @@ -9,7 +9,7 @@ COPY .docker-bin/gateway ./gateway # Sample queue configuration; the gateway reads it on startup via # QUEUE_CONFIG_PATH (set in docker-compose.yml). -COPY example/submitqueue/gateway/server/queues.yaml ./queues.yaml +COPY service/submitqueue/gateway/server/queues.yaml ./queues.yaml EXPOSE 8080 diff --git a/example/submitqueue/gateway/server/docker-compose.yml b/service/submitqueue/gateway/server/docker-compose.yml similarity index 95% rename from example/submitqueue/gateway/server/docker-compose.yml rename to service/submitqueue/gateway/server/docker-compose.yml index 08a2f24d..6cc666c8 100644 --- a/example/submitqueue/gateway/server/docker-compose.yml +++ b/service/submitqueue/gateway/server/docker-compose.yml @@ -3,7 +3,7 @@ # IMPORTANT: Before running docker-compose, build the Linux binary: # make build-gateway-linux # OR -# bazel build --platforms=@rules_go//go/toolchain:linux_amd64 //example/submitqueue/gateway/server +# bazel build --platforms=@rules_go//go/toolchain:linux_amd64 //service/submitqueue/gateway/server # # Quick start: # make docker-gateway (builds binary + starts compose) @@ -44,7 +44,7 @@ services: gateway-service: build: context: ${REPO_ROOT} - dockerfile: example/submitqueue/gateway/server/Dockerfile + dockerfile: service/submitqueue/gateway/server/Dockerfile ports: - "8080" # Random ephemeral port to avoid conflicts environment: diff --git a/example/submitqueue/gateway/server/main.go b/service/submitqueue/gateway/server/main.go similarity index 100% rename from example/submitqueue/gateway/server/main.go rename to service/submitqueue/gateway/server/main.go diff --git a/example/submitqueue/gateway/server/queues.yaml b/service/submitqueue/gateway/server/queues.yaml similarity index 100% rename from example/submitqueue/gateway/server/queues.yaml rename to service/submitqueue/gateway/server/queues.yaml diff --git a/example/submitqueue/orchestrator/client/BUILD.bazel b/service/submitqueue/orchestrator/client/BUILD.bazel similarity index 88% rename from example/submitqueue/orchestrator/client/BUILD.bazel rename to service/submitqueue/orchestrator/client/BUILD.bazel index b02201de..a1345ca6 100644 --- a/example/submitqueue/orchestrator/client/BUILD.bazel +++ b/service/submitqueue/orchestrator/client/BUILD.bazel @@ -3,7 +3,7 @@ load("@rules_go//go:def.bzl", "go_binary", "go_library") go_library( name = "orchestrator_lib", srcs = ["main.go"], - importpath = "github.com/uber/submitqueue/example/submitqueue/orchestrator/client", + importpath = "github.com/uber/submitqueue/service/submitqueue/orchestrator/client", visibility = ["//visibility:private"], deps = [ "//api/submitqueue/orchestrator/protopb", diff --git a/example/submitqueue/orchestrator/client/main.go b/service/submitqueue/orchestrator/client/main.go similarity index 100% rename from example/submitqueue/orchestrator/client/main.go rename to service/submitqueue/orchestrator/client/main.go diff --git a/example/submitqueue/orchestrator/server/BUILD.bazel b/service/submitqueue/orchestrator/server/BUILD.bazel similarity index 97% rename from example/submitqueue/orchestrator/server/BUILD.bazel rename to service/submitqueue/orchestrator/server/BUILD.bazel index caacd86e..a2c81a22 100644 --- a/example/submitqueue/orchestrator/server/BUILD.bazel +++ b/service/submitqueue/orchestrator/server/BUILD.bazel @@ -8,7 +8,7 @@ exports_files( go_library( name = "orchestrator_lib", srcs = ["main.go"], - importpath = "github.com/uber/submitqueue/example/submitqueue/orchestrator/server", + importpath = "github.com/uber/submitqueue/service/submitqueue/orchestrator/server", visibility = ["//visibility:private"], deps = [ "//api/runway/messagequeue", diff --git a/example/submitqueue/orchestrator/server/Dockerfile b/service/submitqueue/orchestrator/server/Dockerfile similarity index 100% rename from example/submitqueue/orchestrator/server/Dockerfile rename to service/submitqueue/orchestrator/server/Dockerfile diff --git a/example/submitqueue/orchestrator/server/docker-compose.yml b/service/submitqueue/orchestrator/server/docker-compose.yml similarity index 95% rename from example/submitqueue/orchestrator/server/docker-compose.yml rename to service/submitqueue/orchestrator/server/docker-compose.yml index c7ebac0d..90db979f 100644 --- a/example/submitqueue/orchestrator/server/docker-compose.yml +++ b/service/submitqueue/orchestrator/server/docker-compose.yml @@ -3,7 +3,7 @@ # IMPORTANT: Before running docker-compose, build the Linux binary: # make build-orchestrator-linux # OR -# bazel build --platforms=@rules_go//go/toolchain:linux_amd64 //example/submitqueue/orchestrator/server +# bazel build --platforms=@rules_go//go/toolchain:linux_amd64 //service/submitqueue/orchestrator/server # # Quick start: # make docker-orchestrator (builds binary + starts compose) @@ -44,7 +44,7 @@ services: orchestrator-service: build: context: ${REPO_ROOT} - dockerfile: example/submitqueue/orchestrator/server/Dockerfile + dockerfile: service/submitqueue/orchestrator/server/Dockerfile ports: - "8080" # Random ephemeral port to avoid conflicts environment: diff --git a/example/submitqueue/orchestrator/server/main.go b/service/submitqueue/orchestrator/server/main.go similarity index 100% rename from example/submitqueue/orchestrator/server/main.go rename to service/submitqueue/orchestrator/server/main.go diff --git a/submitqueue/orchestrator/controller/dlq/README.md b/submitqueue/orchestrator/controller/dlq/README.md index 551138b7..3e12f432 100644 --- a/submitqueue/orchestrator/controller/dlq/README.md +++ b/submitqueue/orchestrator/controller/dlq/README.md @@ -1,6 +1,6 @@ # DLQ Reconciliation Controllers -This package contains the controllers that drain each primary pipeline topic's `{topic}_dlq` companion and reconcile the affected request or batch into a terminal failed state. They are wired alongside the primary controllers in `example/submitqueue/orchestrator/server/main.go`. +This package contains the controllers that drain each primary pipeline topic's `{topic}_dlq` companion and reconcile the affected request or batch into a terminal failed state. They are wired alongside the primary controllers in `service/submitqueue/orchestrator/server/main.go`. ## Design principles diff --git a/test/e2e/submitqueue/BUILD.bazel b/test/e2e/submitqueue/BUILD.bazel index 94d1d5da..903166ce 100644 --- a/test/e2e/submitqueue/BUILD.bazel +++ b/test/e2e/submitqueue/BUILD.bazel @@ -6,9 +6,9 @@ go_test( data = [ "//:MODULE.bazel", "//:go.mod", - "//example/submitqueue:docker-compose.yml", "//platform/extension/counter/mysql/schema", "//platform/extension/messagequeue/mysql/schema", + "//service/submitqueue:docker-compose.yml", "//submitqueue/extension/storage/mysql/schema", ], tags = [ diff --git a/test/e2e/submitqueue/suite_test.go b/test/e2e/submitqueue/suite_test.go index e124452a..69a254ff 100644 --- a/test/e2e/submitqueue/suite_test.go +++ b/test/e2e/submitqueue/suite_test.go @@ -16,7 +16,7 @@ package e2e_test // E2E Integration Tests // -// These tests use docker-compose from example/submitqueue/docker-compose.yml +// These tests use docker-compose from service/submitqueue/docker-compose.yml // which requires pre-built Linux binaries. // // Run with make target (builds binaries + runs test): @@ -80,9 +80,9 @@ func (s *E2EIntegrationSuite) SetupSuite() { repoRoot := testutil.FindRepoRoot(t) t.Setenv("REPO_ROOT", repoRoot) - // Use docker-compose from example/submitqueue (full stack) + // Use docker-compose from service/submitqueue (full stack) // NOTE: Assumes Linux binaries are pre-built via make target - composeFile := filepath.Join(repoRoot, "example/submitqueue/docker-compose.yml") + composeFile := filepath.Join(repoRoot, "service/submitqueue/docker-compose.yml") s.stack = testutil.NewComposeStack(t, s.log, s.ctx, composeFile, "e2e-submitqueue") // Start the compose stack (Gateway + Orchestrator + 2 MySQL DBs) diff --git a/test/integration/stovepipe/BUILD.bazel b/test/integration/stovepipe/BUILD.bazel index 8f8acfef..9cc1b43d 100644 --- a/test/integration/stovepipe/BUILD.bazel +++ b/test/integration/stovepipe/BUILD.bazel @@ -6,8 +6,8 @@ go_test( data = [ "//:MODULE.bazel", "//:go.mod", - "//example/stovepipe:docker-compose.yml", "//platform/extension/messagequeue/mysql/schema", + "//service/stovepipe:docker-compose.yml", "//stovepipe/extension/storage/mysql/schema", ], tags = [ diff --git a/test/integration/stovepipe/suite_test.go b/test/integration/stovepipe/suite_test.go index 79f9edc3..d9f4e0e8 100644 --- a/test/integration/stovepipe/suite_test.go +++ b/test/integration/stovepipe/suite_test.go @@ -16,7 +16,7 @@ package stovepipe // Stovepipe integration tests // -// These tests use compose from example/stovepipe/docker-compose.yml and require +// These tests use compose from service/stovepipe/docker-compose.yml and require // a pre-built Linux binary (make integration-test runs //test/integration/... // and builds all Linux binaries via build-all-linux). The stack runs the // Stovepipe gRPC service plus a storage MySQL (request, request_uri) and a queue @@ -66,7 +66,7 @@ func (s *StovepipeIntegrationSuite) SetupSuite() { repoRoot := testutil.FindRepoRoot(t) t.Setenv("REPO_ROOT", repoRoot) - composeFile := filepath.Join(repoRoot, "example/stovepipe/docker-compose.yml") + composeFile := filepath.Join(repoRoot, "service/stovepipe/docker-compose.yml") s.stack = testutil.NewComposeStack(t, s.log, s.ctx, composeFile, "svc-stovepipe") err := s.stack.Up() diff --git a/test/integration/submitqueue/gateway/BUILD.bazel b/test/integration/submitqueue/gateway/BUILD.bazel index 25aebc41..d6d8990d 100644 --- a/test/integration/submitqueue/gateway/BUILD.bazel +++ b/test/integration/submitqueue/gateway/BUILD.bazel @@ -6,9 +6,9 @@ go_test( data = [ "//:MODULE.bazel", "//:go.mod", - "//example/submitqueue/gateway/server:docker-compose.yml", "//platform/extension/counter/mysql/schema", "//platform/extension/messagequeue/mysql/schema", + "//service/submitqueue/gateway/server:docker-compose.yml", "//submitqueue/extension/storage/mysql/schema", ], tags = [ diff --git a/test/integration/submitqueue/gateway/suite_test.go b/test/integration/submitqueue/gateway/suite_test.go index eab37662..20e22820 100644 --- a/test/integration/submitqueue/gateway/suite_test.go +++ b/test/integration/submitqueue/gateway/suite_test.go @@ -16,7 +16,7 @@ package gateway // Gateway Integration Tests // -// These tests use docker-compose from example/submitqueue/gateway/server/docker-compose.yml +// These tests use docker-compose from service/submitqueue/gateway/server/docker-compose.yml // which requires pre-built Linux binaries. // // Run with make target (builds binary + runs test): @@ -85,9 +85,9 @@ func (s *GatewayIntegrationSuite) SetupSuite() { repoRoot := testutil.FindRepoRoot(t) t.Setenv("REPO_ROOT", repoRoot) - // Use docker-compose from example/submitqueue/gateway/server + // Use docker-compose from service/submitqueue/gateway/server // NOTE: Assumes Linux binary is pre-built via make target - composeFile := filepath.Join(repoRoot, "example/submitqueue/gateway/server/docker-compose.yml") + composeFile := filepath.Join(repoRoot, "service/submitqueue/gateway/server/docker-compose.yml") s.stack = testutil.NewComposeStack(t, s.log, s.ctx, composeFile, "svc-submitqueue-gateway") // Start the compose stack (Gateway + 2 MySQL DBs) diff --git a/test/integration/submitqueue/orchestrator/BUILD.bazel b/test/integration/submitqueue/orchestrator/BUILD.bazel index 67e8e57e..45d66841 100644 --- a/test/integration/submitqueue/orchestrator/BUILD.bazel +++ b/test/integration/submitqueue/orchestrator/BUILD.bazel @@ -6,9 +6,9 @@ go_test( data = [ "//:MODULE.bazel", "//:go.mod", - "//example/submitqueue/orchestrator/server:docker-compose.yml", "//platform/extension/counter/mysql/schema", "//platform/extension/messagequeue/mysql/schema", + "//service/submitqueue/orchestrator/server:docker-compose.yml", "//submitqueue/extension/storage/mysql/schema", ], tags = [ diff --git a/test/integration/submitqueue/orchestrator/suite_test.go b/test/integration/submitqueue/orchestrator/suite_test.go index 2d8cfd26..bcdba170 100644 --- a/test/integration/submitqueue/orchestrator/suite_test.go +++ b/test/integration/submitqueue/orchestrator/suite_test.go @@ -16,7 +16,7 @@ package orchestrator // Orchestrator Integration Tests // -// These tests use docker-compose from example/submitqueue/orchestrator/server/docker-compose.yml +// These tests use docker-compose from service/submitqueue/orchestrator/server/docker-compose.yml // which requires pre-built Linux binaries. // // Run with make target (builds binary + runs test): @@ -64,9 +64,9 @@ func (s *OrchestratorIntegrationSuite) SetupSuite() { repoRoot := testutil.FindRepoRoot(t) t.Setenv("REPO_ROOT", repoRoot) - // Use docker-compose from example/submitqueue/orchestrator/server + // Use docker-compose from service/submitqueue/orchestrator/server // NOTE: Assumes Linux binary is pre-built via make target - composeFile := filepath.Join(repoRoot, "example/submitqueue/orchestrator/server/docker-compose.yml") + composeFile := filepath.Join(repoRoot, "service/submitqueue/orchestrator/server/docker-compose.yml") s.stack = testutil.NewComposeStack(t, s.log, s.ctx, composeFile, "svc-submitqueue-orchestrator") // Start the compose stack (Orchestrator + 2 MySQL DBs)