docker-compose: add otel-collector override, dev override (#838)

Author: bobheadxi

.gitignore

@@ -7,6 +7,6 @@ nginx/sourcegraph.key
 
 .vagrant
 *.log
-.env
+/.env
 
 /right.dhall

docker-compose/db-only-migrate.docker-compose.yaml

@@ -13,7 +13,7 @@ services:
   #
   pgsql:
     container_name: pgsql
-    image: 'index.docker.io/sourcegraph/postgres-12-alpine:135107_2022-03-03_9498a8bd3366@sha256:e26b159dc7c0c47d136886390c899816e669a3c2c1ead689bdad0b610364e45e'
+    image: 'index.docker.io/sourcegraph/postgres-12-alpine:161449_2022-07-18_e8a013edc325@sha256:cc79c0db11e07e0a5c1dde27c130384bf2f131ce5c51a4d5cba32820ef69fc1a'
     cpus: 4
     mem_limit: '2g'
     healthcheck:
@@ -30,7 +30,7 @@ services:
 
   codeintel-db:
     container_name: codeintel-db
-    image: 'index.docker.io/sourcegraph/codeintel-db:insiders@sha256:cd901b1180229101cc8c6deebda24c14f6224295cd2153cafe0665199499c342'
+    image: 'index.docker.io/sourcegraph/codeintel-db:161449_2022-07-18_e8a013edc325@sha256:61a858e02eb368aa7ffd1264c23db6b1ce5fc94d4cff38921633ff17cf860eff'
     cpus: 4
     mem_limit: '2g'
     healthcheck:

docker-compose/dev/README.md

@@ -0,0 +1,13 @@
+# Development overlay
+
+For use developing Sourcegraph's docker-compose deployment only.
+For deployments, refer to the official [Sourcegraph with Docker Compose](https://docs.sourcegraph.com/admin/install/docker-compose) documentation.
+
+Example usage:
+
+```sh
+docker-compose \
+    -f docker-compose/docker-compose.yaml \
+    -f docker-compose/otel/docker-compose.yaml \
+    -f docker-compose/dev/docker-compose.yaml up
+```

docker-compose/dev/docker-compose.yaml

@@ -0,0 +1,12 @@
+# Development overlay - has reduced resource requirements, and exposes the deployment on
+# 127.0.0.1:8080
+
+version: '2.4'
+services:
+  zoekt-webserver-0:
+    cpus: 1
+  zoekt-indexserver-0:
+    cpus: 1
+  caddy:
+    ports:
+      - '0.0.0.0:8080:80'

docker-compose/docker-compose.yaml

@@ -13,7 +13,7 @@ services:
   # for this container will need to be updated to reflect the new connection information.
   migrator:
     container_name: migrator
-    image: 'index.docker.io/sourcegraph/migrator:insiders@sha256:05e820b0694f0a5008e8069a50089ebc9dc7cc464075c2c166b0ef29dba60ac5'
+    image: 'index.docker.io/sourcegraph/migrator:161449_2022-07-18_e8a013edc325@sha256:5cc21674ecd4914a11aad157cec2f4a94eef8a71157c89295766fcc384e21b34'
     cpus: 0.5
     mem_limit: '500m'
     command:
@@ -125,7 +125,7 @@ services:
   # service.
   sourcegraph-frontend-0:
     container_name: sourcegraph-frontend-0
-    image: 'index.docker.io/sourcegraph/frontend:insiders@sha256:42bdb4d375adff51c00195d435c836f5066fe9ef15fd9c24b567c659c56c2099'
+    image: 'index.docker.io/sourcegraph/frontend:161449_2022-07-18_e8a013edc325@sha256:0c5d48081d65fd6b95eb0eb66f8fb0a1155c05768e6bdb2193ed281aef4c4b6b'
     cpus: 4
     mem_limit: '8g'
     environment:
@@ -168,7 +168,7 @@ services:
   #
   sourcegraph-frontend-internal:
     container_name: sourcegraph-frontend-internal
-    image: 'index.docker.io/sourcegraph/frontend:insiders@sha256:42bdb4d375adff51c00195d435c836f5066fe9ef15fd9c24b567c659c56c2099'
+    image: 'index.docker.io/sourcegraph/frontend:161449_2022-07-18_e8a013edc325@sha256:0c5d48081d65fd6b95eb0eb66f8fb0a1155c05768e6bdb2193ed281aef4c4b6b'
     cpus: 4
     mem_limit: '8g'
     environment:
@@ -187,6 +187,7 @@ services:
       - 'JAEGER_SERVER_URL=http://jaeger:16686'
       - 'GITHUB_BASE_URL=http://github-proxy:3180'
       - 'PROMETHEUS_URL=http://prometheus:9090'
+      - JAEGER_AGENT_HOST=jaeger
     volumes:
       - 'sourcegraph-frontend-internal-0:/mnt/cache'
     networks:
@@ -214,7 +215,7 @@ services:
   #
   gitserver-0:
     container_name: gitserver-0
-    image: 'index.docker.io/sourcegraph/gitserver:insiders@sha256:21fe84d3dc706c010a4d286eef3272c88d1b996521e881805231de0b0d4f5390'
+    image: 'index.docker.io/sourcegraph/gitserver:161449_2022-07-18_e8a013edc325@sha256:57b8b7b61d5394e16c81e8d68064a7413ed4815316be23a23990be4ab25ac450'
     cpus: 4
     mem_limit: '8g'
     environment:
@@ -237,7 +238,7 @@ services:
   #
   zoekt-indexserver-0:
     container_name: zoekt-indexserver-0
-    image: 'index.docker.io/sourcegraph/search-indexer:insiders@sha256:461625546e3636a1abd38b94b3accea6b8d1ae92924b738cc7b41f5e8f74bb45'
+    image: 'index.docker.io/sourcegraph/search-indexer:161449_2022-07-18_e8a013edc325@sha256:39448b944001e012ac405c25aebaec93f17f73a04866cfb25f4fee6fcbf0c621'
     cpus: 8
     mem_limit: '16g'
     environment:
@@ -258,7 +259,7 @@ services:
   #
   zoekt-webserver-0:
     container_name: zoekt-webserver-0
-    image: 'index.docker.io/sourcegraph/indexed-searcher:insiders@sha256:9d7f9a49ba16dae3c2e0ae27972b02421d65c60682ea107633dac6ab286228fb'
+    image: 'index.docker.io/sourcegraph/indexed-searcher:161449_2022-07-18_e8a013edc325@sha256:ccce9c20c8ad5c3b738801eb79b7339b0ffe6f23d0eefbc5dc3c613525561908'
     cpus: 8
     mem_limit: '50g'
     environment:
@@ -283,7 +284,7 @@ services:
   #
   searcher-0:
     container_name: searcher-0
-    image: 'index.docker.io/sourcegraph/searcher:insiders@sha256:936192a6308c409db340fa13c4872cc6facab02247c3b3d6fea773ec64fc87ea'
+    image: 'index.docker.io/sourcegraph/searcher:161449_2022-07-18_e8a013edc325@sha256:ba5a7ca85c07457f4f7e36afe4bd1026db61e8f7b32b69975ab3f48d91252ccc'
     cpus: 2
     mem_limit: '2g'
     environment:
@@ -310,7 +311,7 @@ services:
   #
   github-proxy:
     container_name: github-proxy
-    image: 'index.docker.io/sourcegraph/github-proxy:insiders@sha256:1ee9534f48d4855a429a2611bf1ebbfe40f3dd9d7a1db2281ae20dd64ee0e243'
+    image: 'index.docker.io/sourcegraph/github-proxy:161449_2022-07-18_e8a013edc325@sha256:7eae3165665fda5aed2b68fabea8291a2176c43892bc71a86dfa09904cea6c19'
     cpus: 1
     mem_limit: '1g'
     environment:
@@ -327,7 +328,7 @@ services:
   #
   precise-code-intel-worker:
     container_name: precise-code-intel-worker
-    image: 'index.docker.io/sourcegraph/precise-code-intel-worker:insiders@sha256:a83643eb396d707e62ebbc5c3457747440576edaed013b5c2bf663a03eb56315'
+    image: 'index.docker.io/sourcegraph/precise-code-intel-worker:161449_2022-07-18_e8a013edc325@sha256:5cca4107afdb22a87433d8c424724615bd7ca5751a6fbdaea133da8402210062'
     cpus: 2
     mem_limit: '4g'
     environment:
@@ -351,7 +352,7 @@ services:
   #
   repo-updater:
     container_name: repo-updater
-    image: 'index.docker.io/sourcegraph/repo-updater:insiders@sha256:db4934978c82d1986710a22e79e7dbdcbf19382f8ae0c3d98e84908fde83f3dd'
+    image: 'index.docker.io/sourcegraph/repo-updater:161449_2022-07-18_e8a013edc325@sha256:e636918befb1f115e468e4ed46bd15e4635d8dd8431deeb33dd210c53e5d4ce4'
     cpus: 4
     mem_limit: '4g'
     environment:
@@ -372,7 +373,7 @@ services:
   #
   worker:
     container_name: worker
-    image: 'index.docker.io/sourcegraph/worker:insiders@sha256:60bb1263a5a4b18ee1de32aa5e9151102ab0680e84b263cca687e01fd1eefc45'
+    image: 'index.docker.io/sourcegraph/worker:161449_2022-07-18_e8a013edc325@sha256:96e634a7b01930f97ba281a74751965b28f8baff98c38e944d56744924206e0f'
     cpus: 4
     mem_limit: '4g'
     environment:
@@ -397,7 +398,7 @@ services:
   #
   syntect-server:
     container_name: syntect-server
-    image: 'index.docker.io/sourcegraph/syntax-highlighter:insiders@sha256:c7154c611a62a900c79b49bf1ca99af75ec2db779b7fb4303f0abf20022ba60c'
+    image: 'index.docker.io/sourcegraph/syntax-highlighter:161449_2022-07-18_e8a013edc325@sha256:53c60479f575b1c2eca3cf7c5fd2a500c52244ab024e455a82cb2c7c5811a598'
     cpus: 4
     mem_limit: '6g'
     healthcheck:
@@ -418,7 +419,7 @@ services:
   #
   symbols-0:
     container_name: symbols-0
-    image: 'index.docker.io/sourcegraph/symbols:insiders@sha256:28ee85a4ad008ba1d54125d70c75a59269fee048966a32c235a7c8ab5a88bd3b'
+    image: 'index.docker.io/sourcegraph/symbols:161449_2022-07-18_e8a013edc325@sha256:c2d9ebd61b1d73b3324f562ffdcd5d6498b81a7037a2b3e45e7e3c73ad903601'
     cpus: 2
     mem_limit: '4g'
     environment:
@@ -444,7 +445,7 @@ services:
   #
   prometheus:
     container_name: prometheus
-    image: 'index.docker.io/sourcegraph/prometheus:insiders@sha256:b36ad28d24e010cfacb6c21d8b95656f78e2f1cd74ab43fdddfa35b181d9fe39'
+    image: 'index.docker.io/sourcegraph/prometheus:161449_2022-07-18_e8a013edc325@sha256:c51292acfacb1f4c2cdc0196f64c7f90d3f7aca506d59c2456b5b4ccf67f53c0'
     cpus: 4
     mem_limit: '8g'
     volumes:
@@ -471,7 +472,7 @@ services:
   # 'GF_SERVER_ROOT_URL='https://grafana.example.com'
   grafana:
     container_name: grafana
-    image: 'index.docker.io/sourcegraph/grafana:insiders@sha256:00064d24e19e1159dd70d12bc328db64e8c45fb53062b9b0e3db93984371836d'
+    image: 'index.docker.io/sourcegraph/grafana:161449_2022-07-18_e8a013edc325@sha256:4dbd1b0b3c555aa6738c155a74b2fae4f87008ef99f981ad9d5b647c3ac39662'
     cpus: 1
     mem_limit: '1g'
     volumes:
@@ -492,7 +493,7 @@ services:
   #
   cadvisor:
     container_name: cadvisor
-    image: 'index.docker.io/sourcegraph/cadvisor:insiders@sha256:87571e704d066e9edaa6034cf73dd32042e24a14505c281db0505622940baab3'
+    image: 'index.docker.io/sourcegraph/cadvisor:161449_2022-07-18_e8a013edc325@sha256:d9a60000963ba5258aea6ab818e2d8b36b8baba51624a60bac30810b9e7822dc'
     cpus: 1
     mem_limit: '1g'
     # You may set `privileged` to `false and `cadvisor` will run with reduced privileges.
@@ -527,7 +528,7 @@ services:
   #
   jaeger:
     container_name: jaeger
-    image: 'index.docker.io/sourcegraph/jaeger-all-in-one:insiders@sha256:5ddf4be454010327b9aa79c33cd447e5fb1ec64e7ad4add9702731dfdd497c6d'
+    image: 'index.docker.io/sourcegraph/jaeger-all-in-one:161449_2022-07-18_e8a013edc325@sha256:4c7c3c0ca278fc3ae27546025441a1a286d4ab0eafb059e9f1d85aa734fc4aae'
     cpus: 0.5
     mem_limit: '512m'
     ports:
@@ -554,7 +555,7 @@ services:
   #
   pgsql:
     container_name: pgsql
-    image: 'index.docker.io/sourcegraph/postgres-12-alpine:insiders@sha256:e8f8d8fda4802e1e9f93441ad15b30045add5ebef200086548c232ee3351ad70'
+    image: 'index.docker.io/sourcegraph/postgres-12-alpine:161449_2022-07-18_e8a013edc325@sha256:cc79c0db11e07e0a5c1dde27c130384bf2f131ce5c51a4d5cba32820ef69fc1a'
     cpus: 4
     mem_limit: '4g'
     shm_size: '1g'
@@ -582,7 +583,7 @@ services:
   # for this container will need to be updated to reflect the new connection information.
   pgsql-exporter:
     container_name: pgsql-exporter
-    image: 'index.docker.io/sourcegraph/postgres_exporter:insiders@sha256:9c7eb0c76afcd2d564e9fd9a71d2a4443285aca409521e778ceddf455e2c4d6a'
+    image: 'index.docker.io/sourcegraph/postgres_exporter:161449_2022-07-18_e8a013edc325@sha256:b7d11ad87f3aa471659f6ad0620f21f5f7be5070dd86f608b08d81a0f67bb9b7'
     cpus: 0.1
     mem_limit: '50m'
     networks:
@@ -601,7 +602,7 @@ services:
   #
   codeintel-db:
     container_name: codeintel-db
-    image: 'index.docker.io/sourcegraph/codeintel-db:insiders@sha256:f24ab8dd55fa7c1017cbf235f93e51d4d38be0231f9488f3fd11883184b0969e'
+    image: 'index.docker.io/sourcegraph/codeintel-db:161449_2022-07-18_e8a013edc325@sha256:61a858e02eb368aa7ffd1264c23db6b1ce5fc94d4cff38921633ff17cf860eff'
     cpus: 4
     mem_limit: '4g'
     shm_size: '1g'
@@ -629,7 +630,7 @@ services:
   # for this container will need to be updated to reflect the new connection information.
   codeintel-db-exporter:
     container_name: codeintel-db-exporter
-    image: 'index.docker.io/sourcegraph/postgres_exporter:insiders@sha256:9c7eb0c76afcd2d564e9fd9a71d2a4443285aca409521e778ceddf455e2c4d6a'
+    image: 'index.docker.io/sourcegraph/postgres_exporter:161449_2022-07-18_e8a013edc325@sha256:b7d11ad87f3aa471659f6ad0620f21f5f7be5070dd86f608b08d81a0f67bb9b7'
     cpus: 0.1
     mem_limit: '50m'
     networks:
@@ -648,7 +649,7 @@ services:
   #
   codeinsights-db:
     container_name: codeinsights-db
-    image: 'index.docker.io/sourcegraph/codeinsights-db:insiders@sha256:7c016ad0893bf00b95ebf98f59b772358b0f638f5ec3361e95148a42d641559b'
+    image: 'index.docker.io/sourcegraph/codeinsights-db:161449_2022-07-18_e8a013edc325@sha256:f0708e765064e4421f074c9b0b67d194774d534c8f420155e68368acff2aebe9'
     cpus: 4
     mem_limit: '2g'
     shm_size: '1g'
@@ -681,7 +682,7 @@ services:
   # for this container will need to be updated to reflect the new connection information.
   codeinsights-db-exporter:
     container_name: codeinsights-db-exporter
-    image: 'index.docker.io/sourcegraph/postgres_exporter:insiders@sha256:9c7eb0c76afcd2d564e9fd9a71d2a4443285aca409521e778ceddf455e2c4d6a'
+    image: 'index.docker.io/sourcegraph/postgres_exporter:161449_2022-07-18_e8a013edc325@sha256:b7d11ad87f3aa471659f6ad0620f21f5f7be5070dd86f608b08d81a0f67bb9b7'
     cpus: 0.1
     mem_limit: '50m'
     networks:
@@ -700,7 +701,7 @@ services:
   #
   minio:
     container_name: minio
-    image: 'index.docker.io/sourcegraph/minio:insiders@sha256:0daf4c0c821634eeefbf90f48d467eece09597aff5d9f582685c8c875f03e6fa'
+    image: 'index.docker.io/sourcegraph/minio:161449_2022-07-18_e8a013edc325@sha256:d5377e0c6e4ed742c551f253591d4a1a7f3a104ca17c51d46d6324206577f209'
     cpus: 1
     mem_limit: '1g'
     environment:
@@ -727,7 +728,7 @@ services:
   #
   redis-cache:
     container_name: redis-cache
-    image: 'index.docker.io/sourcegraph/redis-cache:insiders@sha256:793b73f1cd185682109dc3adb235d74aab9a3152c2f6afccf0913a0cf9210879'
+    image: 'index.docker.io/sourcegraph/redis-cache:161449_2022-07-18_e8a013edc325@sha256:e7f6df111948765d29dac831a65953bb225d87c0adb28754be688775a12129b6'
     cpus: 1
     mem_limit: '7g'
     volumes:
@@ -743,7 +744,7 @@ services:
   #
   redis-store:
     container_name: redis-store
-    image: 'index.docker.io/sourcegraph/redis-store:insiders@sha256:130abf3b12d6b4be117aaf0f4ab49e1e00d9d80b8c9b94b0b4594e84fd46d010'
+    image: 'index.docker.io/sourcegraph/redis-store:161449_2022-07-18_e8a013edc325@sha256:4300a1869b5ffad3591f7e95fd3f9ca2221599d07e04432f57471c3065abf02f'
     cpus: 1
     mem_limit: '7g'
     volumes:

docker-compose/otel/docker-compose.yaml

@@ -0,0 +1,67 @@
+version: '2.4'
+services:
+  # Description: Ingests OpenTelemetry data.
+  #
+  # Disk: 128GB / non-persistent SSD
+  # Ports exposed to other Sourcegraph services: 4317 (receiver), 55679 (zpages)
+  # Ports exposed to the public internet: none
+  otel-collector:
+    container_name: otel-collector
+    image: otel/opentelemetry-collector:0.54.0
+    volumes:
+      - '../otel-collector/config.yaml:/etc/otel-collector/config.yaml'
+    cpus: 0.5
+    mem_limit: '512m'
+    networks:
+      - sourcegraph
+    restart: always
+    environment:
+      - JAEGER_AGENT_HOST=jaeger
+    command: ['--config', '/etc/otel-collector/config.yaml']
+
+  # Apply exporter configuration - to learn more, see:
+  # https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md#configuration-options
+
+  sourcegraph-frontend-0:
+    environment:
+      - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
+
+  sourcegraph-frontend-internal:
+    environment:
+      - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
+
+  gitserver-0:
+    environment:
+      - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
+  
+  zoekt-indexserver-0:
+    environment:
+      - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
+
+  zoekt-webserver-0:
+    environment:
+      - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
+
+  searcher-0:
+    environment:
+      - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
+
+  github-proxy:
+    environment:
+      - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
+
+  precise-code-intel-worker:
+    environment:
+      - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
+
+  repo-updater:
+    environment:
+      - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
+  
+  worker:
+    environment:
+      - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
+
+  symbols-0:
+    environment:
+      - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317

otel-collector/config.yaml

@@ -0,0 +1,24 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+
+exporters:
+  jaeger:
+    # Default Jaeger GRPC server
+    endpoint: "$JAEGER_AGENT_HOST:14250"
+    tls:
+      insecure: true
+
+extensions:
+  health_check:
+    port: 13133
+  zpages:
+   endpoint: "localhost:55679"
+
+service:
+  extensions: [health_check,zpages]
+  pipelines:
+    traces:
+      receivers: [otlp]
+      exporters: [jaeger]