diff --git a/.github/jsonnet/GIT_VERSION b/.github/jsonnet/GIT_VERSION index fa23250d..c94d00f2 100644 --- a/.github/jsonnet/GIT_VERSION +++ b/.github/jsonnet/GIT_VERSION @@ -1 +1 @@ -4e950af19b855ed027f1d5b6df12bd3acfdd0a6c +5d9c576c6fbbcd81178560862e084035c0e72ceb diff --git a/.github/jsonnet/clusters.jsonnet b/.github/jsonnet/clusters.jsonnet index 10c9b657..c1483e39 100644 --- a/.github/jsonnet/clusters.jsonnet +++ b/.github/jsonnet/clusters.jsonnet @@ -6,7 +6,8 @@ local misc = import 'misc.jsonnet'; name: 'test', zone: 'europe-west4-b', secret: misc.secret('GCE_NEW_TEST_JSON'), - jobNodeSelectorType: 'preemptible', + jobNodeSelectorKey: 'type', + jobNodeSelectorValue: 'preemptible', }, prod: { @@ -14,7 +15,8 @@ local misc = import 'misc.jsonnet'; name: 'prod-europe-west4', zone: 'europe-west4', secret: misc.secret('GCE_JSON'), - jobNodeSelectorType: 'worker', + jobNodeSelectorKey: 'worker', + jobNodeSelectorValue: 'true', }, 'gynzy-intern': { @@ -22,6 +24,7 @@ local misc = import 'misc.jsonnet'; name: 'gynzy-intern', zone: 'europe-west4', secret: misc.secret('CONTINUOUS_DEPLOYMENT_GCE_JSON'), - jobNodeSelectorType: 'worker', + jobNodeSelectorKey: 'type', + jobNodeSelectorValue: 'worker', }, } diff --git a/.github/jsonnet/docker.jsonnet b/.github/jsonnet/docker.jsonnet index a23ddc14..2fe4ba4e 100644 --- a/.github/jsonnet/docker.jsonnet +++ b/.github/jsonnet/docker.jsonnet @@ -3,14 +3,28 @@ local images = import 'images.jsonnet'; local misc = import 'misc.jsonnet'; { + /** + * Builds a Docker image using the specified configuration. + * + * @param {string} imageName - The name of the Docker image to be built. + * @param {string} [imageTag] - The tag to be used for the Docker image. Defaults to "deploy-" of the current pull request. + * @param {boolean|null} [isPublic] - whether the image should be public or private. + * If null the image name will need to be prefixed with artifact registry repository name + * @param {string} [context] - The context path for the Docker build. defaults to: . + * @param {string} [dockerfile] - The path to the Dockerfile to be used for the build. + * @param {object} [env] - Environment variables to be passed to the Docker build. + * @param {object} [build_args] - Build arguments to be passed to the Docker build. + * @param {string} [project] - The GCP project where the image will be store/pushed to. + * @returns {[object]} - a github actions step to build the docker image + */ buildDocker( imageName, imageTag='deploy-${{ github.event.pull_request.head.sha }}', + isPublic=false, context='.', dockerfile=null, env={}, build_args=null, - registry='eu.gcr.io', project='unicorn-985', ):: base.action( @@ -19,10 +33,10 @@ local misc = import 'misc.jsonnet'; with={ context: context, gcloud_service_key: misc.secret('docker_gcr_io_base64'), - image_name: imageName, + image_name: (if isPublic == null then '' else if isPublic == true then 'public-images/' else if isPublic == false then 'private-images/') + imageName, image_tag: imageTag, project_id: project, - registry: registry, + registry: 'europe-docker.pkg.dev', } + (if build_args != null then { build_args: build_args } else {}) + (if dockerfile != null then { dockerfile: dockerfile } else {}), diff --git a/.github/jsonnet/images.jsonnet b/.github/jsonnet/images.jsonnet index 3f71b806..c0d4e024 100644 --- a/.github/jsonnet/images.jsonnet +++ b/.github/jsonnet/images.jsonnet @@ -1,18 +1,17 @@ { - jsonnet_bin_image: 'eu.gcr.io/unicorn-985/docker-images_jsonnet:v1', + jsonnet_bin_image: 'europe-docker.pkg.dev/unicorn-985/private-images/docker-images_jsonnet:v1', helm_action_image: 'docker://europe-docker.pkg.dev/gynzy-test-project/public-images/helm-action:v2', - mysql_action_image: 'docker://europe-docker.pkg.dev/gynzy-test-project/public-images/docker-images_mysql-cloner-action:v1', + mysql_action_image: 'docker://europe-docker.pkg.dev/unicorn-985/public-images/docker-images_mysql-cloner-action:v1', docker_action_image: 'docker://europe-docker.pkg.dev/gynzy-test-project/public-images/push-to-gcr-github-action:v1', default_job_image: 'mirror.gcr.io/alpine:3.20.0', - default_mysql57_image: 'eu.gcr.io/unicorn-985/docker-images_mysql57_utf8mb4:v1', - default_mysql8_image: 'eu.gcr.io/unicorn-985/docker-images_mysql8_utf8mb4:v1', - default_cloudsql_image: 'eu.gcr.io/unicorn-985/docker-images_cloudsql-sidecar:v1', + default_mysql8_image: 'europe-docker.pkg.dev/unicorn-985/private-images/docker-images_mysql8_utf8mb4:v1', + default_cloudsql_image: 'europe-docker.pkg.dev/unicorn-985/private-images/docker-images_cloudsql-sidecar:v1', default_redis_image: 'mirror.gcr.io/redis:5.0.6', default_unicorns_image: 'mirror.gcr.io/node:18.15', default_pubsub_image: 'mirror.gcr.io/messagebird/gcloud-pubsub-emulator:latest', - default_mongodb_image: 'eu.gcr.io/unicorn-985/docker-images_mongo6-replicated:v1', - mongo_job_image: 'europe-docker.pkg.dev/gynzy-test-project/public-images/docker-images_mongo-cloner-job:v1', + default_mongodb_image: 'europe-docker.pkg.dev/unicorn-985/private-images/docker-images_mongo6-replicated:v1', + mongo_job_image: 'europe-docker.pkg.dev/unicorn-985/public-images/docker-images_mongo-cloner-job:v1', default_python_image: 'mirror.gcr.io/python:3.12.1', default_pulumi_node_image: 'mirror.gcr.io/node:18', - job_poster_image: 'europe-docker.pkg.dev/gynzy-test-project/public-images/docker-images_job-poster:v1', + job_poster_image: 'europe-docker.pkg.dev/unicorn-985/public-images/docker-images_job-poster:v2', } diff --git a/.github/jsonnet/misc.jsonnet b/.github/jsonnet/misc.jsonnet index 385cbb7a..c00512c8 100644 --- a/.github/jsonnet/misc.jsonnet +++ b/.github/jsonnet/misc.jsonnet @@ -424,19 +424,23 @@ local images = import 'images.jsonnet'; // // Parameters: // name: the name of the github job - // job_name: the name of the job to be posted + // jobName: the name of the job to be posted // cluster: the cluster to post the job to. This should be an object from the clusters module // image: the image to use for the job // environment: a map of environment variables to pass to the job // command: the command to run in the job (optional) // ifClause: the condition under which to run the job (optional) - postJob(name, job_name, cluster, image, environment, command='', ifClause=null):: + // memory: the memory requested for the job (optional) + // memoryLimit: the memory limit for the job (optional) + // cpu: the cpu requested for the job (optional) + // cpuLimit: the cpu limit for the job (optional) + postJob(name, jobName, cluster, image, environment, command='', ifClause=null, memory='100Mi', memoryLimit='100Mi', cpu='100m', cpuLimit='100m'):: base.action( name, 'docker://' + images.job_poster_image, ifClause=ifClause, env={ - JOB_NAME: job_name, + JOB_NAME: jobName, IMAGE: image, COMMAND: command, ENVIRONMENT: std.join(' ', std.objectFields(environment)), @@ -444,7 +448,12 @@ local images = import 'images.jsonnet'; GKE_PROJECT: cluster.project, GKE_ZONE: cluster.zone, GKE_CLUSTER: cluster.name, - NODESELECTOR_TYPE: cluster.jobNodeSelectorType, + NODESELECTOR_KEY: cluster.jobNodeSelectorKey, + NODESELECTOR_VALUE: cluster.jobNodeSelectorValue, + JOB_REQUEST_MEM: memory, + JOB_REQUEST_MEM_LIMIT: memoryLimit, + JOB_REQUEST_CPU: cpu, + JOB_REQUEST_CPU_LIMIT: cpuLimit, } + environment, ), diff --git a/.github/jsonnet/mongo.jsonnet b/.github/jsonnet/mongo.jsonnet index ba7da6bb..862ef2c5 100644 --- a/.github/jsonnet/mongo.jsonnet +++ b/.github/jsonnet/mongo.jsonnet @@ -103,7 +103,7 @@ local prodProjectSettings = { misc.postJob( name='copy-mongo-db', - job_name='mongo-copy-' + service + '-${{ github.run_number }}-${{ github.run_attempt }}', + jobName='mongo-copy-' + service + '-${{ github.run_number }}-${{ github.run_attempt }}', cluster=mongoCluster.gke_cluster, image=images.mongo_job_image, environment={ @@ -113,7 +113,15 @@ local prodProjectSettings = { MONGO_HOST: mongoCluster.connectionString, MONGO_USER: mongoCluster.CIUsername, MONGO_PASS: mongoCluster.CIPassword, + JOB_REQUEST_CPU: "500m", + JOB_REQUEST_CPU_LIMIT: "1", + JOB_REQUEST_MEM: "512Mi", + JOB_REQUEST_MEM_LIMIT: "1Gi", }, + memory='1Gi', + memoryLimit='1Gi', + cpu='500m', + cpuLimit='1', ), // Delete a MongoDB PR database. @@ -131,7 +139,7 @@ local prodProjectSettings = { misc.postJob( name='delete-mongo-db', - job_name='mongo-delete-' + service + '-${{ github.run_number }}-${{ github.run_attempt }}', + jobName='mongo-delete-' + service + '-${{ github.run_number }}-${{ github.run_attempt }}', cluster=mongoCluster.gke_cluster, image=images.mongo_job_image, environment={ @@ -140,6 +148,8 @@ local prodProjectSettings = { MONGO_HOST: mongoCluster.connectionString, MONGO_USER: mongoCluster.CIUsername, MONGO_PASS: mongoCluster.CIPassword, + JOB_REQUEST_MEM: '200Mi', + JOB_REQUEST_MEM_LIMIT: '200Mi', }, ), // Sync the indexes of a MongoDB database with the current codebase. @@ -156,7 +166,7 @@ local prodProjectSettings = { mongoSyncIndexes(service, image, mongoCluster, database, ifClause=null):: misc.postJob( name='sync-mongo-indexes-' + mongoCluster.name + '-' + database, - job_name='mongo-sync-' + service + '-${{ github.run_number }}-${{ github.run_attempt }}', + jobName='mongo-sync-' + service + '-${{ github.run_number }}-${{ github.run_attempt }}', cluster=mongoCluster.gke_cluster, image=image, environment={ @@ -167,6 +177,9 @@ local prodProjectSettings = { MONGO_USER: mongoCluster.CIUsername, MONGO_PASS: mongoCluster.CIPassword, IS_ATLAS_MONGO: 'true', + JOB_REQUEST_MEM: '200Mi', + JOB_REQUEST_MEM_LIMIT: '200Mi', + JOB_REQUEST_CPU_LIMIT: '1', }, ifClause=ifClause, command='docker/mongo-sync-indexes.sh', @@ -188,7 +201,7 @@ local prodProjectSettings = { misc.postJob( name='diff-mongo-indexes-' + mongoCluster.name + '-' + database, - job_name='mongo-diff-' + service + '-${{ github.run_number }}-${{ github.run_attempt }}', + jobName='mongo-diff-' + service + '-${{ github.run_number }}-${{ github.run_attempt }}', cluster=mongoCluster.gke_cluster, image=image, environment={ @@ -203,6 +216,9 @@ local prodProjectSettings = { MONGO_CLUSTER: mongoCluster.name, MONGO_DEEPLINK: mongoDBLink, IS_ATLAS_MONGO: 'true', + JOB_REQUEST_MEM: '200Mi', + JOB_REQUEST_MEM_LIMIT: '200Mi', + JOB_REQUEST_CPU_LIMIT: '1', }, command='docker/mongo-sync-indexes.sh', ), diff --git a/.github/jsonnet/ruby.jsonnet b/.github/jsonnet/ruby.jsonnet index ad0dc85d..82819c2a 100644 --- a/.github/jsonnet/ruby.jsonnet +++ b/.github/jsonnet/ruby.jsonnet @@ -9,7 +9,7 @@ local servicesImport = import 'services.jsonnet'; { rubyDeployPRPipeline( serviceName, - dockerImageName='github-gynzy-docker-' + serviceName, + dockerImageName='backend-' + serviceName, helmDeployOptions={ ingress: { enabled: true }, cronjob: { enabled: true }, diff --git a/.github/jsonnet/services.jsonnet b/.github/jsonnet/services.jsonnet index cb52902e..2f3c879a 100644 --- a/.github/jsonnet/services.jsonnet +++ b/.github/jsonnet/services.jsonnet @@ -2,24 +2,6 @@ local images = import 'images.jsonnet'; local misc = import 'misc.jsonnet'; { - mysql57service(database=null, password=null, root_password=null, username=null, port='3306'):: - { - image: images.default_mysql57_image, - credentials: { - username: '_json_key', - password: misc.secret('docker_gcr_io'), - }, - env: { - MYSQL_DATABASE: database, - MYSQL_PASSWORD: password, - MYSQL_ROOT_PASSWORD: root_password, - MYSQL_USER: username, - MYSQL_TCP_PORT: port, - }, - options: '--health-cmd="mysqladmin ping" --health-interval=1s --health-timeout=1s --health-retries=40', - ports: [port + ':' + port], - }, - mysql8service(database=null, password=null, root_password=null, username=null, port='3306'):: { image: images.default_mysql8_image, diff --git a/.github/workflows/misc.yml b/.github/workflows/misc.yml index a9cf3476..a09b81fa 100644 --- a/.github/workflows/misc.yml +++ b/.github/workflows/misc.yml @@ -4,7 +4,7 @@ "credentials": "password": "${{ secrets.docker_gcr_io }}" "username": "_json_key" - "image": "eu.gcr.io/unicorn-985/docker-images_jsonnet:v1" + "image": "europe-docker.pkg.dev/unicorn-985/private-images/docker-images_jsonnet:v1" "runs-on": "ubuntu-latest" "steps": - "id": "check-binaries"