From ce7673cfe65bd5830277e2ea9e086aa280ac8c53 Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Mon, 18 Jul 2022 10:41:24 +0530 Subject: [PATCH 01/18] updated with github deploy workflow --- .../setup-deploy-pipeline.asciidoc | 6 + .../setup-deploy-pipeline.asciidoc | 64 +++++++++++ .../src/github/setup-deploy-pipeline.asciidoc | 6 + .../templates/deploy/deploy-pipeline.cfg | 30 +++++ .../deploy/deploy-pipeline.yml.template | 103 ++++++++++++++++++ .../github/templates/deploy/deploy.sh | 9 ++ .../github/templates/deploy/secrets.sh | 12 ++ 7 files changed, 230 insertions(+) create mode 100644 documentation/src/azure-devops/setup-deploy-pipeline.asciidoc create mode 100644 documentation/src/common_templates/setup-deploy-pipeline.asciidoc create mode 100644 documentation/src/github/setup-deploy-pipeline.asciidoc create mode 100644 scripts/pipelines/github/templates/deploy/deploy-pipeline.cfg create mode 100644 scripts/pipelines/github/templates/deploy/deploy-pipeline.yml.template create mode 100644 scripts/pipelines/github/templates/deploy/deploy.sh create mode 100644 scripts/pipelines/github/templates/deploy/secrets.sh diff --git a/documentation/src/azure-devops/setup-deploy-pipeline.asciidoc b/documentation/src/azure-devops/setup-deploy-pipeline.asciidoc new file mode 100644 index 000000000..cb95ffe42 --- /dev/null +++ b/documentation/src/azure-devops/setup-deploy-pipeline.asciidoc @@ -0,0 +1,6 @@ +:provider: Azure Devops +:pipeline_type: Pipeline +:trigger_sentence: This pipeline will be configured to be triggered every time there is a commit to the Azure Devops repository +:pipeline_type2: pipeline +:path_provider: azure-devops +include::../common_templates/setup-build-pipeline.asciidoc[] \ No newline at end of file diff --git a/documentation/src/common_templates/setup-deploy-pipeline.asciidoc b/documentation/src/common_templates/setup-deploy-pipeline.asciidoc new file mode 100644 index 000000000..a2a669bb9 --- /dev/null +++ b/documentation/src/common_templates/setup-deploy-pipeline.asciidoc @@ -0,0 +1,64 @@ += Setting up a Deploy Pipeline on {provider} + +In this section we will create a deploy {pipeline_type} on {provider} to deploy the project application on an already provisioned Kubernetes cluster. This pipeline will be configured in order to be triggered every time package {pipeline_type} is executed successfully on a commit for `release/*` and `develop` branches, requiring manual launch for other branches but still enforcing that package {pipeline_type} has passed. By default, it depends on the environment provisioning {pipeline_type} being successfully run on beforehand and, depending on the Kubernetes provider, it consumes the artifact produced by that. It also consumes variable groups created by package and environment provisioning {pipeline_type}. + +The creation of the pipeline will follow the project workflow, so a new branch named `feature/deploy-pipeline` will be created and the YAML file for the pipeline will be pushed to it. + +Then, a Pull Request (PR) will be created in order to merge the new branch into the appropriate branch (provided in `-b` flag). The PR will be automatically merged if the repository policies are met. If the merge is not possible, either the PR URL will be shown as output, or it will be opened in your web browser if using `-w` flag. + +The script located at `/scripts/pipelines/{path_provider}/pipeline_generator.sh` will automatically create the new branch, create a deploy pipeline based on a YAML template appropriate for the project manifests files, create the Pull Request, and if it is possible, merge this new branch into the specified branch. + +== Prerequisites + +* This script will commit and push the corresponding YAML template into your repository, so please be sure your local repository is up-to-date (i.e you have pulled the latest changes with `git pull`). + +* The YAML containing Deployment object should be named `application-deployment.yaml` for allowing automated `imagePullSecret` injection. + +== Creating the {pipeline_type} using provided script + +=== Usage +``` +pipeline_generator.sh \ + -c \ + -n \ + -d \ + --package-pipeline-name \ + --env-provision-pipeline-name ] \ + --k8s-provider \ + --k8s-namespace \ + --k8s-deploy-files-path \ + [--k8s-image-pull-secret-name ] \ + [-b ] \ + [-w] +``` +NOTE: The config file for the deploy pipeline is located at `/scripts/pipelines/{path_provider}/templates/deploy/deploy-pipeline.cfg`. + +=== Flags +``` +-c --config-file [Required] Configuration file containing pipeline definition. +-n --pipeline-name [Required] Name that will be set to the pipeline. +-d --local-directory [Required] Local directory of your project. + --package-pipeline-name [Required] Package pipeline name. + --env-provision-pipeline-name [Required] Environment provisioning pipeline name. + --k8s-provider [Required] Kubernetes cluster provider name. Accepted values: EKS, AKS. + --k8s-namespace [Required] Kubernetes namespace where the application will be deployed. + --k8s-deploy-files-path [Required] Path from the root of the project to the YAML manifests directory. + --k8s-image-pull-secret-name Name for the generated secret containing registry credentials. Required when using a private registry to host images. +-b --target-branch Name of the branch to which the Pull Request will target. PR is not created if the flag is not provided. +-w Open the Pull Request on the web browser if it cannot be automatically merged. Requires -b flag. +``` + +=== Examples + +``` +./pipeline_generator.sh -c ./templates/deploy/deploy-pipeline.cfg -n quarkus-project-deploy -d C:/Users/$USERNAME/Desktop/quarkus-project --package-pipeline-name quarkus-project-package --env-provision-pipeline-name eks-provisioning --k8s-provider EKS --k8s-namespace hangar --k8s-deploy-files-path k8s -b develop -w +``` + + +=== Appendix: accessing the application + +Once the {pipeline_type} is executed and your application is deployed, you can list the hostname to access it with: + +``` +kubectl get ingress -n +``` \ No newline at end of file diff --git a/documentation/src/github/setup-deploy-pipeline.asciidoc b/documentation/src/github/setup-deploy-pipeline.asciidoc new file mode 100644 index 000000000..ce00cbc6e --- /dev/null +++ b/documentation/src/github/setup-deploy-pipeline.asciidoc @@ -0,0 +1,6 @@ +:provider: Github +:pipeline_type: workflow +:trigger_sentence: This workflow will be configures to be executed inside a CI pipeline for the build workflow +:pipeline_type2: Github action +:path_provider: github +include::../common_templates/setup-build-pipeline.asciidoc[] \ No newline at end of file diff --git a/scripts/pipelines/github/templates/deploy/deploy-pipeline.cfg b/scripts/pipelines/github/templates/deploy/deploy-pipeline.cfg new file mode 100644 index 000000000..10ad9114b --- /dev/null +++ b/scripts/pipelines/github/templates/deploy/deploy-pipeline.cfg @@ -0,0 +1,30 @@ +# List of the flages we used in different stages for deploy pipeline. +# Comma separated list of flags. Make sure to end with a comma, Eg: $name,$xyz, +mandatoryFlags="$pipelineName,$localDirectory,$packagePipelineName,$envProvisionPipelineName,$k8sProvider,$k8sNamespace,$k8sDeployFiles," +# Path to the templates. +templatesPath="scripts/pipelines/github/templates/deploy" +# YAML file name. +yamlFile="deploy-pipeline.yml" +# Source branch. +sourceBranch="feature/deploy-pipeline" + +# Function that copies the extra yaml files into the directory. +function copyScript { + # Copy the deploy script. + cp "${hangarPath}/${templatesPath}/deploy.sh" "${localDirectory}/${scriptFilePath}" + + # Copy the script for generating imagePullSecrets for private registries. + cp "${hangarPath}/${templatesPath}/secrets.sh" "${localDirectory}/${scriptFilePath}" +} + +function addPipelineVariables { + # We cannot use a variable in the definition of resource in the pipeline so we have to use a placeholder to replace it with the value we need + export packagePipelineName + export envProvisionPipelineName + export k8sProvider + export k8sNamespace + export k8sDeployFiles + export k8sImagePullSecret + specificEnvSubstList='${packagePipelineName} ${envProvisionPipelineName} ${k8sProvider} ${k8sNamespace} ${k8sDeployFiles} ${k8sImagePullSecret}' +} + diff --git a/scripts/pipelines/github/templates/deploy/deploy-pipeline.yml.template b/scripts/pipelines/github/templates/deploy/deploy-pipeline.yml.template new file mode 100644 index 000000000..6b85bec3f --- /dev/null +++ b/scripts/pipelines/github/templates/deploy/deploy-pipeline.yml.template @@ -0,0 +1,103 @@ +name: $pipelineName + +on: + workflow_run: + workflows: $packagePipelineName + types: completed + branches: + - release/* + - develop + workflow_dispatch: + inputs: + k8sProvider: + description: 'Deploy Cluster Name.' + required: true + type: string + default: '$k8sProvider' + secretsName: + description: 'ImagePullSecret Name.' + type: string + default: '$k8sImagePullSecret' +# mark to insert additional artifact input # + +env: + deploymentYamlPath: "${{ github.workspace }}/$k8sDeployFiles/application-deployment.yaml" + ingressYamlPath: "${{ github.workspace }}/$k8sDeployFiles/ingress.yaml" + deployFilesPath: "${{ github.workspace }}/$k8sDeployFiles" + provisionPipeline: "$envProvisionPipelineName.yml" + packagePipelineName: "$packagePipelineName.yml" +# mark to insert additional artifact env var # + +jobs: + Deploy_to_AKS: + runs-on: ubuntu-latest + if: ${{ github.event.inputs.k8sProvider == 'AKS' || github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} + steps: + - name: Checkout the repository + uses: actions/checkout@master + + - name: Download artifact from AKS provision pipeline workflow + uses: dawidd6/action-download-artifact@v2 + continue-on-error: true + with: + workflow: ${{ env.provisionPipeline }} + check_artifacts: true + search_artifacts: true + name: kubeconfig + + - name: Create secrets for AKS + if: ${{ github.event.inputs.k8sProvider == 'AKS' && github.event.inputs.secretsName != '' }} + env: + KUBECONFIG: "${{ github.workspace }}/kubeconfig" + run: .github/workflows/scripts/secrets.sh "${{ env.k8sNamespace }}" "${{ github.event.inputs.secretsName }}" "${{ secrets.DOCKER_USER }}" "${{ secrets.DOCKER_PASSWORD }}" "${{ env.registry }}" "${{ env.deploymentYamlPath }}" + + - name: Set EKS Environment Variables + uses: tw3lveparsecs/github-actions-setvars@v0.1 + with: + envFilePath: ".github/vars/*" + + - name: Deploy app to AKS + env: + KUBECONFIG: "${{ github.workspace }}/kubeconfig" + run: .github/workflows/scripts/deploy.sh "${{ env.k8sNamespace }}" "${{ env.imageName }}" "${{ env.dns_name }}" "${{ env.deploymentYamlPath }}" "${{ env.ingressYamlPath }}" "${{ env.deployFilesPath }}" + + + Deploy_to_EKS: + runs-on: ubuntu-latest + if: ${{ github.event.inputs.k8sProvider == 'EKS' || github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} + steps: + - name: Checkout the repository + uses: actions/checkout@v3 + + - name: Set EKS Environment Variables + uses: tw3lveparsecs/github-actions-setvars@v0.1 + with: + envFilePath: ".github/vars/*" + + - name: Create secrets for EKS + if: ${{ github.event.inputs.k8sProvider == 'EKS' && github.event.inputs.secretsName != '' }} + run: | + aws eks update-kubeconfig --name ${{ env.cluster_name }} --region ${{ env.aws_region }} + .github/workflows/scripts/secrets.sh "${{ env.k8sNamespace }}" "${{ github.event.inputs.secretsName }}" "${{ secrets.AWS_ACCESS_KEY_ID }}" "${{ secrets.AWS_SECRET_ACCESS_KEY }}" "${{ env.registry }}" "${{ env.deploymentYamlPath }}" + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.aws_region }} + + - name: Deploy app to EKS + run: | + aws eks update-kubeconfig --name ${{ env.cluster_name }} --region ${{ env.aws_region }} + .github/workflows/scripts/deploy.sh "${{ env.k8sNamespace }}" "${{ env.imageName }}" "${{ env.dns_name }}" "${{ env.deploymentYamlPath }}" "${{ env.ingressYamlPath }}" "${{ env.deployFilesPath }}" + + # mark to insert step for additional artifact # + on-failure: + name: CI Workflow Failed + runs-on: ubuntu-latest + if: ${{ github.event.workflow_run.conclusion == 'failure' }} + steps: + - name: Cancel Workflow + uses: andymckay/cancel-action@0.2 + \ No newline at end of file diff --git a/scripts/pipelines/github/templates/deploy/deploy.sh b/scripts/pipelines/github/templates/deploy/deploy.sh new file mode 100644 index 000000000..3e256f1cf --- /dev/null +++ b/scripts/pipelines/github/templates/deploy/deploy.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -e +# Add image name and ingress DNS name. +export image="$2" dns="$3" +yq eval '.spec.template.spec.containers[0].image = "'"$image"'"' -i "$4" +yq eval '.spec.rules[0].host = "'"$dns"'"' -i "$5" +# Deploy apps in exists namespace, If not exists Create new namespace and apply manifest files. +kubectl get namespace | grep -q "^$1" || kubectl create namespace "$1" +kubectl apply -f "$6" --namespace="$1" \ No newline at end of file diff --git a/scripts/pipelines/github/templates/deploy/secrets.sh b/scripts/pipelines/github/templates/deploy/secrets.sh new file mode 100644 index 000000000..e4d4eef84 --- /dev/null +++ b/scripts/pipelines/github/templates/deploy/secrets.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -e +# Deploy apps in exists namespace, If not exists Create new namespace. +kubectl get namespace | grep -q "^$1" || kubectl create namespace "$1" +# Command to existed secrets and create new secrets to pull image from private registry. +kubectl delete secret "$2" --namespace="$1" --ignore-not-found +kubectl create secret docker-registry "$2" --docker-server="$5" --docker-username="$3" --docker-password="$4" --namespace="$1" +export secrets="$2" +yq e '.spec.template.spec."imagePullSecrets"=[{"name":"secrets"}]' -i "$6" +yq e '.spec.template.spec.imagePullSecrets[0].name = "'"$secrets"'"' -i "$6" +# Apply the changes. +kubectl apply -f "$6" --namespace="$1" \ No newline at end of file From 59cb072f348d260fc66354a5e4dc5661279856a1 Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Mon, 18 Jul 2022 11:21:01 +0530 Subject: [PATCH 02/18] updated --- documentation/src/azure-devops/setup-deploy-pipeline.asciidoc | 4 ++-- documentation/src/github/setup-deploy-pipeline.asciidoc | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/documentation/src/azure-devops/setup-deploy-pipeline.asciidoc b/documentation/src/azure-devops/setup-deploy-pipeline.asciidoc index cb95ffe42..b7a58985b 100644 --- a/documentation/src/azure-devops/setup-deploy-pipeline.asciidoc +++ b/documentation/src/azure-devops/setup-deploy-pipeline.asciidoc @@ -1,6 +1,6 @@ :provider: Azure Devops :pipeline_type: Pipeline -:trigger_sentence: This pipeline will be configured to be triggered every time there is a commit to the Azure Devops repository +:trigger_sentence: :pipeline_type2: pipeline :path_provider: azure-devops -include::../common_templates/setup-build-pipeline.asciidoc[] \ No newline at end of file +include::../common_templates/setup-deploy-pipeline.asciidoc[] \ No newline at end of file diff --git a/documentation/src/github/setup-deploy-pipeline.asciidoc b/documentation/src/github/setup-deploy-pipeline.asciidoc index ce00cbc6e..1ca440ee0 100644 --- a/documentation/src/github/setup-deploy-pipeline.asciidoc +++ b/documentation/src/github/setup-deploy-pipeline.asciidoc @@ -1,6 +1,6 @@ :provider: Github :pipeline_type: workflow -:trigger_sentence: This workflow will be configures to be executed inside a CI pipeline for the build workflow +:trigger_sentence: :pipeline_type2: Github action :path_provider: github -include::../common_templates/setup-build-pipeline.asciidoc[] \ No newline at end of file +include::../common_templates/setup-deploy-pipeline.asciidoc[] \ No newline at end of file From 58f222169ef8d0f1514d6f85caef2572d6e802ef Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Mon, 18 Jul 2022 05:52:02 +0000 Subject: [PATCH 03/18] Automatic generation of documentation --- .../setup-deploy-pipeline.asciidoc | 21 +++--- .../github/setup-deploy-pipeline.asciidoc | 69 +++++++++++++++++++ 2 files changed, 82 insertions(+), 8 deletions(-) create mode 100644 documentation/github/setup-deploy-pipeline.asciidoc diff --git a/documentation/azure-devops/setup-deploy-pipeline.asciidoc b/documentation/azure-devops/setup-deploy-pipeline.asciidoc index 2d77b3284..cadd66064 100644 --- a/documentation/azure-devops/setup-deploy-pipeline.asciidoc +++ b/documentation/azure-devops/setup-deploy-pipeline.asciidoc @@ -1,12 +1,17 @@ -= Setting up a Deploy Pipeline on Azure DevOps +:provider: Azure Devops +:pipeline_type: Pipeline +:trigger_sentence: +:pipeline_type2: pipeline +:path_provider: azure-devops += Setting up a Deploy Pipeline on {provider} -In this section we will create a deploy pipeline on Azure DevOps to deploy the project application on an already provisioned Kubernetes cluster. This pipeline will be configured in order to be triggered every time package pipeline is executed successfully on a commit for `release/*` and `develop` branches, requiring manual launch for other branches but still enforcing that package pipeline has passed. By default, it depends on the environment provisioning pipeline being successfully run on beforehand and, depending on the Kubernetes provider, it consumes the artifact produced by that. It also consumes variable groups created by package and environment provisioning pipelines. +In this section we will create a deploy {pipeline_type} on {provider} to deploy the project application on an already provisioned Kubernetes cluster. This pipeline will be configured in order to be triggered every time package {pipeline_type} is executed successfully on a commit for `release/*` and `develop` branches, requiring manual launch for other branches but still enforcing that package {pipeline_type} has passed. By default, it depends on the environment provisioning {pipeline_type} being successfully run on beforehand and, depending on the Kubernetes provider, it consumes the artifact produced by that. It also consumes variable groups created by package and environment provisioning {pipeline_type}. The creation of the pipeline will follow the project workflow, so a new branch named `feature/deploy-pipeline` will be created and the YAML file for the pipeline will be pushed to it. Then, a Pull Request (PR) will be created in order to merge the new branch into the appropriate branch (provided in `-b` flag). The PR will be automatically merged if the repository policies are met. If the merge is not possible, either the PR URL will be shown as output, or it will be opened in your web browser if using `-w` flag. -The script located at `/scripts/pipelines/azure-devops/pipeline_generator.sh` will automatically create the new branch, create a deploy pipeline based on a YAML template appropriate for the project manifests files, create the Pull Request, and if it is possible, merge this new branch into the specified branch. +The script located at `/scripts/pipelines/{path_provider}/pipeline_generator.sh` will automatically create the new branch, create a deploy pipeline based on a YAML template appropriate for the project manifests files, create the Pull Request, and if it is possible, merge this new branch into the specified branch. == Prerequisites @@ -14,7 +19,7 @@ The script located at `/scripts/pipelines/azure-devops/pipeline_generator.sh` wi * The YAML containing Deployment object should be named `application-deployment.yaml` for allowing automated `imagePullSecret` injection. -== Creating the pipeline using provided script +== Creating the {pipeline_type} using provided script === Usage ``` @@ -26,12 +31,12 @@ pipeline_generator.sh \ --env-provision-pipeline-name ] \ --k8s-provider \ --k8s-namespace \ - --k8s-deploy-files-path \ + --k8s-deploy-files-path \ [--k8s-image-pull-secret-name ] \ [-b ] \ [-w] ``` -NOTE: The config file for the deploy pipeline is located at `/scripts/pipelines/azure-devops/templates/deploy/deploy-pipeline.cfg`. +NOTE: The config file for the deploy pipeline is located at `/scripts/pipelines/{path_provider}/templates/deploy/deploy-pipeline.cfg`. === Flags ``` @@ -43,7 +48,7 @@ NOTE: The config file for the deploy pipeline is located at `/scripts/pipelines --k8s-provider [Required] Kubernetes cluster provider name. Accepted values: EKS, AKS. --k8s-namespace [Required] Kubernetes namespace where the application will be deployed. --k8s-deploy-files-path [Required] Path from the root of the project to the YAML manifests directory. - --k8s-image-pull-secret-name Name for the generated secret containing registry credentials. Required when using a private registry to host images. + --k8s-image-pull-secret-name Name for the generated secret containing registry credentials. Required when using a private registry to host images. -b --target-branch Name of the branch to which the Pull Request will target. PR is not created if the flag is not provided. -w Open the Pull Request on the web browser if it cannot be automatically merged. Requires -b flag. ``` @@ -57,7 +62,7 @@ NOTE: The config file for the deploy pipeline is located at `/scripts/pipelines === Appendix: accessing the application -Once the pipeline is executed and your application is deployed, you can list the hostname to access it with: +Once the {pipeline_type} is executed and your application is deployed, you can list the hostname to access it with: ``` kubectl get ingress -n diff --git a/documentation/github/setup-deploy-pipeline.asciidoc b/documentation/github/setup-deploy-pipeline.asciidoc new file mode 100644 index 000000000..c9866504b --- /dev/null +++ b/documentation/github/setup-deploy-pipeline.asciidoc @@ -0,0 +1,69 @@ +:provider: Github +:pipeline_type: workflow +:trigger_sentence: +:pipeline_type2: Github action +:path_provider: github += Setting up a Deploy Pipeline on {provider} + +In this section we will create a deploy {pipeline_type} on {provider} to deploy the project application on an already provisioned Kubernetes cluster. This pipeline will be configured in order to be triggered every time package {pipeline_type} is executed successfully on a commit for `release/*` and `develop` branches, requiring manual launch for other branches but still enforcing that package {pipeline_type} has passed. By default, it depends on the environment provisioning {pipeline_type} being successfully run on beforehand and, depending on the Kubernetes provider, it consumes the artifact produced by that. It also consumes variable groups created by package and environment provisioning {pipeline_type}. + +The creation of the pipeline will follow the project workflow, so a new branch named `feature/deploy-pipeline` will be created and the YAML file for the pipeline will be pushed to it. + +Then, a Pull Request (PR) will be created in order to merge the new branch into the appropriate branch (provided in `-b` flag). The PR will be automatically merged if the repository policies are met. If the merge is not possible, either the PR URL will be shown as output, or it will be opened in your web browser if using `-w` flag. + +The script located at `/scripts/pipelines/{path_provider}/pipeline_generator.sh` will automatically create the new branch, create a deploy pipeline based on a YAML template appropriate for the project manifests files, create the Pull Request, and if it is possible, merge this new branch into the specified branch. + +== Prerequisites + +* This script will commit and push the corresponding YAML template into your repository, so please be sure your local repository is up-to-date (i.e you have pulled the latest changes with `git pull`). + +* The YAML containing Deployment object should be named `application-deployment.yaml` for allowing automated `imagePullSecret` injection. + +== Creating the {pipeline_type} using provided script + +=== Usage +``` +pipeline_generator.sh \ + -c \ + -n \ + -d \ + --package-pipeline-name \ + --env-provision-pipeline-name ] \ + --k8s-provider \ + --k8s-namespace \ + --k8s-deploy-files-path \ + [--k8s-image-pull-secret-name ] \ + [-b ] \ + [-w] +``` +NOTE: The config file for the deploy pipeline is located at `/scripts/pipelines/{path_provider}/templates/deploy/deploy-pipeline.cfg`. + +=== Flags +``` +-c --config-file [Required] Configuration file containing pipeline definition. +-n --pipeline-name [Required] Name that will be set to the pipeline. +-d --local-directory [Required] Local directory of your project. + --package-pipeline-name [Required] Package pipeline name. + --env-provision-pipeline-name [Required] Environment provisioning pipeline name. + --k8s-provider [Required] Kubernetes cluster provider name. Accepted values: EKS, AKS. + --k8s-namespace [Required] Kubernetes namespace where the application will be deployed. + --k8s-deploy-files-path [Required] Path from the root of the project to the YAML manifests directory. + --k8s-image-pull-secret-name Name for the generated secret containing registry credentials. Required when using a private registry to host images. +-b --target-branch Name of the branch to which the Pull Request will target. PR is not created if the flag is not provided. +-w Open the Pull Request on the web browser if it cannot be automatically merged. Requires -b flag. +``` + +=== Examples + +``` +./pipeline_generator.sh -c ./templates/deploy/deploy-pipeline.cfg -n quarkus-project-deploy -d C:/Users/$USERNAME/Desktop/quarkus-project --package-pipeline-name quarkus-project-package --env-provision-pipeline-name eks-provisioning --k8s-provider EKS --k8s-namespace hangar --k8s-deploy-files-path k8s -b develop -w +``` + + +=== Appendix: accessing the application + +Once the {pipeline_type} is executed and your application is deployed, you can list the hostname to access it with: + +``` +kubectl get ingress -n +``` From ee16702e6e88dc592f022117ac0250a5848d0b75 Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Tue, 19 Jul 2022 09:09:59 +0530 Subject: [PATCH 04/18] update with tag --- .../github/templates/deploy/deploy-pipeline.yml.template | 8 ++++---- scripts/pipelines/github/templates/deploy/deploy.sh | 7 ++++--- scripts/pipelines/github/templates/deploy/secrets.sh | 6 ++---- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/scripts/pipelines/github/templates/deploy/deploy-pipeline.yml.template b/scripts/pipelines/github/templates/deploy/deploy-pipeline.yml.template index 6b85bec3f..907099860 100644 --- a/scripts/pipelines/github/templates/deploy/deploy-pipeline.yml.template +++ b/scripts/pipelines/github/templates/deploy/deploy-pipeline.yml.template @@ -26,6 +26,7 @@ env: deployFilesPath: "${{ github.workspace }}/$k8sDeployFiles" provisionPipeline: "$envProvisionPipelineName.yml" packagePipelineName: "$packagePipelineName.yml" + path: ".github/workflows/scripts/package-extra.sh" # mark to insert additional artifact env var # jobs: @@ -59,7 +60,7 @@ jobs: - name: Deploy app to AKS env: KUBECONFIG: "${{ github.workspace }}/kubeconfig" - run: .github/workflows/scripts/deploy.sh "${{ env.k8sNamespace }}" "${{ env.imageName }}" "${{ env.dns_name }}" "${{ env.deploymentYamlPath }}" "${{ env.ingressYamlPath }}" "${{ env.deployFilesPath }}" + run: .github/workflows/scripts/deploy.sh "${{ env.k8sNamespace }}" "${{ env.imageName }}" "${{ env.aks_dns_name }}" "${{ env.deploymentYamlPath }}" "${{ env.ingressYamlPath }}" "${{ env.deployFilesPath }}" "${{ env.path }}" Deploy_to_EKS: @@ -90,7 +91,7 @@ jobs: - name: Deploy app to EKS run: | aws eks update-kubeconfig --name ${{ env.cluster_name }} --region ${{ env.aws_region }} - .github/workflows/scripts/deploy.sh "${{ env.k8sNamespace }}" "${{ env.imageName }}" "${{ env.dns_name }}" "${{ env.deploymentYamlPath }}" "${{ env.ingressYamlPath }}" "${{ env.deployFilesPath }}" + .github/workflows/scripts/deploy.sh "${{ env.k8sNamespace }}" "${{ env.imageName }}" "${{ env.eks_dns_name }}" "${{ env.deploymentYamlPath }}" "${{ env.ingressYamlPath }}" "${{ env.deployFilesPath }}" "${{ env.path }}" # mark to insert step for additional artifact # on-failure: @@ -99,5 +100,4 @@ jobs: if: ${{ github.event.workflow_run.conclusion == 'failure' }} steps: - name: Cancel Workflow - uses: andymckay/cancel-action@0.2 - \ No newline at end of file + uses: andymckay/cancel-action@0.2 \ No newline at end of file diff --git a/scripts/pipelines/github/templates/deploy/deploy.sh b/scripts/pipelines/github/templates/deploy/deploy.sh index 3e256f1cf..03fdf5bee 100644 --- a/scripts/pipelines/github/templates/deploy/deploy.sh +++ b/scripts/pipelines/github/templates/deploy/deploy.sh @@ -1,8 +1,9 @@ #!/bin/bash set -e -# Add image name and ingress DNS name. -export image="$2" dns="$3" -yq eval '.spec.template.spec.containers[0].image = "'"$image"'"' -i "$4" +# Add image name, dns_name and tag. +source "$7" +export image="$2" tag="${tag}" dns="$3" +yq eval '.spec.template.spec.containers[0].image = "'"$image:$tag"'"' -i "$4" yq eval '.spec.rules[0].host = "'"$dns"'"' -i "$5" # Deploy apps in exists namespace, If not exists Create new namespace and apply manifest files. kubectl get namespace | grep -q "^$1" || kubectl create namespace "$1" diff --git a/scripts/pipelines/github/templates/deploy/secrets.sh b/scripts/pipelines/github/templates/deploy/secrets.sh index e4d4eef84..0d68cf1f6 100644 --- a/scripts/pipelines/github/templates/deploy/secrets.sh +++ b/scripts/pipelines/github/templates/deploy/secrets.sh @@ -2,11 +2,9 @@ set -e # Deploy apps in exists namespace, If not exists Create new namespace. kubectl get namespace | grep -q "^$1" || kubectl create namespace "$1" -# Command to existed secrets and create new secrets to pull image from private registry. -kubectl delete secret "$2" --namespace="$1" --ignore-not-found -kubectl create secret docker-registry "$2" --docker-server="$5" --docker-username="$3" --docker-password="$4" --namespace="$1" +# Command to create secrets to pull image from private registry. export secrets="$2" yq e '.spec.template.spec."imagePullSecrets"=[{"name":"secrets"}]' -i "$6" yq e '.spec.template.spec.imagePullSecrets[0].name = "'"$secrets"'"' -i "$6" # Apply the changes. -kubectl apply -f "$6" --namespace="$1" \ No newline at end of file +kubectl create secret docker-registry "$2" --docker-server="$5" --docker-username="$3" --docker-password="$4" --namespace="$1" --save-config --dry-run=none -o yaml | kubectl apply -f "$6" --namespace="$1" \ No newline at end of file From 919ecc5e4d8a5b68cd8f561c4f940851279c7047 Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Tue, 19 Jul 2022 09:29:11 +0530 Subject: [PATCH 05/18] Update deploy.sh --- scripts/pipelines/github/templates/deploy/deploy.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/pipelines/github/templates/deploy/deploy.sh b/scripts/pipelines/github/templates/deploy/deploy.sh index 03fdf5bee..114978b4b 100644 --- a/scripts/pipelines/github/templates/deploy/deploy.sh +++ b/scripts/pipelines/github/templates/deploy/deploy.sh @@ -1,10 +1,11 @@ #!/bin/bash set -e # Add image name, dns_name and tag. -source "$7" +# Run package-extra.sh to ${tag} +. "$7" export image="$2" tag="${tag}" dns="$3" yq eval '.spec.template.spec.containers[0].image = "'"$image:$tag"'"' -i "$4" yq eval '.spec.rules[0].host = "'"$dns"'"' -i "$5" # Deploy apps in exists namespace, If not exists Create new namespace and apply manifest files. kubectl get namespace | grep -q "^$1" || kubectl create namespace "$1" -kubectl apply -f "$6" --namespace="$1" \ No newline at end of file +kubectl apply -f "$6" --namespace="$1" From cf4ed3a8c7b70b89fbf69feb0bc23aa13c07f55e Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Tue, 19 Jul 2022 11:26:54 +0530 Subject: [PATCH 06/18] Update deploy-pipeline.yml.template --- .../github/templates/deploy/deploy-pipeline.yml.template | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/pipelines/github/templates/deploy/deploy-pipeline.yml.template b/scripts/pipelines/github/templates/deploy/deploy-pipeline.yml.template index 907099860..a5ede01ca 100644 --- a/scripts/pipelines/github/templates/deploy/deploy-pipeline.yml.template +++ b/scripts/pipelines/github/templates/deploy/deploy-pipeline.yml.template @@ -26,7 +26,7 @@ env: deployFilesPath: "${{ github.workspace }}/$k8sDeployFiles" provisionPipeline: "$envProvisionPipelineName.yml" packagePipelineName: "$packagePipelineName.yml" - path: ".github/workflows/scripts/package-extra.sh" + packageTagScript: ".github/workflows/scripts/package-extra.sh" # mark to insert additional artifact env var # jobs: @@ -60,7 +60,7 @@ jobs: - name: Deploy app to AKS env: KUBECONFIG: "${{ github.workspace }}/kubeconfig" - run: .github/workflows/scripts/deploy.sh "${{ env.k8sNamespace }}" "${{ env.imageName }}" "${{ env.aks_dns_name }}" "${{ env.deploymentYamlPath }}" "${{ env.ingressYamlPath }}" "${{ env.deployFilesPath }}" "${{ env.path }}" + run: .github/workflows/scripts/deploy.sh "${{ env.k8sNamespace }}" "${{ env.imageName }}" "${{ env.aks_dns_name }}" "${{ env.deploymentYamlPath }}" "${{ env.ingressYamlPath }}" "${{ env.deployFilesPath }}" "${{ env.packageTagScript }}" Deploy_to_EKS: @@ -91,7 +91,7 @@ jobs: - name: Deploy app to EKS run: | aws eks update-kubeconfig --name ${{ env.cluster_name }} --region ${{ env.aws_region }} - .github/workflows/scripts/deploy.sh "${{ env.k8sNamespace }}" "${{ env.imageName }}" "${{ env.eks_dns_name }}" "${{ env.deploymentYamlPath }}" "${{ env.ingressYamlPath }}" "${{ env.deployFilesPath }}" "${{ env.path }}" + .github/workflows/scripts/deploy.sh "${{ env.k8sNamespace }}" "${{ env.imageName }}" "${{ env.eks_dns_name }}" "${{ env.deploymentYamlPath }}" "${{ env.ingressYamlPath }}" "${{ env.deployFilesPath }}" "${{ env.packageTagScript }}" # mark to insert step for additional artifact # on-failure: @@ -100,4 +100,4 @@ jobs: if: ${{ github.event.workflow_run.conclusion == 'failure' }} steps: - name: Cancel Workflow - uses: andymckay/cancel-action@0.2 \ No newline at end of file + uses: andymckay/cancel-action@0.2 From 1155ab355694179acc52a4f3710883fa26d321ab Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Wed, 20 Jul 2022 18:06:44 +0530 Subject: [PATCH 07/18] Update deploy-pipeline.yml.template --- .../deploy/deploy-pipeline.yml.template | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/scripts/pipelines/github/templates/deploy/deploy-pipeline.yml.template b/scripts/pipelines/github/templates/deploy/deploy-pipeline.yml.template index a5ede01ca..4c353ce93 100644 --- a/scripts/pipelines/github/templates/deploy/deploy-pipeline.yml.template +++ b/scripts/pipelines/github/templates/deploy/deploy-pipeline.yml.template @@ -27,6 +27,7 @@ env: provisionPipeline: "$envProvisionPipelineName.yml" packagePipelineName: "$packagePipelineName.yml" packageTagScript: ".github/workflows/scripts/package-extra.sh" + k8sNamespace: "$k8sNamespace" # mark to insert additional artifact env var # jobs: @@ -45,22 +46,22 @@ jobs: check_artifacts: true search_artifacts: true name: kubeconfig - - - name: Create secrets for AKS - if: ${{ github.event.inputs.k8sProvider == 'AKS' && github.event.inputs.secretsName != '' }} - env: - KUBECONFIG: "${{ github.workspace }}/kubeconfig" - run: .github/workflows/scripts/secrets.sh "${{ env.k8sNamespace }}" "${{ github.event.inputs.secretsName }}" "${{ secrets.DOCKER_USER }}" "${{ secrets.DOCKER_PASSWORD }}" "${{ env.registry }}" "${{ env.deploymentYamlPath }}" - + - name: Set EKS Environment Variables uses: tw3lveparsecs/github-actions-setvars@v0.1 with: envFilePath: ".github/vars/*" + - name: Create secrets for AKS + if: ${{ github.event.inputs.k8sProvider == 'AKS' && github.event.inputs.secretsName != '' }} + env: + KUBECONFIG: "${{ github.workspace }}/kubeconfig" + run: .github/workflows/scripts/secrets.sh "${{ env.k8sNamespace }}" "${{ github.event.inputs.secretsName }}" "${{ secrets.DOCKER_USER }}" "${{ secrets.DOCKER_PASSWORD }}" "${{ env.registry }}" "${{ env.deploymentYamlPath }}" + - name: Deploy app to AKS env: KUBECONFIG: "${{ github.workspace }}/kubeconfig" - run: .github/workflows/scripts/deploy.sh "${{ env.k8sNamespace }}" "${{ env.imageName }}" "${{ env.aks_dns_name }}" "${{ env.deploymentYamlPath }}" "${{ env.ingressYamlPath }}" "${{ env.deployFilesPath }}" "${{ env.packageTagScript }}" + run: .github/workflows/scripts/deploy.sh "${{ env.k8sNamespace }}" "${{ env.imageName }}" "${{ env.aks_dns_name }}" "${{ env.deploymentYamlPath }}" "${{ env.ingressYamlPath }}" "${{ env.deployFilesPath }}" "${{ env.packageTagScript }}" "${{ github.event.workflow_run.head_branch || github.ref }}" Deploy_to_EKS: @@ -69,29 +70,29 @@ jobs: steps: - name: Checkout the repository uses: actions/checkout@v3 - + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.aws_region }} + - name: Set EKS Environment Variables uses: tw3lveparsecs/github-actions-setvars@v0.1 with: envFilePath: ".github/vars/*" - + - name: Create secrets for EKS if: ${{ github.event.inputs.k8sProvider == 'EKS' && github.event.inputs.secretsName != '' }} run: | aws eks update-kubeconfig --name ${{ env.cluster_name }} --region ${{ env.aws_region }} .github/workflows/scripts/secrets.sh "${{ env.k8sNamespace }}" "${{ github.event.inputs.secretsName }}" "${{ secrets.AWS_ACCESS_KEY_ID }}" "${{ secrets.AWS_SECRET_ACCESS_KEY }}" "${{ env.registry }}" "${{ env.deploymentYamlPath }}" - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ env.aws_region }} - - name: Deploy app to EKS run: | aws eks update-kubeconfig --name ${{ env.cluster_name }} --region ${{ env.aws_region }} - .github/workflows/scripts/deploy.sh "${{ env.k8sNamespace }}" "${{ env.imageName }}" "${{ env.eks_dns_name }}" "${{ env.deploymentYamlPath }}" "${{ env.ingressYamlPath }}" "${{ env.deployFilesPath }}" "${{ env.packageTagScript }}" + .github/workflows/scripts/deploy.sh "${{ env.k8sNamespace }}" "${{ env.imageName }}" "${{ env.eks_dns_name }}" "${{ env.deploymentYamlPath }}" "${{ env.ingressYamlPath }}" "${{ env.deployFilesPath }}" "${{ env.packageTagScript }}" "${{ github.event.workflow_run.head_branch || github.ref }}" # mark to insert step for additional artifact # on-failure: From 0ec4aa60e34f3e96ce507854ae4acc14ac32b6b7 Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Wed, 20 Jul 2022 18:21:20 +0530 Subject: [PATCH 08/18] Update pipeline_generator.sh --- scripts/pipelines/github/pipeline_generator.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/pipelines/github/pipeline_generator.sh b/scripts/pipelines/github/pipeline_generator.sh index 556f6f248..a121b7fd5 100644 --- a/scripts/pipelines/github/pipeline_generator.sh +++ b/scripts/pipelines/github/pipeline_generator.sh @@ -1,6 +1,6 @@ #!/bin/bash set -e -FLAGS=$(getopt -a --options c:n:d:a:b:l:i:u:p:hw --long "config-file:,pipeline-name:,local-directory:,artifact-path:,target-branch:,language:,build-pipeline-name:,sonar-url:,sonar-token:,image-name:,registry-user:,registry-password:,resource-group:,storage-account:,storage-container:,cluster-name:,s3-bucket:,s3-key-path:,quality-pipeline-name:,dockerfile:,test-pipeline-name:,aws-access-key:,aws-secret-access-key:,aws-region:,ci-pipeline-name:,help" -- "$@") +FLAGS=$(getopt -a --options c:n:d:a:b:l:i:u:p:hw --long "config-file:,pipeline-name:,local-directory:,artifact-path:,target-branch:,language:,build-pipeline-name:,sonar-url:,sonar-token:,image-name:,registry-user:,registry-password:,resource-group:,storage-account:,storage-container:,cluster-name:,s3-bucket:,s3-key-path:,quality-pipeline-name:,dockerfile:,test-pipeline-name:,aws-access-key:,aws-secret-access-key:,aws-region:,rancher:,ci-pipeline-name:,package-pipeline-name:,env-provision-pipeline-name:,k8s-provider:,k8s-namespace:,k8s-deploy-files-path:,k8s-image-pull-secret-name:,help" -- "$@") eval set -- "$FLAGS" while true; do @@ -30,6 +30,12 @@ while true; do --aws-access-key) awsAccessKey="$2"; shift 2;; --aws-secret-access-key) awsSecretAccessKey="$2"; shift 2;; --aws-region) awsRegion="$2"; shift 2;; + --package-pipeline-name) export packagePipelineName=$2; shift 2;; + --env-provision-pipeline-name) envProvisionPipelineName="$2"; shift 2;; + --k8s-provider) k8sProvider=$2; shift 2;; + --k8s-namespace) k8sNamespace="$2"; shift 2;; + --k8s-deploy-files-path) k8sDeployFiles=$2; shift 2;; + --k8s-image-pull-secret-name) k8sImagePullSecret=$2; shift 2;; -h | --help) help="true"; shift 1;; -w) webBrowser="true"; shift 1;; --) shift; break;; From c43b52a87cb055a42db3fd92edfc74250c7971ed Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Wed, 20 Jul 2022 20:05:23 +0530 Subject: [PATCH 09/18] Update deploy.sh --- scripts/pipelines/github/templates/deploy/deploy.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/pipelines/github/templates/deploy/deploy.sh b/scripts/pipelines/github/templates/deploy/deploy.sh index 114978b4b..b7ae5ebcf 100644 --- a/scripts/pipelines/github/templates/deploy/deploy.sh +++ b/scripts/pipelines/github/templates/deploy/deploy.sh @@ -3,6 +3,13 @@ set -e # Add image name, dns_name and tag. # Run package-extra.sh to ${tag} . "$7" +# we get what is located after the last '/' in the branch name, so it removes /ref/head or /ref/head/ if your branche is named correctly" +branch_short=$(echo "$8" | awk -F '/' '{ print $NF }') + +# We change the name of the tag depending if it is a release or another branch +echo "tag_completed: $8" | grep release && tag_completed="${tag}" +echo "tag_completed_branch: $8" | grep release || tag_completed="${tag}_${branch_short}" + export image="$2" tag="${tag}" dns="$3" yq eval '.spec.template.spec.containers[0].image = "'"$image:$tag"'"' -i "$4" yq eval '.spec.rules[0].host = "'"$dns"'"' -i "$5" From 075cfb38f44211b0c3c380d9aeaf838a20da0f05 Mon Sep 17 00:00:00 2001 From: ultymatom <90399827+ultymatom@users.noreply.github.com> Date: Thu, 21 Jul 2022 08:49:37 +0200 Subject: [PATCH 10/18] tag_completed was unused --- scripts/pipelines/github/templates/deploy/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pipelines/github/templates/deploy/deploy.sh b/scripts/pipelines/github/templates/deploy/deploy.sh index b7ae5ebcf..402aaa25f 100644 --- a/scripts/pipelines/github/templates/deploy/deploy.sh +++ b/scripts/pipelines/github/templates/deploy/deploy.sh @@ -11,7 +11,7 @@ echo "tag_completed: $8" | grep release && tag_completed="${tag}" echo "tag_completed_branch: $8" | grep release || tag_completed="${tag}_${branch_short}" export image="$2" tag="${tag}" dns="$3" -yq eval '.spec.template.spec.containers[0].image = "'"$image:$tag"'"' -i "$4" +yq eval '.spec.template.spec.containers[0].image = "'"$image:$tag_completed"'"' -i "$4" yq eval '.spec.rules[0].host = "'"$dns"'"' -i "$5" # Deploy apps in exists namespace, If not exists Create new namespace and apply manifest files. kubectl get namespace | grep -q "^$1" || kubectl create namespace "$1" From 2a706b5f5316f20dec71eca4a34b8d6b51905423 Mon Sep 17 00:00:00 2001 From: ultymatom <90399827+ultymatom@users.noreply.github.com> Date: Thu, 21 Jul 2022 09:17:13 +0200 Subject: [PATCH 11/18] Update deploy.sh --- scripts/pipelines/github/templates/deploy/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pipelines/github/templates/deploy/deploy.sh b/scripts/pipelines/github/templates/deploy/deploy.sh index 402aaa25f..7da63aa37 100644 --- a/scripts/pipelines/github/templates/deploy/deploy.sh +++ b/scripts/pipelines/github/templates/deploy/deploy.sh @@ -10,7 +10,7 @@ branch_short=$(echo "$8" | awk -F '/' '{ print $NF }') echo "tag_completed: $8" | grep release && tag_completed="${tag}" echo "tag_completed_branch: $8" | grep release || tag_completed="${tag}_${branch_short}" -export image="$2" tag="${tag}" dns="$3" +export image="$2" tag_completed="${tag_completed}" dns="$3" yq eval '.spec.template.spec.containers[0].image = "'"$image:$tag_completed"'"' -i "$4" yq eval '.spec.rules[0].host = "'"$dns"'"' -i "$5" # Deploy apps in exists namespace, If not exists Create new namespace and apply manifest files. From ce4379d22c798a04ef5395d9ea95a17d1564de23 Mon Sep 17 00:00:00 2001 From: Timothe Date: Thu, 25 Aug 2022 12:12:52 +0200 Subject: [PATCH 12/18] commit intermediaire --- .../pipelines/common/pipeline_generator.lib | 2 +- .../deploy/deploy-pipeline.yml.template | 113 +++++++++--------- 2 files changed, 56 insertions(+), 59 deletions(-) diff --git a/scripts/pipelines/common/pipeline_generator.lib b/scripts/pipelines/common/pipeline_generator.lib index 1c6f092dc..158b75f30 100644 --- a/scripts/pipelines/common/pipeline_generator.lib +++ b/scripts/pipelines/common/pipeline_generator.lib @@ -209,7 +209,7 @@ function commitCommonFiles { # changing all files to be executable find "$pipelinePath" -type f -name '*.sh' -exec git update-index --chmod=+x {} \; - git commit -m "Adding the source YAML" + git commit -m "Adding the source YAML [skip ci]" git push -u origin ${sourceBranch} } diff --git a/scripts/pipelines/github/templates/deploy/deploy-pipeline.yml.template b/scripts/pipelines/github/templates/deploy/deploy-pipeline.yml.template index 4c353ce93..7a01ca24c 100644 --- a/scripts/pipelines/github/templates/deploy/deploy-pipeline.yml.template +++ b/scripts/pipelines/github/templates/deploy/deploy-pipeline.yml.template @@ -4,15 +4,12 @@ on: workflow_run: workflows: $packagePipelineName types: completed - branches: - - release/* - - develop workflow_dispatch: inputs: k8sProvider: description: 'Deploy Cluster Name.' required: true - type: string + type: string default: '$k8sProvider' secretsName: description: 'ImagePullSecret Name.' @@ -21,78 +18,78 @@ on: # mark to insert additional artifact input # env: - deploymentYamlPath: "${{ github.workspace }}/$k8sDeployFiles/application-deployment.yaml" - ingressYamlPath: "${{ github.workspace }}/$k8sDeployFiles/ingress.yaml" + k8sProvider: ${{ github.event.inputs.k8sProvider || '$k8sProvider' }} + deploymentYamlPath: "${{ github.workspace }}/$k8sDeployFiles/application-deployment.yaml" + ingressYamlPath: "${{ github.workspace }}/$k8sDeployFiles/ingress.yaml" deployFilesPath: "${{ github.workspace }}/$k8sDeployFiles" - provisionPipeline: "$envProvisionPipelineName.yml" - packagePipelineName: "$packagePipelineName.yml" + aksProvisionPipeline: 'aks-provisioning.yml' + packagePipelineYamlName: "package-pipeline.yml" packageTagScript: ".github/workflows/scripts/package-extra.sh" k8sNamespace: "$k8sNamespace" + kubeConfigPath: "~/.kube/config" # mark to insert additional artifact env var # jobs: - Deploy_to_AKS: + Deploy: runs-on: ubuntu-latest - if: ${{ github.event.inputs.k8sProvider == 'AKS' || github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} + if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} steps: - - name: Checkout the repository - uses: actions/checkout@master - - - name: Download artifact from AKS provision pipeline workflow + - name: get info from package uses: dawidd6/action-download-artifact@v2 - continue-on-error: true + id: downloadFromBuild + if: ${{ github.event_name == 'workflow_run' }} with: - workflow: ${{ env.provisionPipeline }} + workflow: ${{ env.packagePipelineYamlName }} + run_number: ${{ github.event.workflow_run.run_number }} check_artifacts: true - search_artifacts: true - name: kubeconfig - - - name: Set EKS Environment Variables + name: package_workflow_info + + - name: Get env var from the package-pipeline uses: tw3lveparsecs/github-actions-setvars@v0.1 + if: ${{ github.event_name == 'workflow_run' }} with: - envFilePath: ".github/vars/*" - - - name: Create secrets for AKS - if: ${{ github.event.inputs.k8sProvider == 'AKS' && github.event.inputs.secretsName != '' }} - env: - KUBECONFIG: "${{ github.workspace }}/kubeconfig" - run: .github/workflows/scripts/secrets.sh "${{ env.k8sNamespace }}" "${{ github.event.inputs.secretsName }}" "${{ secrets.DOCKER_USER }}" "${{ secrets.DOCKER_PASSWORD }}" "${{ env.registry }}" "${{ env.deploymentYamlPath }}" - - - name: Deploy app to AKS - env: - KUBECONFIG: "${{ github.workspace }}/kubeconfig" - run: .github/workflows/scripts/deploy.sh "${{ env.k8sNamespace }}" "${{ env.imageName }}" "${{ env.aks_dns_name }}" "${{ env.deploymentYamlPath }}" "${{ env.ingressYamlPath }}" "${{ env.deployFilesPath }}" "${{ env.packageTagScript }}" "${{ github.event.workflow_run.head_branch || github.ref }}" - - - Deploy_to_EKS: - runs-on: ubuntu-latest - if: ${{ github.event.inputs.k8sProvider == 'EKS' || github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }} - steps: + envFilePath: package_workflow_info.env + + - name: Cancel Workflow + if: ${{ github.event_name == 'workflow_run' && env.branch_package_workflow != 'develop' && !(startsWith(env.branch_package_workflow, 'release/')) }} + uses: andymckay/cancel-action@0.2 + - name: Checkout the repository - uses: actions/checkout@v3 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 + uses: actions/checkout@master with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ env.aws_region }} - - - name: Set EKS Environment Variables + ref: ${{ env.commit_package_workflow || github.sha }} + + - name: Set additional Environment Variables uses: tw3lveparsecs/github-actions-setvars@v0.1 with: envFilePath: ".github/vars/*" - - - name: Create secrets for EKS - if: ${{ github.event.inputs.k8sProvider == 'EKS' && github.event.inputs.secretsName != '' }} - run: | - aws eks update-kubeconfig --name ${{ env.cluster_name }} --region ${{ env.aws_region }} - .github/workflows/scripts/secrets.sh "${{ env.k8sNamespace }}" "${{ github.event.inputs.secretsName }}" "${{ secrets.AWS_ACCESS_KEY_ID }}" "${{ secrets.AWS_SECRET_ACCESS_KEY }}" "${{ env.registry }}" "${{ env.deploymentYamlPath }}" - - - name: Deploy app to EKS + + - name: Download artifact from AKS provision pipeline workflow + if: ${{ env.k8sProvider == 'AKS' }} + uses: dawidd6/action-download-artifact@v2 + with: + workflow: ${{ env.aksProvisionPipeline }} + check_artifacts: true + search_artifacts: true + name: kubeconfig + + - name: Download artifact from EKS provision pipeline workflow + if: ${{ env.k8sProvider == 'EKS' }} run: | - aws eks update-kubeconfig --name ${{ env.cluster_name }} --region ${{ env.aws_region }} - .github/workflows/scripts/deploy.sh "${{ env.k8sNamespace }}" "${{ env.imageName }}" "${{ env.eks_dns_name }}" "${{ env.deploymentYamlPath }}" "${{ env.ingressYamlPath }}" "${{ env.deployFilesPath }}" "${{ env.packageTagScript }}" "${{ github.event.workflow_run.head_branch || github.ref }}" + aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} + aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws eks update-kubeconfig --name ${{ env.cluster_name }} --region ${{ secrets.AWS_REGION }} + + - name: Create secrets + if: ${{ github.event.inputs.secretsName != '' }} + env: + KUBECONFIG: ${{ env.kubeConfigPath }} + run: .github/workflows/scripts/secrets.sh "${{ env.k8sNamespace }}" "${{ github.event.inputs.secretsName }}" "${{ secrets.DOCKER_USER }}" "${{ secrets.DOCKER_PASSWORD }}" "${{ env.registry }}" "${{ env.deploymentYamlPath }}" + + - name: Deploy app + env: + KUBECONFIG: ${{ env.kubeConfigPath }} + run: .github/workflows/scripts/deploy.sh "${{ env.k8sNamespace }}" "${{ env.imageName }}" "${{ env.aeks_dns_name }}" "${{ env.deploymentYamlPath }}" "${{ env.ingressYamlPath }}" "${{ env.deployFilesPath }}" "${{ env.packageTagScript }}" "${{ github.event.workflow_run.head_branch || github.ref }}" # mark to insert step for additional artifact # on-failure: @@ -101,4 +98,4 @@ jobs: if: ${{ github.event.workflow_run.conclusion == 'failure' }} steps: - name: Cancel Workflow - uses: andymckay/cancel-action@0.2 + uses: andymckay/cancel-action@0.2 From e51397ec91fffa4c5dcf68eba84e84500ddecad9 Mon Sep 17 00:00:00 2001 From: Timothe Date: Wed, 31 Aug 2022 10:46:16 +0200 Subject: [PATCH 13/18] works well ! --- .../deploy/deploy-pipeline.yml.template | 26 ++++++++++++++----- .../package/package-pipeline.yml.template | 11 ++++++++ 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/scripts/pipelines/github/templates/deploy/deploy-pipeline.yml.template b/scripts/pipelines/github/templates/deploy/deploy-pipeline.yml.template index 7a01ca24c..93154cca4 100644 --- a/scripts/pipelines/github/templates/deploy/deploy-pipeline.yml.template +++ b/scripts/pipelines/github/templates/deploy/deploy-pipeline.yml.template @@ -26,7 +26,8 @@ env: packagePipelineYamlName: "package-pipeline.yml" packageTagScript: ".github/workflows/scripts/package-extra.sh" k8sNamespace: "$k8sNamespace" - kubeConfigPath: "~/.kube/config" + kubeConfigPath: "/home/runner/.kube/config" + kubeConfigFolder: "/home/runner/.kube" # mark to insert additional artifact env var # jobs: @@ -50,7 +51,7 @@ jobs: with: envFilePath: package_workflow_info.env - - name: Cancel Workflow + - name: Cancel Workflow (if the branch are not develop or release) if: ${{ github.event_name == 'workflow_run' && env.branch_package_workflow != 'develop' && !(startsWith(env.branch_package_workflow, 'release/')) }} uses: andymckay/cancel-action@0.2 @@ -73,12 +74,16 @@ jobs: search_artifacts: true name: kubeconfig + - name: MV kubeconfig AKS + if: ${{ env.k8sProvider == 'AKS' }} + run: mkdir -p ${{ env.kubeConfigFolder }}; mv ${{ github.workspace }}/kubeconfig ${{ env.kubeConfigPath }} + - name: Download artifact from EKS provision pipeline workflow if: ${{ env.k8sProvider == 'EKS' }} run: | aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws eks update-kubeconfig --name ${{ env.cluster_name }} --region ${{ secrets.AWS_REGION }} + aws eks update-kubeconfig --name ${{ env.eks_cluster_name }} --region ${{ secrets.AWS_REGION }} - name: Create secrets if: ${{ github.event.inputs.secretsName != '' }} @@ -86,16 +91,23 @@ jobs: KUBECONFIG: ${{ env.kubeConfigPath }} run: .github/workflows/scripts/secrets.sh "${{ env.k8sNamespace }}" "${{ github.event.inputs.secretsName }}" "${{ secrets.DOCKER_USER }}" "${{ secrets.DOCKER_PASSWORD }}" "${{ env.registry }}" "${{ env.deploymentYamlPath }}" - - name: Deploy app + - name: Deploy app AKS + if: ${{ env.k8sProvider == 'AKS' }} env: KUBECONFIG: ${{ env.kubeConfigPath }} - run: .github/workflows/scripts/deploy.sh "${{ env.k8sNamespace }}" "${{ env.imageName }}" "${{ env.aeks_dns_name }}" "${{ env.deploymentYamlPath }}" "${{ env.ingressYamlPath }}" "${{ env.deployFilesPath }}" "${{ env.packageTagScript }}" "${{ github.event.workflow_run.head_branch || github.ref }}" + run: .github/workflows/scripts/deploy.sh "${{ env.k8sNamespace }}" "${{ env.imageName }}" "${{ env.aks_dns_name }}" "${{ env.deploymentYamlPath }}" "${{ env.ingressYamlPath }}" "${{ env.deployFilesPath }}" "${{ env.packageTagScript }}" "${{ env.branch_package_workflow || github.ref }}" - # mark to insert step for additional artifact # + - name: Deploy app EKS + if: ${{ env.k8sProvider == 'EKS' }} + env: + KUBECONFIG: ${{ env.kubeConfigPath }} + run: .github/workflows/scripts/deploy.sh "${{ env.k8sNamespace }}" "${{ env.imageName }}" "${{ env.eks_dns_name }}" "${{ env.deploymentYamlPath }}" "${{ env.ingressYamlPath }}" "${{ env.deployFilesPath }}" "${{ env.packageTagScript }}" "${{ env.branch_package_workflow || github.ref }}" + +# mark to insert step for additional artifact # on-failure: name: CI Workflow Failed runs-on: ubuntu-latest if: ${{ github.event.workflow_run.conclusion == 'failure' }} steps: - name: Cancel Workflow - uses: andymckay/cancel-action@0.2 + uses: andymckay/cancel-action@0.2 \ No newline at end of file diff --git a/scripts/pipelines/github/templates/package/package-pipeline.yml.template b/scripts/pipelines/github/templates/package/package-pipeline.yml.template index 80d0443b0..6130eb239 100644 --- a/scripts/pipelines/github/templates/package/package-pipeline.yml.template +++ b/scripts/pipelines/github/templates/package/package-pipeline.yml.template @@ -75,6 +75,17 @@ jobs: run: .github/workflows/scripts/package.sh -f "$GITHUB_WORKSPACE/${{ env.relativeDockerfilePath }}" -c "$GITHUB_WORKSPACE" -a "${{ secrets.AWS_ACCESS_KEY }}" -s "${{ secrets.AWS_SECRET_ACCESS_KEY }}" -l "${{ env.region }}" -r "${{ env.registry }}" -i "${{ env.imageName }}" -b "${{ github.event.workflow_run.head_branch || github.ref }}" if: ${{ env.isECR == 'true' }} + - name: store workflow_run info + run: | + echo "branch_package_workflow=${{ github.event.workflow_run.head_branch || github.ref }}" >> package_workflow_info.env + echo "commit_package_workflow=${{ github.event.workflow_run.head_commit.id || github.sha }}" >> package_workflow_info.env + + - name: Publish workflow_run info + uses: actions/upload-artifact@v3 + with: + name: package_workflow_info + path: package_workflow_info.env + # mark to insert step for additional artifact # on-failure: name: CI Workflow Failed From 821d9bc0bf350427e1b0e3566774ba591df57ce9 Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Thu, 22 Sep 2022 14:11:23 +0530 Subject: [PATCH 14/18] Update setup-deploy-pipeline.asciidoc --- .../src/common_templates/setup-deploy-pipeline.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/src/common_templates/setup-deploy-pipeline.asciidoc b/documentation/src/common_templates/setup-deploy-pipeline.asciidoc index a2a669bb9..5bc290298 100644 --- a/documentation/src/common_templates/setup-deploy-pipeline.asciidoc +++ b/documentation/src/common_templates/setup-deploy-pipeline.asciidoc @@ -23,7 +23,7 @@ pipeline_generator.sh \ -n \ -d \ --package-pipeline-name \ - --env-provision-pipeline-name ] \ + --env-provision-pipeline-name \ --k8s-provider \ --k8s-namespace \ --k8s-deploy-files-path \ @@ -61,4 +61,4 @@ Once the {pipeline_type} is executed and your application is deployed, you can l ``` kubectl get ingress -n -``` \ No newline at end of file +``` From 451a7c30e9d0f12875560a0fad565264413daf2d Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Thu, 22 Sep 2022 08:41:42 +0000 Subject: [PATCH 15/18] Automatic generation of documentation --- documentation/azure-devops/setup-deploy-pipeline.asciidoc | 2 +- documentation/github/setup-deploy-pipeline.asciidoc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/azure-devops/setup-deploy-pipeline.asciidoc b/documentation/azure-devops/setup-deploy-pipeline.asciidoc index cadd66064..c6a5b8e80 100644 --- a/documentation/azure-devops/setup-deploy-pipeline.asciidoc +++ b/documentation/azure-devops/setup-deploy-pipeline.asciidoc @@ -28,7 +28,7 @@ pipeline_generator.sh \ -n \ -d \ --package-pipeline-name \ - --env-provision-pipeline-name ] \ + --env-provision-pipeline-name \ --k8s-provider \ --k8s-namespace \ --k8s-deploy-files-path \ diff --git a/documentation/github/setup-deploy-pipeline.asciidoc b/documentation/github/setup-deploy-pipeline.asciidoc index c9866504b..16ff6f7ba 100644 --- a/documentation/github/setup-deploy-pipeline.asciidoc +++ b/documentation/github/setup-deploy-pipeline.asciidoc @@ -28,7 +28,7 @@ pipeline_generator.sh \ -n \ -d \ --package-pipeline-name \ - --env-provision-pipeline-name ] \ + --env-provision-pipeline-name \ --k8s-provider \ --k8s-namespace \ --k8s-deploy-files-path \ From cf8fcfd26d38845fb1a5bcb7eed0c0c9a9316c1e Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Thu, 22 Sep 2022 14:48:26 +0530 Subject: [PATCH 16/18] Update setup-deploy-pipeline.asciidoc --- documentation/src/github/setup-deploy-pipeline.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/src/github/setup-deploy-pipeline.asciidoc b/documentation/src/github/setup-deploy-pipeline.asciidoc index 1ca440ee0..22177fdb8 100644 --- a/documentation/src/github/setup-deploy-pipeline.asciidoc +++ b/documentation/src/github/setup-deploy-pipeline.asciidoc @@ -1,6 +1,6 @@ :provider: Github :pipeline_type: workflow -:trigger_sentence: +:trigger_sentence_github: :pipeline_type2: Github action :path_provider: github -include::../common_templates/setup-deploy-pipeline.asciidoc[] \ No newline at end of file +include::../common_templates/setup-deploy-pipeline.asciidoc[] From 1998d19f4809087173ea919fb658e05d24455d10 Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Thu, 22 Sep 2022 09:18:45 +0000 Subject: [PATCH 17/18] Automatic generation of documentation --- documentation/github/setup-deploy-pipeline.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/github/setup-deploy-pipeline.asciidoc b/documentation/github/setup-deploy-pipeline.asciidoc index 16ff6f7ba..ff4064029 100644 --- a/documentation/github/setup-deploy-pipeline.asciidoc +++ b/documentation/github/setup-deploy-pipeline.asciidoc @@ -1,6 +1,6 @@ :provider: Github :pipeline_type: workflow -:trigger_sentence: +:trigger_sentence_github: :pipeline_type2: Github action :path_provider: github = Setting up a Deploy Pipeline on {provider} From 584b15c4ec72653c537e2e5865464342af609253 Mon Sep 17 00:00:00 2001 From: prathibhapadma Date: Tue, 18 Oct 2022 11:05:46 +0530 Subject: [PATCH 18/18] fixed with line ending checks --- .../pipelines/common/pipeline_generator.lib | 484 +++++++++--------- 1 file changed, 242 insertions(+), 242 deletions(-) diff --git a/scripts/pipelines/common/pipeline_generator.lib b/scripts/pipelines/common/pipeline_generator.lib index b7719ddf0..5ac0473f0 100644 --- a/scripts/pipelines/common/pipeline_generator.lib +++ b/scripts/pipelines/common/pipeline_generator.lib @@ -1,242 +1,242 @@ -#!/bin/bash -function help { - echo "" - echo "Generates a $pipeline_type on $provider based on the given definition." - echo "" - echo "Common flags:" - echo " -c, --config-file [Required] Configuration file containing $pipeline_type definition." - echo " -n, --pipeline-name [Required] Name that will be set to the $pipeline_type." - echo " -d, --local-directory [Required] Local directory of your project." - echo " -a, --artifact-path Path to be persisted as an artifact after $pipeline_type execution, e.g. where the application stores logs or any other blob on runtime." - echo " -b, --target-branch Name of the branch to which the Pull Request will target. PR is not created if the flag is not provided." - [[ "$provider" != "gcloud" ]] && echo " -w Open the Pull Request on the web browser if it cannot be automatically merged. Requires -b flag." - echo "" - echo "Build $pipeline_type flags:" - echo " -l, --language [Required] Language or framework of the project." - echo " -t, --target-directory Target directory of build process. Takes precedence over the language/framework default one." - echo "" - echo "Test $pipeline_type flags:" - echo " -l, --language [Required] Language or framework of the project." - [ "$provider" == "azure-devops" ] && echo " --build-pipeline-name [Required] Build $pipeline_type name." - echo "" - echo "Quality $pipeline_type flags:" - echo " -l, --language [Required] Language or framework of the project." - echo " --sonar-url [Required] Sonarqube URL." - echo " --sonar-token [Required] Sonarqube token." - [ "$provider" == "azure-devops" ] && echo " --build-pipeline-name [Required] Build $pipeline_type name." - [ "$provider" == "azure-devops" ] && echo " --test-pipeline-name [Required] Test $pipeline_type name." - echo "" - [ "$provider" == "github" ] && echo "" - [ "$provider" == "github" ] && echo "CI pipeline flags" - [ "$provider" == "github" ] && echo " --build-pipeline-name [Required] Name of the job calling the build $pipeline_type." - [ "$provider" == "github" ] && echo " --test-pipeline-name Name of the job calling the test $pipeline_type." - [ "$provider" == "github" ] && echo " --quality-pipeline-name Name of the job calling the quality $pipeline_type." - [ "$provider" == "github" ] && echo "" - echo "Package pipeline flags:" - echo " -l, --language [Required, if dockerfile not set] Language or framework of the project." - echo " --dockerfile [Required, if language not set] Path from the root of the project to its Dockerfile. Takes precedence over the language/framework default one." - [ "$provider" == "azure-devops" ] && echo " --build-pipeline-name [Required] Build $pipeline_type name." - [ "$provider" == "azure-devops" ] && echo " --quality-pipeline-name [Required] Quality $pipeline_type name." - [ "$provider" == "github" ] && echo " --ci-pipeline-name [Required] CI $pipeline_type name." - echo " -i, --image-name [Required] Name (excluding tag) for the generated container image." - echo " -u, --registry-user [Required, unless AWS] Container registry login user." - echo " -p, --registry-password [Required, unless AWS] Container registry login password." - echo " --aws-access-key [Required, if AWS] AWS account access key ID. Takes precedence over registry credentials." - echo " --aws-secret-access-key [Required, if AWS] AWS account secret access key." - echo " --aws-region [Required, if AWS] AWS region for ECR." - echo "" - echo "Library package $pipeline_type flags:" - echo " -l, --language [Required] Language or framework of the project." - echo "" - echo "Azure AKS provisioning $pipeline_type flags:" - echo " --resource-group [Required] Name of the resource group for the cluster." - echo " --storage-account [Required] Name of the storage account for the cluster." - echo " --storage-container [Required] Name of the storage container where the Terraform state of the cluster will be stored." - echo " --rancher Install Rancher to manage the cluster." - echo "" - echo "AWS EKS provisioning $pipeline_type flags:" - echo " --cluster-name [Required] Name for the cluster." - echo " --s3-bucket [Required] Name of the S3 bucket where the Terraform state of the cluster will be stored." - echo " --s3-key-path [Required] Path within the S3 bucket where the Terraform state of the cluster will be stored." - echo " --aws-access-key [Required, on first run] AWS account access key ID." - echo " --aws-secret-access-key [Required, on first run] AWS account secret access key." - echo " --aws-region [Required, on first run] AWS region for provisioning resources." - - echo " --rancher Install Rancher to manage the cluster." - echo "" - echo "Deploy $pipeline_type flags:" - echo "" - echo " --package-pipeline-name [Required] Package $pipeline_type name." - echo " --env-provision-pipeline-name [Required] Environment provisioning $pipeline_type name." - echo " --k8s-provider [Required] Kubernetes cluster provider name. Accepted values: EKS, AKS." - echo " --k8s-namespace [Required] Kubernetes namespace where the application will be deployed." - echo " --k8s-deploy-files-path [Required] Path from the root of the project to the YAML manifests directory." - echo " --k8s-image-pull-secret-name Name for the generated secret containing registry credentials. Required when using a private registry to host images." - echo "" - - exit -} - -function validateRegistryLoginCredentials { - # if the user chose to push to a registry and the user has not already given a password - # then prompt the user - if [ -v dockerUser ] && [ ! -v dockerPassword ] - then - read -rsp "Please enter Docker registry password..." dockerPassword - fi - - if [ -v awsRegion ] && [ -v awsAccessKey ] && [ ! -v awsSecretAccessKey ] - then - read -rsp "Please enter AWS secret access key..." awsSecretAccessKey - fi -} - -function ensurePathFormat { - currentDirectory=$(pwd) - - # When necessary, converts a relative path into an absolute path, and a Windows-style path (e.g. "C:\Users" or C:/Users) into a - # Unix-style path using forward slashes (e.g. "/c/Users"). - localDirectory=${localDirectory//'\'/"/"} - cd "${localDirectory}" || { echo -e "${red}Error: Local directory '${localDirectory}' does not exist. Check provided path (missing quotes?)."; exit 1; } - localDirectory=$(pwd) - - # Return to initial directory - cd "$currentDirectory" -} - -function importConfigFile { - # Import config file. - source $configFile - IFS=, read -ra flags <<< "$mandatoryFlags" - - # Check if the config file was supplied. - if test -z "$configFile" - then - echo -e "${red}Error: $pipeline_type definition configuration file not specified." >&2 - exit 2 - fi - - # Check if the required flags in the config file have been activated. - for flag in "${flags[@]}" - do - if test -z $flag - then - echo -e "${red}Error: Missing parameters, some flags are mandatory." >&2 - echo -e "${red}Use -h or --help flag to display help." >&2 - exit 2 - fi - done -} - -function checkInstallations { - # Check if Git is installed - if ! [ -x "$(command -v git)" ]; then - echo -e "${red}Error: Git is not installed." >&2 - exit 127 - fi - - # Check if the CLI tool for the provider is installed - if ([ "$provider" == "github" ] && ! [ -x "$(command -v gh)" ]); then - echo -e "${red}Error: Github CLI is not installed." >&2 - exit 127 - elif ([ "$provider" == "azure-devops" ] && ! [ -x "$(command -v az)" ]); then - echo -e "${red}Error: Azure CLI is not installed." >&2 - exit 127 - fi - - # Check if Python is installed - if ! [ -x "$(command -v python)" ]; then - echo -e "${red}Error: Python is not installed." >&2 - exit 127 - fi -} - -function createNewBranch { - echo -e "${green}Creating the new branch: ${sourceBranch}..." - echo -ne ${white} - - # Create the new branch. - cd "${localDirectory}" - git checkout -b ${sourceBranch} -} - -function copyYAMLFile { - echo -e "${green}Copying the corresponding files into your directory..." - echo -ne ${white} - - # Create .pipelines and scripts if they do not exist. - mkdir -p "${localDirectory}/$scriptFilePath" - - # Generate pipeline YAML from template and put it in the repository. - # We cannot use a variable in the definition of resource in the pipeline so we have to use a placeholder to replace it with the value we need - commonEnvSubstList='${buildPipelineName} ${testPipelineName} ${qualityPipelineName} ${pipelineName} ${ciPipelineName} ${packagePipelineName}' - envsubst "${commonEnvSubstList} ${specificEnvSubstList}" < "${hangarPath}/${templatesPath}/${yamlFile}.template" > "${localDirectory}/${pipelinePath}/${yamlFile}" - - # Check if an extra artifact to store is supplied. - if test -n "$artifactPath" - then - # Add the extra step to the YAML. - if [ "$provider" == "azure-devops" ] - then - cat "${hangarPath}/${commonTemplatesPath}/store-extra-path.yml" >> "${localDirectory}/${pipelinePath}/${yamlFile}" - elif [ "$provider" == "github" ] - then - # (in case of -a flag set) Here we replace a comment inside the template file by a step in Github Actions to upload an artifact - storeExtraPathContent="\n - name: Publish Additional Output Artifact\n uses: actions\/upload-artifact@v3\n with:\n name: additional-pipeline-output\n path: \"\${{ env.artifactPath }}\"" - sed -i "s/# mark to insert step for additonal artifact #/$storeExtraPathContent\n/" "${localDirectory}/${pipelinePath}/${yamlFile}" - elif [ "$provider" == "gcloud" ] - then - # (in case of -a flag set) We first check if an artifact is already uploaded in this template, - # to know if we need to add an artifact to the list of uploaded artifact because there is/are already artifact(s) uploaded, - # or if we need to add the artifact section because there is not artifact uploaded yet - grep -e "^artifacts:" "${localDirectory}/${pipelinePath}/${yamlFile}" > /dev/null && storeExtraPathContent=" - additional_output_\${BUILD_ID}.tar" || storeExtraPathContent="\nartifacts:\n objects:\n location: gs:\/\/\${PROJECT_ID}_cloudbuild\/pipelinesArtifacts\/${pipelineName}\/commit_\${SHORT_SHA}\n paths:\n - additional_output_\${BUILD_ID}.tar" - # We replace the comment used to know where to add the lines defined is the line above - sed -i "s/# mark to insert entry in artifact for additonal artifact #/$storeExtraPathContent\n/" "${localDirectory}/${pipelinePath}/${yamlFile}" - # Then we add a step to the pipeline to create a tar of the additional artiact, we create a tar for 2 reasons: - # - Keep the permissions of the files as they are - # - In Gcloud, if you try to upload a folder and not a file, it fails - stepCreateTarArtifact="- name: \${_DOCKER_IMAGE_STEPS}\n args: ['tar', '-cf', 'additional_output_\${BUILD_ID}.tar', '\${_ARTIFACT_PATH}']" - sed -i "s/# mark to insert step for additonal artifact #/$stepCreateTarArtifact\n/" "${localDirectory}/${pipelinePath}/${yamlFile}" - fi - else - if [ "$provider" == "github" ] || [ "$provider" == "gcloud" ] - then - # If no -a flag set, we just remove the mark used to place the additional entries and step in the yaml - sed -i '/# mark to insert step for additonal artifact #/d' "${localDirectory}/${pipelinePath}/${yamlFile}" - sed -i '/# mark to insert entry in artifact for additonal artifact #/d' "${localDirectory}/${pipelinePath}/${yamlFile}" - fi - fi -} - -function copyCommonScript { - echo -e "${green}Copying the script(s) common to any $pipeline_type files into your directory..." - echo -ne ${white} - - ! (ls "${hangarPath}/${commonTemplatesPath}"/*.sh) &> /dev/null || cp "${hangarPath}/${commonTemplatesPath}"/*.sh "${localDirectory}/${scriptFilePath}" -} - -function commitCommonFiles { - echo -e "${green}Commiting and pushing into Git remote..." - echo -ne ${white} - - # Move into the project's directory and pushing the template into the Azure DevOps repository. - cd "${localDirectory}" - - # Add the YAML files. - git add "$pipelinePath" -f - - # Git commit and push it into the repository. - # changing all files to be executable - find "$pipelinePath" -type f -name '*.sh' -exec git update-index --chmod=+x {} \; - - git commit -m "Adding the source YAML" - git push -u origin ${sourceBranch} -} - -function setTargetDirectory { - case $language in - node | angular) targetDirectory="./" ;; - quarkus*) targetDirectory="./target/" ;; - *) echo -e "${red}Error: Specified language '${language}' is not supported." >&2; exit 1 - esac -} +#!/bin/bash +function help { + echo "" + echo "Generates a $pipeline_type on $provider based on the given definition." + echo "" + echo "Common flags:" + echo " -c, --config-file [Required] Configuration file containing $pipeline_type definition." + echo " -n, --pipeline-name [Required] Name that will be set to the $pipeline_type." + echo " -d, --local-directory [Required] Local directory of your project." + echo " -a, --artifact-path Path to be persisted as an artifact after $pipeline_type execution, e.g. where the application stores logs or any other blob on runtime." + echo " -b, --target-branch Name of the branch to which the Pull Request will target. PR is not created if the flag is not provided." + [[ "$provider" != "gcloud" ]] && echo " -w Open the Pull Request on the web browser if it cannot be automatically merged. Requires -b flag." + echo "" + echo "Build $pipeline_type flags:" + echo " -l, --language [Required] Language or framework of the project." + echo " -t, --target-directory Target directory of build process. Takes precedence over the language/framework default one." + echo "" + echo "Test $pipeline_type flags:" + echo " -l, --language [Required] Language or framework of the project." + [ "$provider" == "azure-devops" ] && echo " --build-pipeline-name [Required] Build $pipeline_type name." + echo "" + echo "Quality $pipeline_type flags:" + echo " -l, --language [Required] Language or framework of the project." + echo " --sonar-url [Required] Sonarqube URL." + echo " --sonar-token [Required] Sonarqube token." + [ "$provider" == "azure-devops" ] && echo " --build-pipeline-name [Required] Build $pipeline_type name." + [ "$provider" == "azure-devops" ] && echo " --test-pipeline-name [Required] Test $pipeline_type name." + echo "" + [ "$provider" == "github" ] && echo "" + [ "$provider" == "github" ] && echo "CI pipeline flags" + [ "$provider" == "github" ] && echo " --build-pipeline-name [Required] Name of the job calling the build $pipeline_type." + [ "$provider" == "github" ] && echo " --test-pipeline-name Name of the job calling the test $pipeline_type." + [ "$provider" == "github" ] && echo " --quality-pipeline-name Name of the job calling the quality $pipeline_type." + [ "$provider" == "github" ] && echo "" + echo "Package pipeline flags:" + echo " -l, --language [Required, if dockerfile not set] Language or framework of the project." + echo " --dockerfile [Required, if language not set] Path from the root of the project to its Dockerfile. Takes precedence over the language/framework default one." + [ "$provider" == "azure-devops" ] && echo " --build-pipeline-name [Required] Build $pipeline_type name." + [ "$provider" == "azure-devops" ] && echo " --quality-pipeline-name [Required] Quality $pipeline_type name." + [ "$provider" == "github" ] && echo " --ci-pipeline-name [Required] CI $pipeline_type name." + echo " -i, --image-name [Required] Name (excluding tag) for the generated container image." + echo " -u, --registry-user [Required, unless AWS] Container registry login user." + echo " -p, --registry-password [Required, unless AWS] Container registry login password." + echo " --aws-access-key [Required, if AWS] AWS account access key ID. Takes precedence over registry credentials." + echo " --aws-secret-access-key [Required, if AWS] AWS account secret access key." + echo " --aws-region [Required, if AWS] AWS region for ECR." + echo "" + echo "Library package $pipeline_type flags:" + echo " -l, --language [Required] Language or framework of the project." + echo "" + echo "Azure AKS provisioning $pipeline_type flags:" + echo " --resource-group [Required] Name of the resource group for the cluster." + echo " --storage-account [Required] Name of the storage account for the cluster." + echo " --storage-container [Required] Name of the storage container where the Terraform state of the cluster will be stored." + echo " --rancher Install Rancher to manage the cluster." + echo "" + echo "AWS EKS provisioning $pipeline_type flags:" + echo " --cluster-name [Required] Name for the cluster." + echo " --s3-bucket [Required] Name of the S3 bucket where the Terraform state of the cluster will be stored." + echo " --s3-key-path [Required] Path within the S3 bucket where the Terraform state of the cluster will be stored." + echo " --aws-access-key [Required, on first run] AWS account access key ID." + echo " --aws-secret-access-key [Required, on first run] AWS account secret access key." + echo " --aws-region [Required, on first run] AWS region for provisioning resources." + + echo " --rancher Install Rancher to manage the cluster." + echo "" + echo "Deploy $pipeline_type flags:" + echo "" + echo " --package-pipeline-name [Required] Package $pipeline_type name." + echo " --env-provision-pipeline-name [Required] Environment provisioning $pipeline_type name." + echo " --k8s-provider [Required] Kubernetes cluster provider name. Accepted values: EKS, AKS." + echo " --k8s-namespace [Required] Kubernetes namespace where the application will be deployed." + echo " --k8s-deploy-files-path [Required] Path from the root of the project to the YAML manifests directory." + echo " --k8s-image-pull-secret-name Name for the generated secret containing registry credentials. Required when using a private registry to host images." + echo "" + + exit +} + +function validateRegistryLoginCredentials { + # if the user chose to push to a registry and the user has not already given a password + # then prompt the user + if [ -v dockerUser ] && [ ! -v dockerPassword ] + then + read -rsp "Please enter Docker registry password..." dockerPassword + fi + + if [ -v awsRegion ] && [ -v awsAccessKey ] && [ ! -v awsSecretAccessKey ] + then + read -rsp "Please enter AWS secret access key..." awsSecretAccessKey + fi +} + +function ensurePathFormat { + currentDirectory=$(pwd) + + # When necessary, converts a relative path into an absolute path, and a Windows-style path (e.g. "C:\Users" or C:/Users) into a + # Unix-style path using forward slashes (e.g. "/c/Users"). + localDirectory=${localDirectory//'\'/"/"} + cd "${localDirectory}" || { echo -e "${red}Error: Local directory '${localDirectory}' does not exist. Check provided path (missing quotes?)."; exit 1; } + localDirectory=$(pwd) + + # Return to initial directory + cd "$currentDirectory" +} + +function importConfigFile { + # Import config file. + source $configFile + IFS=, read -ra flags <<< "$mandatoryFlags" + + # Check if the config file was supplied. + if test -z "$configFile" + then + echo -e "${red}Error: $pipeline_type definition configuration file not specified." >&2 + exit 2 + fi + + # Check if the required flags in the config file have been activated. + for flag in "${flags[@]}" + do + if test -z $flag + then + echo -e "${red}Error: Missing parameters, some flags are mandatory." >&2 + echo -e "${red}Use -h or --help flag to display help." >&2 + exit 2 + fi + done +} + +function checkInstallations { + # Check if Git is installed + if ! [ -x "$(command -v git)" ]; then + echo -e "${red}Error: Git is not installed." >&2 + exit 127 + fi + + # Check if the CLI tool for the provider is installed + if ([ "$provider" == "github" ] && ! [ -x "$(command -v gh)" ]); then + echo -e "${red}Error: Github CLI is not installed." >&2 + exit 127 + elif ([ "$provider" == "azure-devops" ] && ! [ -x "$(command -v az)" ]); then + echo -e "${red}Error: Azure CLI is not installed." >&2 + exit 127 + fi + + # Check if Python is installed + if ! [ -x "$(command -v python)" ]; then + echo -e "${red}Error: Python is not installed." >&2 + exit 127 + fi +} + +function createNewBranch { + echo -e "${green}Creating the new branch: ${sourceBranch}..." + echo -ne ${white} + + # Create the new branch. + cd "${localDirectory}" + git checkout -b ${sourceBranch} +} + +function copyYAMLFile { + echo -e "${green}Copying the corresponding files into your directory..." + echo -ne ${white} + + # Create .pipelines and scripts if they do not exist. + mkdir -p "${localDirectory}/$scriptFilePath" + + # Generate pipeline YAML from template and put it in the repository. + # We cannot use a variable in the definition of resource in the pipeline so we have to use a placeholder to replace it with the value we need + commonEnvSubstList='${buildPipelineName} ${testPipelineName} ${qualityPipelineName} ${pipelineName} ${ciPipelineName} ${packagePipelineName}' + envsubst "${commonEnvSubstList} ${specificEnvSubstList}" < "${hangarPath}/${templatesPath}/${yamlFile}.template" > "${localDirectory}/${pipelinePath}/${yamlFile}" + + # Check if an extra artifact to store is supplied. + if test -n "$artifactPath" + then + # Add the extra step to the YAML. + if [ "$provider" == "azure-devops" ] + then + cat "${hangarPath}/${commonTemplatesPath}/store-extra-path.yml" >> "${localDirectory}/${pipelinePath}/${yamlFile}" + elif [ "$provider" == "github" ] + then + # (in case of -a flag set) Here we replace a comment inside the template file by a step in Github Actions to upload an artifact + storeExtraPathContent="\n - name: Publish Additional Output Artifact\n uses: actions\/upload-artifact@v3\n with:\n name: additional-pipeline-output\n path: \"\${{ env.artifactPath }}\"" + sed -i "s/# mark to insert step for additonal artifact #/$storeExtraPathContent\n/" "${localDirectory}/${pipelinePath}/${yamlFile}" + elif [ "$provider" == "gcloud" ] + then + # (in case of -a flag set) We first check if an artifact is already uploaded in this template, + # to know if we need to add an artifact to the list of uploaded artifact because there is/are already artifact(s) uploaded, + # or if we need to add the artifact section because there is not artifact uploaded yet + grep -e "^artifacts:" "${localDirectory}/${pipelinePath}/${yamlFile}" > /dev/null && storeExtraPathContent=" - additional_output_\${BUILD_ID}.tar" || storeExtraPathContent="\nartifacts:\n objects:\n location: gs:\/\/\${PROJECT_ID}_cloudbuild\/pipelinesArtifacts\/${pipelineName}\/commit_\${SHORT_SHA}\n paths:\n - additional_output_\${BUILD_ID}.tar" + # We replace the comment used to know where to add the lines defined is the line above + sed -i "s/# mark to insert entry in artifact for additonal artifact #/$storeExtraPathContent\n/" "${localDirectory}/${pipelinePath}/${yamlFile}" + # Then we add a step to the pipeline to create a tar of the additional artiact, we create a tar for 2 reasons: + # - Keep the permissions of the files as they are + # - In Gcloud, if you try to upload a folder and not a file, it fails + stepCreateTarArtifact="- name: \${_DOCKER_IMAGE_STEPS}\n args: ['tar', '-cf', 'additional_output_\${BUILD_ID}.tar', '\${_ARTIFACT_PATH}']" + sed -i "s/# mark to insert step for additonal artifact #/$stepCreateTarArtifact\n/" "${localDirectory}/${pipelinePath}/${yamlFile}" + fi + else + if [ "$provider" == "github" ] || [ "$provider" == "gcloud" ] + then + # If no -a flag set, we just remove the mark used to place the additional entries and step in the yaml + sed -i '/# mark to insert step for additonal artifact #/d' "${localDirectory}/${pipelinePath}/${yamlFile}" + sed -i '/# mark to insert entry in artifact for additonal artifact #/d' "${localDirectory}/${pipelinePath}/${yamlFile}" + fi + fi +} + +function copyCommonScript { + echo -e "${green}Copying the script(s) common to any $pipeline_type files into your directory..." + echo -ne ${white} + + ! (ls "${hangarPath}/${commonTemplatesPath}"/*.sh) &> /dev/null || cp "${hangarPath}/${commonTemplatesPath}"/*.sh "${localDirectory}/${scriptFilePath}" +} + +function commitCommonFiles { + echo -e "${green}Commiting and pushing into Git remote..." + echo -ne ${white} + + # Move into the project's directory and pushing the template into the Azure DevOps repository. + cd "${localDirectory}" + + # Add the YAML files. + git add "$pipelinePath" -f + + # Git commit and push it into the repository. + # changing all files to be executable + find "$pipelinePath" -type f -name '*.sh' -exec git update-index --chmod=+x {} \; + + git commit -m "Adding the source YAML" + git push -u origin ${sourceBranch} +} + +function setTargetDirectory { + case $language in + node | angular) targetDirectory="./" ;; + quarkus*) targetDirectory="./target/" ;; + *) echo -e "${red}Error: Specified language '${language}' is not supported." >&2; exit 1 + esac +}