diff --git a/.pipelines/ci-aks-prod-release.yaml b/.pipelines/ci-aks-prod-release.yaml index fa303618b..42c961bb9 100644 --- a/.pipelines/ci-aks-prod-release.yaml +++ b/.pipelines/ci-aks-prod-release.yaml @@ -475,3 +475,39 @@ extends: displayName: Ev2 - Monitoring inputs: Ev2MonintoringUrl: $(Ev2MonintoringUrl) + + + # ============================================================================= + # Stage 3: Deploy ama-logs to AKS Clusters via Helm + # ============================================================================= + # To add a new cluster, simply add an entry to the template list below. + # Each cluster only needs: clusterName, resourceGroup, region, subscriptionId, workspaceId, and imageTag. + # ============================================================================= + - stage: Stage_3 + displayName: Deploy ama-logs to CI AKS Prod Clusters via Helm + dependsOn: Stage_2 + pool: + name: Azure-Pipelines-CI-Test-EO + image: ci-1es-managed-ubuntu-2204 + os: linux + jobs: + # Monitoring-Model-Cluster-WCUS + - template: .pipelines/helm-deploy-templates/ama-logs-helm-deploy.yaml@self + parameters: + clusterName: 'Monitoring-Model-Cluster-WCUS' + resourceGroup: 'monitoring-model-cluster-wcus' + region: 'westcentralus' + subscriptionId: '9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb' + workspaceId: '22f38e11-4f59-480c-b4b8-2573156b6e06' # "Monitoring-Model-Cluster-WCUS" + imageTag: '$(AgentImageTagSuffix)' + environment: 'CI-Deploy-To-Prod-Cluster-1' + # Monitoring-Model-Cluster-WEU + - template: .pipelines/helm-deploy-templates/ama-logs-helm-deploy.yaml@self + parameters: + clusterName: 'Monitoring-Model-Cluster-WEU' + resourceGroup: 'monitoring-model-cluster-weu' + region: 'westeurope' + subscriptionId: '9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb' + workspaceId: '5c269467-32a9-4468-a1d6-ec1cac551e74' # "Monitoring-Model-Cluster-WEU" + imageTag: '$(AgentImageTagSuffix)' + environment: 'CI-Deploy-To-Prod-Cluster-2' \ No newline at end of file diff --git a/.pipelines/helm-deploy-templates/ama-logs-helm-deploy.yaml b/.pipelines/helm-deploy-templates/ama-logs-helm-deploy.yaml new file mode 100644 index 000000000..b585dc926 --- /dev/null +++ b/.pipelines/helm-deploy-templates/ama-logs-helm-deploy.yaml @@ -0,0 +1,258 @@ +# Template for deploying ama-logs to an AKS cluster via Helm +# Usage: +# - template: helm-deploy-templates/ama-logs-helm-deploy.yaml +# parameters: +# clusterName: 'my-cluster' +# resourceGroup: 'my-rg' +# region: 'eastus' +# subscriptionId: '9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb' +# workspaceId: 'your-workspace-id' +# imageTag: '$(AgentImageTagSuffix)' # e.g., 3.1.32 +# environment: 'deployment environment' + +parameters: +# Required cluster parameters +- name: clusterName + type: string +- name: resourceGroup + type: string +- name: region + type: string +- name: subscriptionId + type: string +- name: workspaceId + type: string + displayName: 'Log Analytics Workspace ID' +- name: imageTag + type: string + displayName: 'Image tag suffix (e.g., 3.1.32)' +- name: environment + type: string + displayName: 'Azure DevOps Environment name' + +# Optional parameters with defaults +- name: cloudEnvironment + type: string + default: 'azurepubliccloud' +- name: kubernetesVersion + type: string + default: '1.32.7' +- name: azureSubscription + type: string + default: 'ContainerInsights_Build_Subscription_CI' +- name: namespace + type: string + default: 'default' +- name: releaseName + type: string + default: 'azuremonitor-containers' +- name: helmVersion + type: string + default: '3.12.3' +- name: dependsOn + type: object + default: [] + +jobs: +- deployment: Deploy_AmaLogs_${{ replace(parameters.clusterName, '-', '_') }} + displayName: 'Deploy: ama-logs to ${{ parameters.clusterName }}' + environment: ${{ parameters.environment }} + dependsOn: ${{ parameters.dependsOn }} + variables: + - name: OneESPT + value: true + readonly: true + - name: OneESPT.BuildType + value: Official + readonly: true + - name: OneESPT.OS + value: linux + readonly: true + - name: skipComponentGovernanceDetection + value: true + - name: Codeql.SkipTaskAutoInjection + value: true + - name: AKS_RESOURCE_ID + value: '/subscriptions/${{ parameters.subscriptionId }}/resourceGroups/${{ parameters.resourceGroup }}/providers/Microsoft.ContainerService/managedClusters/${{ parameters.clusterName }}' + strategy: + runOnce: + deploy: + steps: + - checkout: self + displayName: Checkout repository + - task: HelmInstaller@1 + displayName: Install Helm + inputs: + helmVersionToInstall: '${{ parameters.helmVersion }}' + - task: HelmDeploy@0 + displayName: 'Helm Deploy: ama-logs to ${{ parameters.clusterName }}' + inputs: + connectionType: 'Azure Resource Manager' + azureSubscription: '${{ parameters.azureSubscription }}' + azureResourceGroup: '${{ parameters.resourceGroup }}' + kubernetesCluster: '${{ parameters.clusterName }}' + useClusterAdmin: true + namespace: '${{ parameters.namespace }}' + command: 'upgrade' + chartType: 'FilePath' + # The following chart is copied over from feature branch https://github.com/microsoft/Docker-Provider/tree/longw/addon-to-extension-quick-Test + # TODO: When it is merged to ci_prod, the following chartPath will be updated accordingly. + chartPath: '$(Build.SourcesDirectory)/charts/azuremonitor-containerinsights-for-prod-clusters' + releaseName: '${{ parameters.releaseName }}' + overrideValues: 'global.commonGlobals.CloudEnvironment=${{ parameters.cloudEnvironment }},global.commonGlobals.Region=${{ parameters.region }},OmsAgent.aksResourceID=$(AKS_RESOURCE_ID),OmsAgent.workspaceID=${{ parameters.workspaceId }},OmsAgent.imageTagLinux=${{ parameters.imageTag }},OmsAgent.imageTagWindows=win-${{ parameters.imageTag }}' + waitForExecution: false + arguments: '--timeout 10m --install' + - task: AzureCLI@2 + displayName: Verify Deployment + inputs: + azureSubscription: '${{ parameters.azureSubscription }}' + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + echo "==========================================" + echo "Deployment Summary" + echo "==========================================" + echo "Cluster: ${{ parameters.clusterName }}" + echo "Resource Group: ${{ parameters.resourceGroup }}" + echo "Region: ${{ parameters.region }}" + echo "Linux Image Tag: ${{ parameters.imageTag }}" + echo "Windows Image Tag: win-${{ parameters.imageTag }}" + echo "" + + echo "Getting AKS credentials..." + az aks get-credentials --resource-group ${{ parameters.resourceGroup }} --name ${{ parameters.clusterName }} --overwrite-existing --admin + + echo "" + echo "==========================================" + echo "Waiting for pods to be ready (60s max)..." + echo "==========================================" + + # Wait for daemonset pods (pods are always deployed to kube-system namespace) + kubectl rollout status daemonset/ama-logs -n kube-system --timeout=60s || echo "Warning: Daemonset rollout not complete within timeout" + + # Wait for replicaset pod + kubectl rollout status deployment/ama-logs-rs -n kube-system --timeout=60s 2>/dev/null || \ + kubectl rollout status replicaset -l rsName=ama-logs-rs -n kube-system --timeout=60s 2>/dev/null || \ + echo "Warning: ReplicaSet rollout status check skipped" + + echo "" + echo "==========================================" + echo "Pod Status" + echo "==========================================" + kubectl get pods -n kube-system | grep ama-logs || echo "No ama-logs pods found" + + echo "" + echo "==========================================" + echo "Helm Release Status" + echo "==========================================" + helm status ${{ parameters.releaseName }} -n ${{ parameters.namespace }} 2>/dev/null || echo "Helm release status not available" + + echo "" + echo "==========================================" + echo "Image Verification" + echo "==========================================" + EXPECTED_LINUX_TAG="${{ parameters.imageTag }}" + EXPECTED_WINDOWS_TAG="win-${{ parameters.imageTag }}" + VERIFICATION_PASSED=true + + echo "Expected Linux image tag: $EXPECTED_LINUX_TAG" + echo "Expected Windows image tag: $EXPECTED_WINDOWS_TAG" + echo "" + + # ---- 1. Linux DaemonSet Pod Verification (ama-logs + ama-logs-prometheus containers) ---- + # Note: Pods are always deployed to kube-system namespace regardless of helm release namespace + echo "--- 1. Linux DaemonSet Verification ---" + + # Check ama-logs container + LINUX_AMA_LOGS_IMAGE=$(kubectl get pods -n kube-system -l component=ama-logs-agent -o jsonpath='{.items[0].spec.containers[?(@.name=="ama-logs")].image}' 2>/dev/null) + + if [ -z "$LINUX_AMA_LOGS_IMAGE" ]; then + echo "❌ ERROR: Could not retrieve ama-logs container image from Linux DaemonSet" + VERIFICATION_PASSED=false + else + echo "ama-logs container image: $LINUX_AMA_LOGS_IMAGE" + LINUX_AMA_LOGS_TAG=$(echo "$LINUX_AMA_LOGS_IMAGE" | cut -d':' -f2) + if [ "$LINUX_AMA_LOGS_TAG" == "$EXPECTED_LINUX_TAG" ]; then + echo "✅ ama-logs container: PASSED" + else + echo "❌ ama-logs container MISMATCH! Expected: $EXPECTED_LINUX_TAG, Actual: $LINUX_AMA_LOGS_TAG" + VERIFICATION_PASSED=false + fi + fi + + # Check ama-logs-prometheus container + LINUX_PROM_IMAGE=$(kubectl get pods -n kube-system -l component=ama-logs-agent -o jsonpath='{.items[0].spec.containers[?(@.name=="ama-logs-prometheus")].image}' 2>/dev/null) + + if [ -z "$LINUX_PROM_IMAGE" ]; then + echo "❌ ERROR: Could not retrieve ama-logs-prometheus container image from Linux DaemonSet" + VERIFICATION_PASSED=false + else + echo "ama-logs-prometheus container image: $LINUX_PROM_IMAGE" + LINUX_PROM_TAG=$(echo "$LINUX_PROM_IMAGE" | cut -d':' -f2) + if [ "$LINUX_PROM_TAG" == "$EXPECTED_LINUX_TAG" ]; then + echo "✅ ama-logs-prometheus container: PASSED" + else + echo "❌ ama-logs-prometheus container MISMATCH! Expected: $EXPECTED_LINUX_TAG, Actual: $LINUX_PROM_TAG" + VERIFICATION_PASSED=false + fi + fi + + echo "" + + # ---- 2. ReplicaSet Pod Verification (ama-logs container) ---- + echo "--- 2. ReplicaSet Verification ---" + + RS_AMA_LOGS_IMAGE=$(kubectl get pods -n kube-system -l rsName=ama-logs-rs -o jsonpath='{.items[0].spec.containers[?(@.name=="ama-logs")].image}' 2>/dev/null) + + if [ -z "$RS_AMA_LOGS_IMAGE" ]; then + echo "❌ ERROR: Could not retrieve ama-logs container image from ReplicaSet" + VERIFICATION_PASSED=false + else + echo "ama-logs container image: $RS_AMA_LOGS_IMAGE" + RS_AMA_LOGS_TAG=$(echo "$RS_AMA_LOGS_IMAGE" | cut -d':' -f2) + if [ "$RS_AMA_LOGS_TAG" == "$EXPECTED_LINUX_TAG" ]; then + echo "✅ ReplicaSet ama-logs container: PASSED" + else + echo "❌ ReplicaSet ama-logs container MISMATCH! Expected: $EXPECTED_LINUX_TAG, Actual: $RS_AMA_LOGS_TAG" + VERIFICATION_PASSED=false + fi + fi + + echo "" + + # ---- 3. Windows DaemonSet Pod Verification (ama-logs-windows container) ---- + echo "--- 3. Windows DaemonSet Verification ---" + + WINDOWS_POD_EXISTS=$(kubectl get pods -n kube-system -l component=ama-logs-agent-windows -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) + + if [ -z "$WINDOWS_POD_EXISTS" ]; then + echo "INFO: No Windows ama-logs pods found (cluster may not have Windows nodes)" + echo "Skipping Windows verification" + else + WINDOWS_IMAGE=$(kubectl get pods -n kube-system -l component=ama-logs-agent-windows -o jsonpath='{.items[0].spec.containers[?(@.name=="ama-logs-windows")].image}' 2>/dev/null) + + if [ -z "$WINDOWS_IMAGE" ]; then + echo "❌ ERROR: Could not retrieve ama-logs-windows container image" + VERIFICATION_PASSED=false + else + echo "ama-logs-windows container image: $WINDOWS_IMAGE" + WINDOWS_TAG=$(echo "$WINDOWS_IMAGE" | cut -d':' -f2) + if [ "$WINDOWS_TAG" == "$EXPECTED_WINDOWS_TAG" ]; then + echo "✅ Windows ama-logs-windows container: PASSED" + else + echo "❌ Windows ama-logs-windows container MISMATCH! Expected: $EXPECTED_WINDOWS_TAG, Actual: $WINDOWS_TAG" + VERIFICATION_PASSED=false + fi + fi + fi + + echo "" + echo "==========================================" + echo "Final Verification Result" + echo "==========================================" + if [ "$VERIFICATION_PASSED" = true ]; then + echo "✅ SUCCESS: All image tag verifications PASSED" + else + echo "❌ FAILED: One or more image tag verifications failed" + exit 1 + fi diff --git a/charts/azuremonitor-containerinsights-for-prod-clusters/Chart.yaml b/charts/azuremonitor-containerinsights-for-prod-clusters/Chart.yaml new file mode 100644 index 000000000..cc83a72cd --- /dev/null +++ b/charts/azuremonitor-containerinsights-for-prod-clusters/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v2 +description: azure-monitor-containers helm chart +name: azuremonitor-containers +version: 3.2.1-dev-test diff --git a/charts/azuremonitor-containerinsights-for-prod-clusters/templates/_aks_addon-images.tpl b/charts/azuremonitor-containerinsights-for-prod-clusters/templates/_aks_addon-images.tpl new file mode 100644 index 000000000..623f2472d --- /dev/null +++ b/charts/azuremonitor-containerinsights-for-prod-clusters/templates/_aks_addon-images.tpl @@ -0,0 +1,377 @@ +{{/* Auto-generated by versioning tooling, do not edit. See /toolkit/versioning/README.md for more information. */}} +{{- define "get.addonImageTag" -}} + {{- if eq .component "aci-connector-linux" -}} + {{- if semverCompare ">=1.26.0" .version -}}1.6.2 + {{- else if semverCompare ">=1.25.0" .version -}}1.6.1 + {{- else if semverCompare ">=1.24.0" .version -}}1.6.0 + {{- else -}}1.4.16 + {{- end -}} + {{- else if eq .component "addon-resizer" -}} +v1.8.23-4 + {{- else if eq .component "ai-toolchain-operator" -}} +0.6.0 + {{- else if eq .component "aks-windows-gpu-device-plugin" -}} +0.0.19 + {{- else if eq .component "ama-logs-linux" -}} +3.1.28 + {{- else if eq .component "ama-logs-win" -}} +win-3.1.28 + {{- else if eq .component "app-routing-operator" -}} +0.0.3 + {{- else if eq .component "azure-monitor-metrics-cfg-reader" -}} +6.21.1-main-08-15-2025-f5f679d6-cfg + {{- else if eq .component "azure-monitor-metrics-ksm" -}} +v2.15.0-4 + {{- else if eq .component "azure-monitor-metrics-linux" -}} +6.21.1-main-08-15-2025-f5f679d6 + {{- else if eq .component "azure-monitor-metrics-target-allocator" -}} +6.21.1-main-08-15-2025-f5f679d6-targetallocator + {{- else if eq .component "azure-monitor-metrics-windows" -}} +6.21.1-main-08-15-2025-f5f679d6-win + {{- else if eq .component "azure-npm-image" -}} +v1.6.33 + {{- else if eq .component "azure-npm-image-windows" -}} +v1.5.5 + {{- else if eq .component "azure-policy" -}} + {{- if semverCompare ">=1.27.0" .version -}}1.13.0 + {{- else if semverCompare ">=1.25.0" .version -}}1.4.0 + {{- else if semverCompare ">=1.24.0" .version -}}1.0.1 + {{- else if semverCompare ">=1.21.0" .version -}}0.0.3 + {{- else -}}0.0.1 + {{- end -}} + {{- else if eq .component "azure-policy-webhook" -}} + {{- if semverCompare ">=1.27.0" .version -}}1.13.0 + {{- else if semverCompare ">=1.25.0" .version -}}1.4.0 + {{- else if semverCompare ">=1.24.0" .version -}}1.0.1 + {{- else if semverCompare ">=1.21.0" .version -}}0.0.3 + {{- else if semverCompare ">=1.18.0" .version -}}0.0.2 + {{- else -}}0.0.1 + {{- end -}} + {{- else if eq .component "certgen" -}} +v0.1.9 + {{- else if eq .component "cilium-agent" -}} + {{- if semverCompare ">=1.29.0" .version -}}1.14.10-1 + {{- else if semverCompare ">=1.27.0" .version -}}1.13.13-3 + {{- else -}}1.12.10-5 + {{- end -}} + {{- else if eq .component "cilium-envoy" -}} +v1.31.5-250218 + {{- else if eq .component "cilium-operator-generic" -}} + {{- if semverCompare ">=1.29.0" .version -}}1.14.10 + {{- else if semverCompare ">=1.27.0" .version -}}1.13.13 + {{- else -}}1.12.10 + {{- end -}} + {{- else if eq .component "cloud-provider-node-manager-linux" -}} + {{- if semverCompare ">=1.33.0" .version -}}v1.33.0 + {{- else if semverCompare ">=1.32.0" .version -}}v1.32.5 + {{- else if semverCompare ">=1.31.0" .version -}}v1.31.6 + {{- else if semverCompare ">=1.30.0" .version -}}v1.30.12 + {{- else if semverCompare ">=1.29.0" .version -}}v1.29.15 + {{- else if semverCompare ">=1.28.0" .version -}}v1.28.14 + {{- else if semverCompare ">=1.27.0" .version -}}v1.27.21 + {{- else if semverCompare ">=1.26.0" .version -}}v1.26.22 + {{- else if semverCompare ">=1.25.0" .version -}}v1.25.24 + {{- else if semverCompare ">=1.24.0" .version -}}v1.24.21 + {{- else if semverCompare ">=1.23.0" .version -}}v1.23.24 + {{- else if semverCompare ">=1.22.0" .version -}}v1.1.14 + {{- else if semverCompare ">=1.21.0" .version -}}v1.0.18 + {{- else if semverCompare ">=1.20.0" .version -}}v0.7.21 + {{- else if semverCompare ">=1.19.0" .version -}}v0.6.0 + {{- else -}}v0.5.1.4 + {{- end -}} + {{- else if eq .component "cloud-provider-node-manager-windows" -}} + {{- if semverCompare ">=1.33.0" .version -}}v1.33.0 + {{- else if semverCompare ">=1.32.0" .version -}}v1.32.5 + {{- else if semverCompare ">=1.31.0" .version -}}v1.31.6 + {{- else if semverCompare ">=1.30.0" .version -}}v1.30.12 + {{- else if semverCompare ">=1.29.0" .version -}}v1.29.15 + {{- else if semverCompare ">=1.28.0" .version -}}v1.28.14 + {{- else if semverCompare ">=1.27.0" .version -}}v1.27.21 + {{- else if semverCompare ">=1.26.0" .version -}}v1.26.22 + {{- else if semverCompare ">=1.25.0" .version -}}v1.25.24 + {{- else if semverCompare ">=1.24.0" .version -}}v1.24.21 + {{- else if semverCompare ">=1.23.0" .version -}}v1.23.24 + {{- else if semverCompare ">=1.22.0" .version -}}v1.1.14 + {{- else if semverCompare ">=1.21.0" .version -}}v1.0.18 + {{- else if semverCompare ">=1.20.0" .version -}}v0.7.21 + {{- else if semverCompare ">=1.19.0" .version -}}v0.6.0 + {{- else -}}v0.5.1 + {{- end -}} + {{- else if eq .component "cluster-proportional-autoscaler" -}} + {{- if semverCompare ">=1.32.0" .version -}}v1.9.0-2 + {{- else if semverCompare ">=1.27.0" .version -}}v1.8.11-5 + {{- else if semverCompare ">=1.22.0" .version -}}v1.8.8 + {{- else if semverCompare ">=1.18.0" .version -}}1.8.3 + {{- else -}}1.7.1-hotfix.20200403 + {{- end -}} + {{- else if eq .component "container-networking-cilium-agent" -}} + {{- if semverCompare ">=1.31.0" .version -}}v1.16.6-250129 + {{- else if semverCompare ">=1.29.0" .version -}}v1.14.19-250129 + {{- else if semverCompare ">=1.27.0" .version -}}v1.13.18-241024 + {{- else -}}v1.14.19-250129 + {{- end -}} + {{- else if eq .component "container-networking-cilium-operator-generic" -}} + {{- if semverCompare ">=1.31.0" .version -}}v1.16.6-250129 + {{- else if semverCompare ">=1.29.0" .version -}}v1.14.19-250129 + {{- else if semverCompare ">=1.27.0" .version -}}v1.13.18-241024 + {{- else -}}v1.14.19-250129 + {{- end -}} + {{- else if eq .component "coredns" -}} + {{- if semverCompare ">=1.33.0" .version -}}v1.12.1-2 + {{- else if semverCompare ">=1.32.0" .version -}}v1.11.3-8 + {{- else if semverCompare ">=1.24.0" .version -}}v1.9.4-6 + {{- else if semverCompare ">=1.20.0" .version -}}v1.8.7 + {{- else -}}1.6.6 + {{- end -}} + {{- else if eq .component "cost-analysis-agent" -}} +v0.0.24 + {{- else if eq .component "cost-analysis-opencost" -}} +v1.111.0 + {{- else if eq .component "cost-analysis-prometheus" -}} +v2.54.1 + {{- else if eq .component "cost-analysis-victoria-metrics" -}} +v1.103.0 + {{- else if eq .component "extension-config-agent" -}} +1.28.0 + {{- else if eq .component "extension-manager" -}} +1.28.0 + {{- else if eq .component "fqdn-policy" -}} + {{- if semverCompare ">=1.31.0" .version -}}v1.16.6-250129 + {{- else -}}v1.14.19-250129 + {{- end -}} + {{- else if eq .component "gpu-provisioner" -}} +0.3.5 + {{- else if eq .component "health-probe-proxy" -}} +v1.29.1 + {{- else if eq .component "hubble-relay" -}} +v1.15.0 + {{- else if eq .component "identity-binding-workload-identity-webhook" -}} +v1.6.0-alpha.1 + {{- else if eq .component "image-cleaner" -}} +v1.4.0-4 + {{- else if eq .component "ingress-appgw" -}} + {{- if semverCompare ">=1.27.0" .version -}}1.8.1 + {{- else if semverCompare ">=1.19.0" .version -}}1.5.3 + {{- else -}}1.4.0 + {{- end -}} + {{- else if eq .component "ip-masq-agent-v2" -}} +v0.1.15-2 + {{- else if eq .component "ipv6-hp-bpf" -}} + {{- if semverCompare ">=1.29.0" .version -}}v0.0.1 + {{- else -}}v0.0.1 + {{- end -}} + {{- else if eq .component "keda" -}} + {{- if semverCompare ">=1.33.0" .version -}}2.17.1 + {{- else if semverCompare ">=1.32.0" .version -}}v2.16.1 + {{- else if semverCompare ">=1.30.0" .version -}}2.14.1 + {{- else if semverCompare ">=1.27.0" .version -}}2.11.2 + {{- else if semverCompare ">=1.26.0" .version -}}2.10.1 + {{- else if semverCompare ">=1.23.0" .version -}}2.9.3 + {{- else -}}2.8.1 + {{- end -}} + {{- else if eq .component "keda-admission-webhooks" -}} + {{- if semverCompare ">=1.33.0" .version -}}2.17.1 + {{- else if semverCompare ">=1.32.0" .version -}}v2.16.1 + {{- else if semverCompare ">=1.30.0" .version -}}2.14.1 + {{- else if semverCompare ">=1.27.0" .version -}}2.11.2 + {{- else -}}2.10.1 + {{- end -}} + {{- else if eq .component "keda-metrics-apiserver" -}} + {{- if semverCompare ">=1.33.0" .version -}}2.17.1 + {{- else if semverCompare ">=1.32.0" .version -}}v2.16.1 + {{- else if semverCompare ">=1.30.0" .version -}}2.14.1 + {{- else if semverCompare ">=1.27.0" .version -}}2.11.2 + {{- else if semverCompare ">=1.26.0" .version -}}2.10.1 + {{- else if semverCompare ">=1.23.0" .version -}}2.9.3 + {{- else -}}2.8.1 + {{- end -}} + {{- else if eq .component "kube-egress-gateway-cni" -}} + {{- if semverCompare ">=1.34.0" .version -}}v0.1.1 + {{- else -}}v0.0.21 + {{- end -}} + {{- else if eq .component "kube-egress-gateway-cni-ipam" -}} + {{- if semverCompare ">=1.34.0" .version -}}v0.1.1 + {{- else -}}v0.0.21 + {{- end -}} + {{- else if eq .component "kube-egress-gateway-cnimanager" -}} + {{- if semverCompare ">=1.34.0" .version -}}v0.1.1 + {{- else -}}v0.0.21 + {{- end -}} + {{- else if eq .component "kube-egress-gateway-daemon" -}} + {{- if semverCompare ">=1.34.0" .version -}}v0.1.1 + {{- else -}}v0.0.21 + {{- end -}} + {{- else if eq .component "kube-egress-gateway-daemon-init" -}} + {{- if semverCompare ">=1.34.0" .version -}}v0.1.1 + {{- else -}}v0.0.21 + {{- end -}} + {{- else if eq .component "local-csi-driver" -}} +v0.2.4 + {{- else if eq .component "local-csi-driver-csi-provisioner" -}} +v5.2.0 + {{- else if eq .component "local-csi-driver-csi-resizer" -}} +v1.13.2 + {{- else if eq .component "local-csi-driver-registrar" -}} +v2.13.0 + {{- else if eq .component "metrics-server" -}} + {{- if semverCompare ">=1.32.0" .version -}}v0.7.2-7 + {{- else if semverCompare ">=1.24.0" .version -}}v0.6.3-6 + {{- else if semverCompare ">=1.22.0" .version -}}v0.5.2 + {{- else if semverCompare ">=1.21.0" .version -}}v0.4.5 + {{- else if semverCompare ">=1.8.0" .version -}}v0.3.6 + {{- else -}}v0.2.1 + {{- end -}} + {{- else if eq .component "microsoft-defender-admission-controller" -}} +20250706.3 + {{- else if eq .component "microsoft-defender-low-level-collector" -}} + {{- if semverCompare ">=1.25.0" .version -}}2.0.221 + {{- else -}}1.3.81 + {{- end -}} + {{- else if eq .component "microsoft-defender-low-level-init" -}} +1.3.81 + {{- else if eq .component "microsoft-defender-old-file-cleaner" -}} +1.0.273 + {{- else if eq .component "microsoft-defender-pod-collector" -}} +1.0.202 + {{- else if eq .component "microsoft-defender-security-publisher" -}} +1.0.273 + {{- else if eq .component "open-policy-agent-gatekeeper" -}} + {{- if semverCompare ">=1.27.0" .version -}}v3.20.0-1 + {{- else if semverCompare ">=1.25.0" .version -}}v3.14.2 + {{- else if semverCompare ">=1.24.0" .version -}}v3.11.1 + {{- else if semverCompare ">=1.21.0" .version -}}v3.8.1 + {{- else if semverCompare ">=1.18.0" .version -}}v3.7.1 + {{- else -}}v3.4.1 + {{- end -}} + {{- else if eq .component "osm-bootstrap" -}} + {{- if semverCompare ">=1.24.0" .version -}}v1.2.9 + {{- else if semverCompare ">=1.23.5" .version -}}v1.1.3 + {{- else -}}v1.0.0 + {{- end -}} + {{- else if eq .component "osm-controller" -}} + {{- if semverCompare ">=1.24.0" .version -}}v1.2.9 + {{- else if semverCompare ">=1.23.5" .version -}}v1.1.3 + {{- else -}}v1.0.0 + {{- end -}} + {{- else if eq .component "osm-crds" -}} + {{- if semverCompare ">=1.24.0" .version -}}v1.2.9 + {{- else if semverCompare ">=1.23.5" .version -}}v1.1.3 + {{- else -}}v1.0.0 + {{- end -}} + {{- else if eq .component "osm-healthcheck" -}} + {{- if semverCompare ">=1.24.0" .version -}}v1.2.9 + {{- else if semverCompare ">=1.23.5" .version -}}v1.1.3 + {{- else -}}v1.1.0 + {{- end -}} + {{- else if eq .component "osm-init" -}} + {{- if semverCompare ">=1.24.0" .version -}}v1.2.9 + {{- else if semverCompare ">=1.23.5" .version -}}v1.1.3 + {{- else -}}v1.0.0 + {{- end -}} + {{- else if eq .component "osm-injector" -}} + {{- if semverCompare ">=1.24.0" .version -}}v1.2.9 + {{- else if semverCompare ">=1.23.5" .version -}}v1.1.3 + {{- else -}}v1.0.0 + {{- end -}} + {{- else if eq .component "osm-sidecar" -}} + {{- if semverCompare ">=1.25.0" .version -}}v1.32.2-hotfix.20241216 + {{- else if semverCompare ">=1.24.0" .version -}}v1.25.9-hotfix.20231002 + {{- else -}}v1.19.1 + {{- end -}} + {{- else if eq .component "overlay-vpa" -}} + {{- if semverCompare ">=1.31.0" .version -}}v1.2.1-1 + {{- else if semverCompare ">=1.27.0" .version -}}v1.0.0-1 + {{- else if semverCompare ">=1.25.0" .version -}}0.13.0 + {{- else -}}0.11.0 + {{- end -}} + {{- else if eq .component "overlay-vpa-webhook-generation" -}} +master.250827.1 + {{- else if eq .component "ratify-base" -}} +v1.2.3 + {{- else if eq .component "retina-agent" -}} +v1.0.0-rc2 + {{- else if eq .component "retina-agent-enterprise" -}} +v0.1.11 + {{- else if eq .component "retina-agent-win" -}} +v1.0.0-rc2 + {{- else if eq .component "retina-operator" -}} +v0.1.11 + {{- else if eq .component "secrets-store-csi-driver" -}} + {{- if semverCompare ">=1.26.0" .version -}}v1.5.3 + {{- else if semverCompare ">=1.24.0" .version -}}v1.3.4-1 + {{- else -}}v1.3.0.3 + {{- end -}} + {{- else if eq .component "secrets-store-csi-driver-windows" -}} + {{- if semverCompare ">=1.26.0" .version -}}v1.5.3 + {{- else if semverCompare ">=1.24.0" .version -}}v1.3.4 + {{- else -}}v1.3.0 + {{- end -}} + {{- else if eq .component "secrets-store-driver-registrar-linux" -}} + {{- if semverCompare ">=1.26.0" .version -}}v2.13.0 + {{- else if semverCompare ">=1.24.0" .version -}}v2.8.0 + {{- else -}}v2.6.2 + {{- end -}} + {{- else if eq .component "secrets-store-driver-registrar-windows" -}} + {{- if semverCompare ">=1.26.0" .version -}}v2.13.0 + {{- else if semverCompare ">=1.24.0" .version -}}v2.8.0 + {{- else -}}v2.6.2 + {{- end -}} + {{- else if eq .component "secrets-store-livenessprobe-linux" -}} + {{- if semverCompare ">=1.26.0" .version -}}v2.15.0 + {{- else if semverCompare ">=1.24.0" .version -}}v2.10.0 + {{- else -}}v2.8.0 + {{- end -}} + {{- else if eq .component "secrets-store-livenessprobe-windows" -}} + {{- if semverCompare ">=1.26.0" .version -}}v2.15.0 + {{- else if semverCompare ">=1.24.0" .version -}}v2.10.0 + {{- else -}}v2.8.0 + {{- end -}} + {{- else if eq .component "secrets-store-provider-azure" -}} + {{- if semverCompare ">=1.26.0" .version -}}v1.7.0 + {{- else if semverCompare ">=1.24.0" .version -}}v1.4.1 + {{- else -}}v1.4.0 + {{- end -}} + {{- else if eq .component "secrets-store-provider-azure-windows" -}} + {{- if semverCompare ">=1.26.0" .version -}}v1.7.0 + {{- else if semverCompare ">=1.24.0" .version -}}v1.4.1 + {{- else -}}v1.4.0 + {{- end -}} + {{- else if eq .component "sgx-attestation" -}} +3.3.1 + {{- else if eq .component "sgx-plugin" -}} +1.0.0 + {{- else if eq .component "sgx-webhook" -}} +1.2.2 + {{- else if eq .component "tigera-operator" -}} + {{- if semverCompare ">=1.33.0" .version -}}v1.38.3 + {{- else if semverCompare ">=1.32.0" .version -}}v1.36.11 + {{- else if semverCompare ">=1.30.0" .version -}}v1.34.13 + {{- else if semverCompare ">=1.29.0" .version -}}v1.30.11 + {{- else if semverCompare ">=1.24.0" .version -}}v1.28.13 + {{- else -}}v1.23.8 + {{- end -}} + {{- else if eq .component "windows-gmsa-webhook-image" -}} +v0.12.1-2 + {{- else if eq .component "workload-identity-webhook" -}} +v1.5.1 + {{- end -}} +{{- end -}} + +{{/* Auto-generated by servicemesh tooling, do not edit. See /toolkit/servicemesh/README.md for more information. */}} +{{- define "get.istioImageTag" -}} + {{- if eq .component "azure-service-mesh-istio" -}} + {{- if eq "asm-1-27" .revision -}}1.27.0-1 + {{- else if eq "asm-1-26" .revision -}}1.26.3-2 + {{- else if eq "asm-1-25" .revision -}}1.25.3-4 + {{- else if eq "asm-1-24" .revision -}}1.24.6 + {{- else if eq "asm-1-23" .revision -}}1.23.6-hotfix.20250515 + {{- else if eq "asm-1-22" .revision -}}1.22.7 + {{- else if eq "asm-1-21" .revision -}}1.21.6 + {{- else if eq "asm-1-20" .revision -}}1.20.8 + {{- else if eq "asm-1-19" .revision -}}1.19.10-hotfix.20240528 + {{- else if eq "asm-1-18" .revision -}}1.18.7-hotfix.20240210 + {{- else if eq "asm-1-17" .revision -}}1.17.8 + {{- else -}}not-in-use-9.99.9 + {{- end -}} + {{- end -}} +{{- end -}} diff --git a/charts/azuremonitor-containerinsights-for-prod-clusters/templates/_aks_common.tpl b/charts/azuremonitor-containerinsights-for-prod-clusters/templates/_aks_common.tpl new file mode 100644 index 000000000..29c0c4610 --- /dev/null +++ b/charts/azuremonitor-containerinsights-for-prod-clusters/templates/_aks_common.tpl @@ -0,0 +1,153 @@ +{{/* MCR repository template for adapter charts */}} +{{- define "mcr_repository_base_adapter_chart" }} +{{- $cloud_environment := ((index .Values.v1 "commonGlobals").CloudEnvironment | default "AZUREPUBLICCLOUD") }} +{{- if (eq $cloud_environment "AZURECHINACLOUD") }} +{{- "mcr.azk8s.cn" }} +{{- else if (eq $cloud_environment "USNat") }} +{{- "mcr.microsoft.eaglex.ic.gov" }} +{{- else if (eq $cloud_environment "USSec") }} +{{- "mcr.microsoft.scloud" }} +{{- else }} +{{- "mcr.microsoft.com" }} +{{- end }} +{{- end }} + +{{/* MCR repository template for addon charts */}} +{{- define "mcr_repository_base" }} +{{- $cloud_environment := (.Values.global.commonGlobals.CloudEnvironment| default "AZUREPUBLICCLOUD") }} +{{- if (eq $cloud_environment "AZURECHINACLOUD") }} +{{- "mcr.azk8s.cn" }} +{{- else if (eq $cloud_environment "USNat") }} +{{- "mcr.microsoft.eaglex.ic.gov" }} +{{- else if (eq $cloud_environment "USSec") }} +{{- "mcr.microsoft.scloud" }} +{{- else }} +{{- "mcr.microsoft.com" }} +{{- end }} +{{- end }} + +{{- define "addon_mcr_repository_base" }} +{{- template "mcr_repository_base" . }} +{{- end }} + +{{/* ccp_image_repository_base_by_component returns the image repository to use for a ccp component. + Caller should provide the "component" (the ccp component name), "version" (the ccp k8s version) and "Values" (the helm values object) parameters: + + {{- with $image_settings := (dict "component" "kube-apiserver" "version" .Values.global.commonGlobals.Versions.Kubernetes "Values" .Values) }} + {{ include "ccp_image_repository_base_by_component" $image_settings }} + {{- end }} + + The component name and k8s version will be concatenated as "-" to look up the override in the toggle. + + When the `use-internal-container-image-override-component` toggle is enabled for the specified component and k8s version, a cloud based + private repository will be used, otherwise, the value will fallback to `mcr_repoistory_base`. + Components that expect to be included in the embargo process should use this ACR repository. */}} +{{- define "ccp_image_repository_base_by_component" }} + {{- $key := (print .component "-" .version) }} + {{- if (hasKey .Values.global.commonGlobals.InternalContainerRegistry.enabledComponentOverrides $key) }} + {{- template "ccp_image_repository_base" . }} + {{- else }} + {{- template "mcr_repository_base" . }} + {{- end }} +{{- end }} + +{{/* ccp_image_repository_base returns the ACR repository for embargoed CVE images. + This template is intended to be called by ccp_image_repository_base_by_component and acr pull template only. + Caller should use ccp_image_repository_base_by_component for component based value. */}} +{{- define "ccp_image_repository_base" }} + {{- $cloud_environment := (.Values.global.commonGlobals.CloudEnvironment | upper | default "AZUREPUBLICCLOUD") }} + {{- if (or (eq $cloud_environment "AZUREUSGOVCLOUD") (eq $cloud_environment "AZUREUSGOVERNMENTCLOUD")) }} + {{- "acsdeployment.azurecr.us"}} + {{- else if (eq $cloud_environment "AZURECHINACLOUD") }} + {{- "acsdeployment.azurecr.cn" }} + {{- else if (eq $cloud_environment "USNAT") }} + {{- "acsdeployment.azurecr.eaglex.ic.gov" }} + {{- else if (eq $cloud_environment "USSEC") }} + {{- "acsdeployment.azurecr.microsoft.scloud" }} + {{- else }} + {{- "acsproddeployment.azurecr.io" }} + {{- end }} +{{- end }} + +{{/* ccp_get_imagetag_by_component returns the image tag to use for a ccp component. + Caller should provide the "component" (the ccp component name), "version" (the ccp k8s version) and "Values" (the helm values object) parameters: + + {{- with $image_settings := (dict "component" "kube-apiserver" "version" .Values.global.commonGlobals.Versions.Kubernetes "Values" .Values) }} + {{ include "ccp_get_imagetag_by_component" $image_settings }} + {{- end }} + + When the `use-internal-container-image-override-component` toggle is enabled for the specified component and k8s version, + the override tag will be used, otherwise, the value will fallback to `get.imagetag`. + + See also: ccp_image_repository_base_by_component */}} +{{- define "ccp_get_imagetag_by_component" }} + {{- $key := (print .component "-" .version) }} + {{- if (hasKey .Values.global.commonGlobals.InternalContainerRegistry.enabledComponentOverrides $key) }} + {{- (index .Values.global.commonGlobals.InternalContainerRegistry.enabledComponentOverrides $key) }} + {{- else }} + {{- template "get.imagetag" . }} + {{- end }} +{{- end }} + +{{/* ccp_get_ccpImageTag_by_component uses "get.ccpImageTag" as fallback. + + See also: ccp_get_imagetag_by_component */}} +{{- define "ccp_get_ccpImageTag_by_component" }} + {{- $key := (print .component "-" .version) }} + {{- if (hasKey .Values.global.commonGlobals.InternalContainerRegistry.enabledComponentOverrides $key) }} + {{- (index .Values.global.commonGlobals.InternalContainerRegistry.enabledComponentOverrides $key) }} + {{- else }} + {{- template "get.ccpImageTag" . }} + {{- end }} +{{- end }} + +{{/* nodeaffinity on nodepool */}} +{{- define "nodepool_affinity" -}} +{{- if .Values.global.commonGlobals.requireDedicatedNodepool -}} +preferredDuringSchedulingIgnoredDuringExecution: +- weight: 100 + preference: + matchExpressions: + - key: agentpool + operator: In + values: + - cx-{{ .Values.global.CCPID }} +{{- else -}} +requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: agentpool + operator: In + values: + - agentpool1 +{{- end -}} +{{- end -}} + +{{- define "addon_nodepool_mode_affinity_hard" -}} +{{- if .Values.global.commonGlobals.addonRequireSystemPool }} +- key: kubernetes.azure.com/mode + operator: In + values: + - system +{{- end -}} +{{- end -}} + +{{- define "addon_nodepool_mode_affinity_soft" -}} +{{- if not .Values.global.commonGlobals.addonRequireSystemPool }} +- weight: 100 + preference: + matchExpressions: + - key: kubernetes.azure.com/mode + operator: In + values: + - system +{{- end -}} +{{- end -}} + +{{/* tolerations on nodepool */}} +{{- define "nodepool_toleration" -}} +- key: "agentpool" + operator: "Equal" + value: "cx-{{ .Values.global.CCPID }}" + effect: "NoExecute" +{{- end }} diff --git a/charts/azuremonitor-containerinsights-for-prod-clusters/templates/_aks_helpers.tpl b/charts/azuremonitor-containerinsights-for-prod-clusters/templates/_aks_helpers.tpl new file mode 100644 index 000000000..f14bd9147 --- /dev/null +++ b/charts/azuremonitor-containerinsights-for-prod-clusters/templates/_aks_helpers.tpl @@ -0,0 +1,303 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "fullname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" .Values.global.commonGlobals.CCPID $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* Both formats are needed because the template is used by other adapter charts */}} +{{- define "enableKonnectivity" -}} +{{- $commonGlobals := "" }} +{{- if .Values.v1 }} +{{- $commonGlobals = (index .Values.v1 "commonGlobals") }} +{{- else }} +{{- $commonGlobals = .Values.global.commonGlobals }} +{{- end -}} +{{- if $commonGlobals.Konnectivity -}} +{{- if kindIs "invalid" $commonGlobals.Konnectivity.Enabled -}} +true +{{- else if $commonGlobals.Konnectivity.Enabled -}} +true +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* apiserver endpoint */}} +{{- define "apiserver_endpoint" }} +{{- if .Values.global.commonGlobals.PrivateConnect.enabled }} +{{- .Values.global.commonGlobals.PrivateConnect.privateIP }} +{{- else }} +{{- .Values.global.commonGlobals.endpointFQDN }} +{{- end }} +{{- end }} + +{{- define "enableApiserverProxyForKms" -}} +{{- if and .Values.global.commonGlobals.PrivateConnect.enabled (ne .Values.global.AzureKeyVaultKms.keyVaultNetworkAccess "Private") -}} +true +{{- else if not (or .Values.global.commonGlobals.TunnelOpenVPN.Enabled (include "enableKonnectivityWithEgressSelector" .)) -}} +true +{{- end -}} +{{- end -}} + +{{- define "enableAzureKmsProviderProxy" -}} +{{- if and .Values.global.AzureKeyVaultKms.enabled (include "enableKonnectivityWithEgressSelector" .) -}} +{{- if eq .Values.global.AzureKeyVaultKms.keyVaultNetworkAccess "Private" -}} +true +{{- else if .Values.global.AzureKeyVaultKms.previousKey -}} +{{- if eq .Values.global.AzureKeyVaultKms.previousKey.keyVaultNetworkAccess "Private" -}} +true +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{- define "enableKonnectivityProxyPodAndSvcCIDROnly" -}} +{{- if (include "enableKonnectivity" .) -}} +{{- if .Values.global.commonGlobals.Konnectivity.ProxyPodAndSvcCIDROnly -}} +true +{{- end -}} +{{- end -}} +{{- end -}} + +{{- define "enableKonnectivityWithEgressSelector" -}} +{{- if (include "enableKonnectivity" .) -}} +{{- if not .Values.global.commonGlobals.Konnectivity.ProxyPodAndSvcCIDROnly -}} +true +{{- end -}} +{{- end -}} +{{- end -}} + +{{- define "enableKonnectivityServerPreStop" -}} +{{- if (include "enableKonnectivity" .) -}} +{{- if .Values.global.commonGlobals.Konnectivity.enableKonnectivityServerPreStop -}} +{{- if semverCompare ">=1.28.0" .Values.global.commonGlobals.Versions.Kubernetes -}} +true +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{- define "enableKonnectivityServerSeparateCert" -}} + {{- if (include "enableKonnectivity" .) -}} + {{- if .Values.global.commonGlobals.Konnectivity.EnableSeparateServerCert -}} + {{- if semverCompare (printf ">=%s" .Values.global.commonGlobals.Konnectivity.EnableSeparateServerCertFromK8sVersion) .Values.global.commonGlobals.Versions.Kubernetes -}} + true + {{- end -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{- define "loggingResourceId" -}} +{{- if .Values.global.commonGlobals.FleetHubProfile.isHubCluster }} +{{- .Values.global.commonGlobals.FleetHubProfile.fleetResourceID }} +{{- else }} +{{- .Values.global.commonGlobals.Customer.AzureResourceID }} +{{- end }} +{{- end }} + +{{/* +Get the value of override update mode annotation, +default is "disabled" and only support "enabled" and "disabled" currently. +Return none and fall back to "disabled" if the value is not supported or current VPA is not existed. +*/}} +{{- define "getOverrideUpdateModeAnnotation" -}} +{{- if .current }} + {{- if eq (index .current.metadata.annotations "kubernetes.azure.com/override-update-mode") "enabled" }} + {{- "enabled" }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +Try to get the override updateMode value if the override update mode annotation is enabled, +and the current VPA cr is existed. If not, return none and use the default updateMode "Initial" +*/}} +{{- define "getUpdateMode" -}} +{{- if .current }} + {{- if eq (index .current.metadata.annotations "kubernetes.azure.com/override-update-mode") "enabled" }} + {{- dict "current" .current | include "getOverrideUpdateMode" }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +Get the value of override VPA update mode, user can override the updateMode in VPA cr +when the override update mode annotation is enabled, return none and use the default +updateMode value if the user input is invalid or any property is not existed +*/}} +{{- define "getOverrideUpdateMode" -}} +{{- /* +Use parentheses () to check the nested values existed due to the limitation of Helm +https://github.com/helm/helm/issues/8026 +*/}} +{{- if ((((.current).spec).updatePolicy).updateMode) }} + {{- if (dict "updateMode" .current.spec.updatePolicy.updateMode | include "isValidUpdateMode" ) }} + {{- .current.spec.updatePolicy.updateMode | quote }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +Check if the update mode is valid, +only support "Off", "Initial" and "Auto" update mode currently +*/}} +{{- define "isValidUpdateMode" -}} +{{- if not (has .updateMode (list "Recreate")) }} +true +{{- end }} +{{- end -}} + +{{/* +Get the value of override min/max annotation, +default is "disabled" and only support "enabled" and "disabled" currently. +Return none and fall back to "disabled" if the value is not supported. +*/}} +{{- define "getOverrideMinMaxAnnotation" -}} +{{- if .current }} + {{- if eq (index .current.metadata.annotations "kubernetes.azure.com/override-min-max") "enabled" }} + {{- "enabled" }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +Try to get the user override vpa min/max allowed value if the override min/max allowed annotation is enabled, +and the current VPA cr is existed. +If not, return none and use the default min/max allowed value. +*/}} +{{- define "getAllowedValue" -}} +{{- if .current }} + {{- if eq (index .current.metadata.annotations "kubernetes.azure.com/override-min-max") "enabled" }} + {{- (dict "current" .current "containerName" .containerName "resource" .resource) | include "getOverrideAllowedValue" }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +Find the target container policy in VPA containerPolicies array +*/}} +{{- define "getVpaContainer" -}} + {{- $name := .containerName }} + {{- range $container := .containerPolicies }} + {{- if eq $name $container.containerName }} + {{- toYaml $container }} + {{- end }} + {{- end }} +{{- end -}} + +{{/* +Get the user override vpa min/max allowed value from target container in current existing vpa cr +*/}} +{{- define "getOverrideAllowedValue" -}} +{{- /* +Use parentheses () to check the nested values existed due to the limitation of Helm +https://github.com/helm/helm/issues/8026 +*/}} +{{- $container := (dict "containerName" .containerName "containerPolicies" .current.spec.resourcePolicy.containerPolicies) | include "getVpaContainer" | fromYaml }} +{{- if eq .resource "maxCPU" }} + {{- if ((($container).maxAllowed).cpu) }} + {{- $container.maxAllowed.cpu }} + {{- end }} +{{- end }} +{{- if eq .resource "maxMemory" }} + {{- if ((($container).maxAllowed).memory) }} + {{- $container.maxAllowed.memory }} + {{- end }} +{{- end }} +{{- if eq .resource "minCPU" }} + {{- if ((($container).minAllowed).cpu) }} + {{- $container.minAllowed.cpu }} + {{- end }} +{{- end }} +{{- if eq .resource "minMemory" }} + {{- if ((($container).minAllowed).memory) }} + {{- $container.minAllowed.memory }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +Get the value of override requests limits annotation, +default is "disabled" and only support "enabled" and "disabled" currently. +Return none and fall back to "disabled" if the value is not supported. +*/}} +{{- define "getOverrideRequestsLimitsAnnotation" -}} +{{- if .current }} + {{- if eq (index .current.metadata.annotations "kubernetes.azure.com/override-requests-limits") "enabled" }} + {{- "enabled" }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +Find target container in deployment / daemonset containers property +*/}} +{{- define "getContainer" -}} + {{- $name := .containerName }} + {{- range $container := .containers }} + {{- if eq $name $container.name }} + {{- toYaml $container }} + {{- end }} + {{- end }} +{{- end -}} + +{{/* +Get user override resource requests/limits value from target container in existing deployment / daemonset +*/}} +{{- define "getOverrideRequestsLimitsValue" -}} +{{- $container := (dict "containerName" .containerName "containers" .current.spec.template.spec.containers) | include "getContainer" | fromYaml }} +{{- if eq .resource "requestCPU" }} + {{- if (((($container).resources).requests).cpu) }} + {{- $container.resources.requests.cpu }} + {{- end }} +{{- end }} +{{- if eq .resource "requestMemory" }} + {{- if (((($container).resources).requests).memory) }} + {{- $container.resources.requests.memory }} + {{- end }} +{{- end }} +{{- if eq .resource "limitCPU" }} + {{- if (((($container).resources).limits).cpu) }} + {{- $container.resources.limits.cpu }} + {{- end }} +{{- end }} +{{- if eq .resource "limitMemory" }} + {{- if (((($container).resources).limits).memory) }} + {{- $container.resources.limits.memory }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +Get user override requests/limits value when current deployment/daemonset and override annotation is existed, +if not, this function will return none and caller should set the default/fallback resource requests/limits value. +*/}} +{{- define "getRequestsLimitsValue" -}} +{{- if .current }} + {{- if eq (index .current.metadata.annotations "kubernetes.azure.com/override-requests-limits") "enabled" }} + {{- (dict "current" .current "containerName" .containerName "resource" .resource) | include "getOverrideRequestsLimitsValue" }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* should use AzureStackCloud */}} +{{- define "should_use_azurestackcloud" -}} + {{- $cloud_environment := (.Values.global.commonGlobals.CloudEnvironment | default "azurepubliccloud" | lower) }} + {{- has $cloud_environment (list "usnat" "ussec" "azurebleucloud") -}} +{{- end }} + +{{/* should mount ca certs from host */}} +{{- define "should_mount_hostca" -}} + {{- $cloud_environment := (.Values.global.commonGlobals.CloudEnvironment | default "azurepubliccloud" | lower) }} + {{- has $cloud_environment (list "usnat" "ussec" "azurebleucloud") -}} +{{- end }} \ No newline at end of file diff --git a/charts/azuremonitor-containerinsights-for-prod-clusters/templates/_aks_images.tpl b/charts/azuremonitor-containerinsights-for-prod-clusters/templates/_aks_images.tpl new file mode 100644 index 000000000..86380c455 --- /dev/null +++ b/charts/azuremonitor-containerinsights-for-prod-clusters/templates/_aks_images.tpl @@ -0,0 +1,655 @@ +{{- define "get.imagetag" -}} +{{- if eq .component "kube-addon-manager" -}} + {{- if semverCompare "<1.7.0" .version -}}v6.5 + {{- else if semverCompare "<1.10.0" .version -}}v8.6 + {{- else if semverCompare "<1.13.0" .version -}}v8.9.1 + {{- else -}}v9.0.2_v0.0.5.9 + {{- end -}} +{{- else if eq .component "kube-apiserver" -}} + {{- if semverCompare "=1.17.3" .version -}}v1.17.3-hotfix.20200601 + {{- else if semverCompare "=1.17.7" .version -}}v1.17.7-hotfix.20200624 + {{- else if semverCompare "=1.17.9" .version -}}v1.17.9-hotfix.20200714 + {{- else if semverCompare "=1.17.11" .version -}}v1.17.11-hotfix.20200901 + {{- else if semverCompare "=1.17.13" .version -}}v1.17.13-hotfix.20210310.1 + {{- else if semverCompare "=1.17.16" .version -}}v1.17.16-hotfix.20210310.1 + {{- else if semverCompare "=1.18.4" .version -}}v1.18.4-hotfix.20200624 + {{- else if semverCompare "=1.18.6" .version -}}v1.18.6-hotfix.20200723 + {{- else if semverCompare "=1.18.8" .version -}}v1.18.8-hotfix.20200924 + {{- else if semverCompare "=1.18.10" .version -}}v1.18.10-hotfix.20210310.1 + {{- else if semverCompare "=1.18.14" .version -}}v1.18.14-hotfix.20210322.1 + {{- else if semverCompare "=1.18.17" .version -}}v1.18.17-hotfix.20210525.3 + {{- else if semverCompare "=1.18.19" .version -}}v1.18.19-hotfix.20210522.3 + {{- else if semverCompare "=1.19.1" .version -}}v1.19.1-hotfix.20200923 + {{- else if semverCompare "=1.19.6" .version -}}v1.19.6-hotfix.20210310 + {{- else if semverCompare "=1.19.7" .version -}}v1.19.7-hotfix.20210310 + {{- else if semverCompare "=1.19.9" .version -}}v1.19.9-hotfix.20210526 + {{- else if semverCompare "=1.19.11" .version -}}v1.19.11-hotfix.20211101 + {{- else if semverCompare "=1.19.13" .version -}}v1.19.13-hotfix.20211101 + {{- else if semverCompare "=1.20.2" .version -}}v1.20.2-hotfix.20210310 + {{- else if semverCompare "=1.20.5" .version -}}v1.20.5-hotfix.20210603 + {{- else if semverCompare "=1.20.7" .version -}}v1.20.7-hotfix.20211115 + {{- else if semverCompare "=1.20.9" .version -}}v1.20.9-hotfix.20211115 + {{- else if semverCompare "=1.20.13" .version -}}v1.20.13-hotfix.20220210 + {{- else if semverCompare "=1.20.15" .version -}}v1.20.15-hotfix.20220201 + {{- else if semverCompare "=1.21.1" .version -}}v1.21.1-hotfix.20211115 + {{- else if semverCompare "=1.21.2" .version -}}v1.21.2-hotfix.20211115 + {{- else if semverCompare "=1.21.7" .version -}}v1.21.7-hotfix.20220601 + {{- else if semverCompare "=1.21.9" .version -}}v1.21.9-hotfix.20220601 + {{- else if semverCompare "=1.21.14" .version -}}v1.21.14-hotfix.20220620 + {{- else if semverCompare "=1.22.1" .version -}}v1.22.1-hotfix.20211115 + {{- else if semverCompare "=1.22.2" .version -}}v1.22.2-hotfix.20211115 + {{- else if semverCompare "=1.22.4" .version -}}v1.22.4-hotfix.20220615 + {{- else if semverCompare "=1.22.6" .version -}}v1.22.6-hotfix.20220916 + {{- else if semverCompare "=1.22.11" .version -}}v1.22.11-hotfix.20221109 + {{- else if semverCompare "=1.22.15" .version -}}v1.22.15-hotfix.20221109 + {{- else if semverCompare "=1.23.3" .version -}}v1.23.3-hotfix.20220615 + {{- else if semverCompare "=1.23.5" .version -}}v1.23.5-hotfix.20220916 + {{- else if semverCompare "=1.23.8" .version -}}v1.23.8-hotfix.20221109 + {{- else if semverCompare "=1.23.12" .version -}}v1.23.12-hotfix.20230208 + {{- else if semverCompare "=1.23.15" .version -}}v1.23.15-hotfix.20230208 + {{- else if semverCompare "=1.24.0" .version -}}v1.24.0-hotfix.20220916 + {{- else if semverCompare "=1.24.3" .version -}}v1.24.3-hotfix.20221216 + {{- else if semverCompare "=1.24.6" .version -}}v1.24.6-hotfix.20230208 + {{- else if semverCompare "=1.24.9" .version -}}v1.24.9-hotfix.20230612 + {{- else if semverCompare "=1.24.10" .version -}}v1.24.10-hotfix.20230612 + {{- else if semverCompare "=1.25.2" .version -}}v1.25.2-hotfix.20221216 + {{- else if semverCompare "=1.25.4" .version -}}v1.25.4-hotfix.20230208 + {{- else if semverCompare "=1.25.5" .version -}}v1.25.5-hotfix.20230612 + {{- else if semverCompare "=1.25.6" .version -}}v1.25.6-hotfix.20231009 + {{- else if semverCompare "=1.25.11" .version -}}v1.25.11-hotfix.20231102 + {{- else if semverCompare "=1.25.15" .version -}}v1.25.15-hotfix.20231103 + {{- else if semverCompare "=1.26.0" .version -}}v1.26.0-hotfix.20230612 + {{- else if semverCompare "=1.26.3" .version -}}v1.26.3-hotfix.20231009 + {{- else if semverCompare "=1.26.6" .version -}}v1.26.6-hotfix.20231102 + {{- else if semverCompare "=1.26.10" .version -}}v1.26.10-hotfix.20231103 + {{- else if semverCompare "=1.26.12" .version -}}v1.26.12 + {{- else if semverCompare "=1.27.1" .version -}}v1.27.1-hotfix.20231009 + {{- else if semverCompare "=1.27.3" .version -}}v1.27.3-hotfix.20231102 + {{- else if semverCompare "=1.27.7" .version -}}v1.27.7-hotfix.20240411 + {{- else if semverCompare "=1.27.9" .version -}}v1.27.9-hotfix.20240411 + {{- else if semverCompare "=1.27.13" .version -}}v1.27.13 + {{- else if semverCompare "=1.28.0" .version -}}v1.28.0-hotfix.20240712-1 + {{- else if semverCompare "=1.28.3" .version -}}v1.28.3-hotfix.20240712-1 + {{- else if semverCompare "=1.28.5" .version -}}v1.28.5-hotfix.20240712-1 + {{- else if semverCompare "=1.28.9" .version -}}v1.28.9-hotfix.20240712-1 + {{- else if semverCompare "=1.28.10" .version -}}v1.28.10-hotfix.20240712-1 + {{- else if semverCompare "=1.28.11" .version -}}v1.28.11-hotfix.20240712-1 + {{- else if semverCompare "=1.29.0" .version -}}v1.29.0-hotfix.20240712 + {{- else if semverCompare "=1.29.2" .version -}}v1.29.2-hotfix.20240712 + {{- else if semverCompare "=1.29.4" .version -}}v1.29.4-hotfix.20240712 + {{- else if semverCompare "=1.29.14" .version -}}v1.29.14-hotfix.20250703 + {{- else if semverCompare "=1.29.15" .version -}}v1.29.15-hotfix.20250703 + {{- else if semverCompare "=1.30.0" .version -}}v1.30.0-hotfix.20240712 + {{- else if semverCompare "=1.30.2" .version -}}v1.30.2-hotfix.20240712 + {{- else if and (semverCompare ">=1.30.11" .version) (semverCompare "<=1.30.14" .version) -}}v{{.version}}-hotfix.20250703 + {{- else if and (semverCompare ">=1.31.0" .version) (semverCompare "<=1.31.11" .version) -}}v{{.version}}-hotfix.20250703 + {{- else if and (semverCompare ">=1.32.0" .version) (semverCompare "<=1.32.7" .version) -}}v{{.version}}-hotfix.20250703 + {{- else if and (semverCompare ">=1.33.0" .version) (semverCompare "<=1.33.3" .version) -}}v{{.version}}-hotfix.20250703 + {{- else if semverCompare ">=1.34.0" .version -}}v{{ .version }}-1 + {{- else if and (semverCompare ">=1.28.100" .version) (semverCompare "<=1.28.101" .version) -}}v{{.version}}-akslts-hotfix.20250703 + {{- else if and (semverCompare ">=1.27.0" .version) (ge ((semver .version).Patch) 100) -}}v{{.version}}-akslts + {{- else -}}v{{ .version }} + {{- end -}} +{{- else if eq .component "kube-scheduler" -}} + {{- if semverCompare "=1.17.3" .version -}}v1.17.3-hotfix.20200601 + {{- else if semverCompare "=1.17.7" .version -}}v1.17.7-hotfix.20200624 + {{- else if semverCompare "=1.17.9" .version -}}v1.17.9-hotfix.20200714 + {{- else if semverCompare "=1.17.11" .version -}}v1.17.11-hotfix.20200901 + {{- else if semverCompare "=1.17.13" .version -}}v1.17.13-hotfix.20210310.1 + {{- else if semverCompare "=1.17.16" .version -}}v1.17.16-hotfix.20210310.1 + {{- else if semverCompare "=1.18.4" .version -}}v1.18.4-hotfix.20200624 + {{- else if semverCompare "=1.18.6" .version -}}v1.18.6-hotfix.20200723 + {{- else if semverCompare "=1.18.8" .version -}}v1.18.8-hotfix.20200924 + {{- else if semverCompare "=1.18.10" .version -}}v1.18.10-hotfix.20210310.1 + {{- else if semverCompare "=1.18.14" .version -}}v1.18.14-hotfix.20210322.1 + {{- else if semverCompare "=1.18.17" .version -}}v1.18.17-hotfix.20210525.3 + {{- else if semverCompare "=1.18.19" .version -}}v1.18.19-hotfix.20210522.3 + {{- else if semverCompare "=1.19.1" .version -}}v1.19.1-hotfix.20200923 + {{- else if semverCompare "=1.19.6" .version -}}v1.19.6-hotfix.20210310 + {{- else if semverCompare "=1.19.7" .version -}}v1.19.7-hotfix.20210310 + {{- else if semverCompare "=1.19.9" .version -}}v1.19.9-hotfix.20210526 + {{- else if semverCompare "=1.19.11" .version -}}v1.19.11-hotfix.20211101 + {{- else if semverCompare "=1.19.13" .version -}}v1.19.13-hotfix.20211101 + {{- else if semverCompare "=1.20.2" .version -}}v1.20.2-hotfix.20210310 + {{- else if semverCompare "=1.20.5" .version -}}v1.20.5-hotfix.20210603 + {{- else if semverCompare "=1.20.7" .version -}}v1.20.7-hotfix.20211115 + {{- else if semverCompare "=1.20.9" .version -}}v1.20.9-hotfix.20211115 + {{- else if semverCompare "=1.20.13" .version -}}v1.20.13-hotfix.20220210 + {{- else if semverCompare "=1.20.15" .version -}}v1.20.15-hotfix.20220201 + {{- else if semverCompare "=1.21.1" .version -}}v1.21.1-hotfix.20211115 + {{- else if semverCompare "=1.21.2" .version -}}v1.21.2-hotfix.20211115 + {{- else if semverCompare "=1.21.7" .version -}}v1.21.7-hotfix.20220601 + {{- else if semverCompare "=1.21.9" .version -}}v1.21.9-hotfix.20220601 + {{- else if semverCompare "=1.21.14" .version -}}v1.21.14-hotfix.20220620 + {{- else if semverCompare "=1.22.1" .version -}}v1.22.1-hotfix.20211115 + {{- else if semverCompare "=1.22.2" .version -}}v1.22.2-hotfix.20211115 + {{- else if semverCompare "=1.22.4" .version -}}v1.22.4-hotfix.20220615 + {{- else if semverCompare "=1.22.6" .version -}}v1.22.6-hotfix.20220916 + {{- else if semverCompare "=1.22.11" .version -}}v1.22.11-hotfix.20221109 + {{- else if semverCompare "=1.22.15" .version -}}v1.22.15-hotfix.20221109 + {{- else if semverCompare "=1.23.3" .version -}}v1.23.3-hotfix.20220615 + {{- else if semverCompare "=1.23.5" .version -}}v1.23.5-hotfix.20220916 + {{- else if semverCompare "=1.23.8" .version -}}v1.23.8-hotfix.20221109 + {{- else if semverCompare "=1.23.12" .version -}}v1.23.12-hotfix.20230208 + {{- else if semverCompare "=1.23.15" .version -}}v1.23.15-hotfix.20230208 + {{- else if semverCompare "=1.24.0" .version -}}v1.24.0-hotfix.20220916 + {{- else if semverCompare "=1.24.3" .version -}}v1.24.3-hotfix.20221216 + {{- else if semverCompare "=1.24.6" .version -}}v1.24.6-hotfix.20230208 + {{- else if semverCompare "=1.24.9" .version -}}v1.24.9-hotfix.20230612 + {{- else if semverCompare "=1.24.10" .version -}}v1.24.10-hotfix.20230612 + {{- else if semverCompare "=1.25.2" .version -}}v1.25.2-hotfix.20221216 + {{- else if semverCompare "=1.25.4" .version -}}v1.25.4-hotfix.20230208 + {{- else if semverCompare "=1.25.5" .version -}}v1.25.5-hotfix.20230612 + {{- else if semverCompare "=1.25.6" .version -}}v1.25.6-hotfix.20230728 + {{- else if semverCompare "=1.25.11" .version -}}v1.25.11-hotfix.20231102 + {{- else if semverCompare "=1.25.15" .version -}}v1.25.15-hotfix.20231103 + {{- else if semverCompare "=1.26.0" .version -}}v1.26.0-hotfix.20230612 + {{- else if semverCompare "=1.26.3" .version -}}v1.26.3-hotfix.20230728 + {{- else if semverCompare "=1.26.6" .version -}}v1.26.6-hotfix.20231102 + {{- else if semverCompare "=1.26.10" .version -}}v1.26.10-hotfix.20231103 + {{- else if semverCompare "=1.26.12" .version -}}v1.26.12 + {{- else if semverCompare "=1.27.1" .version -}}v1.27.1-hotfix.20230728 + {{- else if semverCompare "=1.27.3" .version -}}v1.27.3-hotfix.20231102 + {{- else if semverCompare "=1.27.7" .version -}}v1.27.7-hotfix.20240411 + {{- else if semverCompare "=1.27.9" .version -}}v1.27.9-hotfix.20240411 + {{- else if semverCompare "=1.27.14" .version -}}v1.27.15 + {{- else if semverCompare "=1.28.0" .version -}}v1.28.0-hotfix.20240712-1 + {{- else if semverCompare "=1.28.3" .version -}}v1.28.3-hotfix.20240712-1 + {{- else if semverCompare "=1.28.5" .version -}}v1.28.5-hotfix.20240712-1 + {{- else if semverCompare "=1.28.10" .version -}}v1.28.11-hotfix.20240712-1 + {{- else if semverCompare "=1.28.11" .version -}}v1.28.11-hotfix.20240712-1 + {{- else if semverCompare "=1.29.0" .version -}}v1.29.0-hotfix.20240712 + {{- else if semverCompare "=1.29.2" .version -}}v1.29.2-hotfix.20240712 + {{- else if semverCompare "=1.29.5" .version -}}v1.29.6-hotfix.20240712 + {{- else if semverCompare "=1.29.6" .version -}}v1.29.6-hotfix.20240712 + {{- else if semverCompare "=1.30.0" .version -}}v1.30.0-hotfix.20240712 + {{- else if semverCompare "=1.30.2" .version -}}v1.30.2-hotfix.20240712 + {{- else if and (semverCompare ">=1.27.0" .version) (ge ((semver .version).Patch | int) 100) -}}v{{.version}}-akslts + {{- else if semverCompare ">=1.34.0" .version -}}v{{ .version }}-1 + {{- else -}}v{{ .version }} + {{- end -}} +{{- else if eq .component "kube-controller-manager" -}} + {{- if semverCompare "=1.17.3" .version -}}v1.17.3-hotfix.20200601 + {{- else if semverCompare "=1.17.7" .version -}}v1.17.7-hotfix.20200917 + {{- else if semverCompare "=1.17.9" .version -}}v1.17.9-hotfix.20200917 + {{- else if semverCompare "=1.17.11" .version -}}v1.17.11-hotfix.20200901 + {{- else if semverCompare "=1.17.13" .version -}}v1.17.13-hotfix.20210310.1 + {{- else if semverCompare "=1.17.16" .version -}}v1.17.16-hotfix.20210310.1 + {{- else if semverCompare "=1.18.4" .version -}}v1.18.4-hotfix.20200624 + {{- else if semverCompare "=1.18.6" .version -}}v1.18.6-hotfix.20200917 + {{- else if semverCompare "=1.18.8" .version -}}v1.18.8-hotfix.20200924 + {{- else if semverCompare "=1.18.10" .version -}}v1.18.10-hotfix.20210310.1 + {{- else if semverCompare "=1.18.14" .version -}}v1.18.14-hotfix.20210525 + {{- else if semverCompare "=1.18.17" .version -}}v1.18.17-hotfix.20210525.3 + {{- else if semverCompare "=1.18.19" .version -}}v1.18.19-hotfix.20210522.3 + {{- else if semverCompare "=1.19.1" .version -}}v1.19.1-hotfix.20200923 + {{- else if semverCompare "=1.19.6" .version -}}v1.19.6-hotfix.20210310 + {{- else if semverCompare "=1.19.7" .version -}}v1.19.7-hotfix.20210525 + {{- else if semverCompare "=1.19.9" .version -}}v1.19.9-hotfix.20210526 + {{- else if semverCompare "=1.19.11" .version -}}v1.19.11-hotfix.20211101 + {{- else if semverCompare "=1.19.13" .version -}}v1.19.13-hotfix.20211101 + {{- else if semverCompare "=1.20.2" .version -}}v1.20.2-hotfix.20210525 + {{- else if semverCompare "=1.20.5" .version -}}v1.20.5-hotfix.20210603 + {{- else if semverCompare "=1.20.7" .version -}}v1.20.7-hotfix.20211115 + {{- else if semverCompare "=1.20.9" .version -}}v1.20.9-hotfix.20220126 + {{- else if semverCompare "=1.20.13" .version -}}v1.20.13-hotfix.20220210 + {{- else if semverCompare "=1.20.15" .version -}}v1.20.15-hotfix.20220201 + {{- else if semverCompare "=1.21.1" .version -}}v1.21.1-hotfix.20211115 + {{- else if semverCompare "=1.21.2" .version -}}v1.21.2-hotfix.20220126 + {{- else if semverCompare "=1.21.7" .version -}}v1.21.7-hotfix.20220601 + {{- else if semverCompare "=1.21.9" .version -}}v1.21.9-hotfix.20220601 + {{- else if semverCompare "=1.21.14" .version -}}v1.21.14-hotfix.20220620 + {{- else if semverCompare "=1.22.1" .version -}}v1.22.1-hotfix.20211115 + {{- else if semverCompare "=1.22.2" .version -}}v1.22.2-hotfix.20211115 + {{- else if semverCompare "=1.22.4" .version -}}v1.22.4-hotfix.20220615 + {{- else if semverCompare "=1.22.6" .version -}}v1.22.6-hotfix.20220916 + {{- else if semverCompare "=1.22.11" .version -}}v1.22.11-hotfix.20221109 + {{- else if semverCompare "=1.22.15" .version -}}v1.22.15-hotfix.20221109 + {{- else if semverCompare "=1.23.3" .version -}}v1.23.3-hotfix.20220615 + {{- else if semverCompare "=1.23.5" .version -}}v1.23.5-hotfix.20220916 + {{- else if semverCompare "=1.23.8" .version -}}v1.23.8-hotfix.20221109 + {{- else if semverCompare "=1.23.12" .version -}}v1.23.12-hotfix.20230208 + {{- else if semverCompare "=1.23.15" .version -}}v1.23.15-hotfix.20230208 + {{- else if semverCompare "=1.24.0" .version -}}v1.24.0-hotfix.20220916 + {{- else if semverCompare "=1.24.3" .version -}}v1.24.3-hotfix.20221216 + {{- else if semverCompare "=1.24.6" .version -}}v1.24.6-hotfix.20230208 + {{- else if semverCompare "=1.24.9" .version -}}v1.24.9-hotfix.20230612 + {{- else if semverCompare "=1.24.10" .version -}}v1.24.10-hotfix.20230612 + {{- else if semverCompare "=1.25.2" .version -}}v1.25.2-hotfix.20221216 + {{- else if semverCompare "=1.25.4" .version -}}v1.25.4-hotfix.20230208 + {{- else if semverCompare "=1.25.5" .version -}}v1.25.5-hotfix.20230612 + {{- else if semverCompare "=1.25.6" .version -}}v1.25.6-hotfix.20230728 + {{- else if semverCompare "=1.25.11" .version -}}v1.25.11-hotfix.20231102 + {{- else if semverCompare "=1.25.15" .version -}}v1.25.15-hotfix.20231103 + {{- else if semverCompare "=1.26.0" .version -}}v1.26.0-hotfix.20230612 + {{- else if semverCompare "=1.26.3" .version -}}v1.26.3-hotfix.20230728 + {{- else if semverCompare "=1.26.6" .version -}}v1.26.6-hotfix.20231102 + {{- else if semverCompare "=1.26.10" .version -}}v1.26.10-hotfix.20231103 + {{- else if semverCompare "=1.26.12" .version -}}v1.26.12 + {{- else if semverCompare "=1.27.1" .version -}}v1.27.1-hotfix.20230728 + {{- else if semverCompare "=1.27.3" .version -}}v1.27.3-hotfix.20231102 + {{- else if semverCompare "=1.27.7" .version -}}v1.27.7-hotfix.20240411 + {{- else if semverCompare "=1.27.9" .version -}}v1.27.9-hotfix.20240411 + {{- else if semverCompare "=1.27.13" .version -}}v1.27.13 + {{- else if semverCompare "=1.28.0" .version -}}v1.28.0-hotfix.20240712-1 + {{- else if semverCompare "=1.28.3" .version -}}v1.28.3-hotfix.20240712-1 + {{- else if semverCompare "=1.28.5" .version -}}v1.28.5-hotfix.20240712-1 + {{- else if semverCompare "=1.28.9" .version -}}v1.28.9-hotfix.20240712-1 + {{- else if semverCompare "=1.28.10" .version -}}v1.28.10-hotfix.20240712-1 + {{- else if semverCompare "=1.28.11" .version -}}v1.28.11-hotfix.20240712-1 + {{- else if semverCompare "=1.29.0" .version -}}v1.29.0-hotfix.20240712 + {{- else if semverCompare "=1.29.2" .version -}}v1.29.2-hotfix.20240712 + {{- else if semverCompare "=1.29.4" .version -}}v1.29.4-hotfix.20240712 + {{- else if semverCompare "=1.30.0" .version -}}v1.30.0-hotfix.20240712 + {{- else if semverCompare "=1.30.2" .version -}}v1.30.2-hotfix.20240712 + {{- else if and (semverCompare ">=1.27.0" .version) (ge ((semver .version).Patch) 100) -}}v{{.version}}-akslts + {{- else if semverCompare ">=1.34.0" .version -}}v{{ .version }}-1 + {{- else -}}v{{ .version }} + {{- end -}} +{{- else if eq .component "hyperkube" -}} + {{- if semverCompare "=1.12.8" .version -}}v1.12.8_v0.0.5 + {{- else if semverCompare "=1.13.10" .version -}}v1.13.10_v0.0.5 + {{- else if semverCompare "=1.13.11" .version -}}v1.13.11_v0.0.5 + {{- else if semverCompare "=1.13.12" .version -}}v1.13.12_v0.0.5 + {{- else if semverCompare "=1.14.6" .version -}}v1.14.6_v0.0.5 + {{- else if semverCompare "=1.14.7" .version -}}v1.14.7-hotfix.20200408.1 + {{- else if semverCompare "=1.14.8" .version -}}v1.14.8-hotfix.20200529.1 + {{- else if semverCompare "=1.15.3" .version -}}v1.15.3_v0.0.5 + {{- else if semverCompare "=1.15.4" .version -}}v1.15.4_v0.0.5 + {{- else if semverCompare "=1.15.5" .version -}}v1.15.5_v0.0.5 + {{- else if semverCompare "=1.15.7" .version -}}v1.15.7-hotfix.20200408.1 + {{- else if semverCompare "=1.15.10" .version -}}v1.15.10-hotfix.20200408.1 + {{- else if semverCompare "=1.15.11" .version -}}v1.15.11-hotfix.20201203 + {{- else if semverCompare "=1.15.12" .version -}}v1.15.12-hotfix.20200824.2 + {{- else if semverCompare "=1.16.0" .version -}}v1.16.0_v0.0.5 + {{- else if semverCompare "=1.16.7" .version -}}v1.16.7-hotfix.20200601.3 + {{- else if semverCompare "=1.16.8" .version -}}v1.16.8.2 + {{- else if semverCompare "=1.16.9" .version -}}v1.16.9-hotfix.20200529.7 + {{- else if semverCompare "=1.16.10" .version -}}v1.16.10-hotfix.20200917.3 + {{- else if semverCompare "=1.16.13" .version -}}v1.16.13-hotfix.20210118.2 + {{- else if semverCompare "=1.16.14" .version -}}v1.16.14-hotfix.20200901.4 + {{- else if semverCompare "=1.16.15" .version -}}v1.16.15-hotfix.20210118.4 + {{- else if semverCompare "=1.17.3" .version -}}v1.17.3-hotfix.20200601.3 + {{- else if semverCompare "=1.17.4" .version -}}v1.17.4.2 + {{- else if semverCompare "=1.17.5" .version -}}v1.17.5.2 + {{- else if semverCompare "=1.17.7" .version -}}v1.17.7-hotfix.20200917.3 + {{- else if semverCompare "=1.17.9" .version -}}v1.17.9-hotfix.20200917.3 + {{- else if semverCompare "=1.17.11" .version -}}v1.17.11-hotfix.20200901.4 + {{- else if semverCompare "=1.17.13" .version -}}v1.17.13-hotfix.20210310.2 + {{- else if semverCompare "=1.17.16" .version -}}v1.17.16-hotfix.20210310.2 + {{- else if semverCompare "=1.18.1" .version -}}v1.18.1.6 + {{- else if semverCompare "=1.18.2" .version -}}v1.18.2-hotfix.20200626.7 + {{- else if semverCompare "=1.18.4" .version -}}v1.18.4-hotfix.20200626.7 + {{- else if semverCompare "=1.18.6" .version -}}v1.18.6-hotfix.20200917.5 + {{- else if semverCompare "=1.18.8" .version -}}v1.18.8-hotfix.20201112.4 + {{- else if semverCompare "=1.18.10" .version -}}v1.18.10-hotfix.20210310.4 + {{- else if semverCompare "=1.18.14" .version -}}v1.18.14-hotfix.20210525.2 + {{- else if semverCompare "=1.18.17" .version -}}v1.18.17-hotfix.20210525.2 + {{- else if semverCompare "=1.18.19" .version -}}v1.18.19-hotfix.20210522.2 + {{- else -}}v{{ .version }} + {{- end -}} +{{- else if eq .component "kubectl" -}} + {{- if semverCompare "=1.17.3" .version -}}v1.17.3-hotfix.20200601 + {{- else if semverCompare "=1.17.7" .version -}}v1.17.7-hotfix.20200624 + {{- else if semverCompare "=1.17.9" .version -}}v1.17.9-hotfix.20200714 + {{- else if semverCompare "=1.17.11" .version -}}v1.17.11-hotfix.20200901 + {{- else if semverCompare "=1.17.13" .version -}}v1.17.13-hotfix.20210310.1 + {{- else if semverCompare "=1.17.16" .version -}}v1.17.16-hotfix.20210310.1 + {{- else if semverCompare "=1.18.4" .version -}}v1.18.4-hotfix.20200624 + {{- else if semverCompare "=1.18.6" .version -}}v1.18.6-hotfix.20200723 + {{- else if semverCompare "=1.18.8" .version -}}v1.18.8-hotfix.20200924 + {{- else if semverCompare "=1.18.10" .version -}}v1.18.10-hotfix.20210310.1 + {{- else if semverCompare "=1.18.14" .version -}}v1.18.14-hotfix.20210322 + {{- else if semverCompare "=1.18.17" .version -}}v1.18.17-hotfix.20210525.2 + {{- else if semverCompare "=1.18.19" .version -}}v1.18.19-hotfix.20210522.2 + {{- else if semverCompare "=1.19.1" .version -}}v1.19.1-hotfix.20200923 + {{- else if semverCompare "=1.19.6" .version -}}v1.19.6-hotfix.20210310.1 + {{- else if semverCompare "=1.19.7" .version -}}v1.19.7-hotfix.20210310.1 + {{- else if semverCompare "=1.19.9" .version -}}v1.19.9-hotfix.20210526.2 + {{- else if semverCompare "=1.19.11" .version -}}v1.19.11-hotfix.20211101.1 + {{- else if semverCompare "=1.19.13" .version -}}v1.19.13-hotfix.20211101.1 + {{- else if semverCompare "=1.20.2" .version -}}v1.20.2-hotfix.20210310.1 + {{- else if semverCompare "=1.20.5" .version -}}v1.20.5-hotfix.20210603.2 + {{- else if semverCompare "=1.20.7" .version -}}v1.20.7-hotfix.20211115 + {{- else if semverCompare "=1.20.9" .version -}}v1.20.9-hotfix.20211115.1 + {{- else if semverCompare "=1.20.13" .version -}}v1.20.13-hotfix.20220210.2 + {{- else if semverCompare "=1.20.15" .version -}}v1.20.15-hotfix.20220201.2 + {{- else if semverCompare "=1.21.1" .version -}}v1.21.1-hotfix.20211115 + {{- else if semverCompare "=1.21.2" .version -}}v1.21.2-hotfix.20211115.1 + {{- else if semverCompare "=1.21.7" .version -}}v1.21.7-hotfix.20220601 + {{- else if semverCompare "=1.21.9" .version -}}v1.21.9-hotfix.20220601.1 + {{- else if semverCompare "=1.21.14" .version -}}v1.21.14-hotfix.20220620.1 + {{- else if semverCompare "=1.22.1" .version -}}v1.22.1-hotfix.20211115.1 + {{- else if semverCompare "=1.22.2" .version -}}v1.22.2-hotfix.20211115.1 + {{- else if semverCompare "=1.22.4" .version -}}v1.22.4-hotfix.20220615 + {{- else if semverCompare "=1.22.6" .version -}}v1.22.6-hotfix.20220916 + {{- else if semverCompare "=1.22.11" .version -}}v1.22.11-hotfix.20221109.1 + {{- else if semverCompare "=1.22.15" .version -}}v1.22.15-hotfix.20221109.1 + {{- else if semverCompare "=1.23.3" .version -}}v1.23.3-hotfix.20220615 + {{- else if semverCompare "=1.23.5" .version -}}v1.23.5-hotfix.20220916 + {{- else if semverCompare "=1.23.8" .version -}}v1.23.8-hotfix.20221109.2 + {{- else if semverCompare "=1.23.12" .version -}}v1.23.12-hotfix.20230208.1 + {{- else if semverCompare "=1.23.15" .version -}}v1.23.15-hotfix.20230208.1 + {{- else if semverCompare "=1.24.0" .version -}}v1.24.0-hotfix.20220916 + {{- else if semverCompare "=1.24.3" .version -}}v1.24.3-hotfix.20221216 + {{- else if semverCompare "=1.24.6" .version -}}v1.24.6-hotfix.20230208.1 + {{- else if semverCompare "=1.24.9" .version -}}v1.24.9-hotfix.20230612 + {{- else if semverCompare "=1.24.10" .version -}}v1.24.10-hotfix.20230612 + {{- else if semverCompare "=1.25.2" .version -}}v1.25.2-hotfix.20221216.1 + {{- else if semverCompare "=1.25.4" .version -}}v1.25.4-hotfix.20230208.1 + {{- else if semverCompare "=1.25.5" .version -}}v1.25.5-hotfix.20230612 + {{- else if semverCompare "=1.25.6" .version -}}v1.25.6-hotfix.20230728 + {{- else if semverCompare "=1.25.11" .version -}}v1.25.11-hotfix.20231102 + {{- else if semverCompare "=1.25.15" .version -}}v1.25.15-hotfix.20231103 + {{- else if semverCompare "=1.26.0" .version -}}v1.26.0-hotfix.20230612 + {{- else if semverCompare "=1.26.3" .version -}}v1.26.3-hotfix.20230728 + {{- else if semverCompare "=1.26.6" .version -}}v1.26.6-hotfix.20231102 + {{- else if semverCompare "=1.26.10" .version -}}v1.26.10-hotfix.20231103-1 + {{- else if semverCompare "=1.26.12" .version -}}v1.26.12-1 + {{- else if semverCompare "=1.27.1" .version -}}v1.27.1-hotfix.20230728 + {{- else if semverCompare "=1.27.3" .version -}}v1.27.3-hotfix.20240125 + {{- else if semverCompare "=1.27.7" .version -}}v1.27.7-hotfix.20240712-4 + {{- else if semverCompare "=1.27.9" .version -}}v1.27.9-hotfix.20240712-4 + {{- else if semverCompare "=1.27.13" .version -}}v1.27.13-2 + {{- else if semverCompare "=1.28.0" .version -}}v1.28.0-hotfix.20240712-4 + {{- else if semverCompare "=1.28.3" .version -}}v1.28.3-hotfix.20240712-4 + {{- else if semverCompare "=1.28.5" .version -}}v1.28.5-hotfix.20240712-4 + {{- else if and (semverCompare ">=1.27.0" .version) (ge ((semver .version).Patch) 100) -}}v{{.version}}-akslts + {{- else if and (semverCompare ">=1.29.0" .version) (semverCompare "<1.30.0" .version) -}}v1.29.13 + {{- else if semverCompare "=1.30.0" .version -}}v1.30.0-1 + {{- else if semverCompare "=1.30.1" .version -}}v1.30.1-1 + {{- else if semverCompare "=1.30.2" .version -}}v1.30.2-hotfix.20240613 + {{- else if semverCompare ">=1.34.0" .version -}}v{{ .version }}-1 + {{- else -}}v{{ .version }} + {{- end -}} +{{- else if eq .component "kube-proxy" -}} + {{- if semverCompare "=1.17.3" .version -}}v1.17.3-hotfix.20200601.3 + {{- else if semverCompare "=1.17.7" .version -}}v1.17.7-hotfix.20200917.3 + {{- else if semverCompare "=1.17.9" .version -}}v1.17.9-hotfix.20200917.3 + {{- else if semverCompare "=1.17.11" .version -}}v1.17.11-hotfix.20200901.2 + {{- else if semverCompare "=1.17.13" .version -}}v1.17.13-hotfix.20210310.2 + {{- else if semverCompare "=1.17.16" .version -}}v1.17.16-hotfix.20210310.2 + {{- else if semverCompare "=1.18.2" .version -}}v1.18.2-hotfix.20200626.4 + {{- else if semverCompare "=1.18.4" .version -}}v1.18.4-hotfix.20200626.5 + {{- else if semverCompare "=1.18.6" .version -}}v1.18.6-hotfix.20200917.4 + {{- else if semverCompare "=1.18.8" .version -}}v1.18.8-hotfix.20201112.2 + {{- else if semverCompare "=1.18.10" .version -}}v1.18.10-hotfix.20210310.2 + {{- else if semverCompare "=1.18.14" .version -}}v1.18.14-hotfix.20210525 + {{- else if semverCompare "=1.18.17" .version -}}v1.18.17-hotfix.20210525.3 + {{- else if semverCompare "=1.18.19" .version -}}v1.18.19-hotfix.20210522.3 + {{- else if semverCompare "=1.19.7" .version -}}v1.19.7-hotfix.20210525 + {{- else if semverCompare "=1.19.9" .version -}}v1.19.9-hotfix.20210526.3 + {{- else if semverCompare "=1.19.11" .version -}}v1.19.11-hotfix.20211101.1 + {{- else if semverCompare "=1.19.13" .version -}}v1.19.13-hotfix.20211101.1 + {{- else if semverCompare "=1.20.2" .version -}}v1.20.2-hotfix.20210525 + {{- else if semverCompare "=1.20.5" .version -}}v1.20.5-hotfix.20210603.3 + {{- else if semverCompare "=1.20.7" .version -}}v1.20.7-hotfix.20211021.1 + {{- else if semverCompare "=1.20.9" .version -}}v1.20.9-hotfix.20211115.2 + {{- else if semverCompare "=1.20.13" .version -}}v1.20.13-hotfix.20220210.3 + {{- else if semverCompare "=1.20.15" .version -}}v1.20.15-hotfix.20220201.3 + {{- else if semverCompare "=1.21.1" .version -}}v1.21.1-hotfix.20211022.1 + {{- else if semverCompare "=1.21.2" .version -}}v1.21.2-hotfix.20211115.2 + {{- else if semverCompare "=1.21.7" .version -}}v1.21.7-hotfix.20220601.1 + {{- else if semverCompare "=1.21.9" .version -}}v1.21.9-hotfix.20220601.2 + {{- else if semverCompare "=1.21.14" .version -}}v1.21.14-hotfix.20220620.3 + {{- else if semverCompare "=1.22.1" .version -}}v1.22.1-hotfix.20211115.1 + {{- else if semverCompare "=1.22.2" .version -}}v1.22.2-hotfix.20211115.1 + {{- else if semverCompare "=1.22.4" .version -}}v1.22.4-hotfix.20220615.1 + {{- else if semverCompare "=1.22.6" .version -}}v1.22.6-hotfix.20220728.2 + {{- else if semverCompare "=1.22.11" .version -}}v1.22.11-hotfix.20221109.1 + {{- else if semverCompare "=1.22.15" .version -}}v1.22.15-hotfix.20221109.1 + {{- else if semverCompare "=1.23.3" .version -}}v1.23.3-hotfix.20220615.1 + {{- else if semverCompare "=1.23.5" .version -}}v1.23.5-hotfix.20220728.4 + {{- else if semverCompare "=1.23.8" .version -}}v1.23.8-hotfix.20221109.3 + {{- else if semverCompare "=1.23.12" .version -}}v1.23.12-hotfix.20230208.2 + {{- else if semverCompare "=1.23.15" .version -}}v1.23.15-hotfix.20230208.2 + {{- else if semverCompare "=1.24.0" .version -}}v1.24.0-hotfix.20220615.4 + {{- else if semverCompare "=1.24.3" .version -}}v1.24.3-hotfix.20221216.1 + {{- else if semverCompare "=1.24.6" .version -}}v1.24.6-hotfix.20230208.2 + {{- else if semverCompare "=1.24.9" .version -}}v1.24.9-hotfix.20230612 + {{- else if semverCompare "=1.24.10" .version -}}v1.24.10-hotfix.20230612-1 + {{- else if semverCompare "=1.25.2" .version -}}v1.25.2-hotfix.20221216 + {{- else if semverCompare "=1.25.4" .version -}}v1.25.4-hotfix.20230208 + {{- else if semverCompare "=1.25.5" .version -}}v1.25.5-hotfix.20230612 + {{- else if semverCompare "=1.25.6" .version -}}v1.25.6-hotfix.20231009-3 + {{- else if semverCompare "=1.25.11" .version -}}v1.25.11-hotfix.20231102-1 + {{- else if semverCompare "=1.25.15" .version -}}v1.25.15-hotfix.20231103-1 + {{- else if semverCompare "=1.26.0" .version -}}v1.26.0-hotfix.20230612 + {{- else if semverCompare "=1.26.3" .version -}}v1.26.3-hotfix.20231009-2 + {{- else if semverCompare "=1.26.6" .version -}}v1.26.6-hotfix.20231102 + {{- else if semverCompare "=1.26.10" .version -}}v1.26.10-hotfix.20231103-8 + {{- else if semverCompare "=1.27.1" .version -}}v1.27.1-hotfix.20231009 + {{- else if semverCompare "=1.27.3" .version -}}v1.27.3-hotfix.20240125 + {{- else if semverCompare "=1.27.7" .version -}}v1.27.7-hotfix.20240411 + {{- else if semverCompare "=1.27.9" .version -}}v1.27.9-hotfix.20240411 + {{- else if semverCompare "=1.27.14" .version -}}v1.27.14-1 + {{- else if semverCompare "=1.28.0" .version -}}v1.28.0-hotfix.20240125 + {{- else if semverCompare "=1.28.3" .version -}}v1.28.3-hotfix.20240411 + {{- else if semverCompare "=1.28.5" .version -}}v1.28.5-hotfix.20240411 + {{- else if semverCompare "=1.28.10" .version -}}v1.28.10-1 + {{- else if semverCompare "=1.29.2" .version -}}v1.29.2-hotfix.20240411 + {{- else if semverCompare "=1.29.5" .version -}}v1.29.5-1 + {{- else if semverCompare "=1.30.0" .version -}}v1.30.0-hotfix.20240712-3 + {{- else if semverCompare "=1.30.1" .version -}}v1.30.1-hotfix.20240712-3 + {{- else if semverCompare "=1.30.2" .version -}}v1.30.2-hotfix.20240712-3 + {{- else if semverCompare "=1.30.6" .version -}}v1.30.6-1 + {{- else if semverCompare "=1.31.1" .version -}}v1.31.1-2 + {{- else if and (semverCompare ">=1.27.0" .version) (ge ((semver .version).Patch) 100) -}}v{{.version}}-akslts + {{- else if semverCompare ">=1.34.0" .version -}}v{{ .version }}-1 + {{- else -}}v{{ .version }} + {{- end -}} +{{- else if eq .component "cloud-provider-controller-manager" -}} + {{- if semverCompare ">=1.33.0" .version -}}v1.33.2 + {{- else if semverCompare ">=1.32.0" .version -}}v1.32.7 + {{- else if semverCompare ">=1.31.0" .version -}}v1.31.8 + {{- else if semverCompare ">=1.30.0" .version -}}v1.30.14 + {{- else if semverCompare ">=1.29.0" .version -}}v1.29.15 + {{- else if semverCompare ">=1.28.0" .version -}}v1.28.14 + {{- else if semverCompare ">=1.27.0" .version -}}v1.27.21 + {{- else if semverCompare ">=1.26.0" .version -}}v1.26.22 + {{- else if semverCompare ">=1.25.0" .version -}}v1.25.24 + {{- else if semverCompare ">=1.24.0" .version -}}v1.24.22 + {{- else if semverCompare ">=1.23.0" .version -}}v1.23.30 + {{- else if semverCompare ">=1.22.0" .version -}}v1.1.26 + {{- else if semverCompare ">=1.21.0" .version -}}v1.0.23 + {{- else if semverCompare ">=1.20.0" .version -}}v0.7.21 + {{- else if semverCompare ">=1.19.0" .version -}}v0.6.0 + {{- else -}}v0.5.1.4 + {{- end -}} +{{- else if eq .component "appmonitoring-webhook" -}} +1.0.0-beta.8 +{{- else if eq .component "tunnel-front" -}} +master.250401.1 +{{- else if eq .component "tunnel-end" -}} +master.250401.1 +{{- else if eq .component "tunnel-openvpn-front" -}} +master.241001.1 +{{- else if eq .component "tunnel-openvpn-end" -}} +master.241001.1 +{{- else if eq .component "apiserver-network-proxy-agent" -}} +v0.30.3-5 +{{- else if eq .component "aad-pod-identity-nmi" -}} +v1.8.18 +{{- else if eq .component "gitops-manager-config-operator" -}} +1.7.0 +{{- else if eq .component "gitops-manager-config-agent" -}} +1.7.0 +{{- else if eq .component "resourcesync-operator" -}} +1.7.1 +{{- else if eq .component "http-application-routing-nginx-ingress-controller" -}} + {{- if semverCompare ">=1.22.0" .version -}}1.2.1 + {{- else if semverCompare ">=1.21.0" .version -}}0.49.3 + {{- else -}}0.19.0 + {{- end -}} +{{- else if eq .component "http-application-routing-external-dns" -}} + {{- if semverCompare ">=1.22.0" .version -}}v0.10.2 + {{- else if semverCompare ">=1.21.0" .version -}}v0.8.0 + {{- else -}}v0.6.0-hotfix-20200228 + {{- end -}} +{{- else if eq .component "http-application-routing-defaultbackend" -}} +1.4 +{{- else if eq .component "ip-masq-agent" -}} +v2.5.0.12 +{{- else if eq .component "azuredisk-csi-v2" -}} +v2.0.0-beta.10 +{{- else if eq .component "azdiskschedulerextender-csi" -}} +v2.0.0-beta.10 +{{- else if eq .component "csi-node-driver-registrar" -}} + {{- if semverCompare ">=1.31.0" .version -}}v2.14.0 + {{- else if semverCompare ">=1.29.0" .version -}}v2.13.0 + {{- else if semverCompare ">=1.28.0" .version -}}v2.12.0 + {{- else if semverCompare ">=1.27.0" .version -}}v2.10.1 + {{- else if semverCompare ">=1.24.0" .version -}}v2.8.0 + {{- else if semverCompare ">=1.21.0" .version -}}v2.5.0 + {{- else -}}v2.3.0 + {{- end -}} +{{- else if eq .component "csi-livenessprobe" -}} + {{- if semverCompare ">=1.31.0" .version -}}v2.16.0 + {{- else if semverCompare ">=1.29.0" .version -}}v2.15.0 + {{- else if semverCompare ">=1.28.0" .version -}}v2.14.0 + {{- else if semverCompare ">=1.27.0" .version -}}v2.12.0 + {{- else if semverCompare ">=1.24.0" .version -}}v2.10.0 + {{- else if semverCompare ">=1.21.0" .version -}}v2.6.0 + {{- else -}}v2.2.0 + {{- end -}} +{{- else if eq .component "azuredisk-csi-linux" -}} + {{- if semverCompare ">=1.33.0" .version -}}v1.33.4-2 + {{- else if semverCompare ">=1.32.0" .version -}}v1.32.10-2 + {{- else if semverCompare ">=1.31.0" .version -}}v1.31.11 + {{- else if semverCompare ">=1.30.0" .version -}}v1.30.12 + {{- else if semverCompare ">=1.28.0" .version -}}v1.29.14 + {{- else if semverCompare ">=1.27.0" .version -}}v1.28.12 + {{- else if semverCompare ">=1.26.0" .version -}}v1.26.9 + {{- else if semverCompare ">=1.24.0" .version -}}v1.26.8 + {{- else if semverCompare ">=1.21.0" .version -}}v1.26.2.2 + {{- else -}}v1.2.2.5 + {{- end -}} +{{- else if eq .component "azuredisk-csi-windows" -}} + {{- if semverCompare ">=1.33.0" .version -}}v1.33.4 + {{- else if semverCompare ">=1.32.0" .version -}}v1.32.10 + {{- else if semverCompare ">=1.31.0" .version -}}v1.31.11 + {{- else if semverCompare ">=1.30.0" .version -}}v1.30.12 + {{- else if semverCompare ">=1.28.0" .version -}}v1.29.14 + {{- else if semverCompare ">=1.27.0" .version -}}v1.28.12 + {{- else if semverCompare ">=1.26.0" .version -}}v1.26.9 + {{- else if semverCompare ">=1.24.0" .version -}}v1.26.8 + {{- else if semverCompare ">=1.21.0" .version -}}v1.26.2 + {{- else -}}v1.2.2.5 + {{- end -}} +{{- else if eq .component "azurefile-csi-linux" -}} + {{- if semverCompare ">=1.33.0" .version -}}v1.33.4-2 + {{- else if semverCompare ">=1.32.0" .version -}}v1.32.5 + {{- else if semverCompare ">=1.31.0" .version -}}v1.31.7 + {{- else if semverCompare ">=1.29.0" .version -}}v1.30.10 + {{- else if semverCompare ">=1.28.0" .version -}}v1.29.12 + {{- else if semverCompare ">=1.27.0" .version -}}v1.28.14 + {{- else if semverCompare ">=1.26.0" .version -}}v1.26.11-2 + {{- else if semverCompare ">=1.24.0" .version -}}v1.24.11 + {{- else if semverCompare ">=1.21.0" .version -}}v1.24.0 + {{- else -}}v1.2.2 + {{- end -}} +{{- else if eq .component "azurefile-csi-windows" -}} + {{- if semverCompare ">=1.33.0" .version -}}v1.33.4 + {{- else if semverCompare ">=1.32.0" .version -}}v1.32.5 + {{- else if semverCompare ">=1.31.0" .version -}}v1.31.7 + {{- else if semverCompare ">=1.29.0" .version -}}v1.30.10 + {{- else if semverCompare ">=1.28.0" .version -}}v1.29.12 + {{- else if semverCompare ">=1.27.0" .version -}}v1.28.14 + {{- else if semverCompare ">=1.26.0" .version -}}v1.26.11 + {{- else if semverCompare ">=1.24.0" .version -}}v1.24.11 + {{- else if semverCompare ">=1.21.0" .version -}}v1.24.0 + {{- else -}}v1.2.2 + {{- end -}} +{{- else if eq .component "blob-csi" -}} + {{- if semverCompare ">=1.33.0" .version -}}v1.26.7 + {{- else if semverCompare ">=1.32.0" .version -}}v1.26.6 + {{- else if semverCompare ">=1.31.0" .version -}}v1.25.9 + {{- else if semverCompare ">=1.30.0" .version -}}v1.24.11 + {{- else if semverCompare ">=1.28.0" .version -}}v1.23.11 + {{- else if semverCompare ">=1.27.0" .version -}}v1.22.9 + {{- else if semverCompare ">=1.26.0" .version -}}v1.21.7-2 + {{- else if semverCompare ">=1.24.0" .version -}}v1.19.5-7 + {{- else -}}v1.19.2 + {{- end -}} +{{- else if eq .component "csi-provisioner" -}} + {{- if semverCompare ">=1.29.0" .version -}}v5.2.0 + {{- else if semverCompare ">=1.28.0" .version -}}v3.6.2 + {{- else if semverCompare ">=1.24.0" .version -}}v3.5.0 + {{- else if semverCompare ">=1.21.0" .version -}}v3.1.0 + {{- else -}}v2.1.1-hotfix.20220128-aks + {{- end -}} +{{- else if eq .component "csi-attacher" -}} + {{- if semverCompare ">=1.32.0" .version -}}v4.9.0 + {{- else if semverCompare ">=1.29.0" .version -}}v4.8.1 + {{- else if semverCompare ">=1.28.0" .version -}}v4.4.2 + {{- else if semverCompare ">=1.27.0" .version -}}v4.3.0 + {{- else if semverCompare ">=1.21.0" .version -}}v3.4.0 + {{- else -}}v3.1.0-hotfix.20220128-aks + {{- end -}} +{{- else if eq .component "csi-resizer" -}} + {{- if semverCompare ">=1.29.0" .version -}}v1.13.2 + {{- else if semverCompare ">=1.28.0" .version -}}v1.9.3 + {{- else if semverCompare ">=1.27.0" .version -}}v1.8.0 + {{- else if semverCompare ">=1.21.0" .version -}}v1.4.0 + {{- else -}}v1.1.0-hotfix.20220128-aks + {{- end -}} +{{- else if eq .component "csi-snapshotter" -}} + {{- if semverCompare ">=1.33.0" .version -}}v8.3.0 + {{- else if semverCompare ">=1.29.0" .version -}}v8.2.0 + {{- else if semverCompare ">=1.27.0" .version -}}v6.2.2 + {{- else if semverCompare ">=1.21.0" .version -}}v5.0.1 + {{- else -}}v3.0.3-hotfix.20220128-aks + {{- end -}} +{{- else if eq .component "snapshot-controller" -}} + {{- if semverCompare ">=1.33.0" .version -}}v8.3.0 + {{- else if semverCompare ">=1.29.0" .version -}}v8.2.0 + {{- else if semverCompare ">=1.27.0" .version -}}v6.2.2 + {{- else if semverCompare ">=1.21.0" .version -}}v5.0.1 + {{- else -}}v3.0.3-hotfix.20220128-aks + {{- end -}} +{{- else if eq .component "azure-cns-image" -}} +v1.4.44.5 +{{- else if eq .component "azure-cns-image-windows" -}} +v1.4.44.5 +{{- else if eq .component "azure-cni-networkmonitor" -}} +v1.1.8_hotfix +{{- else if eq .component "calico-typha-image" -}} +v3.8.9 +{{- else if eq .component "calico-pod2daemon-flexvol-image" -}} +v3.8.9.1 +{{- else if eq .component "calico-cni-image" -}} +v3.8.9.3 +{{- else if eq .component "calico-node-image" -}} +v3.8.9.5 +{{- else if eq .component "ccp-initializer" -}} +master.250807.1 +{{- else if eq .component "ccp-auto-thrust" -}} + {{- if semverCompare ">=1.27.0" .version -}}master.250505.2 + {{- else -}}master.250108.7 + {{- end -}} +{{- else if eq .component "ccp-auto-thrust-csi" -}} + {{- if semverCompare ">=1.27.0" .version -}}master.250307.1 + {{- else -}}master.250108.7 + {{- end -}} +{{- else if eq .component "admissionsenforcer" -}} +master.250822.2 +{{- else if eq .component "msi-adapter" -}} +master.250822.1 +{{- else if eq .component "private-connect-router" -}} +master.250811.1 +{{- else if eq .component "private-connect-balancer" -}} +master.250731.2 +{{- else if eq .component "addon-token-adapter-linux" -}} +master.250902.1 +{{- else if eq .component "addon-token-adapter-windows" -}} +master.250902.1 +{{- else if eq .component "addon-token-reconciler" -}} +master.250819.2 +{{- else if eq .component "aks-kube-addon-manager" -}} +master.250528.2 +{{- else if eq .component "kms-plugin" -}} +v0.8.0 +{{- else if eq .component "ccp-coredns" -}} +v1.12.0-1 +{{- end -}} +{{- end -}} diff --git a/charts/azuremonitor-containerinsights-for-prod-clusters/templates/ama-logs.yaml b/charts/azuremonitor-containerinsights-for-prod-clusters/templates/ama-logs.yaml new file mode 100644 index 000000000..5f7a7d864 --- /dev/null +++ b/charts/azuremonitor-containerinsights-for-prod-clusters/templates/ama-logs.yaml @@ -0,0 +1,1916 @@ +{{- $amalogsLinuxDefaultImageTag := dict "component" "ama-logs-linux" "version" .Values.global.commonGlobals.Versions.Kubernetes | include "get.addonImageTag" }} +{{- $amalogsWindowsDefaultImageTag := dict "component" "ama-logs-win" "version" .Values.global.commonGlobals.Versions.Kubernetes | include "get.addonImageTag" }} +{{- $addonTokenAdapterLinuxDefaultImageTag := dict "component" "addon-token-adapter-linux" "version" .Values.global.commonGlobals.Versions.Kubernetes | include "get.imagetag"}} +{{- $addonTokenAdapterWindowsDefaultImageTag := dict "component" "addon-token-adapter-windows" "version" .Values.global.commonGlobals.Versions.Kubernetes | include "get.imagetag"}} +{{- $amalogsRSVPAImageTag := dict "component" "addon-resizer" "version" .Values.global.commonGlobals.Versions.Kubernetes | include "get.addonImageTag" -}} +{{- $WinImageTag := default $amalogsWindowsDefaultImageTag .Values.OmsAgent.imageTagWindows -}} +{{/* Determine isusingaadauth value from OmsAgent.isUsingAADAuth */}} +{{- $isusingaadauth := false -}} +{{- if hasKey .Values.OmsAgent "isUsingAADAuth" -}} + {{- $isusingaadauth = .Values.OmsAgent.isUsingAADAuth -}} +{{- end -}} +{{/* TODO This needs to be fixed post Canary validation */}} +{{/* Extract cluster information from aksresourceid */}} +{{- $resourceParts := splitList "/" .Values.OmsAgent.aksResourceID -}} +{{- $aksclustername := last $resourceParts -}} +{{- $aksResourceGroup := index $resourceParts 4 -}} +{{- $region := .Values.global.commonGlobals.Region -}} +{{- $aksnoderesourcegroup := printf "MC_%s_%s_%s" $aksResourceGroup $aksclustername $region -}} +apiVersion: v1 +kind: Secret +metadata: + name: ama-logs-secret + namespace: kube-system + labels: +{{- if .Values.legacyAddonDelivery }} + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +{{- end }} +type: Opaque +data: + WSID: {{ .Values.OmsAgent.workspaceID | b64enc | quote }} + KEY: {{ .Values.OmsAgent.workspaceKey | b64enc | quote }} +{{- if .Values.OmsAgent.isMoonCake }} + DOMAIN: {{ b64enc "opinsights.azure.cn" }} +{{- end }} +{{- if .Values.OmsAgent.isFairfax }} + DOMAIN: {{ b64enc "opinsights.azure.us" }} +{{- end }} +{{- if eq (upper .Values.global.commonGlobals.CloudEnvironment) "USNAT" }} + DOMAIN: {{ b64enc "opinsights.azure.eaglex.ic.gov" }} +{{- end }} +{{- if eq (upper .Values.global.commonGlobals.CloudEnvironment) "USSEC" }} + DOMAIN: {{ b64enc "opinsights.azure.microsoft.scloud" }} +{{- end }} +{{- if eq (upper .Values.global.commonGlobals.CloudEnvironment) "AZUREBLEUCLOUD" }} + DOMAIN: {{ b64enc "opinsights.sovcloud-api.fr" }} +{{- end }} +{{- if .Values.OmsAgent.httpsProxy }} + PROXY: {{ .Values.OmsAgent.httpsProxy | b64enc | quote }} +{{- else if .Values.OmsAgent.httpProxy }} + PROXY: {{ .Values.OmsAgent.httpProxy | b64enc | quote }} +{{- end}} +{{- if .Values.OmsAgent.trustedCA }} + PROXYCERT.crt: {{ .Values.OmsAgent.trustedCA | quote }} +{{- end}} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ama-logs + namespace: kube-system + labels: +{{- if .Values.legacyAddonDelivery }} + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +{{- end }} +--- +kind: ClusterRole +{{- if semverCompare ">=1.16.0" .Values.global.commonGlobals.Versions.Kubernetes }} +apiVersion: rbac.authorization.k8s.io/v1 +{{- else }} +apiVersion: rbac.authorization.k8s.io/v1beta1 +{{- end }} +metadata: + name: ama-logs-reader + labels: +{{- if .Values.legacyAddonDelivery }} + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +{{- end }} +rules: +- apiGroups: [""] + resources: ["pods", "events", "nodes", "nodes/stats", "nodes/metrics", "nodes/spec", "nodes/proxy", "namespaces", "services", "persistentvolumes"] + verbs: ["list", "get", "watch"] +- apiGroups: ["apps", "extensions", "autoscaling"] + resources: ["replicasets", "deployments", "horizontalpodautoscalers"] + verbs: ["list"] +{{- if .Values.OmsAgent.isRSVPAEnabled }} +- apiGroups: ["apps"] + resources: ["deployments"] + resourceNames: [ "ama-logs-rs" ] + verbs: ["get", "patch"] +{{- end }} +{{- if $isusingaadauth }} +- apiGroups: [""] + resources: ["secrets"] + resourceNames: [{{ .Values.OmsAgent.accessTokenSecretName | quote }}] + verbs: ["get", "watch"] +{{- end }} +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +--- +kind: ClusterRoleBinding +{{- if semverCompare ">=1.16.0" .Values.global.commonGlobals.Versions.Kubernetes }} +apiVersion: rbac.authorization.k8s.io/v1 +{{- else }} +apiVersion: rbac.authorization.k8s.io/v1beta1 +{{- end }} +metadata: + name: amalogsclusterrolebinding + labels: +{{- if .Values.legacyAddonDelivery }} + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +{{- end }} +subjects: + - kind: ServiceAccount + name: ama-logs + namespace: kube-system +roleRef: + kind: ClusterRole + name: ama-logs-reader + apiGroup: rbac.authorization.k8s.io +--- +kind: ConfigMap +apiVersion: v1 +data: + CLUSTER_RESOURCE_ID: "{{ .Values.OmsAgent.aksResourceID }}" +metadata: + name: container-azm-ms-aks-k8scluster + namespace: kube-system + labels: +{{- if .Values.legacyAddonDelivery }} + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +{{- end }} +--- +kind: ConfigMap +apiVersion: v1 +data: + kube.conf: |- + # Fluentd config file for OMS Docker - cluster components (kubeAPI) + #fluent forward plugin + + type forward + port "#{ENV['HEALTHMODEL_REPLICASET_SERVICE_SERVICE_PORT']}" + bind 0.0.0.0 + chunk_size_limit 4m + + + #Kubernetes pod inventory + + type kubepodinventory + tag oms.containerinsights.KubePodInventory + run_interval 60 + log_level debug + + + #Kubernetes Persistent Volume inventory + + type kubepvinventory + tag oms.containerinsights.KubePVInventory + run_interval 60 + log_level debug + + + #Kubernetes events + + type kubeevents + tag oms.containerinsights.KubeEvents + run_interval 60 + log_level debug + + + #Kubernetes Nodes + + type kubenodeinventory + tag oms.containerinsights.KubeNodeInventory + run_interval 60 + log_level debug + + + #Kubernetes health + + type kubehealth + tag kubehealth.ReplicaSet + run_interval 60 + log_level debug + + + #cadvisor perf- Windows nodes + + type wincadvisorperf + tag oms.api.wincadvisorperf + run_interval 60 + log_level debug + + + #Kubernetes object state - deployments + + type kubestatedeployments + tag oms.containerinsights.KubeStateDeployments + run_interval 60 + log_level debug + + + #Kubernetes object state - HPA + + type kubestatehpa + tag oms.containerinsights.KubeStateHpa + run_interval 60 + log_level debug + + + + type filter_inventory2mdm + log_level info + + + #custom_metrics_mdm filter plugin for perf data from windows nodes + + type filter_cadvisor2mdm + metrics_to_collect cpuUsageNanoCores,memoryWorkingSetBytes,pvUsedBytes + log_level info + + + #health model aggregation filter + + type filter_health_model_builder + + + + type out_oms + log_level debug + num_threads 2 + buffer_chunk_limit 4m + buffer_type file + buffer_path %STATE_DIR_WS%/out_oms_kubepods*.buffer + buffer_queue_limit 20 + buffer_queue_full_action drop_oldest_chunk + flush_interval 20s + retry_limit 10 + retry_wait 5s + max_retry_wait 5m + + + + type out_oms + log_level debug + num_threads 5 + buffer_chunk_limit 4m + buffer_type file + buffer_path %STATE_DIR_WS%/state/out_oms_kubepv*.buffer + buffer_queue_limit 20 + buffer_queue_full_action drop_oldest_chunk + flush_interval 20s + retry_limit 10 + retry_wait 5s + max_retry_wait 5m + + + + type out_oms + log_level debug + num_threads 2 + buffer_chunk_limit 4m + buffer_type file + buffer_path %STATE_DIR_WS%/out_oms_kubeevents*.buffer + buffer_queue_limit 20 + buffer_queue_full_action drop_oldest_chunk + flush_interval 20s + retry_limit 10 + retry_wait 5s + max_retry_wait 5m + + + + type out_oms + log_level debug + num_threads 2 + buffer_chunk_limit 4m + buffer_type file + buffer_path %STATE_DIR_WS%/out_oms_kubeservices*.buffer + buffer_queue_limit 20 + buffer_queue_full_action drop_oldest_chunk + flush_interval 20s + retry_limit 10 + retry_wait 5s + max_retry_wait 5m + + + + type out_oms + log_level debug + num_threads 2 + buffer_chunk_limit 4m + buffer_type file + buffer_path %STATE_DIR_WS%/state/out_oms_kubenodes*.buffer + buffer_queue_limit 20 + buffer_queue_full_action drop_oldest_chunk + flush_interval 20s + retry_limit 10 + retry_wait 5s + max_retry_wait 5m + + + + type out_oms + log_level debug + num_threads 3 + buffer_chunk_limit 4m + buffer_type file + buffer_path %STATE_DIR_WS%/out_oms_containernodeinventory*.buffer + buffer_queue_limit 20 + flush_interval 20s + retry_limit 10 + retry_wait 5s + max_retry_wait 5m + + + + type out_oms + log_level debug + num_threads 2 + buffer_chunk_limit 4m + buffer_type file + buffer_path %STATE_DIR_WS%/out_oms_kubeperf*.buffer + buffer_queue_limit 20 + buffer_queue_full_action drop_oldest_chunk + flush_interval 20s + retry_limit 10 + retry_wait 5s + max_retry_wait 5m + + + + type out_mdm + log_level debug + num_threads 5 + buffer_chunk_limit 4m + buffer_type file + buffer_path %STATE_DIR_WS%/out_mdm_*.buffer + buffer_queue_limit 20 + buffer_queue_full_action drop_oldest_chunk + flush_interval 20s + retry_limit 10 + retry_wait 5s + max_retry_wait 5m + retry_mdm_post_wait_minutes 30 + + + + type out_oms + log_level debug + num_threads 5 + buffer_chunk_limit 4m + buffer_type file + buffer_path %STATE_DIR_WS%/out_oms_api_wincadvisorperf*.buffer + buffer_queue_limit 20 + buffer_queue_full_action drop_oldest_chunk + flush_interval 20s + retry_limit 10 + retry_wait 5s + max_retry_wait 5m + + + + type out_mdm + log_level debug + num_threads 5 + buffer_chunk_limit 4m + buffer_type file + buffer_path %STATE_DIR_WS%/out_mdm_cdvisorperf*.buffer + buffer_queue_limit 20 + buffer_queue_full_action drop_oldest_chunk + flush_interval 20s + retry_limit 10 + retry_wait 5s + max_retry_wait 5m + retry_mdm_post_wait_minutes 30 + + + + type out_oms + log_level debug + num_threads 5 + buffer_chunk_limit 4m + buffer_type file + buffer_path %STATE_DIR_WS%/out_oms_kubehealth*.buffer + buffer_queue_limit 20 + buffer_queue_full_action drop_oldest_chunk + flush_interval 20s + retry_limit 10 + retry_wait 5s + max_retry_wait 5m + + + type out_oms + log_level debug + num_threads 5 + buffer_chunk_limit 4m + buffer_type file + buffer_path %STATE_DIR_WS%/out_oms_insightsmetrics*.buffer + buffer_queue_limit 20 + buffer_queue_full_action drop_oldest_chunk + flush_interval 20s + retry_limit 10 + retry_wait 5s + max_retry_wait 5m + +metadata: + name: ama-logs-rs-config + namespace: kube-system + labels: +{{- if .Values.legacyAddonDelivery }} + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +{{- end }} +--- +{{/* Get sizes */}} +{{- $useDaemonSetSizing := and .Values.global.commonGlobals.isAutomaticSKU .Values.OmsAgent.enableDaemonSetSizing -}} +{{- $singleSize := dict "containers" (dict "addon-token-adapter" (dict "cpuLimit" "100m" "memoryLimit" "100Mi" "cpuRequest" "20m" "memoryRequest" "50Mi") "ama-logs" (dict "cpuLimit" .Values.OmsAgent.omsAgentDsCPULimitLinux "memoryLimit" .Values.OmsAgent.omsAgentDsMemoryLimitLinux "cpuRequest" "75m" "memoryRequest" "325Mi") "ama-logs-prometheus" (dict "cpuLimit" .Values.OmsAgent.omsAgentPrometheusSidecarCPULimit "memoryLimit" .Values.OmsAgent.omsAgentPrometheusSidecarMemoryLimit "cpuRequest" "75m" "memoryRequest" "225Mi")) -}} +{{- $sizes := list ($singleSize) -}} +{{/* - if $useDaemonSetSizing - */}} + {{/* - $singleSize = .Values.OmsAgent.daemonSetSizingValues.singleSize - */}} + {{/* - $sizes = list ($singleSize) - */}} + {{/* - $sizes = prepend .Values.OmsAgent.daemonSetSizingValues.tShirtSizes $singleSize - */}} +{{/* - end - */}} +{{/* Generate DaemonSets */}} +{{- $prevmaxCPU := 0 -}} +{{- range $index, $size := $sizes -}} +{{- if gt $index 0 }} +--- +{{ end -}} +{{- if semverCompare ">=1.16.0" $.Values.global.commonGlobals.Versions.Kubernetes -}} +apiVersion: apps/v1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: DaemonSet +metadata: + labels: + component: ama-logs-agent + tier: node + kubernetes.azure.com/managedby: aks +{{- if $.Values.legacyAddonDelivery }} + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +{{- end }} + {{/* + {{- if and $useDaemonSetSizing $size.name }} + kubernetes.azure.com/ds-tshirt-size: {{ $size.name }} + {{- end }} + */}} + name: ama-logs{{/* {{- if and $useDaemonSetSizing $size.name }}-{{ $size.name }}{{- end }} */}} + namespace: kube-system +spec: + selector: + matchLabels: + component: ama-logs-agent + tier: node + {{/* + {{- if and $useDaemonSetSizing $size.name }} + kubernetes.azure.com/ds-tshirt-size: {{ $size.name }} + {{- end }} + */}} + template: + metadata: + annotations: + agentVersion: "azure-mdsd-1.37.0" + dockerProviderVersion: "18.0.1-0" + schema-versions: "v1" + WSID: {{ $.Values.OmsAgent.workspaceID | b64enc | quote }} + kubernetes.azure.com/no-http-proxy-vars: "true" + labels: + component: ama-logs-agent + tier: node + kubernetes.azure.com/managedby: aks + {{/* + {{- if and $useDaemonSetSizing $size.name }} + kubernetes.azure.com/ds-tshirt-size: {{ $size.name }} + {{- end }} + */}} +{{- if semverCompare "<1.11.0" $.Values.global.commonGlobals.Versions.Kubernetes }} + annotations: + scheduler.alpha.kubernetes.io/critical-pod: "" +{{- end }} + spec: +{{- if semverCompare ">=1.11.0" $.Values.global.commonGlobals.Versions.Kubernetes }} + priorityClassName: system-node-critical +{{- end }} + serviceAccountName: ama-logs + dnsConfig: + options: + - name: ndots + value: "3" + containers: +{{- if $isusingaadauth }} + - name: addon-token-adapter + command: + - /addon-token-adapter + args: + - --secret-namespace=kube-system + - --secret-name={{ $.Values.OmsAgent.accessTokenSecretName }} + - --token-server-listening-port=8888 + - --health-server-listening-port=9999 + - --restart-pod-waiting-minutes-on-broken-connection=240 + image: "{{ template "addon_mcr_repository_base" $ }}/aks/msi/addon-token-adapter:{{- $addonTokenAdapterLinuxDefaultImageTag -}}" + imagePullPolicy: IfNotPresent + env: + - name: AZMON_COLLECT_ENV + value: "false" + livenessProbe: + httpGet: + path: /healthz + port: 9999 + initialDelaySeconds: 10 + periodSeconds: 60 + resources: + limits: + {{- $containerResources := index $size.containers "addon-token-adapter" }} + cpu: {{ $containerResources.cpuLimit }} + memory: {{ $containerResources.memoryLimit }} + requests: + cpu: {{ $containerResources.cpuRequest }} + memory: {{ $containerResources.memoryRequest }} + securityContext: + capabilities: + drop: + - ALL + add: + - NET_ADMIN + - NET_RAW +{{- end }} + - name: ama-logs + image: "{{ template "addon_mcr_repository_base" $ }}/azuremonitor/containerinsights/ciprod:{{- default $amalogsLinuxDefaultImageTag $.Values.OmsAgent.imageTagLinux -}}" + {{- if $.Values.OmsAgent.isImagePullPolicyAlways }} + imagePullPolicy: Always + {{- else }} + imagePullPolicy: IfNotPresent + {{- end }} + resources: + limits: + {{- $containerResources := index $size.containers "ama-logs" }} + cpu: {{ $containerResources.cpuLimit }} + memory: {{ $containerResources.memoryLimit }} + requests: + {{- $containerResources := index $size.containers "ama-logs" }} + cpu: {{ $containerResources.cpuRequest }} + memory: {{ $containerResources.memoryRequest }} + env: + - name: NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: CONTAINER_MEMORY_LIMIT_IN_BYTES + valueFrom: + resourceFieldRef: + containerName: ama-logs + resource: limits.memory + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: FBIT_SERVICE_FLUSH_INTERVAL + value: "15" + - name: FBIT_TAIL_BUFFER_CHUNK_SIZE + value: "1" + - name: FBIT_TAIL_BUFFER_MAX_SIZE + value: "1" + - name: AKS_CLUSTER_NAME + value: "{{ $.Values.OmsAgent.aksClusterName }}" + - name: AKS_RESOURCE_ID + value: "{{ $.Values.OmsAgent.aksResourceID }}" + - name: AKS_NODE_RESOURCE_GROUP + value: "{{ $.Values.OmsAgent.aksNodeResourceGroup }}" + {{/* TODO This needs to be fixed post Canary validation */}} + - name: AKS_REGION + value: "{{ $.Values.global.commonGlobals.Region }}" + - name: CONTROLLER_TYPE + value: "DaemonSet" + - name: USER_ASSIGNED_IDENTITY_CLIENT_ID + value: "{{ $.Values.OmsAgent.identityClientID }}" + - name: AZMON_CONTAINERLOGS_ONEAGENT_REGIONS + value: "koreacentral,norwayeast,eastus2" + - name: APPMONITORING_AUTOINSTRUMENTATION_ENABLED + value: "{{ $.Values.AppmonitoringAgent.enabled }}" + - name: APPMONITORING_OPENTELEMETRYLOGS_ENABLED + value: "{{ $.Values.AppmonitoringAgent.isOpenTelemetryLogsEnabled | default false }}" + - name: APPMONITORING_OPENTELEMETRYLOGS_PORT + value: "{{ $.Values.AppmonitoringAgent.openTelemetryLogsPort | default 28331 }}" + - name: AZMON_OPENTELEMETRYLOGS_CONTAINER_PORT + value: "4319" + - name: PROMETHEUS_METRICS_SCRAPING_DISABLED + value: "{{ $.Values.OmsAgent.isPrometheusMetricsScrapingDisabled }}" + {{- if (eq $.Values.global.commonGlobals.CloudEnvironment "USNat") }} + - name: MCR_URL + value: "https://mcr.microsoft.eaglex.ic.gov/v2/" + {{- end }} + {{- if (eq $.Values.global.commonGlobals.CloudEnvironment "USSec") }} + - name: MCR_URL + value: "https://mcr.microsoft.scloud/v2/" + {{- end }} + {{- if (eq $.Values.global.commonGlobals.CloudEnvironment "AzureBleuCloud") }} + - name: MCR_URL + value: "https://mcr.microsoft.sovcloud-api.fr/v2/" + {{- end }} + {{- if $isusingaadauth }} + - name: USING_AAD_MSI_AUTH + value: "true" + {{- else }} + - name: USING_AAD_MSI_AUTH + value: "false" + {{- end }} + {{- if eq ($.Values.OmsAgent.shouldMountSyslogHostPort | default false) true }} + - name: SYSLOG_HOST_PORT + value: {{ $.Values.OmsAgent.syslogHostPort | default 28330 | quote}} + {{- end }} + - name: AZMON_RETINA_FLOW_LOGS_ENABLED + value: "{{ $.Values.OmsAgent.isRetinaFlowLogsEnabled | default false }}" + - name: AZMON_RESOURCE_OPTIMIZATION_ENABLED + value: "{{ $.Values.OmsAgent.isResourceOptimizationEnabled | default false }}" + - name: AZMON_TELEGRAF_LIVENESSPROBE_ENABLED + value: "{{ $.Values.OmsAgent.isTelegrafLivenessprobeEnabled | default false }}" + - name: CLUSTER_CLOUD_ENVIRONMENT + value: "{{ $.Values.global.commonGlobals.CloudEnvironment | lower }}" + livenessProbe: + exec: + command: + - /bin/bash + - "-c" + - "/opt/livenessprobe.sh" + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 15 + ports: + - containerPort: 25225 + protocol: TCP + - containerPort: 25224 + protocol: UDP + {{- if eq ($.Values.OmsAgent.shouldMountSyslogHostPort | default false) true }} + - name: syslog + containerPort: 28330 + hostPort: {{ $.Values.OmsAgent.syslogHostPort | default 28330 }} + protocol: TCP + {{- end }} + {{- if eq ($.Values.AppmonitoringAgent.isOpenTelemetryLogsEnabled | default false) true }} + - name: otlp-logs + containerPort: 4319 + hostPort: {{ $.Values.AppmonitoringAgent.openTelemetryLogsPort | default 28331 }} + protocol: TCP + {{- end }} + securityContext: + privileged: true + capabilities: + drop: + - ALL + add: + - DAC_OVERRIDE + volumeMounts: + - mountPath: /hostfs + name: host-root + readOnly: true + mountPropagation: HostToContainer + - mountPath: /var/log + name: host-log + {{- if $.Values.OmsAgent.isSyslogEnabled }} + - mountPath: /var/run/mdsd-ci + name: mdsd-sock + {{- end }} + {{- if $.Values.OmsAgent.isRetinaFlowLogsEnabled }} + - mountPath: /var/log/acns/hubble + name: acns-hubble + {{- end }} + - mountPath: /var/run/mdsd-PrometheusSidecar + name: mdsd-prometheus-sock + - mountPath: /var/lib/docker/containers + name: containerlog-path + readOnly: true + - mountPath: /mnt/docker + name: containerlog-path-2 + readOnly: true + - mountPath: /mnt/containers + name: containerlog-path-3 + readOnly: true + - mountPath: /etc/kubernetes/host + name: azure-json-path + - mountPath: /etc/ama-logs-secret + name: ama-logs-secret + readOnly: true + - mountPath: /etc/omsagent-secret + name: ama-logs-secret + readOnly: true + - mountPath: /etc/config/settings + name: settings-vol-config + readOnly: true + - mountPath: /etc/config/settings/adx + name: ama-logs-adx-secret + readOnly: true + {{- if (eq (include "should_mount_hostca" $ ) "true" ) }} + # USSec and USNat have cloud-specific Root and Intermediate CAs that must be mounted into all running containers from the host + # Container Insights has logic to update the trust store in the image based on certs present in /anchors/ mounts below + - mountPath: /anchors/mariner + name: anchors-mariner + readOnly: true + - mountPath: /anchors/ubuntu + name: anchors-ubuntu + readOnly: true + {{- end }} + {{- if $.Values.OmsAgent.trustedCA }} + - mountPath: /etc/ssl/certs/proxy-cert.crt + subPath: PROXYCERT.crt + name: ama-logs-secret + readOnly: true + {{- end }} + {{- if and (not $.Values.OmsAgent.isPrometheusMetricsScrapingDisabled) $.Values.OmsAgent.isSidecarScrapingEnabled }} + - name: ama-logs-prometheus + image: "{{ template "addon_mcr_repository_base" $ }}/azuremonitor/containerinsights/ciprod:{{- default $amalogsLinuxDefaultImageTag $.Values.OmsAgent.imageTagLinux -}}" + {{- if $.Values.OmsAgent.isImagePullPolicyAlways }} + imagePullPolicy: Always + {{- else }} + imagePullPolicy: IfNotPresent + {{- end }} + resources: + limits: + {{- $containerResources := index $size.containers "ama-logs-prometheus" }} + cpu: {{ $containerResources.cpuLimit }} + memory: {{ $containerResources.memoryLimit }} + requests: + {{- $containerResources := index $size.containers "ama-logs-prometheus" }} + cpu: {{ $containerResources.cpuRequest }} + memory: {{ $containerResources.memoryRequest }} + env: + - name: NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: CONTAINER_MEMORY_LIMIT_IN_BYTES + valueFrom: + resourceFieldRef: + containerName: ama-logs-prometheus + resource: limits.memory + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: AKS_CLUSTER_NAME + value: "{{ $.Values.OmsAgent.aksClusterName }}" + - name: AKS_RESOURCE_ID + value: "{{ $.Values.OmsAgent.aksResourceID }}" + - name: AKS_NODE_RESOURCE_GROUP + value: "{{ $.Values.OmsAgent.aksNodeResourceGroup }}" + {{/* TODO This needs to be fixed post Canary validation */}} + - name: AKS_REGION + value: "{{ $.Values.global.commonGlobals.Region }}" + - name: CONTROLLER_TYPE + value: "DaemonSet" + - name: CONTAINER_TYPE + value: "PrometheusSidecar" + - name: USER_ASSIGNED_IDENTITY_CLIENT_ID + value: "{{ $.Values.OmsAgent.identityClientID }}" + {{- if (eq $.Values.global.commonGlobals.CloudEnvironment "USNat") }} + - name: MCR_URL + value: "https://mcr.microsoft.eaglex.ic.gov/v2/" + {{- end }} + {{- if (eq $.Values.global.commonGlobals.CloudEnvironment "USSec") }} + - name: MCR_URL + value: "https://mcr.microsoft.scloud/v2/" + {{- end }} + {{- if (eq $.Values.global.commonGlobals.CloudEnvironment "AzureBleuCloud") }} + - name: MCR_URL + value: "https://mcr.microsoft.sovcloud-api.fr/v2/" + {{- end }} + {{- if $isusingaadauth }} + - name: USING_AAD_MSI_AUTH + value: "true" + {{- else }} + - name: USING_AAD_MSI_AUTH + value: "false" + {{- end }} + {{- if eq ($.Values.OmsAgent.shouldMountSyslogHostPort | default false) true }} + - name: SYSLOG_HOST_PORT + value: {{ $.Values.OmsAgent.syslogHostPort | default 28330 | quote}} + {{- end }} + - name: AZMON_TELEGRAF_LIVENESSPROBE_ENABLED + value: "{{ $.Values.OmsAgent.isTelegrafLivenessprobeEnabled | default false }}" + - name: CLUSTER_CLOUD_ENVIRONMENT + value: "{{ $.Values.global.commonGlobals.CloudEnvironment | lower }}" + securityContext: + privileged: true + capabilities: + drop: + - ALL + add: + - DAC_OVERRIDE + volumeMounts: + - mountPath: /etc/kubernetes/host + name: azure-json-path + - mountPath: /etc/ama-logs-secret + name: ama-logs-secret + readOnly: true + - mountPath: /etc/omsagent-secret + name: ama-logs-secret + readOnly: true + - mountPath: /etc/config/settings + name: settings-vol-config + readOnly: true + - mountPath: /etc/config/osm-settings + name: osm-settings-vol-config + readOnly: true + - mountPath: /var/run/mdsd-PrometheusSidecar + name: mdsd-prometheus-sock + {{- if (eq (include "should_mount_hostca" $ ) "true" ) }} + # USSec and USNat have cloud-specific Root and Intermediate CAs that must be mounted into all running containers from the host + # Container Insights has logic to update the trust store in the image based on certs present in /anchors/ mounts below + - mountPath: /anchors/mariner + name: anchors-mariner + readOnly: true + - mountPath: /anchors/ubuntu + name: anchors-ubuntu + readOnly: true + {{- end }} + {{- if $.Values.OmsAgent.trustedCA }} + - mountPath: /etc/ssl/certs/proxy-cert.crt + subPath: PROXYCERT.crt + name: ama-logs-secret + readOnly: true + {{- end }} + {{- if $.Values.OmsAgent.isSyslogEnabled }} + - mountPath: /var/run/mdsd-ci + name: mdsd-sock + {{- end }} + livenessProbe: + exec: + command: + - /bin/bash + - -c + - /opt/livenessprobe.sh + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 15 + {{- end }} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: +{{- if semverCompare ">=1.16.0" $.Values.global.commonGlobals.Versions.Kubernetes }} + - key: kubernetes.io/os +{{- else }} + - key: beta.kubernetes.io/os +{{- end }} + operator: In + values: + - linux + - key: kubernetes.azure.com/cluster + operator: Exists + - key: type + operator: NotIn + values: + - virtual-kubelet + {{- if $useDaemonSetSizing -}} + {{- if eq $size.name $singleSize.name -}} + {{/* Target non-Karpenter nodes */}} + - key: karpenter.azure.com/aksnodeclass + operator: DoesNotExist + {{- else }} + {{/* Target Karpenter nodes with CPU range */}} + {{- if gt $prevmaxCPU 0 -}} + - key: karpenter.azure.com/sku-cpu + operator: Gt + values: + - "{{ $prevmaxCPU }}" + {{- end -}} + {{/* Add new line. */}} + {{- if and $prevmaxCPU $size.maxCPU }} + {{ end -}} + {{- if $size.maxCPU -}} + - key: karpenter.azure.com/sku-cpu + operator: Lt + values: + - "{{ add ($size.maxCPU | int) 1 }}" + {{- end -}} + {{- end -}} + {{- end }} + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + - operator: "Exists" + effect: PreferNoSchedule + volumes: + - name: host-root + hostPath: + path: / + - name: mdsd-prometheus-sock + emptyDir: {} + - name: container-hostname + hostPath: + path: /etc/hostname + - name: host-log + hostPath: + path: /var/log + {{- if $.Values.OmsAgent.isSyslogEnabled }} + - name: mdsd-sock + hostPath: + path: /var/run/mdsd-ci + {{- end }} + {{- if $.Values.OmsAgent.isRetinaFlowLogsEnabled }} + - name: acns-hubble + hostPath: + path: /var/log/acns/hubble + {{- end }} + - name: containerlog-path + hostPath: + path: /var/lib/docker/containers + - name: containerlog-path-2 + hostPath: + path: /mnt/docker + - name: containerlog-path-3 + hostPath: + path: /mnt/containers + - name: azure-json-path + hostPath: + path: /etc/kubernetes + - name: ama-logs-secret + secret: + secretName: ama-logs-secret + - name: settings-vol-config + configMap: + name: container-azm-ms-agentconfig + optional: true + - name: ama-logs-adx-secret + secret: + secretName: ama-logs-adx-secret + optional: true + - name: osm-settings-vol-config + configMap: + name: container-azm-ms-osmconfig + optional: true + {{- if (eq (include "should_mount_hostca" $ ) "true" ) }} + - name: anchors-ubuntu + hostPath: + path: /usr/local/share/ca-certificates/ + type: DirectoryOrCreate + - name: anchors-mariner + hostPath: + path: /etc/pki/ca-trust/source/anchors + type: DirectoryOrCreate + {{- end }} + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 50% + +{{- if and (ne (default "" $size.name) (default "" $singleSize.name)) $size.maxCPU }} +{{- $prevmaxCPU = $size.maxCPU | int }} +{{- end }} +{{- end }} +--- +{{- if semverCompare ">=1.16.0" .Values.global.commonGlobals.Versions.Kubernetes }} +apiVersion: apps/v1 +{{- else }} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Deployment +metadata: + name: ama-logs-rs + namespace: kube-system + labels: + component: ama-logs-agent + tier: node + kubernetes.azure.com/managedby: aks +{{- if .Values.legacyAddonDelivery }} + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +{{- end }} +spec: + replicas: 1 + revisionHistoryLimit: 2 + paused: false + selector: + matchLabels: + rsName: "ama-logs-rs" + strategy: + type: RollingUpdate + template: + metadata: + labels: + rsName: "ama-logs-rs" + kubernetes.azure.com/managedby: aks + annotations: + agentVersion: "azure-mdsd-1.37.0" + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + dockerProviderVersion: "18.0.1-0" + schema-versions: "v1" + WSID: {{ .Values.OmsAgent.workspaceID | b64enc | quote }} + kubernetes.azure.com/no-http-proxy-vars: "true" +{{- if semverCompare "<1.11.0" .Values.global.commonGlobals.Versions.Kubernetes }} + scheduler.alpha.kubernetes.io/critical-pod: "" +{{- end }} + spec: +{{- if semverCompare ">=1.11.0" .Values.global.commonGlobals.Versions.Kubernetes }} + priorityClassName: system-node-critical +{{- end }} + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + serviceAccountName: ama-logs + containers: +{{- if .Values.OmsAgent.isRSVPAEnabled }} + - name: ama-logs-vpa + image: "{{ template "addon_mcr_repository_base" . }}/oss/v2/kubernetes/autoscaler/addon-resizer:{{- $amalogsRSVPAImageTag -}}" + imagePullPolicy: IfNotPresent + resources: + limits: + cpu: 100m + memory: 300Mi + requests: + cpu: 5m + memory: 30Mi + env: + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: ama-logs-rs-vpa-config-volume + mountPath: /etc/config + command: + - /pod_nanny + - --config-dir=/etc/config + - --cpu=200m + - --extra-cpu=2m + - --memory=300Mi + - --extra-memory=4Mi + - --poll-period=180000 + - --threshold=5 + - --namespace=kube-system + - --deployment=ama-logs-rs + - --container=ama-logs +{{- end }} +{{- if $isusingaadauth }} + - name: addon-token-adapter + command: + - /addon-token-adapter + args: + - --secret-namespace=kube-system + - --secret-name={{ .Values.OmsAgent.accessTokenSecretName }} + - --token-server-listening-port=8888 + - --health-server-listening-port=9999 + - --restart-pod-waiting-minutes-on-broken-connection=240 + image: "{{ template "addon_mcr_repository_base" . }}/aks/msi/addon-token-adapter:{{- $addonTokenAdapterLinuxDefaultImageTag -}}" + imagePullPolicy: IfNotPresent + env: + - name: AZMON_COLLECT_ENV + value: "false" + livenessProbe: + httpGet: + path: /healthz + port: 9999 + initialDelaySeconds: 10 + periodSeconds: 60 + resources: + limits: + cpu: 500m + memory: 500Mi + requests: + cpu: 100m + memory: 100Mi + securityContext: + capabilities: + drop: + - ALL + add: + - NET_ADMIN + - NET_RAW +{{- end }} + - name: ama-logs + image: "{{ template "addon_mcr_repository_base" . }}/azuremonitor/containerinsights/ciprod:{{- default $amalogsLinuxDefaultImageTag .Values.OmsAgent.imageTagLinux -}}" + {{- if .Values.OmsAgent.isImagePullPolicyAlways }} + imagePullPolicy: Always + {{- else }} + imagePullPolicy: IfNotPresent + {{- end }} + {{- if not .Values.OmsAgent.isRSVPAEnabled }} + resources: + limits: + cpu: "{{ .Values.OmsAgent.omsAgentRsCPULimit }}" + memory: "{{ .Values.OmsAgent.omsAgentRsMemoryLimit }}" + requests: + cpu: 150m + memory: 250Mi + {{- end }} + env: + - name: NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: AKS_CLUSTER_NAME + value: "{{ .Values.OmsAgent.aksClusterName }}" + - name: AKS_RESOURCE_ID + value: "{{ .Values.OmsAgent.aksResourceID }}" + - name: AKS_NODE_RESOURCE_GROUP + value: "{{ .Values.OmsAgent.aksNodeResourceGroup }}" + {{/* TODO This needs to be fixed post Canary validation */}} + - name: AKS_REGION + value: "{{ $.Values.global.commonGlobals.Region }}" + - name: CONTROLLER_TYPE + value: "ReplicaSet" + - name: USER_ASSIGNED_IDENTITY_CLIENT_ID + value: "{{ .Values.OmsAgent.identityClientID }}" + - name: NUM_OF_FLUENTD_WORKERS + valueFrom: + resourceFieldRef: + containerName: ama-logs + resource: limits.cpu + - name: CONTAINER_MEMORY_LIMIT_IN_BYTES + valueFrom: + resourceFieldRef: + containerName: ama-logs + resource: limits.memory + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- if .Values.OmsAgent.isSidecarScrapingEnabled }} + - name: SIDECAR_SCRAPING_ENABLED + value: "true" + {{- else }} + - name: SIDECAR_SCRAPING_ENABLED + value: "false" + {{- end }} + {{- if (eq (upper .Values.global.commonGlobals.CloudEnvironment) "USNAT") }} + - name: MCR_URL + value: "https://mcr.microsoft.eaglex.ic.gov/v2/" + {{- end }} + {{- if (eq (upper .Values.global.commonGlobals.CloudEnvironment) "USSEC") }} + - name: MCR_URL + value: "https://mcr.microsoft.scloud/v2/" + {{- end }} + {{- if (eq (upper .Values.global.commonGlobals.CloudEnvironment) "AZUREBLEUCLOUD") }} + - name: MCR_URL + value: "https://mcr.microsoft.sovcloud-api.fr/v2/" + {{- end }} + {{- if $isusingaadauth }} + - name: USING_AAD_MSI_AUTH + value: "true" + {{- else }} + - name: USING_AAD_MSI_AUTH + value: "false" + {{- end }} + {{- if .Values.OmsAgent.isRSVPAEnabled }} + - name: RS_ADDON-RESIZER_VPA_ENABLED + value: "true" + {{- end }} + - name: APPMONITORING_AUTOINSTRUMENTATION_ENABLED + value: "{{ .Values.AppmonitoringAgent.enabled }}" + - name: PROMETHEUS_METRICS_SCRAPING_DISABLED + value: "{{ .Values.OmsAgent.isPrometheusMetricsScrapingDisabled }}" + - name: AZMON_TELEGRAF_LIVENESSPROBE_ENABLED + value: "{{ .Values.OmsAgent.isTelegrafLivenessprobeEnabled | default false }}" + - name: AZMON_WINDOWS_FLUENT_BIT_ENABLED + value: "{{ .Values.OmsAgent.isWindowsAMAFluentBitEnabled | default false }}" + - name: CLUSTER_CLOUD_ENVIRONMENT + value: "{{ .Values.global.commonGlobals.CloudEnvironment | lower }}" + securityContext: + privileged: true + capabilities: + drop: + - ALL + add: + - DAC_OVERRIDE + ports: + - containerPort: 25225 + protocol: TCP + - containerPort: 25224 + protocol: UDP + - containerPort: 25227 + protocol: TCP + name: in-rs-tcp + volumeMounts: + - mountPath: /var/log + name: host-log + - mountPath: /etc/kubernetes/host + name: azure-json-path + - mountPath: /etc/ama-logs-secret + name: ama-logs-secret + readOnly: true + - mountPath: /etc/omsagent-secret + name: ama-logs-secret + readOnly: true + - mountPath: /etc/config + name: ama-logs-rs-config + - mountPath: /etc/config/settings + name: settings-vol-config + readOnly: true + - mountPath: /etc/config/settings/adx + name: ama-logs-adx-secret + readOnly: true + - mountPath: /etc/config/osm-settings + name: osm-settings-vol-config + readOnly: true + {{- if (eq (include "should_mount_hostca" . ) "true" ) }} + # USSec and USNat have cloud-specific Root and Intermediate CAs that must be mounted into all running containers from the host + # Container Insights has logic to update the trust store in the image based on certs present in /anchors/ mounts below + - mountPath: /anchors/mariner + name: anchors-mariner + readOnly: true + - mountPath: /anchors/ubuntu + name: anchors-ubuntu + readOnly: true + {{- end }} + {{- if .Values.OmsAgent.trustedCA }} + - mountPath: /etc/ssl/certs/proxy-cert.crt + subPath: PROXYCERT.crt + name: ama-logs-secret + readOnly: true + {{- end }} + livenessProbe: + exec: + command: + - /bin/bash + - -c + - /opt/livenessprobe.sh + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 15 + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.azure.com/mode + operator: In + values: + - system + - weight: 1 + preference: + matchExpressions: + - key: storageprofile + operator: NotIn + values: + - managed + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: +{{- if semverCompare ">=1.16.0" .Values.global.commonGlobals.Versions.Kubernetes }} + - key: kubernetes.io/os +{{- else }} + - key: beta.kubernetes.io/os +{{- end }} + operator: In + values: + - linux + - key: kubernetes.azure.com/cluster + operator: Exists + - key: type + operator: NotIn + values: + - virtual-kubelet + volumes: + - name: container-hostname + hostPath: + path: /etc/hostname + - name: host-log + hostPath: + path: /var/log + - name: azure-json-path + hostPath: + path: /etc/kubernetes + - name: ama-logs-secret + secret: + secretName: ama-logs-secret + - name: ama-logs-rs-config + configMap: + name: ama-logs-rs-config + - name: settings-vol-config + configMap: + name: container-azm-ms-agentconfig + optional: true + - name: ama-logs-adx-secret + secret: + secretName: ama-logs-adx-secret + optional: true + - name: osm-settings-vol-config + configMap: + name: container-azm-ms-osmconfig + optional: true + {{- if (eq (include "should_mount_hostca" . ) "true" ) }} + - name: anchors-ubuntu + hostPath: + path: /usr/local/share/ca-certificates/ + type: DirectoryOrCreate + - name: anchors-mariner + hostPath: + path: /etc/pki/ca-trust/source/anchors + type: DirectoryOrCreate + {{- end }} + {{- if .Values.OmsAgent.isRSVPAEnabled }} + - name: ama-logs-rs-vpa-config-volume + configMap: + name: ama-logs-rs-vpa-config + optional: true + {{- end }} +--- +{{- if semverCompare ">=1.16.0" .Values.global.commonGlobals.Versions.Kubernetes }} +apiVersion: apps/v1 +{{- else }} +apiVersion: extensions/v1beta1 +{{- end }} +kind: DaemonSet +metadata: + name: ama-logs-windows + namespace: kube-system + labels: + component: ama-logs-agent-windows + tier: node-win + kubernetes.azure.com/managedby: aks +{{- if .Values.legacyAddonDelivery }} + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +{{- end }} +spec: + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 50% + selector: + matchLabels: + component: ama-logs-agent-windows + tier: node-win + template: + metadata: + labels: + component: ama-logs-agent-windows + tier: node-win + kubernetes.azure.com/managedby: aks + annotations: + agentVersion: "46.17.2" + dockerProviderVersion: "18.0.1-0" + schema-versions: "v1" + WSID: {{ .Values.OmsAgent.workspaceID | b64enc | quote }} + kubernetes.azure.com/no-http-proxy-vars: "true" +{{- if semverCompare "<1.11.0" .Values.global.commonGlobals.Versions.Kubernetes }} + scheduler.alpha.kubernetes.io/critical-pod: "" +{{- end }} + spec: +{{- if semverCompare ">=1.11.0" .Values.global.commonGlobals.Versions.Kubernetes }} + priorityClassName: system-node-critical +{{- end }} + serviceAccountName: ama-logs + dnsConfig: + options: + - name: ndots + value: "3" + containers: + - name: ama-logs-windows + image: "{{ template "addon_mcr_repository_base" . }}/azuremonitor/containerinsights/ciprod:{{- default $amalogsWindowsDefaultImageTag .Values.OmsAgent.imageTagWindows -}}" + {{- if .Values.OmsAgent.isImagePullPolicyAlways }} + imagePullPolicy: Always + {{- else }} + imagePullPolicy: IfNotPresent + {{- end }} + {{- if .Values.OmsAgent.isWindowsBurstableQoSEnabled }} + resources: + requests: + cpu: "{{ .Values.OmsAgent.omsAgentDsCPURequestWindows }}" + memory: "{{ .Values.OmsAgent.omsAgentDsMemoryRequestWindows }}" + limits: + cpu: "{{ .Values.OmsAgent.omsAgentDsCPULimitWindows }}" + memory: "{{ .Values.OmsAgent.omsAgentDsMemoryLimitWindows }}" + {{- else }} + resources: + limits: + cpu: "{{ .Values.OmsAgent.omsAgentDsCPULimitWindows }}" + memory: "{{ .Values.OmsAgent.omsAgentDsMemoryLimitWindows }}" + {{- end }} + securityContext: + capabilities: + drop: + - ALL + add: + - DAC_OVERRIDE + env: + - name: FBIT_SERVICE_FLUSH_INTERVAL + value: "15" + - name: FBIT_TAIL_BUFFER_CHUNK_SIZE + value: "1" + - name: FBIT_TAIL_BUFFER_MAX_SIZE + value: "1" + - name: AKS_RESOURCE_ID + value: "{{ .Values.OmsAgent.aksResourceID }}" + {{/* TODO This needs to be fixed post Canary validation */}} + - name: AKS_REGION + value: "{{ $.Values.global.commonGlobals.Region }}" + - name: CONTROLLER_TYPE + value: "DaemonSet" + - name: USER_ASSIGNED_IDENTITY_CLIENT_ID + value: "{{ .Values.OmsAgent.identityClientID }}" + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: PODNAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: CONTAINER_MEMORY_LIMIT_IN_BYTES + valueFrom: + resourceFieldRef: + containerName: ama-logs-windows + resource: limits.memory + {{- if .Values.OmsAgent.isSidecarScrapingEnabled }} + - name: SIDECAR_SCRAPING_ENABLED + value: "true" + {{- else }} + - name: SIDECAR_SCRAPING_ENABLED + value: "false" + {{- end }} + {{- if (eq (upper .Values.global.commonGlobals.CloudEnvironment) "USNAT") }} + - name: REQUIRES_CERT_BOOTSTRAP + value: "true" + {{- end }} + {{- if (eq (upper .Values.global.commonGlobals.CloudEnvironment) "USSEC") }} + - name: REQUIRES_CERT_BOOTSTRAP + value: "true" + {{- end }} + {{- if (eq (upper .Values.global.commonGlobals.CloudEnvironment) "AZUREBLEUCLOUD") }} + - name: REQUIRES_CERT_BOOTSTRAP + value: "true" + {{- end }} + {{- if $isusingaadauth }} + - name: USING_AAD_MSI_AUTH + value: "true" + {{- else }} + - name: USING_AAD_MSI_AUTH + value: "false" + {{- end }} + - name: APPMONITORING_AUTOINSTRUMENTATION_ENABLED + value: "{{ .Values.AppmonitoringAgent.enabled }}" + - name: APPMONITORING_OPENTELEMETRYLOGS_ENABLED + value: "{{ .Values.AppmonitoringAgent.isOpenTelemetryLogsEnabled | default false }}" + - name: APPMONITORING_OPENTELEMETRYLOGS_PORT + value: "{{ .Values.AppmonitoringAgent.openTelemetryLogsPort | default 28331 }}" + - name: PROMETHEUS_METRICS_SCRAPING_DISABLED + value: "{{ .Values.OmsAgent.isPrometheusMetricsScrapingDisabled }}" + - name: AZMON_TELEGRAF_LIVENESSPROBE_ENABLED + value: "{{ .Values.OmsAgent.isTelegrafLivenessprobeEnabled | default false }}" + - name: AZMON_WINDOWS_FLUENT_BIT_ENABLED + value: "{{ .Values.OmsAgent.isWindowsAMAFluentBitEnabled | default false }}" + - name: CLUSTER_CLOUD_ENVIRONMENT + value: "{{ .Values.global.commonGlobals.CloudEnvironment | lower }}" + volumeMounts: + - mountPath: C:\ProgramData\docker\containers + name: docker-windows-containers + readOnly: true + - mountPath: C:\var + name: docker-windows-kuberenetes-container-logs + - mountPath: C:\etc\config\settings + name: settings-vol-config + readOnly: true + - mountPath: C:\etc\ama-logs-secret + name: ama-logs-secret + readOnly: true + - mountPath: C:\etc\omsagent-secret + name: ama-logs-secret + readOnly: true + - mountPath: C:\etc\config\adx + name: ama-logs-adx-secret + readOnly: true + - mountPath: C:\etc\kubernetes\host + name: azure-json-path + readOnly: true + {{- if (eq (include "should_mount_hostca" . ) "true" ) }} + - mountPath: C:\ca + name: ca-certs + readOnly: true + {{- end }} + {{- if $isusingaadauth }} + - mountPath: C:\etc\IMDS-access-token + name: imds-token + readOnly: true + {{- end }} + livenessProbe: + exec: + command: + - cmd + - /c + - C:\opt\amalogswindows\scripts\cmd\livenessprobe.exe + - fluent-bit.exe + - fluentdwinaks + - "C:\\etc\\amalogswindows\\filesystemwatcher.txt" + - "C:\\etc\\amalogswindows\\renewcertificate.txt" + {{- if and $isusingaadauth .Values.OmsAgent.isWindowsAMAEnabled }} + - "MonAgentCore.exe" + {{- end }} + periodSeconds: 60 + initialDelaySeconds: 180 + timeoutSeconds: 15 +{{- if and (and $isusingaadauth .Values.OmsAgent.isWindowsAMAEnabled) (not .Values.OmsAgent.isWindowsAddonTokenAdapterDisabled) }} + - name: addon-token-adapter-win + command: + - addon-token-adapter-win + args: + - --secret-namespace=kube-system + - --secret-name={{ .Values.OmsAgent.accessTokenSecretName }} + - --token-server-listening-port=8888 + - --health-server-listening-port=9999 + - --restart-pod-waiting-minutes-on-broken-connection=240 + image: "{{ template "addon_mcr_repository_base" . }}/aks/msi/addon-token-adapter:{{- $addonTokenAdapterWindowsDefaultImageTag -}}" + imagePullPolicy: IfNotPresent + env: + - name: AZMON_COLLECT_ENV + value: "false" + livenessProbe: + httpGet: + path: /healthz + port: 9999 + initialDelaySeconds: 10 + periodSeconds: 60 + resources: + limits: + cpu: 400m + memory: 400Mi + requests: + cpu: 100m + memory: 100Mi + securityContext: + capabilities: + drop: + - ALL + add: + - NET_ADMIN + - NET_RAW +{{- end}} + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.azure.com/cluster + operator: Exists +{{- if semverCompare ">=1.16.0" .Values.global.commonGlobals.Versions.Kubernetes }} + - key: kubernetes.io/os +{{- else }} + - key: beta.kubernetes.io/os +{{- end }} + operator: In + values: + - windows + - key: type + operator: NotIn + values: + - virtual-kubelet + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + - operator: "Exists" + effect: PreferNoSchedule + volumes: + - name: docker-windows-kuberenetes-container-logs + hostPath: + path: C:\var + - name: azure-json-path + hostPath: + path: C:\k + - name: docker-windows-containers + hostPath: + path: C:\ProgramData\docker\containers + type: DirectoryOrCreate + - name: settings-vol-config + configMap: + name: container-azm-ms-agentconfig + optional: true + - name: ama-logs-secret + secret: + secretName: ama-logs-secret + - name: ama-logs-adx-secret + secret: + secretName: ama-logs-adx-secret + optional: true + {{- if (eq (include "should_mount_hostca" . ) "true" ) }} + - name: ca-certs + hostPath: + path: C:\ca + {{- end }} + {{- if $isusingaadauth }} + - name: imds-token + secret: + secretName: {{ .Values.OmsAgent.accessTokenSecretName }} + {{- end }} +{{- if and $isusingaadauth .Values.OmsAgent.isMultitenancyLogsEnabled }} +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: ama-logs-hpa + namespace: kube-system + labels: +{{- if .Values.legacyAddonDelivery }} + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +{{- end }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: ama-logs-multitenancy + minReplicas: {{ .Values.OmsAgent.omsAgentMultitenancyLogsHPAMinReplicas }} + maxReplicas: {{ .Values.OmsAgent.omsAgentMultitenancyLogsHPAMaxReplicas }} + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.OmsAgent.omsAgentMultitenancyHPAAvgCPUUtilization }} + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: {{ .Values.OmsAgent.omsAgentMultitenancyHPAAvgMemoryUtilization }} + behavior: + scaleDown: + stabilizationWindowSeconds: 1200 + policies: + - type: Percent + value: 5 + periodSeconds: 180 + scaleUp: + stabilizationWindowSeconds: 0 + policies: + - type: Pods + value: 5 + periodSeconds: 5 + - type: Percent + value: 100 + periodSeconds: 5 + selectPolicy: Max +--- +apiVersion: v1 +kind: Service +metadata: + name: ama-logs-service + namespace: kube-system + labels: +{{- if .Values.legacyAddonDelivery }} + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +{{- end }} +spec: + type: ClusterIP + ports: + - port: 24225 + targetPort: 24225 + protocol: TCP + name: fluentbit-fwd + selector: + rsName: "ama-logs-multitenancy" +--- +{{- if semverCompare ">=1.16.0" .Values.global.commonGlobals.Versions.Kubernetes }} +apiVersion: apps/v1 +{{- else }} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Deployment +metadata: + name: ama-logs-multitenancy + namespace: kube-system + labels: + component: ama-logs-agent + tier: node + kubernetes.azure.com/managedby: aks +{{- if .Values.legacyAddonDelivery }} + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +{{- end }} +spec: + replicas: 1 + selector: + matchLabels: + rsName: "ama-logs-multitenancy" + strategy: + type: RollingUpdate + template: + metadata: + labels: + rsName: "ama-logs-multitenancy" + kubernetes.azure.com/managedby: aks + annotations: + agentVersion: "azure-mdsd-1.37.0" + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + dockerProviderVersion: "18.0.1-0" + schema-versions: "v1" + WSID: {{ .Values.OmsAgent.workspaceID | b64enc | quote }} + kubernetes.azure.com/no-http-proxy-vars: "true" +{{- if semverCompare "<1.11.0" .Values.global.commonGlobals.Versions.Kubernetes }} + scheduler.alpha.kubernetes.io/critical-pod: "" +{{- end }} + spec: +{{- if semverCompare ">=1.11.0" .Values.global.commonGlobals.Versions.Kubernetes }} + priorityClassName: system-node-critical +{{- end }} + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + volumes: + - name: ama-logs-secret + secret: + secretName: ama-logs-secret + - name: settings-vol-config + configMap: + name: container-azm-ms-agentconfig + optional: true + {{- if (eq (include "should_mount_hostca" . ) "true" ) }} + - name: anchors-ubuntu + hostPath: + path: /usr/local/share/ca-certificates/ + type: DirectoryOrCreate + - name: anchors-mariner + hostPath: + path: /etc/pki/ca-trust/source/anchors + type: DirectoryOrCreate + {{- end }} + serviceAccountName: ama-logs + containers: + - name: addon-token-adapter + command: + - /addon-token-adapter + args: + - --secret-namespace=kube-system + - --secret-name=aad-msi-auth-token + - --token-server-listening-port=8888 + - --health-server-listening-port=9999 + - --restart-pod-waiting-minutes-on-broken-connection=240 + image: "{{ template "addon_mcr_repository_base" . }}/aks/msi/addon-token-adapter:{{- $addonTokenAdapterLinuxDefaultImageTag -}}" + imagePullPolicy: IfNotPresent + env: + - name: AZMON_COLLECT_ENV + value: "false" + livenessProbe: + httpGet: + path: /healthz + port: 9999 + initialDelaySeconds: 10 + periodSeconds: 60 + resources: + limits: + cpu: 500m + memory: 500Mi + requests: + cpu: 100m + memory: 100Mi + securityContext: + capabilities: + drop: + - ALL + add: + - NET_ADMIN + - NET_RAW + - name: ama-logs + image: "{{ template "addon_mcr_repository_base" . }}/azuremonitor/containerinsights/ciprod:{{- default $amalogsLinuxDefaultImageTag .Values.OmsAgent.imageTagLinux -}}" + {{- if .Values.OmsAgent.isImagePullPolicyAlways }} + imagePullPolicy: Always + {{- else }} + imagePullPolicy: IfNotPresent + {{- end }} + resources: + limits: + cpu: "{{ .Values.OmsAgent.omsAgentMultitenancyCPULimitLinux }}" + memory: "{{ .Values.OmsAgent.omsAgentMultitenancyMemoryLimitLinux }}" + requests: + cpu: "{{ .Values.OmsAgent.omsAgentMultitenancyCPURequestLinux }}" + memory: "{{ .Values.OmsAgent.omsAgentMultitenancyMemoryRequestLinux }}" + env: + - name: AZMON_MULTI_TENANCY_LOG_COLLECTION + value: "true" + - name: AZMON_MULTI_TENANCY_LOGS_SERVICE_MODE + value: "true" + - name: CONTAINER_MEMORY_LIMIT_IN_BYTES + valueFrom: + resourceFieldRef: + containerName: ama-logs + resource: limits.memory + - name: AKS_CLUSTER_NAME + value: "{{ .Values.OmsAgent.aksClusterName }}" + - name: AKS_RESOURCE_ID + value: "{{ .Values.OmsAgent.aksResourceID }}" + - name: AKS_NODE_RESOURCE_GROUP + value: "{{ .Values.OmsAgent.aksNodeResourceGroup }}" + {{/* TODO This needs to be fixed post Canary validation */}} + - name: AKS_REGION + value: "{{ $.Values.global.commonGlobals.Region }}" + - name: CONTROLLER_TYPE + value: "ReplicaSet" + - name: NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if (eq (upper .Values.global.commonGlobals.CloudEnvironment) "USNAT") }} + - name: MCR_URL + value: "https://mcr.microsoft.eaglex.ic.gov/v2/" + {{- end }} + {{- if (eq (upper .Values.global.commonGlobals.CloudEnvironment) "USSEC") }} + - name: MCR_URL + value: "https://mcr.microsoft.scloud/v2/" + {{- end }} + {{- if (eq (upper .Values.global.commonGlobals.CloudEnvironment) "AZUREBLEUCLOUD") }} + - name: MCR_URL + value: "https://mcr.microsoft.sovcloud-api.fr/v2/" + {{- end }} + - name: USING_AAD_MSI_AUTH + value: "true" + - name: APPMONITORING_AUTOINSTRUMENTATION_ENABLED + value: "{{ .Values.AppmonitoringAgent.enabled }}" + - name: CLUSTER_CLOUD_ENVIRONMENT + value: "{{ .Values.global.commonGlobals.CloudEnvironment | lower }}" + securityContext: + privileged: true + capabilities: + drop: + - ALL + add: + - DAC_OVERRIDE + ports: + - name: http + containerPort: 24225 + protocol: TCP + volumeMounts: + - mountPath: /etc/ama-logs-secret + name: ama-logs-secret + readOnly: true + - mountPath: /etc/config/settings + name: settings-vol-config + readOnly: true + {{- if (eq (include "should_mount_hostca" . ) "true" ) }} + # USSec and USNat have cloud-specific Root and Intermediate CAs that must be mounted into all running containers from the host + # Container Insights has logic to update the trust store in the image based on certs present in /anchors/ mounts below + - mountPath: /anchors/mariner + name: anchors-mariner + readOnly: true + - mountPath: /anchors/ubuntu + name: anchors-ubuntu + readOnly: true + {{- end }} + {{- if .Values.OmsAgent.trustedCA }} + - mountPath: /etc/ssl/certs/proxy-cert.crt + subPath: PROXYCERT.crt + name: ama-logs-secret + readOnly: true + {{- end }} + lifecycle: + preStop: + exec: + command: [ + "sh", "-c", + # Introduce a delay to the shutdown sequence to wait for the + # pod eviction event to propagate. Then, gracefully shutdown + "sleep 5" + ] + livenessProbe: + exec: + command: + - /bin/bash + - -c + - /opt/livenessprobe.sh + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 15 + readinessProbe: + tcpSocket: + port: 24225 + initialDelaySeconds: 10 + periodSeconds: 30 + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.azure.com/cluster + operator: Exists + - key: type + operator: NotIn + values: + - virtual-kubelet + - key: kubernetes.io/os + operator: In + values: + - linux + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.azure.com/mode + operator: In + values: + - system +{{- end }} diff --git a/charts/azuremonitor-containerinsights-for-prod-clusters/values.yaml b/charts/azuremonitor-containerinsights-for-prod-clusters/values.yaml new file mode 100644 index 000000000..20e5de3f8 --- /dev/null +++ b/charts/azuremonitor-containerinsights-for-prod-clusters/values.yaml @@ -0,0 +1,201 @@ +# Add this section to fix the AppmonitoringAgent references +AppmonitoringAgent: + enabled: false + isOpenTelemetryLogsEnabled: false + openTelemetryLogsPort: 28331 + +# Add complete global section +global: + commonGlobals: + CloudEnvironment: + isAutomaticSKU: false + Region: + Versions: + Kubernetes: "1.32.7" + +legacyAddonDelivery: false + +# Default values for ama-logs configuration +# omsagent configuration +OmsAgent: + aksResourceID: + enableDaemonSetSizing: false + isAppMonitoringAgentEnabled: false + isOpenTelemetryLogsEnabled: false + isCustomMetricsDisabled: false + isUsingAADAuth: "true" + openTelemetryLogsPort: 28331 + retinaFlowLogsEnabled: false + workspaceID: "" + accessTokenSecretName: "aad-msi-auth-token" + # Cloud environment + isMoonCake: false + isFairfax: false + workspaceKey: "" + + # Image configuration + imageTagLinux: + imageTagWindows: + isImagePullPolicyAlways: false + + # Resource ID and cluster information + # aksResourceID: "" + # aksClusterName: "" + # aksNodeResourceGroup: "" + # aksRegion: "" + + # Resource limits and requests + omsAgentDsCPULimitLinux: "500m" + omsAgentDsMemoryLimitLinux: "1Gi" + omsAgentDsCPULimitWindows: "2" + omsAgentDsMemoryLimitWindows: "2Gi" + omsAgentDsCPURequestWindows: "100m" + omsAgentDsMemoryRequestWindows: "150Mi" + omsAgentRsCPULimit: "1" + omsAgentRsMemoryLimit: "1.5Gi" + omsAgentPrometheusSidecarCPULimit: "500m" + omsAgentPrometheusSidecarMemoryLimit: "1Gi" + + # Multitenancy settings + omsAgentMultitenancyCPULimitLinux: "1" + omsAgentMultitenancyMemoryLimitLinux: "1Gi" + omsAgentMultitenancyCPURequestLinux: "100m" + omsAgentMultitenancyMemoryRequestLinux: "100Mi" + omsAgentMultitenancyLogsHPAMinReplicas: 2 + omsAgentMultitenancyLogsHPAMaxReplicas: 50 + omsAgentMultitenancyHPAAvgCPUUtilization: 700 + omsAgentMultitenancyHPAAvgMemoryUtilization: 700 + + # Feature flags + isSyslogEnabled: true + isPrometheusMetricsScrapingDisabled: false + isSidecarScrapingEnabled: true + isRSVPAEnabled: false + isRetinaFlowLogsEnabled: false + isResourceOptimizationEnabled: false + isWindowsAMAFluentBitEnabled: false + isMultitenancyLogsEnabled: false + isWindowsBurstableQoSEnabled: true + isTelegrafLivenessprobeEnabled: false + isWindowsAMAEnabled: true + isWindowsAddonTokenAdapterDisabled: false + legacyAddonDelivery: false + + # Network settings + syslogHostPort: "28330" + shouldMountSyslogHostPort: true + # httpProxy: "" + # httpsProxy: "" + # trustedCA: "" + + # # Identity settings + # identityClientID: "" + # accessTokenSecretName: "aad-msi-auth-token" + + # # DaemonSet sizing configuration + # enableDaemonSetSizing: false + # daemonSetSizingValues: + # singleSize: + # containers: + # addon-token-adapter: + # cpuLimit: "100m" + # memoryLimit: "100Mi" + # cpuRequest: "20m" + # memoryRequest: "50Mi" + # ama-logs: + # cpuLimit: "150m" + # memoryLimit: "750Mi" + # cpuRequest: "75m" + # memoryRequest: "325Mi" + # ama-logs-prometheus: + # cpuLimit: "500m" + # memoryLimit: "1Gi" + # cpuRequest: "75m" + # memoryRequest: "225Mi" + # tShirtSizes: + # - name: "small" + # maxCPU: 4 + # containers: + # addon-token-adapter: + # cpuLimit: "100m" + # memoryLimit: "100Mi" + # cpuRequest: "20m" + # memoryRequest: "50Mi" + # ama-logs: + # cpuLimit: "150m" + # memoryLimit: "750Mi" + # cpuRequest: "75m" + # memoryRequest: "325Mi" + # ama-logs-prometheus: + # cpuLimit: "500m" + # memoryLimit: "1Gi" + # cpuRequest: "75m" + # memoryRequest: "225Mi" + # - name: "medium" + # maxCPU: 8 + # containers: + # addon-token-adapter: + # cpuLimit: "200m" + # memoryLimit: "200Mi" + # cpuRequest: "40m" + # memoryRequest: "100Mi" + # ama-logs: + # cpuLimit: "300m" + # memoryLimit: "1.5Gi" + # cpuRequest: "150m" + # memoryRequest: "650Mi" + # ama-logs-prometheus: + # cpuLimit: "1" + # memoryLimit: "2Gi" + # cpuRequest: "150m" + # memoryRequest: "450Mi" + # - name: "large" + # maxCPU: 16 + # containers: + # addon-token-adapter: + # cpuLimit: "400m" + # memoryLimit: "400Mi" + # cpuRequest: "80m" + # memoryRequest: "200Mi" + # ama-logs: + # cpuLimit: "600m" + # memoryLimit: "3Gi" + # cpuRequest: "300m" + # memoryRequest: "1.3Gi" + # ama-logs-prometheus: + # cpuLimit: "2" + # memoryLimit: "4Gi" + # cpuRequest: "300m" + # memoryRequest: "900Mi" + +# # Application monitoring settings +# AppmonitoringAgent: +# enabled: false +# isOpenTelemetryLogsEnabled: false +# openTelemetryLogsPort: "28331" + +# # Azure-specific settings +# Azure: +# Cluster: +# Cloud: "" +# Region: "" +# ResourceId: "" +# Extension: +# Name: "" +# ResourceId: "" +# proxySettings: +# isProxyEnabled: false +# httpProxy: "" +# httpsProxy: "" +# noProxy: "" +# proxyCert: "" +# isCustomCert: false +# autonomousFqdn: "" + +# # Global settings +# global: +# commonGlobals: +# CloudEnvironment: "AzurePublicCloud" +# Versions: +# Kubernetes: "1.25.0" +# isAutomaticSKU: false