Merge pull request #1129 from stackhpc/zed-yoga-merge

zed: yoga merge

Author: markgoddard

.github/workflows/stackhpc-container-image-build.yml

@@ -34,11 +34,10 @@ on:
         required: false
         default: true
       push-dirty:
-        description: Push scanned images that have vulnerabilities?
+        description: Push scanned images that have critical vulnerabilities?
         type: boolean
         required: false
-        # NOTE(Alex-Welsh): This default should be flipped once we resolve existing failures
-        default: true
+        default: false
 
 env:
   ANSIBLE_FORCE_COLOR: True
@@ -176,7 +175,7 @@ jobs:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
 
       - name: Create build logs output directory
-        run: mkdir image-build-logs 
+        run: mkdir image-build-logs
 
       - name: Build kolla overcloud images
         id: build_overcloud_images
@@ -235,9 +234,16 @@ jobs:
         run: cp image-build-logs/image-scan-output/clean-images.txt image-build-logs/push-attempt-images.txt
         if: inputs.push
 
+      # NOTE(seunghun1ee): This always appends dirty images with CVEs severity lower than critical.
+      # This should be reverted when it's decided to filter high level CVEs as well.
       - name: Append dirty images to push list
         run: |
           cat image-build-logs/image-scan-output/dirty-images.txt >> image-build-logs/push-attempt-images.txt
+        if: ${{ inputs.push }}
+
+      - name: Append images with critical vulnerabilities to push list
+        run: |
+          cat image-build-logs/image-scan-output/critical-images.txt >> image-build-logs/push-attempt-images.txt
         if: ${{ inputs.push && inputs.push-dirty }}
 
       - name: Push images
@@ -249,7 +255,7 @@ jobs:
 
           while read -r image; do
             # Retries!
-            for i in {1..5}; do 
+            for i in {1..5}; do
               if docker push $image; then
                 echo "Pushed $image"
                 break
@@ -283,8 +289,15 @@ jobs:
         run: if [ $(wc -l < image-build-logs/push-failed-images.txt) -gt 0 ]; then cat image-build-logs/push-failed-images.txt && exit 1; fi
         if: ${{ !cancelled() }}
 
-      - name: Fail when images failed scanning
-        run: if [ $(wc -l < image-build-logs/dirty-images.txt) -gt 0 ]; then cat image-build-logs/dirty-images.txt && exit 1; fi
+      # NOTE(seunghun1ee): Currently we want to mark the job fail only when critical CVEs are detected.
+      # This can be used again instead of "Fail when critical vulnerabilities are found" when it's
+      # decided to fail the job on detecting high CVEs as well.
+      # - name: Fail when images failed scanning
+      #   run: if [ $(wc -l < image-build-logs/image-scan-output/dirty-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/dirty-images.txt && exit 1; fi
+      #   if: ${{ !inputs.push-dirty && !cancelled() }}
+
+      - name: Fail when critical vulnerabilities are found
+        run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/critical-images.txt && exit 1; fi
         if: ${{ !inputs.push-dirty && !cancelled() }}
 
       # NOTE(mgoddard): Trigger another CI workflow in the

etc/kayobe/kolla/config/prometheus/system.rules

@@ -25,7 +25,7 @@ groups:
       description: "Available memory is {{ $value }} GiB."
 
   - alert: LowSwapSpace
-    expr: (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes) < {% endraw %}{{ alertmanager_node_free_swap_warning_threshold_ratio }}{% raw %}
+    expr: node_memory_SwapTotal_bytes > 0 and (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes) < {% endraw %}{{ alertmanager_node_free_swap_warning_threshold_ratio }}{% raw %}
     for: 1m
     labels:
       severity: warning
@@ -34,7 +34,7 @@ groups:
       description: "Available swap space is {{ $value | humanizePercentage }}. Running out of swap space causes OOM Kills."
 
   - alert: LowSwapSpace
-    expr: (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes) < {% endraw %}{{ alertmanager_node_free_swap_critical_threshold_ratio }}{% raw %}
+    expr: node_memory_SwapTotal_bytes > 0 and (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes) < {% endraw %}{{ alertmanager_node_free_swap_critical_threshold_ratio }}{% raw %}
     for: 1m
     labels:
       severity: critical

tools/scan-images.sh

@@ -27,11 +27,13 @@ docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$2" > $1-sca
 images=$(grep --invert-match --no-filename ^REPOSITORY $1-scanned-container-images.txt | sed 's/ \+/:/g' | cut -f 1,2 -d:)
 
 # Ensure output files exist
-touch image-scan-output/clean-images.txt image-scan-output/dirty-images.txt
+touch image-scan-output/clean-images.txt image-scan-output/dirty-images.txt image-scan-output/critical-images.txt
 
 # If Trivy detects no vulnerabilities, add the image name to clean-images.txt.
 # If there are vulnerabilities detected, add it to dirty-images.txt and
 # generate a csv summary
+# If the image contains at least one critical vulnerabilities, add it to
+# critical-images.txt
 for image in $images; do
   filename=$(basename $image | sed 's/:/\./g')
   if $(trivy image \
@@ -49,15 +51,13 @@ for image in $images; do
     # Add the image to the clean list
     echo "${image}" >> image-scan-output/clean-images.txt
   else
-    # Add the image to the dirty list
-    echo "${image}" >> image-scan-output/dirty-images.txt
-    
+
     # Write a header for the summary CSV
     echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > image-scan-output/${filename}.summary.csv
 
     # Write the summary CSV data
-    jq -r '.Results[] 
-            | select(.Vulnerabilities) 
+    jq -r '.Results[]
+            | select(.Vulnerabilities)
             | .Vulnerabilities
             # Ignore packages with "kernel" in the PkgName
             | map(select(.PkgName | test("kernel") | not ))
@@ -72,8 +72,16 @@ for image in $images; do
                     .[0].PrimaryURL,
                     .[0].Severity
                     ]
-                ) 
-            | .[] 
+                )
+            | .[]
             | @csv' image-scan-output/${filename}.json >> image-scan-output/${filename}.summary.csv
+
+    if [ $(grep "CRITICAL" image-scan-output/${filename}.summary.csv -c) -gt 0 ]; then
+      # If the image contains critical vulnerabilities, add the image to critical list
+      echo "${image}" >> image-scan-output/critical-images.txt
+    else
+      # Otherwise, add the image to the dirty list
+      echo "${image}" >> image-scan-output/dirty-images.txt
+    fi
   fi
 done