diff --git a/.github/actions/build-okd/action.yaml b/.github/actions/build-okd/action.yaml index 60e37fa3..0f0eddeb 100644 --- a/.github/actions/build-okd/action.yaml +++ b/.github/actions/build-okd/action.yaml @@ -25,52 +25,77 @@ inputs: token: description: Token for the GitHub Container Registry required: true + cleanup-staging: + description: Only run staging registry cleanup (skip build/test/push steps) + required: false + default: 'false' runs: using: "composite" steps: - name: Detect the CPU architecture + if: ${{ inputs.cleanup-staging != 'true' }} id: detect-cpu-arch uses: ./.github/actions/arch - name: Collect debug information before the build - if: always() + if: ${{ always() && inputs.cleanup-staging != 'true' }} uses: ./.github/actions/debug-info - name: Prepare the build and run environment + if: ${{ inputs.cleanup-staging != 'true' }} uses: ./.github/actions/prebuild - name: Login to GitHub Container Registry + if: ${{ inputs.cleanup-staging != 'true' }} uses: ./.github/actions/podman-login with: token: ${{ inputs.token }} - name: Build OKD images + if: ${{ inputs.cleanup-staging != 'true' }} shell: bash run: | set -euo pipefail cd ${GITHUB_WORKSPACE}/ + # The 'staging' mode builds images locally AND pushes them to staging registry + # Staging registry is automatically derived as: $(dirname target-registry)/okd-staging + # This allows testing before promoting to production TARGET_REGISTRY="${{ inputs.target-registry }}" ./src/okd/build_images.sh \ + staging \ "${{ inputs.okd-version-tag }}" \ "${{ inputs.ushift-gitref }}" \ "${{ inputs.target-arch }}" - - name: Build MicroShift RPMs + - name: Build MicroShift RPMs using staging OKD images + if: ${{ inputs.cleanup-staging != 'true' }} shell: bash run: | # See https://github.com/microshift-io/microshift/blob/main/docs/build.md # for more information about the build process. - # Run the RPM build process. + # Run the RPM build process using images from staging registry + # Staging registry is derived as: $(dirname target-registry)/okd-staging cd ${GITHUB_WORKSPACE}/ + PRODUCTION_REGISTRY="${{ inputs.target-registry }}" + STAGING_REGISTRY="$(dirname "${PRODUCTION_REGISTRY}")/okd-staging" + + # Set the correct architecture-specific variable for staging override + if [ "${{ steps.detect-cpu-arch.outputs.go_arch }}" = "arm64" ]; then + OKD_OVERRIDE="OKD_RELEASE_IMAGE_AARCH64=${STAGING_REGISTRY}/okd-release-arm64" + else + OKD_OVERRIDE="OKD_RELEASE_IMAGE_X86_64=${STAGING_REGISTRY}/okd-release-amd64" + fi + make rpm \ USHIFT_GITREF="${{ inputs.ushift-gitref }}" \ OKD_VERSION_TAG="${{ inputs.okd-version-tag }}" \ - OKD_RELEASE_IMAGE="${{ inputs.target-registry }}/okd-release-${{ steps.detect-cpu-arch.outputs.go_arch }}" \ + "${OKD_OVERRIDE}" \ RPM_OUTDIR=/mnt/rpms - name: Build MicroShift bootc container image + if: ${{ inputs.cleanup-staging != 'true' }} shell: bash run: | # See https://github.com/microshift-io/microshift/blob/main/docs/build.md @@ -84,6 +109,7 @@ runs: BOOTC_IMAGE_TAG="${{ inputs.bootc-image-tag }}" \ - name: Run a test to verify that MicroShift is functioning properly + if: ${{ inputs.cleanup-staging != 'true' }} shell: bash run: | # See https://github.com/microshift-io/microshift/blob/main/docs/run.md @@ -97,15 +123,75 @@ runs: make run-healthy make stop + - name: Push OKD images to production registry + if: ${{ success() && inputs.cleanup-staging != 'true' }} + shell: bash + run: | + set -euo pipefail + + cd ${GITHUB_WORKSPACE}/ + # Only push to production if all tests passed + # This ensures we don't publish broken OKD images to production + TARGET_REGISTRY="${{ inputs.target-registry }}" ./src/okd/build_images.sh \ + production \ + "${{ inputs.okd-version-tag }}" \ + "${{ inputs.ushift-gitref }}" \ + "${{ inputs.target-arch }}" + + - name: Cleanup staging registry + if: ${{ inputs.cleanup-staging == 'true' }} + shell: bash + continue-on-error: true + env: + GH_TOKEN: ${{ inputs.token }} + run: | + set -euo pipefail + + # GitHub Container Registry cleanup using gh CLI + # Deletes known staging packages + echo "Cleaning up staging packages..." + + OWNER="${{ github.repository_owner }}" + + # Detect if owner is an organization or user account + if gh api "/orgs/${OWNER}" --silent 2>/dev/null; then + OWNER_TYPE="orgs" + echo "Detected organization: ${OWNER}" + else + OWNER_TYPE="users" + echo "Detected user account: ${OWNER}" + fi + + # Get list of staging packages from the build script + cd ${GITHUB_WORKSPACE}/ + mapfile -t packages < <(./src/okd/build_images.sh list-packages "${{ inputs.okd-version-tag }}") + + # Delete each package + for package in "${packages[@]}"; do + # URL-encode package name (replace / with %2F) + encoded_package="${package//\//%2F}" + + echo "Deleting package: ${package}" + # Use appropriate endpoint based on owner type + if gh api --method DELETE "/${OWNER_TYPE}/${OWNER}/packages/container/${encoded_package}" \ + -H "Accept: application/vnd.github+json" 2>&1; then + echo " ✓ Deleted successfully" + else + echo " ⚠ Failed to delete (may not exist or already deleted)" + fi + done + + echo "Staging registry cleanup completed" + # Uncomment this to enable tmate-debug on failure # - name: Pause and open tmate debug session # if: failure() # uses: ./.github/actions/tmate-debug - name: Collect debug information after the build - if: always() + if: ${{ always() && inputs.cleanup-staging != 'true' }} uses: ./.github/actions/debug-info - name: Collect MicroShift container sos-report on failure - if: failure() + if: ${{ failure() && inputs.cleanup-staging != 'true' }} uses: ./.github/actions/sos-report diff --git a/.github/workflows/release-okd.yaml b/.github/workflows/release-okd.yaml index ee7fded8..ae4bb834 100644 --- a/.github/workflows/release-okd.yaml +++ b/.github/workflows/release-okd.yaml @@ -30,6 +30,11 @@ jobs: build-okd-release: name: Build OKD release images for ARM runs-on: ubuntu-24.04-arm + # Export the detected OKD version as a job output so the cleanup job can use + # the same version. This prevents version mismatches if the build fails before + # version detection completes. + outputs: + okd-version-tag: ${{ steps.set-version.outputs.okd-version-tag }} steps: - name: Check out MicroShift upstream repository uses: actions/checkout@v4 @@ -42,11 +47,41 @@ jobs: with: check-amd64: "true" + # Determine which OKD version to use (user-specified OR auto-detected) and + # capture it as a step output so it can be exported as a job output. + # This ensures the cleanup job uses the exact same version as the build job, + # preventing cleanup from targeting wrong staging images. + - name: Set OKD version for reuse + id: set-version + run: | + VERSION="${{ env.OKD_VERSION_TAG != 'latest' && env.OKD_VERSION_TAG || steps.detect-okd-version.outputs.okd-version-tag }}" + echo "okd-version-tag=${VERSION}" >> $GITHUB_OUTPUT + echo "Using OKD version: ${VERSION}" + - name: Run the OKD release images build action uses: ./.github/actions/build-okd with: ushift-gitref: ${{ env.USHIFT_GITREF }} - okd-version-tag: ${{ env.OKD_VERSION_TAG != 'latest' && env.OKD_VERSION_TAG || steps.detect-okd-version.outputs.okd-version-tag }} + okd-version-tag: ${{ steps.set-version.outputs.okd-version-tag }} + target-arch: arm64 + target-registry: ${{ env.OKD_TARGET_REGISTRY }} + token: ${{ secrets.GITHUB_TOKEN }} + + cleanup-staging: + name: Cleanup staging registry + needs: build-okd-release + if: success() || failure() + runs-on: ubuntu-latest + steps: + - name: Check out MicroShift upstream repository + uses: actions/checkout@v4 + + - name: Run cleanup of staging OKD images + uses: ./.github/actions/build-okd + with: + ushift-gitref: ${{ env.USHIFT_GITREF }} + okd-version-tag: ${{ needs.build-okd-release.outputs.okd-version-tag }} target-arch: arm64 target-registry: ${{ env.OKD_TARGET_REGISTRY }} token: ${{ secrets.GITHUB_TOKEN }} + cleanup-staging: 'true' diff --git a/Makefile b/Makefile index 3b31759e..5737facf 100644 --- a/Makefile +++ b/Makefile @@ -29,14 +29,9 @@ EXPOSE_KUBEAPI_PORT ?= 1 # Internal variables SHELL := /bin/bash -# Override the default OKD_RELEASE_IMAGE variable based on the architecture +# OKD release image URLs for different architectures OKD_RELEASE_IMAGE_X86_64 ?= quay.io/okd/scos-release OKD_RELEASE_IMAGE_AARCH64 ?= ghcr.io/microshift-io/okd/okd-release-arm64 -ifeq ($(ARCH),aarch64) -OKD_RELEASE_IMAGE ?= $(OKD_RELEASE_IMAGE_AARCH64) -else -OKD_RELEASE_IMAGE ?= $(OKD_RELEASE_IMAGE_X86_64) -endif RPM_IMAGE := microshift-okd-rpm USHIFT_IMAGE := microshift-okd diff --git a/src/okd/build_images.sh b/src/okd/build_images.sh index afd2c40e..80cfe9dd 100755 --- a/src/okd/build_images.sh +++ b/src/okd/build_images.sh @@ -4,15 +4,33 @@ set -euo pipefail export LC_ALL=C.UTF-8 export LANG=C.UTF-8 -TARGET_REGISTRY=${TARGET_REGISTRY:-ghcr.io/microshift-io/okd} +# Production registry - must be provided via TARGET_REGISTRY environment variable +# or defaults to the upstream registry if not specified +PRODUCTION_REGISTRY="${TARGET_REGISTRY:-ghcr.io/microshift-io/okd}" +# Automatically derive staging registry by appending '/okd-staging' subpath +STAGING_REGISTRY="$(dirname "${PRODUCTION_REGISTRY}")/okd-staging" PULL_SECRET=${PULL_SECRET:-~/.pull-secret.json} WORKDIR=$(mktemp -d /tmp/okd-build-images-XXXXXX) trap 'cd ; rm -rf "${WORKDIR}"' EXIT usage() { - echo "Usage: $(basename "$0") " - echo " okd-version: The version of OKD to build (see https://amd64.origin.releases.ci.openshift.org/)" + echo "Usage: $(basename "$0") [options]" + echo "" + echo "Modes:" + echo " staging " + echo " Build OKD images locally and push to staging registry" + echo " (${STAGING_REGISTRY})" + echo "" + echo " production " + echo " Push previously built images to production registry" + echo " (${PRODUCTION_REGISTRY})" + echo "" + echo " list-packages " + echo " Output list of staging package names for cleanup" + echo "" + echo "Arguments:" + echo " okd-version: The version of OKD (see https://amd64.origin.releases.ci.openshift.org/)" echo " ocp-branch: The branch of OCP to build (e.g. release-4.19)" echo " target-arch: The architecture of the target images (amd64 or arm64)" exit 1 @@ -320,17 +338,160 @@ create_new_okd_release() { # "ovn-kubernetes-microshift=${images_sha[ovn-kubernetes-microshift]}" \ } +# Build OKD images locally and populate images_sha array +build_okd_images() { + echo "Building OKD images locally..." + create_images + + for key in "${!images[@]}" ; do + # Skip haproxy-router for non-ARM64 architectures (see TODO at line 99) + # haproxy28 package implementation for amd64 is not yet available + if [ "${TARGET_ARCH}" != "arm64" ] && [ "${key}" = "haproxy-router" ] ; then + continue + fi + images_sha["${key}"]="${images[$key]}" + done + + echo "Build completed successfully" +} + +# Push images and manifests to registry, then create OKD release +push_okd_images() { + echo "Pushing images to registry: ${TARGET_REGISTRY}" + push_image_manifests + create_new_okd_release + echo "Push completed successfully" + echo "OKD release image published to: ${OKD_RELEASE_IMAGE}" +} + +# Retag staging images to production names +retag_staging_to_production() { + local staging_image + local production_image + + echo "Re-tagging staging images to production names..." + + for key in "${!images[@]}" ; do + # Skip haproxy-router for non-ARM64 architectures (see TODO at line 99) + # haproxy28 package implementation for amd64 is not yet available + if [ "${TARGET_ARCH}" != "arm64" ] && [ "${key}" = "haproxy-router" ] ; then + continue + fi + + production_image="${images[$key]}" + staging_image="${production_image/${PRODUCTION_REGISTRY}/${STAGING_REGISTRY}}" + + if ! podman image exists "${staging_image}" ; then + echo "ERROR: Local staging image ${staging_image} not found." + echo "Run staging build first: $0 staging ${OKD_VERSION} ${OCP_BRANCH} ${TARGET_ARCH}" + exit 1 + fi + + echo "Re-tagging ${staging_image} to ${production_image}" + podman tag "${staging_image}" "${production_image}" + images_sha["${key}"]="${production_image}" + done +} + +# Staging mode: build images locally and push to staging registry +push_staging() { + check_podman_login + check_release_image_exists + build_okd_images + push_okd_images + echo "" + echo "Images built and pushed to staging registry: ${STAGING_REGISTRY}" + echo "OKD release image available at: ${OKD_RELEASE_IMAGE}" + echo "After successful testing, push to production with:" + echo " $0 production ${OKD_VERSION} ${OCP_BRANCH} ${TARGET_ARCH}" +} + +# Production mode: retag staging images and push to production registry +push_production() { + check_podman_login + check_release_image_exists + retag_staging_to_production + push_okd_images +} + +# List packages mode: output staging package names for cleanup +list_packages() { + # WARNING: Keep this list synchronized with the 'images' array at line ~520 + # When adding a new image to the build, add its key to BOTH locations: + # 1. Here (image_keys array) + # 2. Line ~520 (images associative array) + local image_keys=( + "base" + "cli" + "haproxy-router-base" + "haproxy-router" + "kube-proxy" + "coredns" + "csi-snapshot-controller" + "kube-rbac-proxy" + "pod" + "service-ca-operator" + "operator-lifecycle-manager" + "operator-registry" + ) + + local packages=() + + # Derive package names from image keys + for key in "${image_keys[@]}"; do + if [[ "${key}" == "base" ]]; then + # base image maps to scos-${OKD_VERSION} + packages+=("okd-staging/scos-${OKD_VERSION}") + else + # All other images use their key as the package name + packages+=("okd-staging/${key}") + fi + done + + # Add release images for both architectures + packages+=( + "okd-staging/okd-release-arm64" + "okd-staging/okd-release-amd64" + ) + + # Output one package per line + printf '%s\n' "${packages[@]}" +} + # # Main # -if [[ $# -ne 3 ]]; then +if [[ $# -eq 0 ]]; then usage fi -OKD_VERSION="$1" -OCP_BRANCH="$2" -TARGET_ARCH="$3" -OKD_RELEASE_IMAGE="${TARGET_REGISTRY}/okd-release-${TARGET_ARCH}:${OKD_VERSION}" +MODE="$1" + +# Handle list-packages mode (only needs 2 arguments: mode + version) +if [[ "${MODE}" == "list-packages" ]]; then + if [[ $# -ne 2 ]]; then + echo "ERROR: 'list-packages' requires 2 arguments: mode and okd-version" + usage + fi + OKD_VERSION="$2" + list_packages + exit 0 +fi + +# Other modes require exactly 4 arguments +if [[ $# -ne 4 ]]; then + usage +fi + +OKD_VERSION="$2" +OCP_BRANCH="$3" +TARGET_ARCH="$4" + +# Validate mode +if [[ "${MODE}" != "staging" ]] && [[ "${MODE}" != "production" ]]; then + echo "ERROR: Invalid mode '${MODE}'. Must be 'staging' or 'production'" + usage +fi # Determine the alternate architecture case "${TARGET_ARCH}" in @@ -346,7 +507,17 @@ case "${TARGET_ARCH}" in ;; esac +# Set target registry based on mode +if [[ "${MODE}" == "staging" ]]; then + TARGET_REGISTRY="${STAGING_REGISTRY}" +elif [[ "${MODE}" == "production" ]]; then + TARGET_REGISTRY="${PRODUCTION_REGISTRY}" +fi + +OKD_RELEASE_IMAGE="${TARGET_REGISTRY}/okd-release-${TARGET_ARCH}:${OKD_VERSION}" + # Populate associative arrays with image names and tags +# WARNING: When adding a new image here, also update list_packages() at line ~418 declare -A images declare -A images_sha images=( @@ -368,10 +539,9 @@ images=( # Check the prerequisites check_prereqs -check_podman_login -check_release_image_exists -# Create and push images -create_images -push_image_manifests -# Create a new OKD release -create_new_okd_release +# Execute based on mode +if [[ "${MODE}" == "staging" ]]; then + push_staging +elif [[ "${MODE}" == "production" ]]; then + push_production +fi