diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index bc92ab648..1ced8b831 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -9,6 +9,9 @@ permissions: contents: write packages: write +env: + Z3_RELEASE_TAG: z3-latest + defaults: run: shell: bash @@ -129,8 +132,12 @@ jobs: - name: Build Python wheels run: | set -euo pipefail - OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" mise run python:build:multiarch - OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" mise run python:build:macos + OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" \ + OPENSHELL_Z3_RELEASE_TAG="${Z3_RELEASE_TAG}" \ + mise run python:build:multiarch + OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" \ + OPENSHELL_Z3_RELEASE_TAG="${Z3_RELEASE_TAG}" \ + mise run python:build:macos ls -la target/wheels/*.whl - name: Capture wheel filenames @@ -151,7 +158,8 @@ jobs: # Build CLI binaries (Linux musl — static, native on each arch) # # Builds run directly on the CI host (glibc Ubuntu). Zig provides musl - # C/C++ toolchains for bundled-z3 and ring, and is also used as the linker. + # C/C++ toolchains for ring and the final musl linker. The downloaded Z3 + # archives were also built with zig c++, so we keep libc++ for compatibility. # --------------------------------------------------------------------------- build-cli-linux: name: Build CLI (Linux ${{ matrix.arch }}) @@ -234,6 +242,16 @@ jobs: # Override z3-sys default (stdc++) so Rust links the matching runtime. echo "CXXSTDLIB=c++" >> "$GITHUB_ENV" + - name: Download prebuilt Z3 + run: | + set -euo pipefail + Z3_ROOT="$(sh tasks/scripts/z3/download-release-artifact.sh \ + --target "${{ matrix.target }}" \ + --release-tag "${Z3_RELEASE_TAG}" \ + --output-dir "${RUNNER_TEMP:-/tmp}/prebuilt-z3")" + echo "PKG_CONFIG_ALLOW_CROSS=1" >> "$GITHUB_ENV" + echo "PKG_CONFIG_PATH=${Z3_ROOT}/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" >> "$GITHUB_ENV" + - name: Scope workspace to CLI crates run: | set -euo pipefail @@ -246,7 +264,7 @@ jobs: sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${{ needs.compute-versions.outputs.cargo_version }}"'"/}' Cargo.toml - name: Build ${{ matrix.target }} - run: mise x -- cargo build --release --target ${{ matrix.target }} -p openshell-cli --features bundled-z3 + run: mise x -- cargo build --release --target ${{ matrix.target }} -p openshell-cli - name: sccache stats if: always() @@ -311,6 +329,7 @@ jobs: --build-arg OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" \ --build-arg OPENSHELL_IMAGE_TAG=dev \ --build-arg CARGO_TARGET_CACHE_SCOPE="${{ github.sha }}" \ + --build-arg Z3_RELEASE_TAG="${Z3_RELEASE_TAG}" \ --target binary \ --output type=local,dest=out/ \ . diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index 564a88d67..bf02c1698 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -19,6 +19,7 @@ permissions: # push-event ref. Every job references this via env.RELEASE_TAG. env: RELEASE_TAG: ${{ inputs.tag || github.ref_name }} + Z3_RELEASE_TAG: z3-latest defaults: run: @@ -150,8 +151,12 @@ jobs: - name: Build Python wheels run: | set -euo pipefail - OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" mise run python:build:multiarch - OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" mise run python:build:macos + OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" \ + OPENSHELL_Z3_RELEASE_TAG="${Z3_RELEASE_TAG}" \ + mise run python:build:multiarch + OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" \ + OPENSHELL_Z3_RELEASE_TAG="${Z3_RELEASE_TAG}" \ + mise run python:build:macos ls -la target/wheels/*.whl - name: Capture wheel filenames @@ -172,7 +177,8 @@ jobs: # Build CLI binaries (Linux musl — static, native on each arch) # # Builds run directly on the CI host (glibc Ubuntu). Zig provides musl - # C/C++ toolchains for bundled-z3 and ring, and is also used as the linker. + # C/C++ toolchains for ring and the final musl linker. The downloaded Z3 + # archives were also built with zig c++, so we keep libc++ for compatibility. # --------------------------------------------------------------------------- build-cli-linux: name: Build CLI (Linux ${{ matrix.arch }}) @@ -256,6 +262,16 @@ jobs: # Override z3-sys default (stdc++) so Rust links the matching runtime. echo "CXXSTDLIB=c++" >> "$GITHUB_ENV" + - name: Download prebuilt Z3 + run: | + set -euo pipefail + Z3_ROOT="$(sh tasks/scripts/z3/download-release-artifact.sh \ + --target "${{ matrix.target }}" \ + --release-tag "${Z3_RELEASE_TAG}" \ + --output-dir "${RUNNER_TEMP:-/tmp}/prebuilt-z3")" + echo "PKG_CONFIG_ALLOW_CROSS=1" >> "$GITHUB_ENV" + echo "PKG_CONFIG_PATH=${Z3_ROOT}/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" >> "$GITHUB_ENV" + - name: Scope workspace to CLI crates run: | set -euo pipefail @@ -268,7 +284,7 @@ jobs: sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${{ needs.compute-versions.outputs.cargo_version }}"'"/}' Cargo.toml - name: Build ${{ matrix.target }} - run: mise x -- cargo build --release --target ${{ matrix.target }} -p openshell-cli --features bundled-z3 + run: mise x -- cargo build --release --target ${{ matrix.target }} -p openshell-cli - name: sccache stats if: always() @@ -334,6 +350,7 @@ jobs: --build-arg OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" \ --build-arg OPENSHELL_IMAGE_TAG="${{ needs.compute-versions.outputs.semver }}" \ --build-arg CARGO_TARGET_CACHE_SCOPE="${{ github.sha }}" \ + --build-arg Z3_RELEASE_TAG="${Z3_RELEASE_TAG}" \ --target binary \ --output type=local,dest=out/ \ . diff --git a/.github/workflows/release-z3.yml b/.github/workflows/release-z3.yml new file mode 100644 index 000000000..4ee36499a --- /dev/null +++ b/.github/workflows/release-z3.yml @@ -0,0 +1,225 @@ +name: Release Z3 + +on: + workflow_dispatch: + inputs: + z3_version: + description: "Z3 version to build (e.g. 4.16.0)" + required: true + type: string + prune_existing_assets: + description: "Delete previously published z3-latest assets before upload" + required: false + default: true + type: boolean + notes: + description: "Optional additional release notes" + required: false + type: string + +permissions: + contents: write + packages: read + +concurrency: + group: z3-latest-release + cancel-in-progress: false + +env: + Z3_RELEASE_TAG: z3-latest + +defaults: + run: + shell: bash + +jobs: + build-z3-linux: + name: Build Z3 (${{ matrix.target }}) + strategy: + matrix: + include: + - runner: build-amd64 + target: x86_64-unknown-linux-musl + zig_target: x86_64-linux-musl + use_zig: true + - runner: build-arm64 + target: aarch64-unknown-linux-musl + zig_target: aarch64-linux-musl + use_zig: true + - runner: build-amd64 + target: x86_64-unknown-linux-gnu + use_zig: false + - runner: build-arm64 + target: aarch64-unknown-linux-gnu + use_zig: false + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + container: + image: ghcr.io/nvidia/openshell/ci:latest + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + options: --privileged + env: + MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + + - name: Mark workspace safe for git + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Install tools + if: matrix.use_zig + run: mise install + + - name: Set up zig wrappers + if: matrix.use_zig + run: | + set -euo pipefail + ZIG="$(mise which zig)" + mkdir -p /tmp/zig-z3 + for tool in cc c++; do + printf '#!/bin/bash\nexec "%s" %s --target=%s "$@"\n' \ + "$ZIG" "$tool" "${{ matrix.zig_target }}" > "/tmp/zig-z3/${tool}" + chmod +x "/tmp/zig-z3/${tool}" + done + + - name: Build artifact + run: | + set -euo pipefail + mkdir -p artifacts + if [ "${{ matrix.use_zig }}" = "true" ]; then + export CC=/tmp/zig-z3/cc + export CXX=/tmp/zig-z3/c++ + fi + sh tasks/scripts/z3/build-artifact.sh \ + --target "${{ matrix.target }}" \ + --z3-version "${{ inputs.z3_version }}" \ + --output "artifacts/z3-${{ matrix.target }}.tar.gz" + ls -lh artifacts/ + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: z3-${{ matrix.target }} + path: artifacts/z3-${{ matrix.target }}.tar.gz + retention-days: 5 + + build-z3-macos: + name: Build Z3 (aarch64-apple-darwin) + runs-on: build-amd64 + timeout-minutes: 60 + container: + image: ghcr.io/nvidia/openshell/ci:latest + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + options: --privileged + volumes: + - /var/run/docker.sock:/var/run/docker.sock + steps: + - uses: actions/checkout@v4 + + - name: Mark workspace safe for git + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Set up Docker Buildx + uses: ./.github/actions/setup-buildx + + - name: Build macOS artifact via Docker + run: | + set -euo pipefail + docker buildx build \ + --file deploy/docker/Dockerfile.z3-macos \ + --build-arg Z3_VERSION="${{ inputs.z3_version }}" \ + --target artifact \ + --output type=local,dest=artifacts/ \ + . + ls -lh artifacts/ + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: z3-aarch64-apple-darwin + path: artifacts/z3-aarch64-apple-darwin.tar.gz + retention-days: 5 + + release-z3: + name: Release Z3 + needs: [build-z3-linux, build-z3-macos] + runs-on: build-amd64 + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + + - name: Download all Z3 artifacts + uses: actions/download-artifact@v4 + with: + pattern: z3-* + path: release/ + merge-multiple: true + + - name: Generate checksums + run: | + set -euo pipefail + cd release + sha256sum z3-*.tar.gz > z3-checksums-sha256.txt + cat z3-checksums-sha256.txt + + - name: Ensure z3-latest tag exists + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git tag -fa "${Z3_RELEASE_TAG}" -m "OpenShell Z3 Latest" "${GITHUB_SHA}" + git push --force origin "${Z3_RELEASE_TAG}" + + - name: Prune stale Z3 assets from z3-latest release + if: inputs.prune_existing_assets + uses: actions/github-script@v7 + with: + script: | + const [owner, repo] = process.env.GITHUB_REPOSITORY.split('/'); + let release; + try { + release = await github.rest.repos.getReleaseByTag({ owner, repo, tag: 'z3-latest' }); + } catch (err) { + if (err.status === 404) { + core.info('No existing z3-latest release; will create fresh.'); + return; + } + throw err; + } + for (const asset of release.data.assets) { + if (asset.name.startsWith('z3-')) { + core.info(`Deleting stale asset: ${asset.name}`); + await github.rest.repos.deleteReleaseAsset({ owner, repo, asset_id: asset.id }); + } + } + + - name: Create / update z3-latest GitHub Release + uses: softprops/action-gh-release@v2 + with: + name: OpenShell Z3 Latest + prerelease: true + tag_name: ${{ env.Z3_RELEASE_TAG }} + target_commitish: ${{ github.sha }} + body: | + Rolling prebuilt static Z3 artifacts for OpenShell release packaging. + + Z3 version: `${{ inputs.z3_version }}` + + Supported assets: + - `z3-x86_64-unknown-linux-musl.tar.gz` + - `z3-aarch64-unknown-linux-musl.tar.gz` + - `z3-x86_64-unknown-linux-gnu.tar.gz` + - `z3-aarch64-unknown-linux-gnu.tar.gz` + - `z3-aarch64-apple-darwin.tar.gz` + + ${{ inputs.notes }} + files: | + release/z3-x86_64-unknown-linux-musl.tar.gz + release/z3-aarch64-unknown-linux-musl.tar.gz + release/z3-x86_64-unknown-linux-gnu.tar.gz + release/z3-aarch64-unknown-linux-gnu.tar.gz + release/z3-aarch64-apple-darwin.tar.gz + release/z3-checksums-sha256.txt diff --git a/architecture/build-containers.md b/architecture/build-containers.md index ee69896e9..4a5bcba93 100644 --- a/architecture/build-containers.md +++ b/architecture/build-containers.md @@ -33,6 +33,12 @@ openshell sandbox create --from This pulls `ghcr.io/nvidia/openshell-community/sandboxes/:latest`. +## CLI and Wheel Release Builds + +Standalone CLI archives and Python wheels do not compile Z3 from source during release packaging. A manually triggered GitHub Actions workflow, `release-z3.yml`, publishes target-specific static Z3 bundles to the rolling `z3-latest` GitHub release in `NVIDIA/OpenShell`. + +Release workflows and packaging Dockerfiles download the matching archive for their target, verify it against `z3-checksums-sha256.txt`, and expose the bundled `z3.pc` via `PKG_CONFIG_PATH` so `z3-sys` links against the prebuilt static library. Local development can still opt into source-built Z3 with the existing `bundled-z3` feature when needed. + ## Local Development `mise run cluster` is the primary development command. It bootstraps a cluster if one doesn't exist, then performs incremental deploys for subsequent runs. @@ -70,4 +76,3 @@ The harness runs isolated scenarios in temporary git worktrees, keeps its own st - auto-detection checks for gateway-only, supervisor-only, shared, Helm-only, unrelated, and explicit-target changes - cold vs warm rebuild comparisons for gateway and supervisor code changes - container-ID invalidation coverage to verify gateway + Helm are retriggered when the cluster container changes - diff --git a/deploy/docker/Dockerfile.cli-macos b/deploy/docker/Dockerfile.cli-macos index f9370691c..86b3fde80 100644 --- a/deploy/docker/Dockerfile.cli-macos +++ b/deploy/docker/Dockerfile.cli-macos @@ -19,6 +19,7 @@ FROM ${OSXCROSS_IMAGE} AS osxcross FROM python:3.12-slim AS builder ARG CARGO_TARGET_CACHE_SCOPE=default +ARG Z3_RELEASE_TAG=z3-latest ENV PATH="/root/.cargo/bin:/usr/local/bin:/osxcross/bin:${PATH}" ENV LD_LIBRARY_PATH="/osxcross/lib" @@ -47,6 +48,15 @@ RUN rustup target add aarch64-apple-darwin WORKDIR /build +COPY tasks/scripts/z3/ tasks/scripts/z3/ + +RUN mkdir -p /opt/prebuilt-z3 && \ + sh tasks/scripts/z3/download-release-artifact.sh \ + --target aarch64-apple-darwin \ + --release-tag "${Z3_RELEASE_TAG}" \ + --output-dir /opt/prebuilt-z3 >/dev/null && \ + printf 'export PKG_CONFIG_ALLOW_CROSS=1\nexport PKG_CONFIG_PATH=/opt/prebuilt-z3/z3-aarch64-apple-darwin/lib/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}\n' > /opt/prebuilt-z3/env.sh + ENV CC_aarch64_apple_darwin=oa64-clang ENV CXX_aarch64_apple_darwin=oa64-clang++ ENV AR_aarch64_apple_darwin=aarch64-apple-darwin25.1-ar @@ -96,7 +106,8 @@ RUN mkdir -p crates/openshell-cli/src \ RUN --mount=type=cache,id=cargo-registry-cli-macos,sharing=locked,target=/root/.cargo/registry \ --mount=type=cache,id=cargo-git-cli-macos,sharing=locked,target=/root/.cargo/git \ --mount=type=cache,id=cargo-target-cli-macos-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \ - cargo build --release --target aarch64-apple-darwin -p openshell-cli --features bundled-z3 2>/dev/null || true + . /opt/prebuilt-z3/env.sh && \ + cargo build --release --target aarch64-apple-darwin -p openshell-cli 2>/dev/null || true # --------------------------------------------------------------------------- # Stage 2: real build @@ -121,10 +132,11 @@ ARG OPENSHELL_IMAGE_TAG RUN --mount=type=cache,id=cargo-registry-cli-macos,sharing=locked,target=/root/.cargo/registry \ --mount=type=cache,id=cargo-git-cli-macos,sharing=locked,target=/root/.cargo/git \ --mount=type=cache,id=cargo-target-cli-macos-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \ + . /opt/prebuilt-z3/env.sh && \ if [ -n "${OPENSHELL_CARGO_VERSION:-}" ]; then \ sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${OPENSHELL_CARGO_VERSION}"'"/}' Cargo.toml; \ fi && \ - cargo build --release --target aarch64-apple-darwin -p openshell-cli --features bundled-z3 && \ + cargo build --release --target aarch64-apple-darwin -p openshell-cli && \ cp target/aarch64-apple-darwin/release/openshell /openshell FROM scratch AS binary diff --git a/deploy/docker/Dockerfile.python-wheels b/deploy/docker/Dockerfile.python-wheels index 91e2223c4..ce00ddca6 100644 --- a/deploy/docker/Dockerfile.python-wheels +++ b/deploy/docker/Dockerfile.python-wheels @@ -31,6 +31,7 @@ FROM base AS builder ARG TARGETARCH ARG BUILDARCH ARG CARGO_TARGET_CACHE_SCOPE=default +ARG Z3_RELEASE_TAG=z3-latest ARG SCCACHE_MEMCACHED_ENDPOINT @@ -38,6 +39,17 @@ WORKDIR /build RUN . cross-build.sh && install_cross_toolchain && install_sccache && add_rust_target +COPY tasks/scripts/z3/ tasks/scripts/z3/ + +RUN mkdir -p /opt/prebuilt-z3 && \ + . cross-build.sh && \ + Z3_TARGET="$(rust_target)" && \ + sh tasks/scripts/z3/download-release-artifact.sh \ + --target "${Z3_TARGET}" \ + --release-tag "${Z3_RELEASE_TAG}" \ + --output-dir /opt/prebuilt-z3 >/dev/null && \ + printf 'export PKG_CONFIG_ALLOW_CROSS=1\nexport PKG_CONFIG_PATH=/opt/prebuilt-z3/z3-%s/lib/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}\n' "${Z3_TARGET}" > /opt/prebuilt-z3/env.sh + # Copy dependency manifests first for better caching. COPY Cargo.toml Cargo.lock ./ COPY crates/openshell-cli/Cargo.toml crates/openshell-cli/Cargo.toml @@ -75,7 +87,9 @@ RUN --mount=type=cache,id=cargo-registry-python-wheels-${TARGETARCH},sharing=loc --mount=type=cache,id=cargo-git-python-wheels-${TARGETARCH},sharing=locked,target=/root/.cargo/git \ --mount=type=cache,id=cargo-target-python-wheels-${TARGETARCH}-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \ --mount=type=cache,id=sccache-python-wheels-${TARGETARCH},sharing=locked,target=/tmp/sccache \ - . cross-build.sh && cargo_cross_build --release -p openshell-cli --features bundled-z3 2>/dev/null || true + . cross-build.sh && \ + . /opt/prebuilt-z3/env.sh && \ + cargo_cross_build --release -p openshell-cli 2>/dev/null || true # Copy actual source code and Python packaging files. COPY crates/ crates/ @@ -103,13 +117,14 @@ RUN --mount=type=cache,id=cargo-registry-python-wheels-${TARGETARCH},sharing=loc --mount=type=cache,id=cargo-target-python-wheels-${TARGETARCH}-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \ --mount=type=cache,id=sccache-python-wheels-${TARGETARCH},sharing=locked,target=/tmp/sccache \ . cross-build.sh && \ + . /opt/prebuilt-z3/env.sh && \ export_cross_env && \ export RUSTC_WRAPPER=sccache && \ export CARGO_BUILD_TARGET="$(rust_target)" && \ if [ -n "${OPENSHELL_CARGO_VERSION:-}" ]; then \ sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${OPENSHELL_CARGO_VERSION}"'"/}' Cargo.toml; \ fi && \ - maturin build --release --target "${CARGO_BUILD_TARGET}" --features bundled-z3 --out /wheels + maturin build --release --target "${CARGO_BUILD_TARGET}" --out /wheels FROM scratch AS wheels COPY --from=builder /wheels/*.whl / diff --git a/deploy/docker/Dockerfile.python-wheels-macos b/deploy/docker/Dockerfile.python-wheels-macos index 79cc6d9b4..3c7eb8e6b 100644 --- a/deploy/docker/Dockerfile.python-wheels-macos +++ b/deploy/docker/Dockerfile.python-wheels-macos @@ -13,6 +13,7 @@ FROM python:${PYTHON_VERSION}-slim AS builder ARG TARGETARCH ARG CARGO_TARGET_CACHE_SCOPE=default +ARG Z3_RELEASE_TAG=z3-latest ENV PATH="/root/.cargo/bin:/usr/local/bin:/osxcross/bin:${PATH}" ENV LD_LIBRARY_PATH="/osxcross/lib" @@ -42,6 +43,15 @@ RUN pip install --no-cache-dir maturin WORKDIR /build +COPY tasks/scripts/z3/ tasks/scripts/z3/ + +RUN mkdir -p /opt/prebuilt-z3 && \ + sh tasks/scripts/z3/download-release-artifact.sh \ + --target aarch64-apple-darwin \ + --release-tag "${Z3_RELEASE_TAG}" \ + --output-dir /opt/prebuilt-z3 >/dev/null && \ + printf 'export PKG_CONFIG_ALLOW_CROSS=1\nexport PKG_CONFIG_PATH=/opt/prebuilt-z3/z3-aarch64-apple-darwin/lib/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}\n' > /opt/prebuilt-z3/env.sh + ENV CC_aarch64_apple_darwin=oa64-clang ENV CXX_aarch64_apple_darwin=oa64-clang++ ENV AR_aarch64_apple_darwin=aarch64-apple-darwin25.1-ar @@ -87,7 +97,8 @@ RUN mkdir -p crates/openshell-cli/src crates/openshell-core/src crates/openshell RUN --mount=type=cache,id=cargo-registry-python-wheels-macos-${TARGETARCH},sharing=locked,target=/root/.cargo/registry \ --mount=type=cache,id=cargo-git-python-wheels-macos-${TARGETARCH},sharing=locked,target=/root/.cargo/git \ --mount=type=cache,id=cargo-target-python-wheels-macos-${TARGETARCH}-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \ - cargo build --release --target aarch64-apple-darwin -p openshell-cli --features bundled-z3 2>/dev/null || true + . /opt/prebuilt-z3/env.sh && \ + cargo build --release --target aarch64-apple-darwin -p openshell-cli 2>/dev/null || true # Copy actual source code and Python packaging files. COPY crates/ crates/ @@ -115,10 +126,11 @@ ARG OPENSHELL_IMAGE_TAG RUN --mount=type=cache,id=cargo-registry-python-wheels-macos-${TARGETARCH},sharing=locked,target=/root/.cargo/registry \ --mount=type=cache,id=cargo-git-python-wheels-macos-${TARGETARCH},sharing=locked,target=/root/.cargo/git \ --mount=type=cache,id=cargo-target-python-wheels-macos-${TARGETARCH}-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \ + . /opt/prebuilt-z3/env.sh && \ if [ -n "${OPENSHELL_CARGO_VERSION:-}" ]; then \ sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${OPENSHELL_CARGO_VERSION}"'"/}' Cargo.toml; \ fi && \ - maturin build --release --target aarch64-apple-darwin --features bundled-z3 --out /wheels + maturin build --release --target aarch64-apple-darwin --out /wheels FROM scratch AS wheels COPY --from=builder /wheels/*.whl / diff --git a/deploy/docker/Dockerfile.z3-macos b/deploy/docker/Dockerfile.z3-macos new file mode 100644 index 000000000..e89dd9263 --- /dev/null +++ b/deploy/docker/Dockerfile.z3-macos @@ -0,0 +1,54 @@ +# syntax=docker/dockerfile:1.6 + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +ARG OSXCROSS_IMAGE=crazymax/osxcross:latest + +FROM ${OSXCROSS_IMAGE} AS osxcross + +FROM python:3.12-slim AS builder + +ARG Z3_VERSION + +ENV PATH="/usr/local/bin:/osxcross/bin:${PATH}" +ENV LD_LIBRARY_PATH="/osxcross/lib" + +COPY --from=osxcross /osxcross /osxcross + +RUN SDKROOT="$(echo /osxcross/SDK/MacOSX*.sdk)" && ln -sfn "${SDKROOT}" /osxcross/SDK/MacOSX.sdk + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + clang \ + cmake \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# Match the linker alias used by the existing macOS release build. +RUN ln -sf /osxcross/bin/arm64-apple-darwin25.1-ld /usr/local/bin/arm64-apple-macosx-ld + +WORKDIR /build + +COPY tasks/scripts/z3/ tasks/scripts/z3/ + +ENV CC=oa64-clang +ENV CXX=oa64-clang++ +ENV AR=aarch64-apple-darwin25.1-ar +ENV SDKROOT=/osxcross/SDK/MacOSX.sdk +ENV MACOSX_DEPLOYMENT_TARGET=13.3 +ENV CFLAGS=--target=arm64-apple-macosx\ -mmacosx-version-min=13.3 +ENV CXXFLAGS=--target=arm64-apple-macosx\ -mmacosx-version-min=13.3 +ENV LDFLAGS=--target=arm64-apple-macosx\ -mmacosx-version-min=13.3 + +RUN mkdir -p /out && \ + Z3_CMAKE_SYSTEM_NAME=Darwin \ + Z3_CMAKE_SYSTEM_PROCESSOR=arm64 \ + sh tasks/scripts/z3/build-artifact.sh \ + --target aarch64-apple-darwin \ + --z3-version "${Z3_VERSION}" \ + --output /out/z3-aarch64-apple-darwin.tar.gz + +FROM scratch AS artifact +COPY --from=builder /out/z3-aarch64-apple-darwin.tar.gz /z3-aarch64-apple-darwin.tar.gz diff --git a/tasks/python.toml b/tasks/python.toml index 211e474c1..0985df45c 100644 --- a/tasks/python.toml +++ b/tasks/python.toml @@ -57,6 +57,7 @@ CARGO_TARGET_CACHE_SCOPE=$(printf '%s' "$CACHE_SCOPE_INPUT" | sha256_16_stdin) VERSION_ARGS=( --build-arg "CARGO_TARGET_CACHE_SCOPE=${CARGO_TARGET_CACHE_SCOPE}" ${OPENSHELL_IMAGE_TAG:+--build-arg "OPENSHELL_IMAGE_TAG=${OPENSHELL_IMAGE_TAG}"} + ${OPENSHELL_Z3_RELEASE_TAG:+--build-arg "Z3_RELEASE_TAG=${OPENSHELL_Z3_RELEASE_TAG}"} ) if [ -n "$CARGO_VERSION" ]; then @@ -192,6 +193,7 @@ docker build \ --build-arg "CARGO_TARGET_CACHE_SCOPE=${CARGO_TARGET_CACHE_SCOPE}" \ ${CARGO_VERSION:+--build-arg "OPENSHELL_CARGO_VERSION=${CARGO_VERSION}"} \ ${OPENSHELL_IMAGE_TAG:+--build-arg "OPENSHELL_IMAGE_TAG=${OPENSHELL_IMAGE_TAG}"} \ + ${OPENSHELL_Z3_RELEASE_TAG:+--build-arg "Z3_RELEASE_TAG=${OPENSHELL_Z3_RELEASE_TAG}"} \ --output type=local,dest=target/wheels \ . diff --git a/tasks/scripts/z3/build-artifact.sh b/tasks/scripts/z3/build-artifact.sh new file mode 100644 index 000000000..d7b84805b --- /dev/null +++ b/tasks/scripts/z3/build-artifact.sh @@ -0,0 +1,142 @@ +#!/bin/sh + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -eu + +SELF_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +. "${SELF_DIR}/common.sh" + +TARGET="" +Z3_VERSION="" +OUTPUT="" + +usage() { + cat <<'EOF' >&2 +Usage: build-artifact.sh --target --z3-version --output + +Build a static Z3 release artifact that contains headers, libz3.a, pkg-config +metadata, and a small manifest for the given target triple. +EOF + exit 2 +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --target) + TARGET="${2:-}" + shift 2 + ;; + --z3-version) + Z3_VERSION="${2:-}" + shift 2 + ;; + --output) + OUTPUT="${2:-}" + shift 2 + ;; + -h|--help) + usage + ;; + *) + echo "unknown argument: $1" >&2 + usage + ;; + esac +done + +[ -n "$TARGET" ] || usage +[ -n "$Z3_VERSION" ] || usage +[ -n "$OUTPUT" ] || usage + +ASSET_NAME=$(z3_asset_name_for_target "$TARGET") +PACKAGE_DIR_NAME=$(z3_package_dir_name "$TARGET") + +WORKDIR=$(mktemp -d) +cleanup() { + rm -rf "$WORKDIR" +} +trap cleanup EXIT INT TERM + +SOURCE_ARCHIVE="${WORKDIR}/z3-source.tar.gz" +SOURCE_ROOT="${WORKDIR}/source" +BUILD_ROOT="${WORKDIR}/build" +PACKAGE_PARENT="${WORKDIR}/package" +PACKAGE_ROOT="${PACKAGE_PARENT}/${PACKAGE_DIR_NAME}" + +mkdir -p "$SOURCE_ROOT" "$BUILD_ROOT" "$PACKAGE_ROOT" "$(dirname "$OUTPUT")" + +SOURCE_URL="https://github.com/Z3Prover/z3/archive/refs/tags/z3-${Z3_VERSION}.tar.gz" +echo "Downloading Z3 ${Z3_VERSION} source from ${SOURCE_URL}" +curl --fail --location --retry 5 --retry-all-errors --silent --show-error \ + --output "$SOURCE_ARCHIVE" \ + "$SOURCE_URL" +tar -xzf "$SOURCE_ARCHIVE" -C "$SOURCE_ROOT" + +set -- "${SOURCE_ROOT}"/* +if [ "$#" -ne 1 ] || [ ! -d "$1" ]; then + echo "expected a single extracted Z3 source directory" >&2 + exit 1 +fi +SOURCE_DIR="$1" + +set -- \ + -S "$SOURCE_DIR" \ + -B "$BUILD_ROOT" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$PACKAGE_ROOT" \ + -DCMAKE_INSTALL_INCLUDEDIR=include \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DZ3_BUILD_LIBZ3_SHARED=false \ + -DZ3_BUILD_EXECUTABLE=false \ + -DZ3_BUILD_TEST_EXECUTABLES=false + +if [ -n "${Z3_CMAKE_SYSTEM_NAME:-}" ]; then + set -- "$@" "-DCMAKE_SYSTEM_NAME=${Z3_CMAKE_SYSTEM_NAME}" +fi +if [ -n "${Z3_CMAKE_SYSTEM_PROCESSOR:-}" ]; then + set -- "$@" "-DCMAKE_SYSTEM_PROCESSOR=${Z3_CMAKE_SYSTEM_PROCESSOR}" +fi +if [ -n "${MACOSX_DEPLOYMENT_TARGET:-}" ]; then + set -- "$@" "-DCMAKE_OSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET}" +fi + +cmake "$@" +cmake --build "$BUILD_ROOT" --parallel "$(z3_cpu_count)" +cmake --install "$BUILD_ROOT" + +HEADER_PATH=$(find "$PACKAGE_ROOT" -path '*/z3.h' -type f | head -n 1 || true) +LIB_PATH=$(find "$PACKAGE_ROOT" -name 'libz3.a' -type f | head -n 1 || true) + +[ -n "$HEADER_PATH" ] || { echo "z3.h missing from packaged artifact" >&2; exit 1; } +[ -n "$LIB_PATH" ] || { echo "libz3.a missing from packaged artifact" >&2; exit 1; } + +INCLUDE_DIR_NAME=$(basename "$(dirname "$HEADER_PATH")") +LIB_DIR=$(dirname "$LIB_PATH") +LIB_DIR_NAME=$(basename "$LIB_DIR") +PKGCONFIG_DIR="${LIB_DIR}/pkgconfig" +mkdir -p "$PKGCONFIG_DIR" + +cat > "${PKGCONFIG_DIR}/z3.pc" < "${PACKAGE_ROOT}/manifest.json" <}" >&2 + return 1 + ;; + esac +} + +z3_package_dir_name() { + case "${1:-}" in + x86_64-unknown-linux-musl|aarch64-unknown-linux-musl|x86_64-unknown-linux-gnu|aarch64-unknown-linux-gnu|aarch64-apple-darwin) + printf 'z3-%s\n' "$1" + ;; + *) + echo "unsupported Z3 target: ${1:-}" >&2 + return 1 + ;; + esac +} + +z3_release_url() { + repo="${1:?repo is required}" + tag="${2:?tag is required}" + asset="${3:?asset is required}" + printf 'https://github.com/%s/releases/download/%s/%s\n' "$repo" "$tag" "$asset" +} + +z3_cpu_count() { + if command -v nproc >/dev/null 2>&1; then + nproc + elif command -v sysctl >/dev/null 2>&1; then + sysctl -n hw.ncpu + else + echo 4 + fi +} diff --git a/tasks/scripts/z3/download-release-artifact.sh b/tasks/scripts/z3/download-release-artifact.sh new file mode 100644 index 000000000..722d42135 --- /dev/null +++ b/tasks/scripts/z3/download-release-artifact.sh @@ -0,0 +1,111 @@ +#!/bin/sh + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -eu + +SELF_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +. "${SELF_DIR}/common.sh" + +TARGET="" +OUTPUT_DIR="" +RELEASE_TAG="${Z3_RELEASE_TAG:-z3-latest}" +REPO="${Z3_RELEASE_REPO:-NVIDIA/OpenShell}" + +usage() { + cat <<'EOF' >&2 +Usage: download-release-artifact.sh --target --output-dir [--release-tag ] [--repo ] + +Download a prebuilt Z3 artifact from an OpenShell GitHub release, verify it +against the published checksum file, extract it, and print the extracted root. +EOF + exit 2 +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --target) + TARGET="${2:-}" + shift 2 + ;; + --output-dir) + OUTPUT_DIR="${2:-}" + shift 2 + ;; + --release-tag) + RELEASE_TAG="${2:-}" + shift 2 + ;; + --repo) + REPO="${2:-}" + shift 2 + ;; + -h|--help) + usage + ;; + *) + echo "unknown argument: $1" >&2 + usage + ;; + esac +done + +[ -n "$TARGET" ] || usage +[ -n "$OUTPUT_DIR" ] || usage + +ASSET_NAME=$(z3_asset_name_for_target "$TARGET") +PACKAGE_DIR_NAME=$(z3_package_dir_name "$TARGET") +CHECKSUMS_NAME="z3-checksums-sha256.txt" + +WORKDIR=$(mktemp -d) +cleanup() { + rm -rf "$WORKDIR" +} +trap cleanup EXIT INT TERM + +ASSET_PATH="${WORKDIR}/${ASSET_NAME}" +CHECKSUMS_PATH="${WORKDIR}/${CHECKSUMS_NAME}" + +curl --fail --location --retry 5 --retry-all-errors --silent --show-error \ + --output "$ASSET_PATH" \ + "$(z3_release_url "$REPO" "$RELEASE_TAG" "$ASSET_NAME")" + +curl --fail --location --retry 5 --retry-all-errors --silent --show-error \ + --output "$CHECKSUMS_PATH" \ + "$(z3_release_url "$REPO" "$RELEASE_TAG" "$CHECKSUMS_NAME")" + +EXPECTED_SHA=$(awk -v asset="$ASSET_NAME" '$2 == asset { print $1 }' "$CHECKSUMS_PATH") +[ -n "$EXPECTED_SHA" ] || { + echo "checksum entry for ${ASSET_NAME} not found in ${CHECKSUMS_NAME}" >&2 + exit 1 +} + +if command -v sha256sum >/dev/null 2>&1; then + ACTUAL_SHA=$(sha256sum "$ASSET_PATH" | awk '{print $1}') +else + ACTUAL_SHA=$(shasum -a 256 "$ASSET_PATH" | awk '{print $1}') +fi + +[ "$EXPECTED_SHA" = "$ACTUAL_SHA" ] || { + echo "checksum mismatch for ${ASSET_NAME}" >&2 + echo "expected: ${EXPECTED_SHA}" >&2 + echo "actual: ${ACTUAL_SHA}" >&2 + exit 1 +} + +mkdir -p "$OUTPUT_DIR" +rm -rf "${OUTPUT_DIR:?}/${PACKAGE_DIR_NAME}" +tar -xzf "$ASSET_PATH" -C "$OUTPUT_DIR" + +EXTRACTED_ROOT="${OUTPUT_DIR}/${PACKAGE_DIR_NAME}" +[ -f "${EXTRACTED_ROOT}/manifest.json" ] || { + echo "manifest missing from extracted Z3 artifact: ${EXTRACTED_ROOT}" >&2 + exit 1 +} +[ -f "${EXTRACTED_ROOT}/lib/pkgconfig/z3.pc" ] || { + echo "pkg-config metadata missing from extracted Z3 artifact: ${EXTRACTED_ROOT}" >&2 + exit 1 +} + +printf '%s\n' "$EXTRACTED_ROOT"