diff --git a/.agents/skills/build-seekdb-python-wheel/SKILL.md b/.agents/skills/build-seekdb-python-wheel/SKILL.md new file mode 100644 index 0000000..da99af3 --- /dev/null +++ b/.agents/skills/build-seekdb-python-wheel/SKILL.md @@ -0,0 +1,160 @@ +--- +name: build-seekdb-python-wheel +description: Build and verify pylibseekdb Python wheels from seekdb source on remote Linux x86_64 or macOS arm64 hosts. Use when a user asks to package seekdb from a commit, tag, or branch; the workflow fetches source through GitHub SSH, runs the seekdb-bindings build, applies release-only compiler policy, handles known remote failures, verifies both supported Python ABI wheels, and returns checksums and artifact paths. +--- + +# Build seekdb Python wheels + +Use the scripts in `oceanbase/seekdb-bindings`. Keep every build in a new isolated directory on the requested remote host. + +## Collect required inputs + +Require all three inputs before changing the remote host: + +- `build_host`: SSH target, such as a host alias, IP, or `user@host`. +- `seekdb_ref`: seekdb commit hash, tag, or branch. +- `wheel_version`: requested PEP 440 Python package version. + +If any are absent, ask the user for the missing values. Do not invent, infer, or reuse defaults for them. + +Determine the target platform from the request and remote host. Supported targets are Linux x86_64 and macOS arm64; ask if the requested target remains ambiguous after host inspection. + +Also determine whether this is an official release package. If the request is ambiguous and this changes the build result, ask. Add `-DDEFAULT_LOG_LEVEL=OB_LOG_LEVEL_DBA_WARN` only for an official release or when the user explicitly requests it. Do not add it to ordinary development packages. + +## Prepare the remote build + +1. Verify non-interactive SSH access, platform and architecture, free disk, and Python 3.11 or newer. Linux also requires a usable Docker or Podman runtime; macOS requires Xcode command-line tools and a usable macOS SDK. +2. Verify GitHub SSH access with `ssh -T git@github.com`; its expected authentication success message still returns a nonzero status. +3. Create a new build directory instead of reusing, deleting, or modifying an existing checkout. +4. Clone `git@github.com:oceanbase/seekdb-bindings.git` through SSH. Record its resolved commit. Initialize only its top-level submodules: + + ```bash + git -C bindings -c url.git@github.com:.insteadOf=https://github.com/ \ + submodule update --init + ``` + + Do not use recursive submodule initialization: OpenSSL contains large optional test and fuzzing submodules that this wheel does not need. +5. Fetch seekdb through SSH into a separate checkout and detach at the resolved commit: + + ```bash + git -C seekdb-src init + git -C seekdb-src remote add origin git@github.com:oceanbase/seekdb.git + git -C seekdb-src fetch --depth 1 origin "$seekdb_ref" + git -C seekdb-src checkout --detach FETCH_HEAD + git -C seekdb-src rev-parse HEAD + ``` + +6. Create a dedicated host virtual environment. On pyenv hosts, resolve a real interpreter path such as `$(pyenv prefix 3.12)/bin/python`; do not trust a shim for a version that is not active. + +If a network operation fails with `ProxyError`, HTTP 503 after CONNECT, or an apparent proxy tunnel failure, confirm direct connectivity and retry with every common proxy variable removed: + +```bash +env -u http_proxy -u https_proxy -u HTTP_PROXY -u HTTPS_PROXY \ + -u all_proxy -u ALL_PROXY command ... +``` + +Use the repository's configured package mirror when appropriate. Do not copy a differently versioned cached download under an expected filename; retain checksum integrity. + +## Build on Linux x86_64 + +When the host checkout is mounted as `/seekdb`, current manylinux Git can reject it as dubious ownership. Check the isolated bindings script for an existing fix. If absent, add this line immediately after the inner container's `set -euo pipefail` in `scripts/build-seekdb-glibc228.sh`: + +```bash +git config --global --add safe.directory /seekdb +``` + +Limit this compatibility edit to the isolated build checkout. + +Run: + +```bash +./scripts/build-pylibseekdb-wheel.sh \ + --build-seekdb \ + --seekdb-git-url git@github.com:oceanbase/seekdb.git \ + --seekdb-git-ref "$seekdb_ref" \ + --seekdb-repo "$seekdb_checkout" \ + --wheel-version "$wheel_version" \ + --platform linux \ + --arch x86_64 +``` + +For an official release, append: + +```bash +--seekdb-cmake-arg -DDEFAULT_LOG_LEVEL=OB_LOG_LEVEL_DBA_WARN +``` + +Set `PYTHON` to the dedicated virtualenv interpreter. Preserve the full build log and monitor the remote process through completion. Expect `cp311-cp311` and `cp312-abi3` wheels tagged `manylinux_2_28_x86_64`. + +## Build on macOS arm64 + +Use low parallelism because the native seekdb build is memory-intensive: + +```bash +export SDKROOT="$(xcrun --sdk macosx --show-sdk-path)" +export CMAKE_BUILD_PARALLEL_LEVEL=2 +export MAKEFLAGS=-j2 +export TMPDIR=/tmp +``` + +`TMPDIR=/tmp` is required for tests that create a seekdb Unix socket. The default macOS per-user temporary path can make the socket path exceed the `AF_UNIX` limit and leave the test waiting even though seekdb started successfully. + +Current bindings scripts default both native macOS build stages to `-j2`. Verify +that the isolated checkout still caps these commands before starting: + +- seekdb's `./build.sh ... --make -j...` +- libseekdb's `cmake --build ... -j...` + +seekdb must not dynamically link RE2 or Abseil on macOS. An obsolete `-lre2` +link caused `delocate` to copy RE2's componentized Abseil closure even though +seekdb did not use it. The bindings script checks the staged seekdb, uses the +standard `delocate` repair, and rejects a wheel containing RE2/Abseil or more +than 20 bundled dylibs. If this check fails for an older seekdb ref, do not +rebuild RE2 inside bindings; ask whether to backport the seekdb source fix or +accept a dependency-heavy wheel. + +Run: + +```bash +./scripts/build-pylibseekdb-wheel.sh \ + --build-seekdb \ + --seekdb-git-url git@github.com:oceanbase/seekdb.git \ + --seekdb-git-ref "$seekdb_ref" \ + --wheel-version "$wheel_version" \ + --platform macos \ + --arch arm64 \ + --seekdb-cmake-arg -DCMAKE_OSX_SYSROOT="$SDKROOT" \ + --cmake-arg -DCMAKE_OSX_SYSROOT="$SDKROOT" +``` + +For an official release, append: + +```bash +--seekdb-cmake-arg -DDEFAULT_LOG_LEVEL=OB_LOG_LEVEL_DBA_WARN +``` + +Preserve the full build log and monitor free disk throughout the native build and wheel repair. Expect `cp311-cp311` and `cp312-abi3` wheels tagged `macosx_*_arm64`. + +### Diagnose the CMake 4.2 protoc failure + +If protobuf generation reports `Subprocess killed`, first run the exact vendor `protoc` command directly. If direct `protoc` and `/usr/bin/env ... protoc` succeed but the same command through `cmake -E env` fails, prefer a compatible CMake version. If that is unavailable, patch only the generated `deps/oblib/src/grpc/CMakeFiles/oblib_grpc.dir/build.make` in the isolated seekdb build so the failing command uses `/usr/bin/env` instead of `cmake -E env`, then retry the seekdb target incrementally at `-j1`. Record this workaround in the result. Do not apply it without reproducing this exact distinction. + +## Verify before delivery + +Do not report success merely because wheel files exist. Verify all applicable items: + +1. `git rev-parse HEAD` in seekdb matches the commit resolved from `seekdb_ref`, and record the bindings commit. +2. For an official release, `build_release/CMakeCache.txt` contains `DEFAULT_LOG_LEVEL=OB_LOG_LEVEL_DBA_WARN`; where possible, confirm the macro in an actual compiler command. +3. Confirm the staged seekdb is stripped and retain the debug-symbol path. On Linux, its maximum referenced GLIBC version must be no greater than `GLIBC_2.28`. +4. Confirm `python/pyproject.toml` returned to its original version after the temporary override. +5. Confirm wheel filenames and `METADATA` contain `wheel_version`. +6. Confirm cibuildwheel's built-in test passed for both wheels and the `cp312-abi3` wheel passed strict ABI3 auditing. +7. Run `scripts/verify-wheel.sh` explicitly against both wheels with real Python 3.11 and 3.12 interpreter paths. Also install and run `python/tests/connection_options_test.py` with the `cp312-abi3` wheel on Python 3.13 and 3.14. On macOS, keep `TMPDIR=/tmp` and restrict `PATH` per invocation when necessary so the script actually selects the intended interpreter. +8. On macOS, inspect repaired native dependencies with `delocate-listdeps` and `otool -L`; wheel binaries must not retain absolute Homebrew paths. + The repaired wheel must contain no RE2 or Abseil dylib. For seekdb commit + `aeadfb274c414ccb78ac502c4268b289372fac77`, expect 8 bundled dylibs and 11 + Mach-O files total. Treat more than 20 bundled dylibs as a packaging + regression unless a new direct dependency has been audited. +9. Compute SHA-256 for each wheel and copy the wheels to the requested local destination. If none was provided, use the current bindings checkout's ignored `build/wheelhouse/`. + +Report the platform and architecture, resolved seekdb and bindings commits, release-flag policy, SDK and concurrency on macOS, any workaround used, wheel names, sizes, checksums, verification results, local paths, remote build directory, build log, and debug-symbol path. diff --git a/.agents/skills/build-seekdb-python-wheel/agents/openai.yaml b/.agents/skills/build-seekdb-python-wheel/agents/openai.yaml new file mode 100644 index 0000000..5760981 --- /dev/null +++ b/.agents/skills/build-seekdb-python-wheel/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Build seekdb Python Wheels" + short_description: "Build and verify Linux and macOS seekdb wheels" + default_prompt: "Use $build-seekdb-python-wheel to build and verify seekdb Python wheels from a requested ref." diff --git a/.gitignore b/.gitignore index ea8ed15..f8fdf63 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ build/ build-*/ +!/.agents/skills/build-seekdb-python-wheel/ +!/.agents/skills/build-seekdb-python-wheel/** compile_commands.json *.o *.a diff --git a/README.md b/README.md index 8039547..fc68ad6 100644 --- a/README.md +++ b/README.md @@ -191,6 +191,12 @@ Set-ExecutionPolicy -Scope Process Bypass -Force `--wheel-version` temporarily patches `python/pyproject.toml` for the build (scikit-build-core does not read `CIBW_PROJECT_VERSION`) and restores it afterward. +On macOS, seekdb must not dynamically link RE2 or Abseil. Those unused links +previously caused `delocate` to bundle roughly 64 componentized Abseil dylibs. +The one-shot script now checks the staged seekdb before packaging, uses standard +`delocate`, and then rejects wheels containing RE2/Abseil or more than 20 +bundled dylibs. + Run `./scripts/build-pylibseekdb-wheel.sh --help` for all options. Debug symbols are written to `build/seekdb.debug` by default. **Linux: build seekdb inside manylinux.** The wheel must be ABI-compatible with `manylinux_2_28`, so the `seekdb` binary you point `SEEKDB_BIN` at also needs to be built against glibc 2.28. `scripts/build-seekdb-glibc228.sh` runs `seekdb`'s own `build.sh` inside the manylinux image: diff --git a/scripts/build-pylibseekdb-wheel.sh b/scripts/build-pylibseekdb-wheel.sh index d831116..cc56a3e 100755 --- a/scripts/build-pylibseekdb-wheel.sh +++ b/scripts/build-pylibseekdb-wheel.sh @@ -6,6 +6,7 @@ # 2. Stripping the binary while saving debug symbols # 3. Configuring and building libseekdb for wheel packaging # 4. Running cibuildwheel (manylinux_2_28 on Linux; delocate on macOS) +# 5. On macOS, rejecting accidental RE2/Abseil dependency expansion # # Supported platforms: # - Linux x86_64 / aarch64 (manylinux_2_28 only) @@ -63,6 +64,7 @@ SEEKDB_CMAKE_EXTRA=() PYTHON="${PYTHON:-}" USE_VENV="${USE_VENV:-1}" VENV_DIR="${VENV_DIR:-$ROOT/.venv}" +MACOS_BUILD_JOBS="${MACOS_BUILD_JOBS:-2}" PYPROJECT="$ROOT/python/pyproject.toml" PYPROJECT_BACKUP="" WHEEL_SCRIPT_ARGS=() @@ -105,7 +107,7 @@ Other: Environment variables: SEEKDB_BIN, SEEKDB_URL, SEEKDB_SHA256, SEEKDB_GIT_URL, SEEKDB_GIT_REF, SEEKDB_REPO, BUILD_SEEKDB, WHEEL_VERSION, PLATFORM, ARCH, OUTPUT_DIR, STRIP_SEEKDB, DEBUG_DIR, CIBW_BUILD, SKIP_TESTS, BUILD_TYPE, PYTHON, CONTAINER_RUNTIME, USE_VENV, VENV_DIR, -SEEKDB_CMAKE_ARGS +SEEKDB_CMAKE_ARGS, MACOS_BUILD_JOBS EOF } @@ -252,7 +254,7 @@ build_seekdb_macos() { ( cd "$src_dir" rm -rf "build_$BUILD_TYPE" - ./build.sh "$BUILD_TYPE" --init "${SEEKDB_CMAKE_EXTRA[@]}" --make -j"$(sysctl -n hw.ncpu 2>/dev/null || echo 4)" + ./build.sh "$BUILD_TYPE" --init "${SEEKDB_CMAKE_EXTRA[@]}" --make -j"$MACOS_BUILD_JOBS" ) local bin @@ -353,7 +355,13 @@ configure_bindings() { build_seekdb_library() { echo "==> building libseekdb" - cmake --build "$BUILD_DIR" --target seekdb -j"$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)" + local jobs + if [[ "$(uname -s)" == "Darwin" ]]; then + jobs="$MACOS_BUILD_JOBS" + else + jobs="$(nproc 2>/dev/null || echo 4)" + fi + cmake --build "$BUILD_DIR" --target seekdb -j"$jobs" [[ -f "$BUILD_DIR/seekdb" ]] || die "expected $BUILD_DIR/seekdb after build" [[ -f "$BUILD_DIR/libseekdb.so" || -f "$BUILD_DIR/libseekdb.dylib" ]] \ || die "libseekdb not found under $BUILD_DIR" @@ -374,6 +382,50 @@ restore_wheel_version() { rm -f "$PYPROJECT_BACKUP" } +verify_macos_seekdb_dependencies() { + local seekdb_bin="$1" + local forbidden + need_cmd otool + forbidden="$(otool -L "$seekdb_bin" | grep -E -i 'lib(re2|absl)[^/]*\.dylib' || true)" + if [[ -n "$forbidden" ]]; then + echo "$forbidden" >&2 + die "macOS seekdb dynamically links RE2/Abseil; remove the unused source dependency before packaging" + fi + echo "==> macOS seekdb has no dynamic RE2/Abseil dependency" >&2 +} + +verify_macos_wheel_dependencies() { + local wheel counts dylib_count forbidden_count + local -a wheels=("$OUTPUT_DIR"/*macosx*.whl) + [[ -e "${wheels[0]}" ]] || die "no macOS wheels found in $OUTPUT_DIR" + + for wheel in "${wheels[@]}"; do + counts="$("$PYTHON" -c ' +import sys +import zipfile + +with zipfile.ZipFile(sys.argv[1]) as archive: + dylibs = [ + name for name in archive.namelist() + if (name.startswith(".dylibs/") or "/.dylibs/" in name) and name.endswith(".dylib") + ] + forbidden = [ + name for name in dylibs + if "re2" in name.lower() or "absl" in name.lower() + ] +print(len(dylibs), len(forbidden)) +' "$wheel")" + read -r dylib_count forbidden_count <<< "$counts" + [[ "$dylib_count" =~ ^[0-9]+$ && "$forbidden_count" =~ ^[0-9]+$ ]] \ + || die "could not inspect bundled dylibs in $wheel" + echo "==> $(basename "$wheel"): $dylib_count bundled dylibs, $forbidden_count RE2/Abseil" >&2 + [[ "$forbidden_count" == "0" ]] \ + || die "macOS wheel unexpectedly bundles RE2/Abseil: $wheel" + (( dylib_count <= 20 )) \ + || die "macOS wheel bundles $dylib_count dylibs (limit: 20): $wheel" + done +} + run_cibuildwheel() { local platform="$1" mkdir -p "$OUTPUT_DIR" @@ -480,6 +532,8 @@ main() { WHEEL_SCRIPT_ARGS=("$@") parse_args "$@" load_seekdb_cmake_extra_from_env + [[ "$MACOS_BUILD_JOBS" =~ ^[1-9][0-9]*$ ]] \ + || die "MACOS_BUILD_JOBS must be a positive integer" local host_platform host_platform="$(detect_host_platform)" @@ -522,9 +576,15 @@ main() { raw_bin="$(resolve_seekdb_bin)" prepared_bin="$(prepare_seekdb_binary "$raw_bin")" + if [[ "$target_platform" == "macos" ]]; then + verify_macos_seekdb_dependencies "$prepared_bin" + fi configure_bindings "$prepared_bin" build_seekdb_library run_cibuildwheel "$target_platform" + if [[ "$target_platform" == "macos" ]]; then + verify_macos_wheel_dependencies + fi echo "" echo "done."