From b282bda9965f9b98a945beb6991fc17bdf3e13eb Mon Sep 17 00:00:00 2001 From: seasonyuu Date: Mon, 27 Apr 2026 14:45:06 +0800 Subject: [PATCH 1/5] Bundle Python runtime for Unix releases --- .github/workflows/ci.yml | 8 +- .github/workflows/release.yml | 9 +- README.md | 2 +- crates/dsview-core/src/lib.rs | 5 +- crates/dsview-core/tests/bundle_discovery.rs | 41 ++- crates/dsview-sys/build.rs | 57 +++- crates/dsview-sys/native/CMakeLists.txt | 10 + tools/package-bundle.py | 272 +++++++++++++++++-- tools/validate-bundle.py | 161 ++++++++++- 9 files changed, 529 insertions(+), 36 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3ea2218..f77775f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,13 @@ jobs: with: targets: ${{ matrix.target }} - - name: Set up Python + - name: Set up Python (Unix) + if: ${{ !contains(matrix.target, 'windows') }} + uses: actions/setup-python@v6 + with: + python-version: '3.13' + + - name: Set up Python (Windows) if: contains(matrix.target, 'windows') uses: actions/setup-python@v6 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9953a4e..2d0ef14 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -59,7 +59,13 @@ jobs: with: targets: ${{ matrix.target }} - - name: Set up Python + - name: Set up Python (Unix) + if: ${{ !contains(matrix.target, 'windows') }} + uses: actions/setup-python@v6 + with: + python-version: '3.13' + + - name: Set up Python (Windows) if: contains(matrix.target, 'windows') uses: actions/setup-python@v6 with: @@ -241,6 +247,7 @@ jobs: - `runtime/` directory with the platform-specific runtime library - `decode-runtime/` directory with the platform-specific decode runtime library - `decoders/` directory with bundled DSView decoder scripts + - `python/` directory with the bundled Python runtime used by protocol decoding - `resources/` directory with DSLogic Plus firmware and bitstreams ### Installation diff --git a/README.md b/README.md index d8dba18..9d2d4f0 100644 --- a/README.md +++ b/README.md @@ -288,7 +288,7 @@ The native integration is intentionally isolated behind Rust layers to keep unsa ### Packaging -Release bundles are created using `tools/package-bundle.py` and validated with `tools/validate-bundle.py`. CI uses these Python helpers to ensure consistent bundle structure across all platforms, including the bundled capture runtime, decode runtime, firmware resources, and decoder scripts, without relying on unstable Cargo script support. +Release bundles are created using `tools/package-bundle.py` and validated with `tools/validate-bundle.py`. CI uses these Python helpers to ensure consistent bundle structure across all platforms, including the bundled capture runtime, decode runtime, Python runtime, firmware resources, and decoder scripts, without relying on unstable Cargo script support. ## License diff --git a/crates/dsview-core/src/lib.rs b/crates/dsview-core/src/lib.rs index e77eb66..d42b0a0 100644 --- a/crates/dsview-core/src/lib.rs +++ b/crates/dsview-core/src/lib.rs @@ -1332,10 +1332,7 @@ impl DecodeDiscoveryPaths { developer_decoder_dir() }; - let python_home = if cfg!(windows) - && runtime_library == bundled_runtime - && bundled_python_home.is_dir() - { + let python_home = if runtime_library == bundled_runtime && bundled_python_home.is_dir() { Some(bundled_python_home) } else { None diff --git a/crates/dsview-core/tests/bundle_discovery.rs b/crates/dsview-core/tests/bundle_discovery.rs index 614deca..765dc8b 100644 --- a/crates/dsview-core/tests/bundle_discovery.rs +++ b/crates/dsview-core/tests/bundle_discovery.rs @@ -2,8 +2,10 @@ use std::fs; use std::path::PathBuf; use std::time::{SystemTime, UNIX_EPOCH}; -use dsview_core::RuntimeDiscoveryPaths; -use dsview_sys::{runtime_library_name, source_runtime_library_path}; +use dsview_core::{DecodeDiscoveryPaths, RuntimeDiscoveryPaths}; +use dsview_sys::{ + decode_runtime_library_name, runtime_library_name, source_runtime_library_path, +}; fn temp_dir(name: &str) -> PathBuf { let unique = SystemTime::now() @@ -22,6 +24,11 @@ fn write_valid_resources(dir: &std::path::Path) { fs::write(dir.join("DSLogicPlus-pgl12.bin"), b"bin").unwrap(); } +fn write_valid_decoder_dir(dir: &std::path::Path) { + fs::create_dir_all(dir).unwrap(); + fs::write(dir.join("spi.py"), b"# decoder placeholder").unwrap(); +} + #[test] fn bundle_defaults_resolve_from_executable_layout() { let exe_dir = temp_dir("bundle-defaults"); @@ -177,6 +184,36 @@ fn bundle_layout_matches_packaging_contract() { assert!(resource_dir.join("DSLogicPlus-pgl12.bin").is_file()); } +#[test] +fn decode_bundle_discovery_uses_bundled_python_home_on_all_platforms() { + let exe_dir = temp_dir("decode-python-home"); + let decode_runtime_dir = exe_dir.join("decode-runtime"); + let decoder_dir = exe_dir.join("decoders"); + let python_home = exe_dir.join("python"); + fs::create_dir_all(&decode_runtime_dir).unwrap(); + fs::create_dir_all(&python_home).unwrap(); + fs::write( + decode_runtime_dir.join(decode_runtime_library_name()), + b"decode runtime", + ) + .unwrap(); + write_valid_decoder_dir(&decoder_dir); + + let paths = DecodeDiscoveryPaths::from_executable_dir( + &exe_dir, + None::<&std::path::Path>, + None::<&std::path::Path>, + ) + .expect("decode bundle-relative discovery should succeed"); + + assert_eq!( + paths.runtime_library, + decode_runtime_dir.join(decode_runtime_library_name()) + ); + assert_eq!(paths.decoder_dir, decoder_dir); + assert_eq!(paths.python_home, Some(python_home)); +} + #[test] fn runtime_library_name_helper_is_consistent() { // Verify the helper returns the same value across calls diff --git a/crates/dsview-sys/build.rs b/crates/dsview-sys/build.rs index 084a79b..908b6d3 100644 --- a/crates/dsview-sys/build.rs +++ b/crates/dsview-sys/build.rs @@ -72,6 +72,11 @@ struct SourceRuntimeArtifacts { decode_library_path: PathBuf, } +struct PythonDiscovery { + executable: PathBuf, + root: PathBuf, +} + fn main() { let target = TargetInfo::from_cargo_env(); let manifest_dir = @@ -334,9 +339,8 @@ fn build_source_runtimes( } } - if !command_available("python3") { - return Err("python3 is not available".to_string()); - } + let python = + discover_python().ok_or_else(|| "python3 or python is not available".to_string())?; let capture_library_path = build_source_runtime_variant( repo_root, @@ -344,6 +348,7 @@ fn build_source_runtimes( target, "source-runtime-build", "capture", + None, )?; let decode_library_path = build_source_runtime_variant( repo_root, @@ -351,6 +356,7 @@ fn build_source_runtimes( target, "source-decode-runtime-build", "decode", + Some(&python), )?; Ok(SourceRuntimeArtifacts { @@ -365,6 +371,7 @@ fn build_source_runtime_variant( target: &TargetInfo, build_dir_name: &str, runtime_kind: &str, + python: Option<&PythonDiscovery>, ) -> Result { let out_dir = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR is set by Cargo")); let build_dir = out_dir.join(build_dir_name); @@ -392,6 +399,11 @@ fn build_source_runtime_variant( configure .arg("-DDSVIEW_BUILD_CAPTURE_RUNTIME=OFF") .arg("-DDSVIEW_BUILD_DECODE_RUNTIME=ON"); + if let Some(python) = python { + configure + .arg(format!("-DPython3_EXECUTABLE={}", python.executable.display())) + .arg(format!("-DPython3_ROOT_DIR={}", python.root.display())); + } } _ => return Err(format!("unknown source runtime kind `{runtime_kind}`")), } @@ -627,6 +639,45 @@ fn command_available(command: &str) -> bool { .unwrap_or(false) } +fn discover_python() -> Option { + let mut candidates = Vec::new(); + if let Ok(explicit) = env::var("PYTHON") { + candidates.push(explicit); + } + candidates.extend(["python3".to_string(), "python".to_string()]); + + for candidate in candidates { + if !command_available(&candidate) { + continue; + } + + let Ok(output) = Command::new(&candidate) + .arg("-c") + .arg("import sys; print(sys.executable); print(sys.base_exec_prefix)") + .output() + else { + continue; + }; + if !output.status.success() { + continue; + } + + let Ok(stdout) = String::from_utf8(output.stdout) else { + continue; + }; + let mut lines = stdout.lines(); + let Some(executable) = lines.next().map(PathBuf::from) else { + continue; + }; + let Some(root) = lines.next().map(PathBuf::from) else { + continue; + }; + return Some(PythonDiscovery { executable, root }); + } + + None +} + fn pkg_config_has(package: &str) -> bool { let Some(pkg_config) = pkg_config_command() else { return false; diff --git a/crates/dsview-sys/native/CMakeLists.txt b/crates/dsview-sys/native/CMakeLists.txt index ca2a3b9..b43a0fc 100644 --- a/crates/dsview-sys/native/CMakeLists.txt +++ b/crates/dsview-sys/native/CMakeLists.txt @@ -226,6 +226,16 @@ if(DSVIEW_BUILD_DECODE_RUNTIME) set_target_properties(dsview_decode_runtime PROPERTIES LINK_FLAGS "/DEF:${DSVIEW_NATIVE_WINDOWS_ROOT}/dsview_decode_runtime.def" ) + elseif(APPLE) + set_target_properties(dsview_decode_runtime PROPERTIES + BUILD_RPATH "@loader_path/../python/lib" + INSTALL_RPATH "@loader_path/../python/lib" + ) + else() + set_target_properties(dsview_decode_runtime PROPERTIES + BUILD_RPATH "$ORIGIN/../python/lib" + INSTALL_RPATH "$ORIGIN/../python/lib" + ) endif() set_target_properties(dsview_decode_runtime PROPERTIES OUTPUT_NAME dsview_decode_runtime) diff --git a/tools/package-bundle.py b/tools/package-bundle.py index 22d45db..26f8b99 100644 --- a/tools/package-bundle.py +++ b/tools/package-bundle.py @@ -6,8 +6,12 @@ import argparse import os +import shutil +import subprocess import sys +import sysconfig import tarfile +import tempfile from pathlib import Path @@ -68,10 +72,10 @@ def should_skip_decoder_path(path: Path) -> bool: def should_skip_python_path(path: Path) -> bool: + skipped_directory_names = {"__pycache__", "site-packages", "dist-packages"} return ( - any(part == "__pycache__" for part in path.parts) + any(part in skipped_directory_names for part in path.parts) or path.suffix in {".pyc", ".pyo"} - or path.parts[:1] == ("Lib",) and len(path.parts) > 1 and path.parts[1] == "site-packages" ) @@ -106,6 +110,18 @@ def vcpkg_triplet_for_target(target: str) -> str: raise ValueError(f"unsupported Windows target: {target}") +def is_windows_target(target: str) -> bool: + return "windows" in target + + +def is_darwin_target(target: str) -> bool: + return "darwin" in target or "macos" in target + + +def python_version_dir() -> str: + return f"python{sys.version_info.major}.{sys.version_info.minor}" + + def windows_runtime_dependency_dir(target: str) -> Path: vcpkg_root = os.environ.get("VCPKG_ROOT") or os.environ.get("DSVIEW_VCPKG_ROOT") if not vcpkg_root: @@ -171,6 +187,203 @@ def add_windows_python_runtime(archive: tarfile.TarFile, archive_root: str) -> N add_file(archive, stdlib_zip, f"{archive_root}/python/{stdlib_zip.name}") +def is_python_dependency(path_or_name: str) -> bool: + name = Path(path_or_name).name.lower() + return name == "python" or name.startswith("libpython") + + +def command_stdout(command: list[str]) -> str: + try: + result = subprocess.run( + command, + check=False, + capture_output=True, + text=True, + ) + except FileNotFoundError: + return "" + if result.returncode != 0: + return "" + return result.stdout + + +def linux_linked_python_dependencies(library: Path) -> list[tuple[str, str]]: + dependencies: list[tuple[str, str]] = [] + output = command_stdout(["readelf", "-d", str(library)]) + for line in output.splitlines(): + if "Shared library:" not in line or "[" not in line or "]" not in line: + continue + dependency = line.split("[", 1)[1].split("]", 1)[0] + if is_python_dependency(dependency): + dependencies.append((dependency, Path(dependency).name)) + return dependencies + + +def macos_linked_python_dependencies(library: Path) -> list[tuple[str, str]]: + dependencies: list[tuple[str, str]] = [] + output = command_stdout(["otool", "-L", str(library)]) + for line in output.splitlines()[1:]: + dependency = line.strip().split(" ", 1)[0] + if is_python_dependency(dependency): + dependencies.append((dependency, Path(dependency).name)) + return dependencies + + +def linked_python_dependencies(target: str, library: Path) -> list[tuple[str, str]]: + if is_darwin_target(target): + return macos_linked_python_dependencies(library) + if is_windows_target(target): + return [] + return linux_linked_python_dependencies(library) + + +def python_library_search_dirs() -> list[Path]: + candidates: list[Path] = [] + + def add(path: Path | None) -> None: + if path and path.is_dir() and path not in candidates: + candidates.append(path) + + for variable in ("LIBDIR", "LIBPL"): + value = sysconfig.get_config_var(variable) + if value: + add(Path(value)) + + base_exec_prefix = Path(sys.base_exec_prefix) + add(base_exec_prefix / "lib") + + framework = sysconfig.get_config_var("PYTHONFRAMEWORK") + framework_prefix = sysconfig.get_config_var("PYTHONFRAMEWORKPREFIX") + framework_version = sysconfig.get_config_var("VERSION") or ( + f"{sys.version_info.major}.{sys.version_info.minor}" + ) + if framework and framework_prefix: + framework_root = Path(framework_prefix) / f"{framework}.framework" / "Versions" / framework_version + add(framework_root) + add(framework_root / "lib") + + return candidates + + +def python_shared_library_candidates( + target: str, decode_runtime: Path +) -> list[tuple[Path, str]]: + entries: dict[str, Path] = {} + + def add_candidate(path: Path, archive_name: str | None = None) -> None: + if not path.is_file(): + return + destination_name = archive_name or path.name + if destination_name.endswith(".a"): + return + # Store the real library bytes for each soname/install-name alias the + # loader may request. This avoids absolute or broken symlinks in tarballs. + entries[destination_name] = path.resolve() + + linked_dependencies = linked_python_dependencies(target, decode_runtime) + for dependency, dependency_name in linked_dependencies: + dependency_path = Path(dependency) + if dependency_path.is_absolute(): + add_candidate(dependency_path, dependency_name) + + expected_names = { + value + for value in ( + sysconfig.get_config_var("INSTSONAME"), + sysconfig.get_config_var("LDLIBRARY"), + ) + if value + } + expected_names.update(name for _, name in linked_dependencies) + + for directory in python_library_search_dirs(): + for name in sorted(expected_names): + add_candidate(directory / name, name) + + for pattern in ( + f"libpython{sys.version_info.major}.{sys.version_info.minor}*.so*", + f"libpython{sys.version_info.major}.{sys.version_info.minor}*.dylib*", + "Python", + ): + for path in sorted(directory.glob(pattern)): + if is_python_dependency(path.name): + add_candidate(path) + + return sorted((source, name) for name, source in entries.items()) + + +def add_unix_python_runtime( + archive: tarfile.TarFile, + archive_root: str, + target: str, + decode_runtime: Path, +) -> None: + shared_libraries = python_shared_library_candidates(target, decode_runtime) + if not shared_libraries: + raise FileNotFoundError( + "No Unix Python shared runtime library was found to bundle" + ) + + required_library_names = { + name for _, name in linked_python_dependencies(target, decode_runtime) + } + bundled_library_names = {archive_name for _, archive_name in shared_libraries} + missing_library_names = sorted(required_library_names - bundled_library_names) + if missing_library_names: + raise FileNotFoundError( + "Bundled Python runtime is missing linked libraries: " + + ", ".join(missing_library_names) + ) + + for library, archive_name in shared_libraries: + add_file(archive, library, f"{archive_root}/python/lib/{archive_name}") + + stdlib_dir = Path(sysconfig.get_path("stdlib")) + ensure_directory(stdlib_dir, "Python stdlib directory") + stdlib_destination = f"{archive_root}/python/lib/{python_version_dir()}" + add_directory_filtered(archive, stdlib_dir, stdlib_destination, should_skip_python_path) + + platstdlib_value = sysconfig.get_path("platstdlib") + if platstdlib_value: + platstdlib_dir = Path(platstdlib_value) + if ( + platstdlib_dir.is_dir() + and platstdlib_dir.resolve() != stdlib_dir.resolve() + ): + add_directory_filtered( + archive, + platstdlib_dir, + stdlib_destination, + should_skip_python_path, + ) + + +def prepare_macos_decode_runtime(source: Path, staging_dir: Path) -> Path: + staged_runtime = staging_dir / source.name + shutil.copy2(source, staged_runtime) + + python_dependencies = macos_linked_python_dependencies(source) + if not python_dependencies: + return staged_runtime + + install_name_tool = shutil.which("install_name_tool") + if not install_name_tool: + raise FileNotFoundError( + "install_name_tool is required to make macOS Python runtime links bundle-relative" + ) + + for dependency, dependency_name in python_dependencies: + replacement = f"@loader_path/../python/lib/{dependency_name}" + if dependency == replacement: + continue + subprocess.run( + [install_name_tool, "-change", dependency, replacement, str(staged_runtime)], + check=True, + ) + + return staged_runtime + + def main() -> int: args = parse_args() @@ -181,7 +394,7 @@ def main() -> int: ensure_directory(args.decoder_dir, "Decoder scripts directory") archive_root = f"dsview-cli-{args.version}-{args.target}" - exe_name = "dsview-cli.exe" if "windows" in args.target else "dsview-cli" + exe_name = "dsview-cli.exe" if is_windows_target(args.target) else "dsview-cli" required_resources = [ "DSLogicPlus.fw", @@ -191,28 +404,43 @@ def main() -> int: ] args.output.parent.mkdir(parents=True, exist_ok=True) - with tarfile.open(args.output, "w:gz") as archive: - add_file(archive, args.exe, f"{archive_root}/{exe_name}") - add_file(archive, args.runtime, f"{archive_root}/runtime/{args.runtime.name}") - add_file( - archive, - args.decode_runtime, - f"{archive_root}/decode-runtime/{args.decode_runtime.name}", - ) - add_directory(archive, args.decoder_dir, f"{archive_root}/decoders") - if "windows" in args.target: - for dependency in windows_dependency_dlls(args.target, args.runtime.name): - add_file( + with tempfile.TemporaryDirectory(prefix="dsview-package-") as staging: + decode_runtime = args.decode_runtime + if is_darwin_target(args.target): + decode_runtime = prepare_macos_decode_runtime( + args.decode_runtime, + Path(staging), + ) + + with tarfile.open(args.output, "w:gz") as archive: + add_file(archive, args.exe, f"{archive_root}/{exe_name}") + add_file(archive, args.runtime, f"{archive_root}/runtime/{args.runtime.name}") + add_file( + archive, + decode_runtime, + f"{archive_root}/decode-runtime/{args.decode_runtime.name}", + ) + add_directory(archive, args.decoder_dir, f"{archive_root}/decoders") + if is_windows_target(args.target): + for dependency in windows_dependency_dlls(args.target, args.runtime.name): + add_file( + archive, + dependency, + f"{archive_root}/{dependency.name}", + ) + add_windows_python_runtime(archive, archive_root) + else: + add_unix_python_runtime( archive, - dependency, - f"{archive_root}/{dependency.name}", + archive_root, + args.target, + args.decode_runtime, ) - add_windows_python_runtime(archive, archive_root) - for resource_name in required_resources: - resource_path = args.resources / resource_name - if resource_path.exists(): - add_file(archive, resource_path, f"{archive_root}/resources/{resource_name}") + for resource_name in required_resources: + resource_path = args.resources / resource_name + if resource_path.exists(): + add_file(archive, resource_path, f"{archive_root}/resources/{resource_name}") print(f"Bundle created: {args.output}") return 0 diff --git a/tools/validate-bundle.py b/tools/validate-bundle.py index 1a0f9bc..74b9dc4 100644 --- a/tools/validate-bundle.py +++ b/tools/validate-bundle.py @@ -6,6 +6,7 @@ import argparse import json +import os import shutil import subprocess import sys @@ -51,19 +52,169 @@ def expected_windows_runtime_dependencies() -> list[str]: ] +def is_windows_target(target: str) -> bool: + return "windows" in target + + +def is_darwin_target(target: str) -> bool: + return "darwin" in target or "macos" in target + + +def is_python_dependency(path_or_name: str) -> bool: + name = Path(path_or_name).name.lower() + return name == "python" or name.startswith("libpython") + + def require_exists(path: Path, label: str) -> None: if not path.exists(): raise FileNotFoundError(f"{label} not found: {path}") +def command_stdout(command: list[str]) -> str: + try: + result = subprocess.run( + command, + check=False, + capture_output=True, + text=True, + ) + except FileNotFoundError as error: + raise RuntimeError(f"required validation command not found: {command[0]}") from error + if result.returncode != 0: + raise RuntimeError( + f"validation command failed with exit code {result.returncode}: {' '.join(command)}" + ) + return result.stdout + + +def linux_dynamic_values(library: Path) -> dict[str, list[str]]: + values = {"NEEDED": [], "RPATH": [], "RUNPATH": []} + output = command_stdout(["readelf", "-d", str(library)]) + for line in output.splitlines(): + for tag in values: + if f"({tag})" in line and "[" in line and "]" in line: + values[tag].append(line.split("[", 1)[1].split("]", 1)[0]) + return values + + +def macos_linked_libraries(library: Path) -> list[str]: + output = command_stdout(["otool", "-L", str(library)]) + return [ + line.strip().split(" ", 1)[0] + for line in output.splitlines()[1:] + if line.strip() + ] + + +def macos_rpaths(library: Path) -> list[str]: + output = command_stdout(["otool", "-l", str(library)]) + rpaths: list[str] = [] + in_rpath_command = False + for line in output.splitlines(): + stripped = line.strip() + if stripped == "cmd LC_RPATH": + in_rpath_command = True + continue + if in_rpath_command and stripped.startswith("path "): + rpaths.append(stripped.split(" ", 2)[1]) + in_rpath_command = False + return rpaths + + +def validate_unix_python_runtime( + bundle_root: Path, + target: str, + decode_runtime: Path, +) -> None: + python_home = bundle_root / "python" + python_lib = python_home / "lib" + require_exists(python_home, "Bundled Python runtime directory") + require_exists(python_lib, "Bundled Python lib directory") + + stdlib_dirs = [ + path for path in python_lib.glob("python*") if path.is_dir() + ] + if not any((path / "encodings").is_dir() for path in stdlib_dirs): + raise FileNotFoundError( + "Bundled Python stdlib is missing encodings/ under python/lib/pythonX.Y" + ) + + if is_darwin_target(target): + if not (python_lib / "Python").is_file() and not any( + python_lib.glob("libpython*.dylib*") + ): + raise FileNotFoundError("Bundled macOS Python dynamic library was not found") + validate_macos_python_links(bundle_root, decode_runtime) + else: + if not any(python_lib.glob("libpython*.so*")): + raise FileNotFoundError("Bundled Linux libpython shared library was not found") + validate_linux_python_links(bundle_root, decode_runtime) + + +def validate_linux_python_links(bundle_root: Path, decode_runtime: Path) -> None: + dynamic_values = linux_dynamic_values(decode_runtime) + python_dependencies = [ + dependency + for dependency in dynamic_values["NEEDED"] + if is_python_dependency(dependency) + ] + if not python_dependencies: + raise RuntimeError("Decode runtime does not declare a libpython dependency") + + python_lib = bundle_root / "python" / "lib" + for dependency in python_dependencies: + require_exists( + python_lib / Path(dependency).name, + "Bundled Linux libpython dependency", + ) + + runpath = ":".join(dynamic_values["RPATH"] + dynamic_values["RUNPATH"]) + if "$ORIGIN/../python/lib" not in runpath: + raise RuntimeError( + "Linux decode runtime RUNPATH must include $ORIGIN/../python/lib" + ) + + +def validate_macos_python_links(bundle_root: Path, decode_runtime: Path) -> None: + python_dependencies = [ + dependency + for dependency in macos_linked_libraries(decode_runtime) + if is_python_dependency(dependency) + ] + if not python_dependencies: + raise RuntimeError("Decode runtime does not declare a Python dynamic library dependency") + + rpaths = macos_rpaths(decode_runtime) + if "@loader_path/../python/lib" not in rpaths: + raise RuntimeError( + "macOS decode runtime LC_RPATH must include @loader_path/../python/lib" + ) + + python_lib = bundle_root / "python" / "lib" + for dependency in python_dependencies: + dependency_name = Path(dependency).name + require_exists(python_lib / dependency_name, "Bundled macOS Python dependency") + if dependency.startswith("@loader_path/../python/lib/"): + continue + if dependency.startswith("@rpath/"): + continue + raise RuntimeError( + f"macOS Python dependency is not bundle-relative: {dependency}" + ) + + def run_smoke_test( exe_path: Path, args: list[str], description: str ) -> subprocess.CompletedProcess[str]: + env = os.environ.copy() + env.pop("PYTHONHOME", None) + env.pop("PYTHONPATH", None) result = subprocess.run( [str(exe_path), *args], check=False, capture_output=True, text=True, + env=env, ) if result.returncode != 0: if result.stdout: @@ -96,10 +247,10 @@ def main() -> int: if not bundle_root.is_dir(): raise RuntimeError("Archive root is not a directory") - exe_name = "dsview-cli.exe" if "windows" in args.target else "dsview-cli" + exe_name = "dsview-cli.exe" if is_windows_target(args.target) else "dsview-cli" exe_path = bundle_root / exe_name require_exists(exe_path, "Executable") - if "windows" in args.target: + if is_windows_target(args.target): for dependency in expected_windows_runtime_dependencies(): require_exists(bundle_root / dependency, "Windows runtime dependency") if not any(bundle_root.glob("python*.dll")): @@ -124,6 +275,12 @@ def main() -> int: decode_runtime_dir / decode_runtime_library_name(args.target), "Decode runtime library", ) + if not is_windows_target(args.target): + validate_unix_python_runtime( + bundle_root, + args.target, + decode_runtime_dir / decode_runtime_library_name(args.target), + ) decoders_dir = bundle_root / "decoders" if not decoders_dir.is_dir(): From bc30a0fbbfe882eb8d54c3d5f4a41cd562894724 Mon Sep 17 00:00:00 2001 From: seasonyuu Date: Mon, 27 Apr 2026 15:00:37 +0800 Subject: [PATCH 2/5] Fix macOS bundled Python linkage --- crates/dsview-sys/native/CMakeLists.txt | 4 +- tools/package-bundle.py | 65 +++++++++++++++++++------ tools/validate-bundle.py | 27 ++++++---- 3 files changed, 69 insertions(+), 27 deletions(-) diff --git a/crates/dsview-sys/native/CMakeLists.txt b/crates/dsview-sys/native/CMakeLists.txt index b43a0fc..8cf94b7 100644 --- a/crates/dsview-sys/native/CMakeLists.txt +++ b/crates/dsview-sys/native/CMakeLists.txt @@ -228,8 +228,8 @@ if(DSVIEW_BUILD_DECODE_RUNTIME) ) elseif(APPLE) set_target_properties(dsview_decode_runtime PROPERTIES - BUILD_RPATH "@loader_path/../python/lib" - INSTALL_RPATH "@loader_path/../python/lib" + BUILD_RPATH "@loader_path/../python;@loader_path/../python/lib" + INSTALL_RPATH "@loader_path/../python;@loader_path/../python/lib" ) else() set_target_properties(dsview_decode_runtime PROPERTIES diff --git a/tools/package-bundle.py b/tools/package-bundle.py index 26f8b99..fe827e1 100644 --- a/tools/package-bundle.py +++ b/tools/package-bundle.py @@ -67,6 +67,17 @@ def add_file(archive: tarfile.TarFile, source: Path, destination: str) -> None: archive.add(source, arcname=destination, recursive=False) +def add_regular_file(archive: tarfile.TarFile, source: Path, destination: str) -> None: + resolved = source.resolve() + stat = resolved.stat() + info = tarfile.TarInfo(destination) + info.size = stat.st_size + info.mode = stat.st_mode & 0o777 + info.mtime = stat.st_mtime + with resolved.open("rb") as file: + archive.addfile(info, file) + + def should_skip_decoder_path(path: Path) -> bool: return any(part == "__pycache__" for part in path.parts) or path.suffix in {".pyc", ".pyo"} @@ -237,6 +248,20 @@ def linked_python_dependencies(target: str, library: Path) -> list[tuple[str, st return linux_linked_python_dependencies(library) +def macos_python_archive_path(dependency_or_source: str, fallback_name: str) -> str: + parts = Path(dependency_or_source).parts + for index, part in enumerate(parts): + if part.endswith(".framework"): + return Path(*parts[index:]).as_posix() + return f"lib/{fallback_name}" + + +def python_archive_path(target: str, dependency_or_source: str, fallback_name: str) -> str: + if is_darwin_target(target): + return macos_python_archive_path(dependency_or_source, fallback_name) + return f"lib/{fallback_name}" + + def python_library_search_dirs() -> list[Path]: candidates: list[Path] = [] @@ -270,21 +295,24 @@ def python_shared_library_candidates( ) -> list[tuple[Path, str]]: entries: dict[str, Path] = {} - def add_candidate(path: Path, archive_name: str | None = None) -> None: + def add_candidate(path: Path, archive_path: str | None = None) -> None: if not path.is_file(): return - destination_name = archive_name or path.name - if destination_name.endswith(".a"): + destination_path = archive_path or python_archive_path(target, str(path), path.name) + if destination_path.endswith(".a"): return # Store the real library bytes for each soname/install-name alias the # loader may request. This avoids absolute or broken symlinks in tarballs. - entries[destination_name] = path.resolve() + entries[destination_path] = path.resolve() linked_dependencies = linked_python_dependencies(target, decode_runtime) for dependency, dependency_name in linked_dependencies: dependency_path = Path(dependency) if dependency_path.is_absolute(): - add_candidate(dependency_path, dependency_name) + add_candidate( + dependency_path, + python_archive_path(target, dependency, dependency_name), + ) expected_names = { value @@ -298,7 +326,10 @@ def add_candidate(path: Path, archive_name: str | None = None) -> None: for directory in python_library_search_dirs(): for name in sorted(expected_names): - add_candidate(directory / name, name) + add_candidate( + directory / name, + python_archive_path(target, str(directory / name), name), + ) for pattern in ( f"libpython{sys.version_info.major}.{sys.version_info.minor}*.so*", @@ -309,7 +340,7 @@ def add_candidate(path: Path, archive_name: str | None = None) -> None: if is_python_dependency(path.name): add_candidate(path) - return sorted((source, name) for name, source in entries.items()) + return sorted((source, archive_path) for archive_path, source in entries.items()) def add_unix_python_runtime( @@ -324,19 +355,20 @@ def add_unix_python_runtime( "No Unix Python shared runtime library was found to bundle" ) - required_library_names = { - name for _, name in linked_python_dependencies(target, decode_runtime) + required_library_paths = { + python_archive_path(target, dependency, name) + for dependency, name in linked_python_dependencies(target, decode_runtime) } - bundled_library_names = {archive_name for _, archive_name in shared_libraries} - missing_library_names = sorted(required_library_names - bundled_library_names) - if missing_library_names: + bundled_library_paths = {archive_path for _, archive_path in shared_libraries} + missing_library_paths = sorted(required_library_paths - bundled_library_paths) + if missing_library_paths: raise FileNotFoundError( "Bundled Python runtime is missing linked libraries: " - + ", ".join(missing_library_names) + + ", ".join(missing_library_paths) ) - for library, archive_name in shared_libraries: - add_file(archive, library, f"{archive_root}/python/lib/{archive_name}") + for library, archive_path in shared_libraries: + add_regular_file(archive, library, f"{archive_root}/python/{archive_path}") stdlib_dir = Path(sysconfig.get_path("stdlib")) ensure_directory(stdlib_dir, "Python stdlib directory") @@ -373,7 +405,8 @@ def prepare_macos_decode_runtime(source: Path, staging_dir: Path) -> Path: ) for dependency, dependency_name in python_dependencies: - replacement = f"@loader_path/../python/lib/{dependency_name}" + archive_path = macos_python_archive_path(dependency, dependency_name) + replacement = f"@loader_path/../python/{archive_path}" if dependency == replacement: continue subprocess.run( diff --git a/tools/validate-bundle.py b/tools/validate-bundle.py index 74b9dc4..540aa20 100644 --- a/tools/validate-bundle.py +++ b/tools/validate-bundle.py @@ -65,6 +65,14 @@ def is_python_dependency(path_or_name: str) -> bool: return name == "python" or name.startswith("libpython") +def macos_python_archive_path(dependency: str) -> str: + dependency_path = Path(dependency) + for index, part in enumerate(dependency_path.parts): + if part.endswith(".framework"): + return Path(*dependency_path.parts[index:]).as_posix() + return f"lib/{dependency_path.name}" + + def require_exists(path: Path, label: str) -> None: if not path.exists(): raise FileNotFoundError(f"{label} not found: {path}") @@ -140,9 +148,10 @@ def validate_unix_python_runtime( ) if is_darwin_target(target): - if not (python_lib / "Python").is_file() and not any( - python_lib.glob("libpython*.dylib*") - ): + framework_binaries = list( + python_home.glob("*.framework/Versions/*/Python") + ) + if not framework_binaries and not any(python_lib.glob("libpython*.dylib*")): raise FileNotFoundError("Bundled macOS Python dynamic library was not found") validate_macos_python_links(bundle_root, decode_runtime) else: @@ -185,16 +194,16 @@ def validate_macos_python_links(bundle_root: Path, decode_runtime: Path) -> None raise RuntimeError("Decode runtime does not declare a Python dynamic library dependency") rpaths = macos_rpaths(decode_runtime) - if "@loader_path/../python/lib" not in rpaths: + if "@loader_path/../python" not in rpaths and "@loader_path/../python/lib" not in rpaths: raise RuntimeError( - "macOS decode runtime LC_RPATH must include @loader_path/../python/lib" + "macOS decode runtime LC_RPATH must include a bundled Python path" ) - python_lib = bundle_root / "python" / "lib" + python_home = bundle_root / "python" for dependency in python_dependencies: - dependency_name = Path(dependency).name - require_exists(python_lib / dependency_name, "Bundled macOS Python dependency") - if dependency.startswith("@loader_path/../python/lib/"): + archive_path = macos_python_archive_path(dependency) + require_exists(python_home / archive_path, "Bundled macOS Python dependency") + if dependency == f"@loader_path/../python/{archive_path}": continue if dependency.startswith("@rpath/"): continue From 264c79ff4f52cc6d827f335bd67fd1312f5e8fd9 Mon Sep 17 00:00:00 2001 From: seasonyuu Date: Mon, 27 Apr 2026 15:05:02 +0800 Subject: [PATCH 3/5] Preserve macOS Python framework in bundles --- tools/package-bundle.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/tools/package-bundle.py b/tools/package-bundle.py index fe827e1..b92f6b8 100644 --- a/tools/package-bundle.py +++ b/tools/package-bundle.py @@ -256,6 +256,13 @@ def macos_python_archive_path(dependency_or_source: str, fallback_name: str) -> return f"lib/{fallback_name}" +def macos_framework_root(path: Path) -> Path | None: + for index, part in enumerate(path.parts): + if part.endswith(".framework"): + return Path(*path.parts[: index + 1]) + return None + + def python_archive_path(target: str, dependency_or_source: str, fallback_name: str) -> str: if is_darwin_target(target): return macos_python_archive_path(dependency_or_source, fallback_name) @@ -301,9 +308,7 @@ def add_candidate(path: Path, archive_path: str | None = None) -> None: destination_path = archive_path or python_archive_path(target, str(path), path.name) if destination_path.endswith(".a"): return - # Store the real library bytes for each soname/install-name alias the - # loader may request. This avoids absolute or broken symlinks in tarballs. - entries[destination_path] = path.resolve() + entries[destination_path] = path linked_dependencies = linked_python_dependencies(target, decode_runtime) for dependency, dependency_name in linked_dependencies: @@ -367,7 +372,26 @@ def add_unix_python_runtime( + ", ".join(missing_library_paths) ) + bundled_by_framework: set[str] = set() + if is_darwin_target(target): + framework_roots: dict[str, Path] = {} + for library, archive_path in shared_libraries: + framework_root = macos_framework_root(library) + if framework_root is not None and framework_root.is_dir(): + framework_roots[framework_root.name] = framework_root + bundled_by_framework.add(archive_path) + + for framework_name, framework_root in sorted(framework_roots.items()): + add_directory_filtered( + archive, + framework_root, + f"{archive_root}/python/{framework_name}", + should_skip_python_path, + ) + for library, archive_path in shared_libraries: + if archive_path in bundled_by_framework: + continue add_regular_file(archive, library, f"{archive_root}/python/{archive_path}") stdlib_dir = Path(sysconfig.get_path("stdlib")) From 5aa33b9f0f2374e6b8aaabd2009d75e1f0550cfd Mon Sep 17 00:00:00 2001 From: seasonyuu Date: Mon, 27 Apr 2026 15:48:00 +0800 Subject: [PATCH 4/5] Reduce bundled Python runtime size --- tools/package-bundle.py | 336 ++++++++++++++++++++++++++++++++++++--- tools/validate-bundle.py | 50 ++++++ 2 files changed, 361 insertions(+), 25 deletions(-) diff --git a/tools/package-bundle.py b/tools/package-bundle.py index b92f6b8..7073795 100644 --- a/tools/package-bundle.py +++ b/tools/package-bundle.py @@ -6,13 +6,14 @@ import argparse import os +import posixpath import shutil import subprocess import sys import sysconfig import tarfile import tempfile -from pathlib import Path +from pathlib import Path, PurePosixPath def parse_args() -> argparse.Namespace: @@ -83,10 +84,26 @@ def should_skip_decoder_path(path: Path) -> bool: def should_skip_python_path(path: Path) -> bool: - skipped_directory_names = {"__pycache__", "site-packages", "dist-packages"} + skipped_directory_names = { + "__pycache__", + "site-packages", + "dist-packages", + "test", + "tests", + "ensurepip", + "idlelib", + "tkinter", + "turtledemo", + } return ( - any(part in skipped_directory_names for part in path.parts) - or path.suffix in {".pyc", ".pyo"} + any( + part in skipped_directory_names or part.startswith("config-") + for part in path.parts + ) + or path.suffix in {".a", ".pyc", ".pyo"} + or path.name.startswith("_test") + or path.name.startswith("_tkinter.") + or path.name.startswith("_ctypes_test.") ) @@ -111,6 +128,35 @@ def add_directory_filtered( archive.add(child, arcname=f"{destination}/{child.relative_to(source)}", recursive=False) +def copy_directory_filtered( + source: Path, + destination: Path, + skip_predicate, +) -> None: + destination.mkdir(parents=True, exist_ok=True) + for child in sorted(source.rglob("*")): + relative_path = child.relative_to(source) + if skip_predicate(relative_path): + continue + + copied_path = destination / relative_path + if child.is_symlink(): + resolved = child.resolve() + if not resolved.is_file(): + continue + copied_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(resolved, copied_path) + elif child.is_dir(): + copied_path.mkdir(parents=True, exist_ok=True) + elif child.is_file(): + copied_path.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(child, copied_path) + + +def should_never_skip(_: Path) -> bool: + return False + + def vcpkg_triplet_for_target(target: str) -> str: if "windows" not in target: raise ValueError(f"target is not a Windows target: {target}") @@ -218,6 +264,26 @@ def command_stdout(command: list[str]) -> str: return result.stdout +def checked_command_stdout(command: list[str]) -> str: + try: + result = subprocess.run( + command, + check=True, + capture_output=True, + text=True, + ) + except FileNotFoundError as error: + raise FileNotFoundError(f"required command not found: {command[0]}") from error + return result.stdout + + +def require_tool(name: str) -> str: + tool = shutil.which(name) + if not tool: + raise FileNotFoundError(f"{name} is required for macOS Python runtime bundling") + return tool + + def linux_linked_python_dependencies(library: Path) -> list[tuple[str, str]]: dependencies: list[tuple[str, str]] = [] output = command_stdout(["readelf", "-d", str(library)]) @@ -240,6 +306,15 @@ def macos_linked_python_dependencies(library: Path) -> list[tuple[str, str]]: return dependencies +def macos_linked_libraries(library: Path) -> list[str]: + output = checked_command_stdout(["otool", "-L", str(library)]) + return [ + line.strip().split(" ", 1)[0] + for line in output.splitlines()[1:] + if line.strip() + ] + + def linked_python_dependencies(target: str, library: Path) -> list[tuple[str, str]]: if is_darwin_target(target): return macos_linked_python_dependencies(library) @@ -256,13 +331,27 @@ def macos_python_archive_path(dependency_or_source: str, fallback_name: str) -> return f"lib/{fallback_name}" -def macos_framework_root(path: Path) -> Path | None: +def macos_framework_version_root(path: Path) -> Path | None: for index, part in enumerate(path.parts): - if part.endswith(".framework"): - return Path(*path.parts[: index + 1]) + if not part.endswith(".framework"): + continue + + framework_root = Path(*path.parts[: index + 1]) + if index + 2 < len(path.parts) and path.parts[index + 1] == "Versions": + return Path(*path.parts[: index + 3]) + + framework_version = sysconfig.get_config_var("VERSION") or python_version_dir().removeprefix("python") + version_root = framework_root / "Versions" / framework_version + return version_root if version_root.is_dir() else None + return None +def macos_framework_archive_version_root(version_root: Path) -> PurePosixPath: + framework_root = version_root.parent.parent + return PurePosixPath(framework_root.name) / "Versions" / version_root.name + + def python_archive_path(target: str, dependency_or_source: str, fallback_name: str) -> str: if is_darwin_target(target): return macos_python_archive_path(dependency_or_source, fallback_name) @@ -348,6 +437,212 @@ def add_candidate(path: Path, archive_path: str | None = None) -> None: return sorted((source, archive_path) for archive_path, source in entries.items()) +def copy_file_to_staging(source: Path, destination: Path) -> None: + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source.resolve(), destination) + + +def macos_loader_path_reference(from_archive_path: PurePosixPath, to_archive_path: PurePosixPath) -> str: + relative_path = posixpath.relpath( + to_archive_path.as_posix(), + from_archive_path.parent.as_posix(), + ) + return f"@loader_path/{relative_path}" + + +def macos_framework_dependency_replacement( + dependency: str, + from_archive_path: PurePosixPath, + framework_version_root: Path, + framework_archive_version_root: PurePosixPath, +) -> str | None: + dependency_path = Path(dependency) + if not dependency_path.is_absolute(): + return None + + relative_dependency: PurePosixPath | None = None + try: + relative_dependency = PurePosixPath( + dependency_path.relative_to(framework_version_root).as_posix() + ) + except ValueError: + framework_name = framework_version_root.parent.parent.name + parts = dependency_path.parts + for index, part in enumerate(parts): + if ( + part == framework_name + and index + 2 < len(parts) + and parts[index + 1] == "Versions" + and parts[index + 2] == framework_version_root.name + ): + relative_dependency = PurePosixPath(*parts[index + 3 :]) + break + + if relative_dependency is None: + return None + + bundled_dependency = ( + PurePosixPath("python") + / framework_archive_version_root + / relative_dependency + ) + return macos_loader_path_reference(from_archive_path, bundled_dependency) + + +def rewrite_macos_framework_references( + file: Path, + archive_path: PurePosixPath, + framework_version_root: Path, + framework_archive_version_root: PurePosixPath, +) -> None: + install_name_tool = require_tool("install_name_tool") + + if file.name == "Python" or file.suffix == ".dylib": + install_name = macos_loader_path_reference(archive_path, archive_path) + subprocess.run([install_name_tool, "-id", install_name, str(file)], check=True) + + for dependency in macos_linked_libraries(file): + replacement = macos_framework_dependency_replacement( + dependency, + archive_path, + framework_version_root, + framework_archive_version_root, + ) + if not replacement or dependency == replacement: + continue + + subprocess.run( + [install_name_tool, "-change", dependency, replacement, str(file)], + check=True, + ) + + +def macos_framework_support_libraries(version_root: Path) -> list[Path]: + lib_dir = version_root / "lib" + if not lib_dir.is_dir(): + return [] + return sorted(path for path in lib_dir.glob("*.dylib") if path.is_file() and not path.is_symlink()) + + +def add_macos_framework_runtime_to_staging( + staging_root: Path, + version_root: Path, +) -> PurePosixPath: + framework_archive_version_root = macos_framework_archive_version_root(version_root) + framework_binary = version_root / "Python" + ensure_file(framework_binary, "macOS Python framework binary") + + framework_binary_archive_path = ( + PurePosixPath("python") / framework_archive_version_root / "Python" + ) + staged_framework_binary = staging_root / framework_binary_archive_path + copy_file_to_staging(framework_binary, staged_framework_binary) + rewrite_macos_framework_references( + staged_framework_binary, + framework_binary_archive_path, + version_root, + framework_archive_version_root, + ) + + for library in macos_framework_support_libraries(version_root): + relative_library = library.relative_to(version_root) + library_archive_path = ( + PurePosixPath("python") + / framework_archive_version_root + / PurePosixPath(relative_library.as_posix()) + ) + staged_library = staging_root / library_archive_path + copy_file_to_staging(library, staged_library) + rewrite_macos_framework_references( + staged_library, + library_archive_path, + version_root, + framework_archive_version_root, + ) + + return framework_archive_version_root + + +def rewrite_macos_stdlib_extension_references( + staging_root: Path, + stdlib_destination: Path, + framework_version_root: Path, + framework_archive_version_root: PurePosixPath, +) -> None: + lib_dynload = stdlib_destination / "lib-dynload" + if not lib_dynload.is_dir(): + return + + for extension in sorted(lib_dynload.glob("*.so")): + extension_archive_path = PurePosixPath( + (PurePosixPath("python") / extension.relative_to(staging_root / "python")).as_posix() + ) + rewrite_macos_framework_references( + extension, + extension_archive_path, + framework_version_root, + framework_archive_version_root, + ) + + +def add_macos_python_runtime( + archive: tarfile.TarFile, + archive_root: str, + shared_libraries: list[tuple[Path, str]], +) -> None: + framework_versions: dict[Path, PurePosixPath] = {} + + with tempfile.TemporaryDirectory(prefix="dsview-macos-python-") as staging: + staging_root = Path(staging) + staged_python_root = staging_root / "python" + + for library, archive_path in shared_libraries: + version_root = macos_framework_version_root(library) + if version_root is not None and version_root.is_dir(): + if version_root not in framework_versions: + framework_versions[version_root] = add_macos_framework_runtime_to_staging( + staging_root, + version_root, + ) + continue + + copied_library = staged_python_root / PurePosixPath(archive_path) + copy_file_to_staging(library, copied_library) + + stdlib_dir = Path(sysconfig.get_path("stdlib")) + ensure_directory(stdlib_dir, "Python stdlib directory") + stdlib_destination = staged_python_root / "lib" / python_version_dir() + copy_directory_filtered(stdlib_dir, stdlib_destination, should_skip_python_path) + + platstdlib_value = sysconfig.get_path("platstdlib") + if platstdlib_value: + platstdlib_dir = Path(platstdlib_value) + if ( + platstdlib_dir.is_dir() + and platstdlib_dir.resolve() != stdlib_dir.resolve() + ): + copy_directory_filtered( + platstdlib_dir, + stdlib_destination, + should_skip_python_path, + ) + + for version_root, framework_archive_version_root in framework_versions.items(): + rewrite_macos_stdlib_extension_references( + staging_root, + stdlib_destination, + version_root, + framework_archive_version_root, + ) + + add_directory_filtered( + archive, + staged_python_root, + f"{archive_root}/python", + should_never_skip, + ) + + def add_unix_python_runtime( archive: tarfile.TarFile, archive_root: str, @@ -372,26 +667,17 @@ def add_unix_python_runtime( + ", ".join(missing_library_paths) ) - bundled_by_framework: set[str] = set() - if is_darwin_target(target): - framework_roots: dict[str, Path] = {} - for library, archive_path in shared_libraries: - framework_root = macos_framework_root(library) - if framework_root is not None and framework_root.is_dir(): - framework_roots[framework_root.name] = framework_root - bundled_by_framework.add(archive_path) + required_shared_libraries = [ + (library, archive_path) + for library, archive_path in shared_libraries + if archive_path in required_library_paths + ] - for framework_name, framework_root in sorted(framework_roots.items()): - add_directory_filtered( - archive, - framework_root, - f"{archive_root}/python/{framework_name}", - should_skip_python_path, - ) + if is_darwin_target(target): + add_macos_python_runtime(archive, archive_root, required_shared_libraries) + return - for library, archive_path in shared_libraries: - if archive_path in bundled_by_framework: - continue + for library, archive_path in required_shared_libraries: add_regular_file(archive, library, f"{archive_root}/python/{archive_path}") stdlib_dir = Path(sysconfig.get_path("stdlib")) diff --git a/tools/validate-bundle.py b/tools/validate-bundle.py index 540aa20..65522b9 100644 --- a/tools/validate-bundle.py +++ b/tools/validate-bundle.py @@ -146,6 +146,7 @@ def validate_unix_python_runtime( raise FileNotFoundError( "Bundled Python stdlib is missing encodings/ under python/lib/pythonX.Y" ) + validate_python_stdlib_is_slim(stdlib_dirs) if is_darwin_target(target): framework_binaries = list( @@ -154,12 +155,24 @@ def validate_unix_python_runtime( if not framework_binaries and not any(python_lib.glob("libpython*.dylib*")): raise FileNotFoundError("Bundled macOS Python dynamic library was not found") validate_macos_python_links(bundle_root, decode_runtime) + validate_macos_python_runtime_is_slim(python_home) + validate_macos_python_extension_links(python_home) else: if not any(python_lib.glob("libpython*.so*")): raise FileNotFoundError("Bundled Linux libpython shared library was not found") validate_linux_python_links(bundle_root, decode_runtime) +def validate_python_stdlib_is_slim(stdlib_dirs: list[Path]) -> None: + excluded_directories = ("test", "ensurepip", "idlelib", "tkinter", "turtledemo") + for stdlib_dir in stdlib_dirs: + for directory_name in excluded_directories: + if (stdlib_dir / directory_name).exists(): + raise RuntimeError( + f"Bundled Python stdlib includes unnecessary {directory_name}/ directory" + ) + + def validate_linux_python_links(bundle_root: Path, decode_runtime: Path) -> None: dynamic_values = linux_dynamic_values(decode_runtime) python_dependencies = [ @@ -212,6 +225,43 @@ def validate_macos_python_links(bundle_root: Path, decode_runtime: Path) -> None ) +def validate_macos_python_runtime_is_slim(python_home: Path) -> None: + for framework in python_home.glob("*.framework"): + versions_dir = framework / "Versions" + require_exists(versions_dir, "Bundled macOS Python framework Versions directory") + version_dirs = [ + path + for path in versions_dir.iterdir() + if path.is_dir() and path.name != "Current" + ] + if len(version_dirs) != 1: + raise RuntimeError( + f"Bundled macOS Python framework must include exactly one version, found {len(version_dirs)}" + ) + + version_dir = version_dirs[0] + if (version_dir / "Resources" / "English.lproj" / "Documentation").exists(): + raise RuntimeError("Bundled macOS Python framework includes documentation") + if any((version_dir / "lib").glob("python*")): + raise RuntimeError( + "Bundled macOS Python framework duplicates the stdlib under Versions/*/lib" + ) + + +def validate_macos_python_extension_links(python_home: Path) -> None: + libraries = [ + *python_home.glob("lib/python*/lib-dynload/*.so"), + *python_home.glob("*.framework/Versions/*/lib/*.dylib"), + *python_home.glob("*.framework/Versions/*/Python"), + ] + for library in libraries: + for dependency in macos_linked_libraries(library): + if dependency.startswith("/Library/Frameworks/Python.framework/"): + raise RuntimeError( + f"Bundled macOS Python library has non-relocatable dependency: {library.name} -> {dependency}" + ) + + def run_smoke_test( exe_path: Path, args: list[str], description: str ) -> subprocess.CompletedProcess[str]: From 954859bbd6476c7d52236bea830d4648af373e7d Mon Sep 17 00:00:00 2001 From: seasonyuu Date: Mon, 27 Apr 2026 15:52:29 +0800 Subject: [PATCH 5/5] Ad-hoc sign bundled macOS Python runtime --- tools/package-bundle.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/tools/package-bundle.py b/tools/package-bundle.py index 7073795..2fa9d0a 100644 --- a/tools/package-bundle.py +++ b/tools/package-bundle.py @@ -284,6 +284,11 @@ def require_tool(name: str) -> str: return tool +def codesign_macos_file(path: Path) -> None: + codesign = require_tool("codesign") + subprocess.run([codesign, "--force", "--sign", "-", str(path)], check=True) + + def linux_linked_python_dependencies(library: Path) -> list[tuple[str, str]]: dependencies: list[tuple[str, str]] = [] output = command_stdout(["readelf", "-d", str(library)]) @@ -496,12 +501,19 @@ def rewrite_macos_framework_references( framework_archive_version_root: PurePosixPath, ) -> None: install_name_tool = require_tool("install_name_tool") + changed = False if file.name == "Python" or file.suffix == ".dylib": install_name = macos_loader_path_reference(archive_path, archive_path) subprocess.run([install_name_tool, "-id", install_name, str(file)], check=True) + changed = True + seen_dependencies: set[str] = set() for dependency in macos_linked_libraries(file): + if dependency in seen_dependencies: + continue + seen_dependencies.add(dependency) + replacement = macos_framework_dependency_replacement( dependency, archive_path, @@ -515,6 +527,10 @@ def rewrite_macos_framework_references( [install_name_tool, "-change", dependency, replacement, str(file)], check=True, ) + changed = True + + if changed: + codesign_macos_file(file) def macos_framework_support_libraries(version_root: Path) -> list[Path]: @@ -708,13 +724,15 @@ def prepare_macos_decode_runtime(source: Path, staging_dir: Path) -> Path: if not python_dependencies: return staged_runtime - install_name_tool = shutil.which("install_name_tool") - if not install_name_tool: - raise FileNotFoundError( - "install_name_tool is required to make macOS Python runtime links bundle-relative" - ) + install_name_tool = require_tool("install_name_tool") + changed = False + seen_dependencies: set[str] = set() for dependency, dependency_name in python_dependencies: + if dependency in seen_dependencies: + continue + seen_dependencies.add(dependency) + archive_path = macos_python_archive_path(dependency, dependency_name) replacement = f"@loader_path/../python/{archive_path}" if dependency == replacement: @@ -723,6 +741,10 @@ def prepare_macos_decode_runtime(source: Path, staging_dir: Path) -> Path: [install_name_tool, "-change", dependency, replacement, str(staged_runtime)], check=True, ) + changed = True + + if changed: + codesign_macos_file(staged_runtime) return staged_runtime