From 02f7a03c45254b2ae056e1637d5c00a19223cdbb Mon Sep 17 00:00:00 2001 From: Nicolas Mattia Date: Tue, 3 Dec 2024 13:55:06 +0100 Subject: [PATCH 01/28] use hermetic toolchains --- MODULE.bazel | 19 ++++++++++++++++++- bazel/conf/.bazelrc.build | 5 ----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 6367c2370ad8..be9975ee907d 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -7,13 +7,13 @@ module( ) # General Bazel helpers - bazel_dep(name = "bazel_skylib", version = "1.7.1") bazel_dep(name = "aspect_bazel_lib", version = "2.9.0") # CC dependencies (for C libs like miracl-core, etc) bazel_dep(name = "rules_cc", version = "0.0.13") bazel_dep(name = "platforms", version = "0.0.10") +bazel_dep(name = "hermetic_cc_toolchain", version = "3.1.1") # configure/make dependencies bazel_dep(name = "rules_foreign_cc", version = "0.12.0") @@ -886,3 +886,20 @@ mainnet_versions( name = "mainnet_versions", path = "//:mainnet-subnet-revisions.json", ) + +# ... + +toolchains = use_extension("@hermetic_cc_toolchain//toolchain:ext.bzl", "toolchains") +use_repo(toolchains, "zig_sdk") + +register_toolchains( + # if no `--platform` is specified, these toolchains will be used for + # (linux,darwin,windows)x(amd64,arm64) + "@zig_sdk//toolchain:linux_amd64_gnu.2.28", + + # wasm/wasi toolchains + "@zig_sdk//toolchain:wasip1_wasm", + + # These toolchains are only registered locally. + dev_dependency = True, +) diff --git a/bazel/conf/.bazelrc.build b/bazel/conf/.bazelrc.build index d487c0ff7cd0..b5ec09739b33 100644 --- a/bazel/conf/.bazelrc.build +++ b/bazel/conf/.bazelrc.build @@ -126,11 +126,6 @@ run:sandbox_fuzzing --run_under="ASAN_OPTIONS=detect_leaks=0:allow_user_segv_han query --ui_event_filters=-info,-debug --noshow_progress cquery --ui_event_filters=-info,-debug --noshow_progress -# The default value makes rules_rust pick a dummy toolchain that breaks -# our canister that depend on C deps -# https://github.com/bazelbuild/rules_rust/issues/2764 -common --noincompatible_enable_cc_toolchain_resolution - # This is disabled by default on bazel 7+ some of our targets choke # on this (not yet clear why) common --remote_download_all From dfceca59d09dd4d53b3a4f0d3734f7fedcff314a Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Wed, 11 Dec 2024 19:11:07 +0000 Subject: [PATCH 02/28] Fix infogetty Wrap the dev container's libsystemd. This should be OK because infogetty is only used inside our images, which currently run a version that is equal to or more recent than that of the build container. --- WORKSPACE.bazel | 24 ++++++++++++++++++++++++ cpp/BUILD.bazel | 1 + 2 files changed, 25 insertions(+) diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index 36de3c9816c5..72c04681aed1 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -266,3 +266,27 @@ sns_quill(name = "sns_quill") load("//bazel:idl2json.bzl", "idl_to_json") idl_to_json(name = "idl2json") + +# Use libsystemd from the host environment +# Ideally this is pulled hermetically, but as this is only used by infogetty +# within IC images, we have less compatibility to worry about, and packaging it +# this way is easier. +new_local_repository( + name = "libsystemd", + build_file_content = """ +cc_import( + name = "libsystemd-internal", + hdrs = glob(["include/systemd/*.h"]), + interface_library = "lib/x86_64-linux-gnu/libsystemd.so", + system_provided = True, + visibility = ["//visibility:private"], +) +# Use an extra cc_library to hide the depth of the include folder +cc_library( + name = "libsystemd", + includes = ["include"], + deps = ["libsystemd-internal"], + visibility = ["//visibility:public"], +)""", + path = "/usr", +) diff --git a/cpp/BUILD.bazel b/cpp/BUILD.bazel index 6f0dad10bd30..d90f295ccb3f 100644 --- a/cpp/BUILD.bazel +++ b/cpp/BUILD.bazel @@ -14,6 +14,7 @@ cc_binary( target_compatible_with = [ "@platforms//os:linux", ], + deps = ["@libsystemd"], ) genrule( From d963bd3012522966a3b341c7befcea882362fa9c Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Fri, 17 Jan 2025 23:36:35 +0000 Subject: [PATCH 03/28] Fix wasm! Patch hermetic_cc_toolchain to support a freestanding wasm toolchain. --- MODULE.bazel | 12 +++-- bazel/cc_rs.patch | 18 ++++++++ bazel/conf/.bazelrc.build | 5 +++ bazel/external_crates.bzl | 4 ++ bazel/hermetic_cc_toolchain.patch | 73 +++++++++++++++++++++++++++++++ 5 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 bazel/cc_rs.patch create mode 100644 bazel/hermetic_cc_toolchain.patch diff --git a/MODULE.bazel b/MODULE.bazel index be9975ee907d..e658818ff609 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -14,6 +14,11 @@ bazel_dep(name = "aspect_bazel_lib", version = "2.9.0") bazel_dep(name = "rules_cc", version = "0.0.13") bazel_dep(name = "platforms", version = "0.0.10") bazel_dep(name = "hermetic_cc_toolchain", version = "3.1.1") +single_version_override( + module_name = "hermetic_cc_toolchain", + patch_strip = 1, + patches = ["//bazel:hermetic_cc_toolchain.patch"], +) # configure/make dependencies bazel_dep(name = "rules_foreign_cc", version = "0.12.0") @@ -887,8 +892,7 @@ mainnet_versions( path = "//:mainnet-subnet-revisions.json", ) -# ... - +# Set up hermetic cc toolchains for binaries and canisters toolchains = use_extension("@hermetic_cc_toolchain//toolchain:ext.bzl", "toolchains") use_repo(toolchains, "zig_sdk") @@ -897,8 +901,8 @@ register_toolchains( # (linux,darwin,windows)x(amd64,arm64) "@zig_sdk//toolchain:linux_amd64_gnu.2.28", - # wasm/wasi toolchains - "@zig_sdk//toolchain:wasip1_wasm", + # wasm toolchains + "@zig_sdk//toolchain:none_wasm", # These toolchains are only registered locally. dev_dependency = True, diff --git a/bazel/cc_rs.patch b/bazel/cc_rs.patch new file mode 100644 index 000000000000..b1a1c1bc192f --- /dev/null +++ b/bazel/cc_rs.patch @@ -0,0 +1,18 @@ +# Adjust target naming to match what zig expects. +diff --git a/src/lib.rs b/src/lib.rs +index f75b951..8e770a7 100644 +--- a/src/lib.rs ++++ b/src/lib.rs +@@ -2091,6 +2091,12 @@ impl Build { + target.versioned_llvm_target(None) + }; + ++ let llvm_target = match llvm_target.clone() { ++ std::borrow::Cow::Borrowed("x86_64-unknown-linux-gnu") => "x86_64-linux-gnu", ++ std::borrow::Cow::Borrowed("wasm32-unknown-unknown") => "wasm32-freestanding-musl", ++ _other => &llvm_target, ++ }; ++ + // Pass `--target` with the LLVM target to properly + // configure Clang even when cross-compiling. + cmd.push_cc_arg(format!("--target={llvm_target}").into()); diff --git a/bazel/conf/.bazelrc.build b/bazel/conf/.bazelrc.build index b5ec09739b33..b47dbc52196f 100644 --- a/bazel/conf/.bazelrc.build +++ b/bazel/conf/.bazelrc.build @@ -138,3 +138,8 @@ common --noexperimental_inmemory_dotd_files # convention would also benefit. If the test does not support this, this "almost # certainly" does no harm. test --test_env=CLICOLOR_FORCE=true + +# Always compile with zig cc +# https://github.com/uber/hermetic_cc_toolchain/tree/8e68b7221ca72c10268338eb1734530438e5ccb7?tab=readme-ov-file#use-case-always-compile-with-zig-cc +build --action_env BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 +build --sandbox_add_mount_pair=/tmp diff --git a/bazel/external_crates.bzl b/bazel/external_crates.bzl index 69894ace0988..cfca948e1cdd 100644 --- a/bazel/external_crates.bzl +++ b/bazel/external_crates.bzl @@ -27,6 +27,10 @@ def external_crates_repository(name, cargo_lockfile, lockfile, sanitizers_enable "canbench": [crate.annotation( gen_binaries = True, )], + "cc": [crate.annotation( + patch_args = ["-p1"], + patches = ["@@//bazel:cc_rs.patch"], + )], "libssh2-sys": [crate.annotation( # Patch for determinism issues patch_args = ["-p1"], diff --git a/bazel/hermetic_cc_toolchain.patch b/bazel/hermetic_cc_toolchain.patch new file mode 100644 index 000000000000..afbed7c19d0c --- /dev/null +++ b/bazel/hermetic_cc_toolchain.patch @@ -0,0 +1,73 @@ +# Extend hermetic_cc_toolchain with a wasm target that does not use wasi. +diff --git a/MODULE.bazel b/MODULE.bazel +index 409d626..1265298 100644 +--- a/MODULE.bazel ++++ b/MODULE.bazel +@@ -49,6 +49,7 @@ register_toolchains( + "@zig_sdk//libc_aware/toolchain:linux_arm64_musl", + # wasm/wasi toolchains + "@zig_sdk//toolchain:wasip1_wasm", ++ "@zig_sdk//toolchain:none_wasm", + + # These toolchains are only registered locally. + dev_dependency = True, +diff --git a/toolchain/platform/defs.bzl b/toolchain/platform/defs.bzl +index d4a8344..faafc5b 100644 +--- a/toolchain/platform/defs.bzl ++++ b/toolchain/platform/defs.bzl +@@ -16,6 +16,7 @@ def declare_platforms(): + + # We can support GOARCH=wasm32 after https://github.com/golang/go/issues/63131 + declare_platform("wasm", "wasm32", "wasi", "wasip1") ++ declare_platform("wasm", "wasm32", "none", "none") + + def declare_libc_aware_platforms(): + # create @zig_sdk//{os}_{arch}_platform entries with zig and go conventions +diff --git a/toolchain/private/defs.bzl b/toolchain/private/defs.bzl +index 716a3a3..b88d082 100644 +--- a/toolchain/private/defs.bzl ++++ b/toolchain/private/defs.bzl +@@ -49,6 +49,7 @@ def target_structs(): + for glibc in _GLIBCS: + ret.append(_target_linux_gnu(gocpu, zigcpu, glibc)) + ret.append(_target_wasm()) ++ ret.append(_target_wasm_no_wasi()) + return ret + + def _target_macos(gocpu, zigcpu): +@@ -222,3 +223,22 @@ def _target_wasm(): + ld_zig_subcmd = "wasm-ld", + artifact_name_patterns = [], + ) ++ ++def _target_wasm_no_wasi(): ++ return struct( ++ gotarget = "none_wasm", ++ zigtarget = "wasm32-freestanding-musl", ++ includes = [] + _INCLUDE_TAIL, ++ linkopts = [], ++ dynamic_library_linkopts = [], ++ supports_dynamic_linker = False, ++ copts = [], ++ libc = "musl", ++ bazel_target_cpu = "wasm32", ++ constraint_values = [ ++ "@platforms//os:none", ++ "@platforms//cpu:wasm32", ++ ], ++ ld_zig_subcmd = "wasm-ld", ++ artifact_name_patterns = [], ++ ) +diff --git a/toolchain/zig-wrapper.zig b/toolchain/zig-wrapper.zig +index d1d59f9..5e2984b 100644 +--- a/toolchain/zig-wrapper.zig ++++ b/toolchain/zig-wrapper.zig +@@ -283,7 +283,7 @@ fn getRunMode(self_exe: []const u8, self_base_noexe: []const u8) error{BadParent + return error.BadParent; + + const got_os = it.next() orelse return error.BadParent; +- if (mem.indexOf(u8, "linux,macos,windows,wasi", got_os) == null) ++ if (mem.indexOf(u8, "linux,macos,windows,wasi,freestanding", got_os) == null) + return error.BadParent; + + // ABI triple is too much of a moving target From 148e7f0b055596dd2c7682577f9dda22bd6b25c9 Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Thu, 30 Jan 2025 23:50:22 +0000 Subject: [PATCH 04/28] Fix libfuzzer! Build libfuzzer ourselves to avoid a stdlib mismatch from the hermetic cc toolchains, and adjust to zig conventions. --- MODULE.bazel | 23 +++++++++++++++++++++++ bazel/fuzz_testing.bzl | 15 ++++++++------- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index e658818ff609..33ac109369e4 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -907,3 +907,26 @@ register_toolchains( # These toolchains are only registered locally. dev_dependency = True, ) + +# Build libfuzzer to ensure compatibility with our toolchain, and so that it +# can be optionally linked by the fuzzers. +http_archive( + name = "libfuzzer", + build_file_content = """ +load("@rules_cc//cc:defs.bzl", "cc_library") + +cc_library( + name = "fuzzer", + srcs = glob(["*.cpp"]), + hdrs = glob(["*.h"]), + additional_compiler_inputs = glob(["*.def"]), + cxxopts = ["-g", "-O2", "-fno-omit-frame-pointer", "-std=c++17"], + linkstatic = True, + visibility = ["//visibility:public"], +) + +""", + integrity = "sha256-CLw4JzN3fdo8liWeNzL/lsHfmNBHDE+FsWMnTq5of08=", + strip_prefix = "llvm-project-llvmorg-20.1.0/compiler-rt/lib/fuzzer", + urls = ["https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-20.1.0.tar.gz"], +) diff --git a/bazel/fuzz_testing.bzl b/bazel/fuzz_testing.bzl index 205dc5a3c65f..e83fbeba3407 100644 --- a/bazel/fuzz_testing.bzl +++ b/bazel/fuzz_testing.bzl @@ -32,6 +32,12 @@ DEFAULT_SANITIZERS = [ # This flag will be used by third party crates and internal rust_libraries during fuzzing DEFAULT_RUSTC_FLAGS_FOR_FUZZING = DEFAULT_RUSTC_FLAGS + DEFAULT_SANITIZERS +# zig doesn't like how rustc pushes the sanitizers, so do it ourselves. +ZIG_LINK_ARGS = [ + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a", +] + def rust_fuzz_test_binary(name, srcs, rustc_flags = [], sanitizers = [], crate_features = [], proc_macro_deps = [], deps = [], allow_main = False, **kwargs): """Wrapper for the rust_binary to compile a fuzzing rust_binary @@ -53,18 +59,12 @@ def rust_fuzz_test_binary(name, srcs, rustc_flags = [], sanitizers = [], crate_f # This would only work inside the devcontainer if allow_main: - FUZZER_LIB = [ - "-Clink-arg=/usr/lib/llvm-18/lib/clang/18/lib/linux/libclang_rt.fuzzer_no_main-x86_64.a", - ] TAGS = ["sandbox_libfuzzer"] else: # default - FUZZER_LIB = [ - "-Clink-arg=/usr/lib/llvm-18/lib/clang/18/lib/linux/libclang_rt.fuzzer-x86_64.a", - ] TAGS = [] - RUSTC_FLAGS_LIBFUZZER = DEFAULT_RUSTC_FLAGS + FUZZER_LIB + RUSTC_FLAGS_LIBFUZZER = DEFAULT_RUSTC_FLAGS + ZIG_LINK_ARGS + ["-Clink-arg=$(location @libfuzzer//:fuzzer)"] kwargs.setdefault("testonly", True) @@ -75,6 +75,7 @@ def rust_fuzz_test_binary(name, srcs, rustc_flags = [], sanitizers = [], crate_f crate_features = crate_features + ["fuzzing"], proc_macro_deps = proc_macro_deps, deps = deps, + compile_data = ["@libfuzzer//:fuzzer"], rustc_flags = rustc_flags + RUSTC_FLAGS_LIBFUZZER + sanitizers, tags = [ # Makes sure this target is not run in normal CI builds. It would fail due to non-nightly Rust toolchain. From dd9503c11b4814f337a880420a07587387650d44 Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Sat, 1 Feb 2025 20:41:17 +0000 Subject: [PATCH 05/28] Fix hs Add extra configuration settings to hermetic_cc_toolchains, to allow hs targets to opt out. GHC is very particular about the toolchains it uses, and does not support zig cc. --- MODULE.bazel | 1 + bazel/BUILD.bazel | 12 +++ bazel/conf/.bazelrc.build | 4 +- bazel/hermetic_cc_toolchain.patch | 139 +++++++++++++++++++++++++++++- hs/spec_compliance/BUILD.bazel | 60 ++++++++++++- hs/spec_compliance/defs.bzl | 36 ++++++++ rs/tests/research/BUILD.bazel | 2 +- 7 files changed, 248 insertions(+), 6 deletions(-) create mode 100644 hs/spec_compliance/defs.bzl diff --git a/MODULE.bazel b/MODULE.bazel index 33ac109369e4..8851dda83ecc 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -894,6 +894,7 @@ mainnet_versions( # Set up hermetic cc toolchains for binaries and canisters toolchains = use_extension("@hermetic_cc_toolchain//toolchain:ext.bzl", "toolchains") +toolchains.extra_settings(settings = ["//bazel:use_hermetic_cc"]) use_repo(toolchains, "zig_sdk") register_toolchains( diff --git a/bazel/BUILD.bazel b/bazel/BUILD.bazel index 7126ad660f1e..07b5ee0b4b1b 100644 --- a/bazel/BUILD.bazel +++ b/bazel/BUILD.bazel @@ -40,6 +40,18 @@ config_setting( }, ) +bool_flag( + name = "hermetic_cc", + build_setting_default = True, +) + +config_setting( + name = "use_hermetic_cc", + flag_values = { + ":hermetic_cc": "True", + }, +) + string_flag( name = "timeout_value", build_setting_default = "10m", diff --git a/bazel/conf/.bazelrc.build b/bazel/conf/.bazelrc.build index b47dbc52196f..dea44572e384 100644 --- a/bazel/conf/.bazelrc.build +++ b/bazel/conf/.bazelrc.build @@ -139,7 +139,5 @@ common --noexperimental_inmemory_dotd_files # certainly" does no harm. test --test_env=CLICOLOR_FORCE=true -# Always compile with zig cc -# https://github.com/uber/hermetic_cc_toolchain/tree/8e68b7221ca72c10268338eb1734530438e5ccb7?tab=readme-ov-file#use-case-always-compile-with-zig-cc -build --action_env BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 +# Speed up compilation with zig cc build --sandbox_add_mount_pair=/tmp diff --git a/bazel/hermetic_cc_toolchain.patch b/bazel/hermetic_cc_toolchain.patch index afbed7c19d0c..3c4fa0e02951 100644 --- a/bazel/hermetic_cc_toolchain.patch +++ b/bazel/hermetic_cc_toolchain.patch @@ -1,4 +1,5 @@ -# Extend hermetic_cc_toolchain with a wasm target that does not use wasi. +# Extend hermetic_cc_toolchain with a wasm target that does not use wasi, and +# add a path to put additional restrictions on the generated toolchains. diff --git a/MODULE.bazel b/MODULE.bazel index 409d626..1265298 100644 --- a/MODULE.bazel @@ -11,6 +12,87 @@ index 409d626..1265298 100644 # These toolchains are only registered locally. dev_dependency = True, +diff --git a/toolchain/defs.bzl b/toolchain/defs.bzl +index f6f613e..02d5597 100644 +--- a/toolchain/defs.bzl ++++ b/toolchain/defs.bzl +@@ -60,7 +60,8 @@ def toolchains( + version = VERSION, + url_formats = [], + host_platform_sha256 = HOST_PLATFORM_SHA256, +- host_platform_ext = _HOST_PLATFORM_EXT): ++ host_platform_ext = _HOST_PLATFORM_EXT, ++ extra_settings = []): + """ + Download zig toolchain and declare bazel toolchains. + The platforms are not registered automatically, that should be done by +@@ -83,6 +84,7 @@ def toolchains( + url_formats = url_formats, + host_platform_sha256 = host_platform_sha256, + host_platform_ext = host_platform_ext, ++ extra_settings = extra_settings + ) + + def _quote(s): +@@ -110,6 +112,8 @@ def _zig_repository_impl(repository_ctx): + "host_platform": host_platform, + } + ++ extra_settings = "[" + " ".join([_quote(str(setting)) for setting in repository_ctx.attr.extra_settings]) + "]" ++ + # Fetch Label dependencies before doing download/extract. + # The Bazel docs are not very clear about this behavior but see: + # https://bazel.build/extending/repo#when_is_the_implementation_function_executed +@@ -117,7 +121,6 @@ def _zig_repository_impl(repository_ctx): + # https://github.com/bazelbuild/bazel-gazelle/pull/1206 + for dest, src in { + "platform/BUILD": "//toolchain/platform:BUILD", +- "toolchain/BUILD": "//toolchain/toolchain:BUILD", + "libc/BUILD": "//toolchain/libc:BUILD", + "libc_aware/platform/BUILD": "//toolchain/libc_aware/platform:BUILD", + "libc_aware/toolchain/BUILD": "//toolchain/libc_aware/toolchain:BUILD", +@@ -126,6 +129,7 @@ def _zig_repository_impl(repository_ctx): + + for dest, src in { + "BUILD": "//toolchain:BUILD.sdk.bazel", ++ "toolchain/BUILD": "//toolchain/toolchain:BUILD", + }.items(): + repository_ctx.template( + dest, +@@ -134,6 +138,7 @@ def _zig_repository_impl(repository_ctx): + substitutions = { + "{zig_sdk_path}": _quote("external/zig_sdk"), + "{os}": _quote(os), ++ "{extra_settings}": extra_settings, + }, + ) + +@@ -230,6 +235,7 @@ zig_repository = repository_rule( + "host_platform_sha256": attr.string_dict(), + "url_formats": attr.string_list(allow_empty = False), + "host_platform_ext": attr.string_dict(), ++ "extra_settings": attr.label_list(), + }, + environ = ["HERMETIC_CC_TOOLCHAIN_CACHE_PREFIX"], + implementation = _zig_repository_impl, +diff --git a/toolchain/ext.bzl b/toolchain/ext.bzl +index ebf0ff8..c2e8af3 100644 +--- a/toolchain/ext.bzl ++++ b/toolchain/ext.bzl +@@ -1,6 +1,11 @@ + load("@hermetic_cc_toolchain//toolchain:defs.bzl", zig_toolchains = "toolchains") + + def _toolchains_impl(ctx): +- zig_toolchains() ++ extra_settings = [] ++ for mod in ctx.modules: ++ for tag in mod.tags.extra_settings: ++ extra_settings += tag.settings + +-toolchains = module_extension(implementation = _toolchains_impl) ++ zig_toolchains(extra_settings = extra_settings) ++ ++toolchains = module_extension(implementation = _toolchains_impl, tag_classes = { "extra_settings": tag_class(attrs = { "settings": attr.label_list(doc = "Each setting is added to every toolchain to make them more restrictive.")})}) diff --git a/toolchain/platform/defs.bzl b/toolchain/platform/defs.bzl index d4a8344..faafc5b 100644 --- a/toolchain/platform/defs.bzl @@ -71,3 +153,58 @@ index d1d59f9..5e2984b 100644 return error.BadParent; // ABI triple is too much of a moving target +diff --git a/toolchain/toolchain/BUILD b/toolchain/toolchain/BUILD +index 552fcaa..8f7dba5 100644 +--- a/toolchain/toolchain/BUILD ++++ b/toolchain/toolchain/BUILD +@@ -4,4 +4,4 @@ package( + default_visibility = ["//visibility:public"], + ) + +-declare_toolchains() ++declare_toolchains(extra_settings = {extra_settings}) +diff --git a/toolchain/toolchain/defs.bzl b/toolchain/toolchain/defs.bzl +index 50cc881..0549c26 100644 +--- a/toolchain/toolchain/defs.bzl ++++ b/toolchain/toolchain/defs.bzl +@@ -1,6 +1,6 @@ + load("@hermetic_cc_toolchain//toolchain/private:defs.bzl", "target_structs") + +-def declare_toolchains(): ++def declare_toolchains(extra_settings = []): + for target_config in target_structs(): + gotarget = target_config.gotarget + zigtarget = target_config.zigtarget +@@ -12,7 +12,7 @@ def declare_toolchains(): + if hasattr(target_config, "libc_constraint"): + extra_constraints = ["@zig_sdk//libc:unconstrained"] + +- _declare_toolchain(gotarget, zigtarget, target_config.constraint_values + extra_constraints) ++ _declare_toolchain(gotarget, zigtarget, target_config.constraint_values + extra_constraints, extra_settings) + + def declare_libc_aware_toolchains(): + for target_config in target_structs(): +@@ -25,13 +25,14 @@ def declare_libc_aware_toolchains(): + if hasattr(target_config, "libc_constraint"): + _declare_toolchain(gotarget, zigtarget, target_config.constraint_values + [target_config.libc_constraint]) + +-def _declare_toolchain(gotarget, zigtarget, target_compatible_with): ++def _declare_toolchain(gotarget, zigtarget, target_compatible_with, extra_settings): + # register two kinds of toolchain targets: Go and Zig conventions. + # Go convention: amd64/arm64, linux/darwin + native.toolchain( + name = gotarget, + exec_compatible_with = None, + target_compatible_with = target_compatible_with, ++ target_settings = extra_settings, + toolchain = "@zig_sdk//:%s_cc" % zigtarget, + toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", + ) +@@ -41,6 +42,7 @@ def _declare_toolchain(gotarget, zigtarget, target_compatible_with): + name = zigtarget, + exec_compatible_with = None, + target_compatible_with = target_compatible_with, ++ target_settings = extra_settings, + toolchain = "@zig_sdk//:%s_cc" % zigtarget, + toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", + ) diff --git a/hs/spec_compliance/BUILD.bazel b/hs/spec_compliance/BUILD.bazel index 7dd71a755810..539bc6e755ed 100644 --- a/hs/spec_compliance/BUILD.bazel +++ b/hs/spec_compliance/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_haskell//haskell:defs.bzl", "haskell_binary", "haskell_library") +load("defs.bzl", "disable_hermetic_cc_binary") STACK_DEPS = [ # Keep sorted. @@ -72,6 +73,12 @@ STACK_DEPS = [ "@stackage//:zlib", ] +disable_hermetic_cc_binary( + name = "ic-ref-test-wrapped", + binary = ":ic-ref-test", + visibility = ["//visibility:public"], +) + haskell_binary( name = "ic-ref-test", srcs = [ @@ -81,8 +88,8 @@ haskell_binary( "-threaded", # use the threaded Run Time System (RTS) "-rtsopts", # allow passing RTS options like: +RTS -N -RTS ], + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], - visibility = ["//visibility:public"], deps = STACK_DEPS + [ ":IC-CBOR-Parser", ":IC-CBOR-Patterns", @@ -142,6 +149,7 @@ haskell_library( name = "IC-Crypto-BLS", srcs = ["src/IC/Crypto/BLS.hsc"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Id-Forms", @@ -155,6 +163,7 @@ haskell_library( name = "IC-Id-Fresh", srcs = ["src/IC/Id/Fresh.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Id-Forms", @@ -168,6 +177,7 @@ haskell_library( name = "IC-Id-Forms", srcs = ["src/IC/Id/Forms.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Hash", @@ -180,6 +190,7 @@ haskell_library( name = "IC-Utils", srcs = ["src/IC/Utils.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Constants", @@ -194,6 +205,7 @@ haskell_library( name = "IC-Certificate", srcs = ["src/IC/Certificate.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-HashTree", @@ -206,6 +218,7 @@ haskell_library( name = "IC-Management", srcs = ["src/IC/Management.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Types", @@ -218,6 +231,7 @@ haskell_library( name = "IC-Purify", srcs = ["src/IC/Purify.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ "@haskell-candid//:candid", @@ -229,6 +243,7 @@ haskell_library( name = "IC-Crypto", srcs = ["src/IC/Crypto.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Crypto-BLS", @@ -247,6 +262,7 @@ haskell_library( name = "IC-HTTP-RequestId", srcs = ["src/IC/HTTP/RequestId.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-HTTP-GenR", @@ -261,6 +277,7 @@ haskell_library( name = "IC-HTTP-CBOR", srcs = ["src/IC/HTTP/CBOR.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-HTTP-GenR", @@ -273,6 +290,7 @@ haskell_library( name = "IC-HTTP-GenR", srcs = ["src/IC/HTTP/GenR.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ "@haskell-candid//:candid", @@ -284,6 +302,7 @@ haskell_library( name = "IC-HTTP-GenR-Parse", srcs = ["src/IC/HTTP/GenR/Parse.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-HTTP-GenR", @@ -296,6 +315,7 @@ haskell_library( name = "IC-Types", srcs = ["src/IC/Types.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ "@haskell-candid//:candid", @@ -307,6 +327,7 @@ haskell_library( name = "IC-HashTree-CBOR", srcs = ["src/IC/HashTree/CBOR.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-CBOR-Patterns", @@ -320,6 +341,7 @@ haskell_library( name = "IC-HashTree", srcs = ["src/IC/HashTree.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ "@haskell-candid//:candid", @@ -331,6 +353,7 @@ haskell_library( name = "IC-Constants", srcs = ["src/IC/Constants.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Types", @@ -343,6 +366,7 @@ haskell_library( name = "IC-Certificate-Value", srcs = ["src/IC/Certificate/Value.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Types", @@ -356,6 +380,7 @@ haskell_library( name = "IC-Certificate-Validate", srcs = ["src/IC/Certificate/Validate.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Certificate", @@ -372,6 +397,7 @@ haskell_library( name = "IC-Certificate-CBOR", srcs = ["src/IC/Certificate/CBOR.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-CBOR-Parser", @@ -388,6 +414,7 @@ haskell_library( name = "IC-Crypto-Secp256k1", srcs = ["src/IC/Crypto/Secp256k1.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ "@haskell-candid//:candid", @@ -399,6 +426,7 @@ haskell_library( name = "IC-Crypto-Ed25519", srcs = ["src/IC/Crypto/Ed25519.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ "@haskell-candid//:candid", @@ -410,6 +438,7 @@ haskell_library( name = "IC-Crypto-DER-Decode", srcs = ["src/IC/Crypto/DER/Decode.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ "@haskell-candid//:candid", @@ -421,6 +450,7 @@ haskell_library( name = "IC-Crypto-DER", srcs = ["src/IC/Crypto/DER.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Crypto-DER-Decode", @@ -433,6 +463,7 @@ haskell_library( name = "IC-Crypto-DER_BLS", srcs = ["src/IC/Crypto/DER_BLS.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Crypto-BLS", @@ -446,6 +477,7 @@ haskell_library( name = "IC-Crypto-WebAuthn", srcs = ["src/IC/Crypto/WebAuthn.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-CBOR-Parser", @@ -463,6 +495,7 @@ haskell_library( name = "IC-Crypto-CanisterSig", srcs = ["src/IC/Crypto/CanisterSig.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-CBOR-Parser", @@ -483,6 +516,7 @@ haskell_library( name = "IC-Crypto-ECDSA", srcs = ["src/IC/Crypto/ECDSA.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ "@haskell-candid//:candid", @@ -494,6 +528,7 @@ haskell_library( name = "IC-Version", srcs = ["src/IC/Version.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":SourceId", @@ -506,6 +541,7 @@ haskell_library( name = "IC-Hash", srcs = ["src/IC/Hash.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ "@haskell-candid//:candid", @@ -517,6 +553,7 @@ haskell_library( name = "IC-CBOR-Utils", srcs = ["src/IC/CBOR/Utils.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-HTTP-CBOR", @@ -531,6 +568,7 @@ haskell_library( name = "IC-CBOR-Parser", srcs = ["src/IC/CBOR/Parser.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-CBOR-Patterns", @@ -543,6 +581,7 @@ haskell_library( name = "IC-CBOR-Patterns", srcs = ["src/IC/CBOR/Patterns.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ "@haskell-candid//:candid", @@ -554,6 +593,7 @@ haskell_library( name = "IC-DRun-Parse", srcs = ["src/IC/DRun/Parse.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ "@haskell-candid//:candid", @@ -565,6 +605,7 @@ haskell_library( name = "IC-Test-Secp256k1", srcs = ["src/IC/Test/Secp256k1.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Crypto-Secp256k1", @@ -577,6 +618,7 @@ haskell_library( name = "IC-Test-Spec", srcs = ["src/IC/Test/Spec.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Certificate", @@ -610,6 +652,7 @@ haskell_library( name = "IC-Test-WebAuthn", srcs = ["src/IC/Test/WebAuthn.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Crypto-WebAuthn", @@ -622,6 +665,7 @@ haskell_library( name = "IC-Test-Spec-Utils", srcs = ["src/IC/Test/Spec/Utils.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Crypto", @@ -643,6 +687,7 @@ haskell_library( name = "IC-Test-Spec-CanisterHistory", srcs = ["src/IC/Test/Spec/CanisterHistory.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Hash", @@ -663,6 +708,7 @@ haskell_library( name = "IC-Test-Spec-CanisterVersion", srcs = ["src/IC/Test/Spec/CanisterVersion.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Test-Agent", @@ -680,6 +726,7 @@ haskell_library( name = "IC-Test-Spec-Timer", srcs = ["src/IC/Test/Spec/Timer.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Management", @@ -696,6 +743,7 @@ haskell_library( name = "IC-Test-BLS", srcs = ["src/IC/Test/BLS.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Crypto-BLS", @@ -708,6 +756,7 @@ haskell_library( name = "IC-Test-ECDSA", srcs = ["src/IC/Test/ECDSA.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Crypto-ECDSA", @@ -720,6 +769,7 @@ haskell_library( name = "IC-Test-Agent", srcs = ["src/IC/Test/Agent.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Certificate", @@ -749,6 +799,7 @@ haskell_library( name = "IC-Test-HashTree", srcs = ["src/IC/Test/HashTree.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-HashTree", @@ -762,6 +813,7 @@ haskell_library( name = "IC-Test-Universal", srcs = ["src/IC/Test/Universal.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ "@haskell-candid//:candid", @@ -773,6 +825,7 @@ haskell_library( name = "IC-Test-Options", srcs = ["src/IC/Test/Options.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Constants", @@ -788,6 +841,7 @@ haskell_library( name = "IC-Test-Agent-UnsafeCalls", srcs = ["src/IC/Test/Agent/UnsafeCalls.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Id-Forms", @@ -805,6 +859,7 @@ haskell_library( name = "IC-Test-Agent-SafeCalls", srcs = ["src/IC/Test/Agent/SafeCalls.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Id-Forms", @@ -822,6 +877,7 @@ haskell_library( name = "IC-Test-Agent-Calls", srcs = ["src/IC/Test/Agent/Calls.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Id-Forms", @@ -836,6 +892,7 @@ haskell_library( name = "IC-Test-Agent-UserCalls", srcs = ["src/IC/Test/Agent/UserCalls.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Id-Forms", @@ -851,6 +908,7 @@ haskell_library( name = "SourceId", srcs = ["src/SourceId.hs"], src_strip_prefix = "src", + tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ "@haskell-candid//:candid", diff --git a/hs/spec_compliance/defs.bzl b/hs/spec_compliance/defs.bzl new file mode 100644 index 000000000000..2eba1237365f --- /dev/null +++ b/hs/spec_compliance/defs.bzl @@ -0,0 +1,36 @@ +"""Create transition to disable hermetic_cc_toolchains for hs targets.""" + +def _disable_hermetic_cc_transition(_settings, _attr): + return { + "//bazel:hermetic_cc": False, + } + +disable_hermetic_cc_transition = transition( + implementation = _disable_hermetic_cc_transition, + inputs = [], + outputs = [ + "//bazel:hermetic_cc", + ], +) + +def _disable_hermetic_cc_impl(ctx): + bin = ctx.attr.binary[0] + info = bin[DefaultInfo] + + executable = ctx.actions.declare_file(ctx.label.name) + ctx.actions.symlink(output = executable, target_file = ctx.file.binary) + + return [ + DefaultInfo(files = info.files, runfiles = info.default_runfiles, executable = executable), + ] + +disable_hermetic_cc_binary = rule( + implementation = _disable_hermetic_cc_impl, + executable = True, + attrs = { + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist", + ), + "binary": attr.label(mandatory = True, cfg = disable_hermetic_cc_transition, allow_single_file = True), + }, +) diff --git a/rs/tests/research/BUILD.bazel b/rs/tests/research/BUILD.bazel index eac6ed7e16e0..d3c4438b2f4b 100644 --- a/rs/tests/research/BUILD.bazel +++ b/rs/tests/research/BUILD.bazel @@ -40,7 +40,7 @@ symlink_dirs( name = "ic-hs", target_compatible_with = ["@platforms//os:linux"], targets = { - "//hs/spec_compliance:ic-ref-test": "bin", + "//hs/spec_compliance:ic-ref-test-wrapped": "bin", "//rs/universal_canister/impl:universal_canister_no_heartbeat.wasm.gz": "test-data", "//rs/tests:wabt-tests": "test-data", }, From 19166e3d57cb84b91c273f3a2dc116bc8aeb6ba8 Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Mon, 24 Feb 2025 21:59:51 +0000 Subject: [PATCH 06/28] Fix AFL fuzzers AFL requires a linker wrapper to instrument, and this does not integrate very cleanly with the zig hermetic_cc toolchains. Keep these builds on the existing path for now. --- bazel/conf/.bazelrc.build | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bazel/conf/.bazelrc.build b/bazel/conf/.bazelrc.build index dea44572e384..3f394ed66aa4 100644 --- a/bazel/conf/.bazelrc.build +++ b/bazel/conf/.bazelrc.build @@ -48,6 +48,7 @@ common --flag_alias=s3_endpoint=//ci/src/artifacts:s3_endpoint common --flag_alias=s3_upload=//ci/src/artifacts:s3_upload common --flag_alias=k8s=//rs/tests:k8s common --flag_alias=timeout_value=//bazel:timeout_value +common --flag_alias=hermetic_cc=//bazel:hermetic_cc common:stamped --workspace_status_command='$(pwd)/bazel/workspace_status.sh --stamp' common:release --config=stamped --s3_upload=True @@ -113,6 +114,7 @@ build:afl --action_env="LD=afl-clang-lto++" build:afl --action_env="LLVM_CONFIG=llvm-config-18" build:afl --action_env="RANLIB=llvm-ranlib-18" build:afl --config=fuzzing +build:afl --hermetic_cc=false build:afl --build_tag_filters=afl run:afl --run_under="//bin:afl_wrapper" From 68c6a4b06f31cbfd80d880d1643128d5ca3caf29 Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Fri, 7 Feb 2025 23:27:24 +0000 Subject: [PATCH 07/28] Fix openssl libs --- MODULE.bazel | 56 +++++++++++++++++++++++++++++++++++++++ bazel/external_crates.bzl | 17 ++++++++++++ 2 files changed, 73 insertions(+) diff --git a/MODULE.bazel b/MODULE.bazel index 8851dda83ecc..7a6267b95fb8 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -931,3 +931,59 @@ cc_library( strip_prefix = "llvm-project-llvmorg-20.1.0/compiler-rt/lib/fuzzer", urls = ["https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-20.1.0.tar.gz"], ) + +# openssl-sys does not build with the vendored feature when using +# rules_rust, so we build it ourselves, here. +# https://github.com/bazelbuild/rules_rust/issues/1519 +http_archive( + name = "openssl", + build_file_content = """ +load("@rules_foreign_cc//foreign_cc:defs.bzl", "configure_make") + +filegroup( + name = "all_srcs", + srcs = glob( + include = ["**"], + exclude = ["*.bazel"], + ), +) + +# https://wiki.openssl.org/index.php/Compilation_and_Installation +CONFIGURE_OPTIONS = [ + "no-comp", + "no-idea", + "no-weak-ssl-ciphers", + "no-shared", +] + +MAKE_TARGETS = [ + "build_libs", + "install_dev", +] + +configure_make( + name = "openssl", + args = ["-j12"], + configure_command = "config", + configure_in_place = True, + configure_options = CONFIGURE_OPTIONS, + lib_name = "openssl", + lib_source = ":all_srcs", + out_lib_dir = "lib64", + out_shared_libs = [], + out_static_libs = ["libssl.a"], + targets = MAKE_TARGETS, + visibility = ["//visibility:public"], +) + +filegroup( + name = "gen_dir", + srcs = [":openssl"], + output_group = "gen_dir", + visibility = ["//visibility:public"], +) +""", + integrity = "sha256-4V3agv4v6BOdwqwho21MoB1TE8dfmfRsTooncJtylL8=", + strip_prefix = "openssl-3.4.0", + urls = ["https://github.com/openssl/openssl/releases/download/openssl-3.4.0/openssl-3.4.0.tar.gz"], +) diff --git a/bazel/external_crates.bzl b/bazel/external_crates.bzl index cfca948e1cdd..d17200ddf827 100644 --- a/bazel/external_crates.bzl +++ b/bazel/external_crates.bzl @@ -24,6 +24,17 @@ BUILD_INFO_REV = "701a696844fba5c87df162fbbc1ccef96f27c9d7" def external_crates_repository(name, cargo_lockfile, lockfile, sanitizers_enabled): CRATE_ANNOTATIONS = { + "openssl-sys": [crate.annotation( + build_script_data = [ + "@@_main~_repo_rules~openssl//:gen_dir", + ], + build_script_env = { + "OPENSSL_NO_VENDOR": "1", + "OPENSSL_LIB_DIR": "$(location @@_main~_repo_rules~openssl//:gen_dir)/lib64", + "OPENSSL_INCLUDE_DIR": "$(location @@_main~_repo_rules~openssl//:gen_dir)/include", + "OPENSSL_STATIC": "1", + }, + )], "canbench": [crate.annotation( gen_binaries = True, )], @@ -35,6 +46,12 @@ def external_crates_repository(name, cargo_lockfile, lockfile, sanitizers_enable # Patch for determinism issues patch_args = ["-p1"], patches = ["@@//bazel:libssh2-sys.patch"], + build_script_data = [ + "@@_main~_repo_rules~openssl//:gen_dir", + ], + )], + "libz-sys": [crate.annotation( + crate_features = ["static"], )], "curve25519-dalek": [crate.annotation( rustc_flags = [ From d1d66b672d86e5e5412960785093a62b950e745f Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Wed, 12 Feb 2025 18:48:43 +0000 Subject: [PATCH 08/28] Fix stripping zig handles requests to strip any symbols by stripping all symbols. rules_rust tries to strip debug symbols, so we must override the link args to disable stripping wherever symbols need to be preserved. --- bazel/fuzz_testing.bzl | 1 + cpp/BUILD.bazel | 9 ++++++++- publish/defs.bzl | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/bazel/fuzz_testing.bzl b/bazel/fuzz_testing.bzl index e83fbeba3407..0869177f3f9a 100644 --- a/bazel/fuzz_testing.bzl +++ b/bazel/fuzz_testing.bzl @@ -21,6 +21,7 @@ DEFAULT_RUSTC_FLAGS = [ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", + "-Cstrip=none", # TODO(PSEC): Add configuration to enable only during profiling # "-Cinstrument-coverage", ] diff --git a/cpp/BUILD.bazel b/cpp/BUILD.bazel index d90f295ccb3f..e79d2ed0260f 100644 --- a/cpp/BUILD.bazel +++ b/cpp/BUILD.bazel @@ -1,3 +1,5 @@ +load("//publish:defs.bzl", "release_strip_binary") + package(default_visibility = ["//visibility:public"]) cc_binary( @@ -17,9 +19,14 @@ cc_binary( deps = ["@libsystemd"], ) +release_strip_binary( + name = "infogetty_stripped", + binary = "infogetty_bin", +) + genrule( name = "infogetty_cleaned", - srcs = ["infogetty_bin"], + srcs = ["infogetty_stripped"], outs = ["infogetty"], cmd = "objcopy -R .comment -R .note -R .note.gnu.build-id $< $@", executable = True, diff --git a/publish/defs.bzl b/publish/defs.bzl index 22430897e0d3..55572ede7c00 100644 --- a/publish/defs.bzl +++ b/publish/defs.bzl @@ -8,7 +8,7 @@ def _release_nostrip_transition(_settings, _attr): return { "//command_line_option:compilation_mode": "opt", "//command_line_option:strip": "never", - "@rules_rust//:extra_rustc_flags": ["-Cdebug-assertions=off"], + "@rules_rust//:extra_rustc_flags": ["-Cdebug-assertions=off", "-Cstrip=none"], } release_nostrip_transition = transition( From 4e03ca6d3ca585a7e0254497af5bbf5c0c371a5d Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Fri, 7 Mar 2025 21:21:24 +0000 Subject: [PATCH 09/28] Remove ZIG_LINK_ARGS --- bazel/fuzz_testing.bzl | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/bazel/fuzz_testing.bzl b/bazel/fuzz_testing.bzl index 0869177f3f9a..cead960e5a29 100644 --- a/bazel/fuzz_testing.bzl +++ b/bazel/fuzz_testing.bzl @@ -28,17 +28,14 @@ DEFAULT_RUSTC_FLAGS = [ DEFAULT_SANITIZERS = [ "-Zsanitizer=address", + # zig doesn't like how rustc pushes the sanitizers, so do it ourselves. + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a", ] # This flag will be used by third party crates and internal rust_libraries during fuzzing DEFAULT_RUSTC_FLAGS_FOR_FUZZING = DEFAULT_RUSTC_FLAGS + DEFAULT_SANITIZERS -# zig doesn't like how rustc pushes the sanitizers, so do it ourselves. -ZIG_LINK_ARGS = [ - "-Zexternal-clangrt", - "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a", -] - def rust_fuzz_test_binary(name, srcs, rustc_flags = [], sanitizers = [], crate_features = [], proc_macro_deps = [], deps = [], allow_main = False, **kwargs): """Wrapper for the rust_binary to compile a fuzzing rust_binary @@ -65,7 +62,7 @@ def rust_fuzz_test_binary(name, srcs, rustc_flags = [], sanitizers = [], crate_f # default TAGS = [] - RUSTC_FLAGS_LIBFUZZER = DEFAULT_RUSTC_FLAGS + ZIG_LINK_ARGS + ["-Clink-arg=$(location @libfuzzer//:fuzzer)"] + RUSTC_FLAGS_LIBFUZZER = DEFAULT_RUSTC_FLAGS + ["-Clink-arg=$(location @libfuzzer//:fuzzer)"] kwargs.setdefault("testonly", True) From 899353f5e67a99766d1ecd8ca0df8465fe29f7fd Mon Sep 17 00:00:00 2001 From: IDX GitHub Automation Date: Fri, 7 Mar 2025 21:55:01 +0000 Subject: [PATCH 10/28] Automatically updated Cargo*.lock --- Cargo.Bazel.Fuzzing.json.lock | 69 ++++++++++++++++++++++++++++++----- Cargo.Bazel.json.lock | 34 +++++++++++++++-- 2 files changed, 90 insertions(+), 13 deletions(-) diff --git a/Cargo.Bazel.Fuzzing.json.lock b/Cargo.Bazel.Fuzzing.json.lock index 0498d5290f63..22b609b9bcc7 100644 --- a/Cargo.Bazel.Fuzzing.json.lock +++ b/Cargo.Bazel.Fuzzing.json.lock @@ -1,5 +1,5 @@ { - "checksum": "de37c6d2c2179633b78dc8c680d638821a439d90c378cbeb90a848cac01fdc0f", + "checksum": "a4684b538a85775bbf651bb2bfe3dd05767dabe92bf0505a831fc3d869f40c07", "crates": { "abnf 0.12.0": { "name": "abnf", @@ -6782,7 +6782,10 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Zsanitizer=address" + "-Cstrip=none", + "-Zsanitizer=address", + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" ], "selects": {} }, @@ -7572,7 +7575,10 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Zsanitizer=address" + "-Cstrip=none", + "-Zsanitizer=address", + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" ], "selects": {} }, @@ -7689,7 +7695,10 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Zsanitizer=address" + "-Cstrip=none", + "-Zsanitizer=address", + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" ], "selects": {} }, @@ -7836,7 +7845,10 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Zsanitizer=address" + "-Cstrip=none", + "-Zsanitizer=address", + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" ], "selects": {} }, @@ -11364,7 +11376,10 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Zsanitizer=address" + "-Cstrip=none", + "-Zsanitizer=address", + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" ], "selects": {} }, @@ -11714,7 +11729,13 @@ "repository": { "Http": { "url": "https://static.crates.io/crates/cc/1.1.37/download", - "sha256": "40545c26d092346d8a8dab71ee48e7685a7a9cba76e634790c215b41a4a7b4cf" + "sha256": "40545c26d092346d8a8dab71ee48e7685a7a9cba76e634790c215b41a4a7b4cf", + "patch_args": [ + "-p1" + ], + "patches": [ + "@@//bazel:cc_rs.patch" + ] } }, "targets": [ @@ -34836,7 +34857,10 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Zsanitizer=address" + "-Cstrip=none", + "-Zsanitizer=address", + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" ], "selects": {} }, @@ -42482,6 +42506,12 @@ "compile_data_glob": [ "**" ], + "data": { + "common": [ + "@@_main~_repo_rules~openssl//:gen_dir" + ], + "selects": {} + }, "data_glob": [ "**" ], @@ -42665,7 +42695,8 @@ ], "crate_features": { "common": [ - "libc" + "libc", + "static" ], "selects": {} }, @@ -49688,6 +49719,12 @@ "compile_data_glob": [ "**" ], + "data": { + "common": [ + "@@_main~_repo_rules~openssl//:gen_dir" + ], + "selects": {} + }, "data_glob": [ "**" ], @@ -49708,6 +49745,15 @@ ], "selects": {} }, + "build_script_env": { + "common": { + "OPENSSL_INCLUDE_DIR": "$(location @@_main~_repo_rules~openssl//:gen_dir)/include", + "OPENSSL_LIB_DIR": "$(location @@_main~_repo_rules~openssl//:gen_dir)/lib64", + "OPENSSL_NO_VENDOR": "1", + "OPENSSL_STATIC": "1" + }, + "selects": {} + }, "links": "openssl" }, "license": "MIT", @@ -86670,7 +86716,10 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Zsanitizer=address" + "-Cstrip=none", + "-Zsanitizer=address", + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" ], "selects": {} }, diff --git a/Cargo.Bazel.json.lock b/Cargo.Bazel.json.lock index 628708a59809..f92a81d07609 100644 --- a/Cargo.Bazel.json.lock +++ b/Cargo.Bazel.json.lock @@ -1,5 +1,5 @@ { - "checksum": "57f96e8f7733cd4d6914fb4a02d688caa500bb4143bf7ec581374871daa70b73", + "checksum": "059a9792f5627f5ef135cb3499f5d0a91cb2cba25447542035d17997c382f9aa", "crates": { "abnf 0.12.0": { "name": "abnf", @@ -11610,7 +11610,13 @@ "repository": { "Http": { "url": "https://static.crates.io/crates/cc/1.1.37/download", - "sha256": "40545c26d092346d8a8dab71ee48e7685a7a9cba76e634790c215b41a4a7b4cf" + "sha256": "40545c26d092346d8a8dab71ee48e7685a7a9cba76e634790c215b41a4a7b4cf", + "patch_args": [ + "-p1" + ], + "patches": [ + "@@//bazel:cc_rs.patch" + ] } }, "targets": [ @@ -42316,6 +42322,12 @@ "compile_data_glob": [ "**" ], + "data": { + "common": [ + "@@_main~_repo_rules~openssl//:gen_dir" + ], + "selects": {} + }, "data_glob": [ "**" ], @@ -42499,7 +42511,8 @@ ], "crate_features": { "common": [ - "libc" + "libc", + "static" ], "selects": {} }, @@ -49528,6 +49541,12 @@ "compile_data_glob": [ "**" ], + "data": { + "common": [ + "@@_main~_repo_rules~openssl//:gen_dir" + ], + "selects": {} + }, "data_glob": [ "**" ], @@ -49548,6 +49567,15 @@ ], "selects": {} }, + "build_script_env": { + "common": { + "OPENSSL_INCLUDE_DIR": "$(location @@_main~_repo_rules~openssl//:gen_dir)/include", + "OPENSSL_LIB_DIR": "$(location @@_main~_repo_rules~openssl//:gen_dir)/lib64", + "OPENSSL_NO_VENDOR": "1", + "OPENSSL_STATIC": "1" + }, + "selects": {} + }, "links": "openssl" }, "license": "MIT", From c10f626e7c6362a46ec7982a468e7dbe7351cb74 Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Fri, 7 Mar 2025 22:17:56 +0000 Subject: [PATCH 11/28] Small cleanups and comments --- MODULE.bazel | 1 - WORKSPACE.bazel | 4 +++- publish/defs.bzl | 4 ++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 7a6267b95fb8..1dbaf0bfeca6 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -925,7 +925,6 @@ cc_library( linkstatic = True, visibility = ["//visibility:public"], ) - """, integrity = "sha256-CLw4JzN3fdo8liWeNzL/lsHfmNBHDE+FsWMnTq5of08=", strip_prefix = "llvm-project-llvmorg-20.1.0/compiler-rt/lib/fuzzer", diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index 72c04681aed1..2529c6ad0e85 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -281,12 +281,14 @@ cc_import( system_provided = True, visibility = ["//visibility:private"], ) + # Use an extra cc_library to hide the depth of the include folder cc_library( name = "libsystemd", includes = ["include"], deps = ["libsystemd-internal"], visibility = ["//visibility:public"], -)""", +) +""", path = "/usr", ) diff --git a/publish/defs.bzl b/publish/defs.bzl index 55572ede7c00..bcac2b391b5b 100644 --- a/publish/defs.bzl +++ b/publish/defs.bzl @@ -8,6 +8,10 @@ def _release_nostrip_transition(_settings, _attr): return { "//command_line_option:compilation_mode": "opt", "//command_line_option:strip": "never", + # opt mode will have rules_rust strip debug symbols, regardless of the + # strip setting. Unfortunately zig cc (from hermetic_cc_toolchain) + # strips as "all or nothing", so we lose all symbols, unless we also + # override the strip setting here. "@rules_rust//:extra_rustc_flags": ["-Cdebug-assertions=off", "-Cstrip=none"], } From 5ea2fd47dbc9a90f93b8bfe205f0c9e66f342973 Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Fri, 7 Mar 2025 23:36:21 +0000 Subject: [PATCH 12/28] Simplify openssl references --- bazel/external_crates.bzl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bazel/external_crates.bzl b/bazel/external_crates.bzl index d17200ddf827..5bfa0e5f2615 100644 --- a/bazel/external_crates.bzl +++ b/bazel/external_crates.bzl @@ -26,12 +26,12 @@ def external_crates_repository(name, cargo_lockfile, lockfile, sanitizers_enable CRATE_ANNOTATIONS = { "openssl-sys": [crate.annotation( build_script_data = [ - "@@_main~_repo_rules~openssl//:gen_dir", + "@openssl//:gen_dir", ], build_script_env = { "OPENSSL_NO_VENDOR": "1", - "OPENSSL_LIB_DIR": "$(location @@_main~_repo_rules~openssl//:gen_dir)/lib64", - "OPENSSL_INCLUDE_DIR": "$(location @@_main~_repo_rules~openssl//:gen_dir)/include", + "OPENSSL_LIB_DIR": "$(location @openssl//:gen_dir)/lib64", + "OPENSSL_INCLUDE_DIR": "$(location @openssl//:gen_dir)/include", "OPENSSL_STATIC": "1", }, )], @@ -47,7 +47,7 @@ def external_crates_repository(name, cargo_lockfile, lockfile, sanitizers_enable patch_args = ["-p1"], patches = ["@@//bazel:libssh2-sys.patch"], build_script_data = [ - "@@_main~_repo_rules~openssl//:gen_dir", + "@openssl//:gen_dir", ], )], "libz-sys": [crate.annotation( From 86e120dc4922852cc59d987e05e8c5947f281a0c Mon Sep 17 00:00:00 2001 From: IDX GitHub Automation Date: Fri, 7 Mar 2025 23:53:37 +0000 Subject: [PATCH 13/28] Automatically updated Cargo*.lock --- Cargo.Bazel.Fuzzing.json.lock | 10 +++++----- Cargo.Bazel.json.lock | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.Bazel.Fuzzing.json.lock b/Cargo.Bazel.Fuzzing.json.lock index 22b609b9bcc7..8ebd328c5ade 100644 --- a/Cargo.Bazel.Fuzzing.json.lock +++ b/Cargo.Bazel.Fuzzing.json.lock @@ -1,5 +1,5 @@ { - "checksum": "a4684b538a85775bbf651bb2bfe3dd05767dabe92bf0505a831fc3d869f40c07", + "checksum": "f6e1d9d1d27392c92d6c5c5e0b37f80cc2c63ecc63523bfb61acbfd4da05e9ed", "crates": { "abnf 0.12.0": { "name": "abnf", @@ -42508,7 +42508,7 @@ ], "data": { "common": [ - "@@_main~_repo_rules~openssl//:gen_dir" + "@openssl//:gen_dir" ], "selects": {} }, @@ -49721,7 +49721,7 @@ ], "data": { "common": [ - "@@_main~_repo_rules~openssl//:gen_dir" + "@openssl//:gen_dir" ], "selects": {} }, @@ -49747,8 +49747,8 @@ }, "build_script_env": { "common": { - "OPENSSL_INCLUDE_DIR": "$(location @@_main~_repo_rules~openssl//:gen_dir)/include", - "OPENSSL_LIB_DIR": "$(location @@_main~_repo_rules~openssl//:gen_dir)/lib64", + "OPENSSL_INCLUDE_DIR": "$(location @openssl//:gen_dir)/include", + "OPENSSL_LIB_DIR": "$(location @openssl//:gen_dir)/lib64", "OPENSSL_NO_VENDOR": "1", "OPENSSL_STATIC": "1" }, diff --git a/Cargo.Bazel.json.lock b/Cargo.Bazel.json.lock index f92a81d07609..3009b815c79d 100644 --- a/Cargo.Bazel.json.lock +++ b/Cargo.Bazel.json.lock @@ -1,5 +1,5 @@ { - "checksum": "059a9792f5627f5ef135cb3499f5d0a91cb2cba25447542035d17997c382f9aa", + "checksum": "45dc49005994f7cdaac3fed857333fdf777a569fc4240ea30135f684eb4c7741", "crates": { "abnf 0.12.0": { "name": "abnf", @@ -42324,7 +42324,7 @@ ], "data": { "common": [ - "@@_main~_repo_rules~openssl//:gen_dir" + "@openssl//:gen_dir" ], "selects": {} }, @@ -49543,7 +49543,7 @@ ], "data": { "common": [ - "@@_main~_repo_rules~openssl//:gen_dir" + "@openssl//:gen_dir" ], "selects": {} }, @@ -49569,8 +49569,8 @@ }, "build_script_env": { "common": { - "OPENSSL_INCLUDE_DIR": "$(location @@_main~_repo_rules~openssl//:gen_dir)/include", - "OPENSSL_LIB_DIR": "$(location @@_main~_repo_rules~openssl//:gen_dir)/lib64", + "OPENSSL_INCLUDE_DIR": "$(location @openssl//:gen_dir)/include", + "OPENSSL_LIB_DIR": "$(location @openssl//:gen_dir)/lib64", "OPENSSL_NO_VENDOR": "1", "OPENSSL_STATIC": "1" }, From 48462714c059c8840a0fa27799c70b14a575cc7b Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Tue, 11 Mar 2025 22:00:19 +0000 Subject: [PATCH 14/28] Switch to `archive_override` --- MODULE.bazel | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index 1dbaf0bfeca6..7fbea9d2fdf1 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -14,10 +14,12 @@ bazel_dep(name = "aspect_bazel_lib", version = "2.9.0") bazel_dep(name = "rules_cc", version = "0.0.13") bazel_dep(name = "platforms", version = "0.0.10") bazel_dep(name = "hermetic_cc_toolchain", version = "3.1.1") -single_version_override( +archive_override( module_name = "hermetic_cc_toolchain", + integrity = "sha256-kHdFv5FVX3foI0wLlTNx5srFunFdHPEv9kFJbdG86dE=", patch_strip = 1, patches = ["//bazel:hermetic_cc_toolchain.patch"], + urls = ["https://github.com/uber/hermetic_cc_toolchain/releases/download/v3.1.1/hermetic_cc_toolchain-v3.1.1.tar.gz"], ) # configure/make dependencies From 05b9e948df398d5e0ac47ce959c25a93f3ff6258 Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Tue, 11 Mar 2025 22:12:02 +0000 Subject: [PATCH 15/28] Add macOS toolchains --- MODULE.bazel | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 7fbea9d2fdf1..037b77f97bb7 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -900,10 +900,13 @@ toolchains.extra_settings(settings = ["//bazel:use_hermetic_cc"]) use_repo(toolchains, "zig_sdk") register_toolchains( - # if no `--platform` is specified, these toolchains will be used for - # (linux,darwin,windows)x(amd64,arm64) + # Linux toolchains "@zig_sdk//toolchain:linux_amd64_gnu.2.28", + # macOS toolchains + "@zig_sdk//toolchain:darwin_amd64", + "@zig_sdk//toolchain:darwin_arm64", + # wasm toolchains "@zig_sdk//toolchain:none_wasm", From d3e4c9b681ae15328c9f9241818d85b5131132b8 Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Fri, 14 Mar 2025 21:01:25 +0000 Subject: [PATCH 16/28] Strip binary within target --- cpp/BUILD.bazel | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/cpp/BUILD.bazel b/cpp/BUILD.bazel index e79d2ed0260f..e9ede47c7e08 100644 --- a/cpp/BUILD.bazel +++ b/cpp/BUILD.bazel @@ -1,5 +1,3 @@ -load("//publish:defs.bzl", "release_strip_binary") - package(default_visibility = ["//visibility:public"]) cc_binary( @@ -9,9 +7,9 @@ cc_binary( "infogetty-cpp/network_info.cc", "infogetty-cpp/network_info.h", ], - copts = ["-std=c++17"], - linkopts = [ - "-lsystemd", + copts = [ + "-std=c++17", + "-Wl,--strip-debug", ], target_compatible_with = [ "@platforms//os:linux", @@ -19,14 +17,9 @@ cc_binary( deps = ["@libsystemd"], ) -release_strip_binary( - name = "infogetty_stripped", - binary = "infogetty_bin", -) - genrule( name = "infogetty_cleaned", - srcs = ["infogetty_stripped"], + srcs = ["infogetty_bin"], outs = ["infogetty"], cmd = "objcopy -R .comment -R .note -R .note.gnu.build-id $< $@", executable = True, From 1b6ea6cdf8edd8c42ce5633885a0571d6055df7f Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Fri, 14 Mar 2025 21:05:36 +0000 Subject: [PATCH 17/28] Rename hs targets --- hs/spec_compliance/BUILD.bazel | 4 ++-- rs/tests/research/BUILD.bazel | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hs/spec_compliance/BUILD.bazel b/hs/spec_compliance/BUILD.bazel index 539bc6e755ed..f18b3bf93f1a 100644 --- a/hs/spec_compliance/BUILD.bazel +++ b/hs/spec_compliance/BUILD.bazel @@ -74,13 +74,13 @@ STACK_DEPS = [ ] disable_hermetic_cc_binary( - name = "ic-ref-test-wrapped", + name = "ic-ref-test", binary = ":ic-ref-test", visibility = ["//visibility:public"], ) haskell_binary( - name = "ic-ref-test", + name = "ic-ref-test-inner", srcs = [ "bin/ic-ref-test.hs", ], diff --git a/rs/tests/research/BUILD.bazel b/rs/tests/research/BUILD.bazel index d3c4438b2f4b..eac6ed7e16e0 100644 --- a/rs/tests/research/BUILD.bazel +++ b/rs/tests/research/BUILD.bazel @@ -40,7 +40,7 @@ symlink_dirs( name = "ic-hs", target_compatible_with = ["@platforms//os:linux"], targets = { - "//hs/spec_compliance:ic-ref-test-wrapped": "bin", + "//hs/spec_compliance:ic-ref-test": "bin", "//rs/universal_canister/impl:universal_canister_no_heartbeat.wasm.gz": "test-data", "//rs/tests:wabt-tests": "test-data", }, From 3f99df8bf44b975cd4166b9734419cf97c152c60 Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Sun, 23 Mar 2025 21:29:12 +0000 Subject: [PATCH 18/28] Organize external libs into third_party --- MODULE.bazel | 78 ---------------------------- WORKSPACE.bazel | 6 +++ third_party/libfuzzer/BUILD.bazel | 0 third_party/libfuzzer/repository.bzl | 29 +++++++++++ third_party/openssl/BUILD.bazel | 0 third_party/openssl/repository.bzl | 63 ++++++++++++++++++++++ 6 files changed, 98 insertions(+), 78 deletions(-) create mode 100644 third_party/libfuzzer/BUILD.bazel create mode 100644 third_party/libfuzzer/repository.bzl create mode 100644 third_party/openssl/BUILD.bazel create mode 100644 third_party/openssl/repository.bzl diff --git a/MODULE.bazel b/MODULE.bazel index 037b77f97bb7..18ce4af605fa 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -913,81 +913,3 @@ register_toolchains( # These toolchains are only registered locally. dev_dependency = True, ) - -# Build libfuzzer to ensure compatibility with our toolchain, and so that it -# can be optionally linked by the fuzzers. -http_archive( - name = "libfuzzer", - build_file_content = """ -load("@rules_cc//cc:defs.bzl", "cc_library") - -cc_library( - name = "fuzzer", - srcs = glob(["*.cpp"]), - hdrs = glob(["*.h"]), - additional_compiler_inputs = glob(["*.def"]), - cxxopts = ["-g", "-O2", "-fno-omit-frame-pointer", "-std=c++17"], - linkstatic = True, - visibility = ["//visibility:public"], -) -""", - integrity = "sha256-CLw4JzN3fdo8liWeNzL/lsHfmNBHDE+FsWMnTq5of08=", - strip_prefix = "llvm-project-llvmorg-20.1.0/compiler-rt/lib/fuzzer", - urls = ["https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-20.1.0.tar.gz"], -) - -# openssl-sys does not build with the vendored feature when using -# rules_rust, so we build it ourselves, here. -# https://github.com/bazelbuild/rules_rust/issues/1519 -http_archive( - name = "openssl", - build_file_content = """ -load("@rules_foreign_cc//foreign_cc:defs.bzl", "configure_make") - -filegroup( - name = "all_srcs", - srcs = glob( - include = ["**"], - exclude = ["*.bazel"], - ), -) - -# https://wiki.openssl.org/index.php/Compilation_and_Installation -CONFIGURE_OPTIONS = [ - "no-comp", - "no-idea", - "no-weak-ssl-ciphers", - "no-shared", -] - -MAKE_TARGETS = [ - "build_libs", - "install_dev", -] - -configure_make( - name = "openssl", - args = ["-j12"], - configure_command = "config", - configure_in_place = True, - configure_options = CONFIGURE_OPTIONS, - lib_name = "openssl", - lib_source = ":all_srcs", - out_lib_dir = "lib64", - out_shared_libs = [], - out_static_libs = ["libssl.a"], - targets = MAKE_TARGETS, - visibility = ["//visibility:public"], -) - -filegroup( - name = "gen_dir", - srcs = [":openssl"], - output_group = "gen_dir", - visibility = ["//visibility:public"], -) -""", - integrity = "sha256-4V3agv4v6BOdwqwho21MoB1TE8dfmfRsTooncJtylL8=", - strip_prefix = "openssl-3.4.0", - urls = ["https://github.com/openssl/openssl/releases/download/openssl-3.4.0/openssl-3.4.0.tar.gz"], -) diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index 2529c6ad0e85..1925a455d994 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -6,6 +6,8 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("//bazel:mainnet-canisters.bzl", "canisters") load("//third_party/jemalloc:repository.bzl", "jemalloc_repository") load("//third_party/lmdb:repository.bzl", "lmdb_repository") +load("//third_party/openssl:repository.bzl", "openssl_repository") +load("//third_party/libfuzzer:repository.bzl", "libfuzzer_repository") # We cannot derive the Bazel repository names (e.g. @mainnet_nns_registry_canister) directly # from the canister names because we use inconsistent repo names. Same goes for filenames. @@ -172,6 +174,10 @@ lmdb_repository() jemalloc_repository() +openssl_repository() + +libfuzzer_repository() + http_archive( name = "buildifier_prebuilt", sha256 = "72b5bb0853aac597cce6482ee6c62513318e7f2c0050bc7c319d75d03d8a3875", diff --git a/third_party/libfuzzer/BUILD.bazel b/third_party/libfuzzer/BUILD.bazel new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/third_party/libfuzzer/repository.bzl b/third_party/libfuzzer/repository.bzl new file mode 100644 index 000000000000..bf9164e27633 --- /dev/null +++ b/third_party/libfuzzer/repository.bzl @@ -0,0 +1,29 @@ +""" +Build libfuzzer to ensure compatibility with our toolchain, and so that it can +be optionally linked by the fuzzers. +""" + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") + +def libfuzzer_repository(): + maybe( + http_archive, + name = "libfuzzer", + build_file_content = """ +load("@rules_cc//cc:defs.bzl", "cc_library") + +cc_library( + name = "fuzzer", + srcs = glob(["*.cpp"]), + hdrs = glob(["*.h"]), + additional_compiler_inputs = glob(["*.def"]), + cxxopts = ["-g", "-O2", "-fno-omit-frame-pointer", "-std=c++17"], + linkstatic = True, + visibility = ["//visibility:public"], +) +""", + integrity = "sha256-CLw4JzN3fdo8liWeNzL/lsHfmNBHDE+FsWMnTq5of08=", + strip_prefix = "llvm-project-llvmorg-20.1.0/compiler-rt/lib/fuzzer", + urls = ["https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-20.1.0.tar.gz"], + ) diff --git a/third_party/openssl/BUILD.bazel b/third_party/openssl/BUILD.bazel new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/third_party/openssl/repository.bzl b/third_party/openssl/repository.bzl new file mode 100644 index 000000000000..9f8ab1daffb4 --- /dev/null +++ b/third_party/openssl/repository.bzl @@ -0,0 +1,63 @@ +""" +openssl-sys does not build with the vendored feature when using rules_rust, so +we build it ourselves, here. +https://github.com/bazelbuild/rules_rust/issues/1519 +""" + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") + +def openssl_repository(): + maybe( + http_archive, + name = "openssl", + build_file_content = """ +load("@rules_foreign_cc//foreign_cc:defs.bzl", "configure_make") + +filegroup( + name = "all_srcs", + srcs = glob( + include = ["**"], + exclude = ["*.bazel"], + ), +) + +# https://wiki.openssl.org/index.php/Compilation_and_Installation +CONFIGURE_OPTIONS = [ + "no-comp", + "no-idea", + "no-weak-ssl-ciphers", + "no-shared", +] + +MAKE_TARGETS = [ + "build_libs", + "install_dev", +] + +configure_make( + name = "openssl", + args = ["-j12"], + configure_command = "config", + configure_in_place = True, + configure_options = CONFIGURE_OPTIONS, + lib_name = "openssl", + lib_source = ":all_srcs", + out_lib_dir = "lib64", + out_shared_libs = [], + out_static_libs = ["libssl.a"], + targets = MAKE_TARGETS, + visibility = ["//visibility:public"], +) + +filegroup( + name = "gen_dir", + srcs = [":openssl"], + output_group = "gen_dir", + visibility = ["//visibility:public"], +) +""", + integrity = "sha256-4V3agv4v6BOdwqwho21MoB1TE8dfmfRsTooncJtylL8=", + strip_prefix = "openssl-3.4.0", + urls = ["https://github.com/openssl/openssl/releases/download/openssl-3.4.0/openssl-3.4.0.tar.gz"], + ) From 117b3a524ae0ca0bdc2421ceb31009651f5c745c Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Sun, 23 Mar 2025 21:30:00 +0000 Subject: [PATCH 19/28] Update various comments --- MODULE.bazel | 2 ++ bazel/BUILD.bazel | 2 ++ bazel/cc_rs.patch | 5 +++++ bazel/conf/.bazelrc.build | 4 ++++ bazel/hermetic_cc_toolchain.patch | 7 +++++-- third_party/openssl/repository.bzl | 6 ++++-- 6 files changed, 22 insertions(+), 4 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 18ce4af605fa..f652e986a006 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -896,6 +896,8 @@ mainnet_versions( # Set up hermetic cc toolchains for binaries and canisters toolchains = use_extension("@hermetic_cc_toolchain//toolchain:ext.bzl", "toolchains") +# The use_hermetic_cc setting is added to each toolchain to allow us to opt out +# in favor of system toolchains, when needed. toolchains.extra_settings(settings = ["//bazel:use_hermetic_cc"]) use_repo(toolchains, "zig_sdk") diff --git a/bazel/BUILD.bazel b/bazel/BUILD.bazel index 07b5ee0b4b1b..7484bfbbef5d 100644 --- a/bazel/BUILD.bazel +++ b/bazel/BUILD.bazel @@ -40,6 +40,8 @@ config_setting( }, ) +# Allow targets to opt out of hermetic toolchains, in favor of the one provided +# by the system bool_flag( name = "hermetic_cc", build_setting_default = True, diff --git a/bazel/cc_rs.patch b/bazel/cc_rs.patch index b1a1c1bc192f..7a5f01a33845 100644 --- a/bazel/cc_rs.patch +++ b/bazel/cc_rs.patch @@ -1,4 +1,9 @@ # Adjust target naming to match what zig expects. +# +# zig does not plan to change their target naming: https://github.com/ziglang/zig/issues/4911 +# and cc-rs is waiting on a 1.0 release to support zig's scheme: https://github.com/rust-lang/cc-rs/pull/986 +# Other related links: https://github.com/bazelbuild/rules_rust/issues/2529 +# https://github.com/Asana/cc-rs diff --git a/src/lib.rs b/src/lib.rs index f75b951..8e770a7 100644 --- a/src/lib.rs diff --git a/bazel/conf/.bazelrc.build b/bazel/conf/.bazelrc.build index 3f394ed66aa4..e8ef1a2de2f5 100644 --- a/bazel/conf/.bazelrc.build +++ b/bazel/conf/.bazelrc.build @@ -114,6 +114,10 @@ build:afl --action_env="LD=afl-clang-lto++" build:afl --action_env="LLVM_CONFIG=llvm-config-18" build:afl --action_env="RANLIB=llvm-ranlib-18" build:afl --config=fuzzing +# Note: Instrumenting with AFL is done by overriding the above variables, but +# these are not respected by the hermetic toolchains. Instead, we use +# toolchains from the host system. +# For more context see: https://github.com/dfinity/ic/pull/3508 build:afl --hermetic_cc=false build:afl --build_tag_filters=afl run:afl --run_under="//bin:afl_wrapper" diff --git a/bazel/hermetic_cc_toolchain.patch b/bazel/hermetic_cc_toolchain.patch index 3c4fa0e02951..b9660d75486b 100644 --- a/bazel/hermetic_cc_toolchain.patch +++ b/bazel/hermetic_cc_toolchain.patch @@ -1,5 +1,8 @@ -# Extend hermetic_cc_toolchain with a wasm target that does not use wasi, and -# add a path to put additional restrictions on the generated toolchains. +# Extend hermetic_cc_toolchain with a wasm target that does not use wasi +# (https://github.com/uber/hermetic_cc_toolchain/pull/214), and add a path to +# put additional restrictions on the generated toolchains so that we can +# restrict which targets pick up the toolchains +# (https://github.com/uber/hermetic_cc_toolchain/pull/213). diff --git a/MODULE.bazel b/MODULE.bazel index 409d626..1265298 100644 --- a/MODULE.bazel diff --git a/third_party/openssl/repository.bzl b/third_party/openssl/repository.bzl index 9f8ab1daffb4..6317b80a71b8 100644 --- a/third_party/openssl/repository.bzl +++ b/third_party/openssl/repository.bzl @@ -1,7 +1,9 @@ """ -openssl-sys does not build with the vendored feature when using rules_rust, so -we build it ourselves, here. +With hermetic toolchains, we would like to not depend on the openssl from the +build enviornment. Normally the vendored feature would build the library from +source alongside the crate, but this does not work when using rules_rust: https://github.com/bazelbuild/rules_rust/issues/1519 +Instead, we build it ourselves, here. """ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") From e9ad1a5fd99deb32856ae8816195888340297f7c Mon Sep 17 00:00:00 2001 From: IDX GitHub Automation Date: Sun, 23 Mar 2025 21:53:27 +0000 Subject: [PATCH 20/28] Automatically updated Cargo*.lock --- Cargo.Bazel.Fuzzing.json.lock | 69 ++++++++++++++++++++++++++++++----- Cargo.Bazel.json.lock | 34 +++++++++++++++-- 2 files changed, 90 insertions(+), 13 deletions(-) diff --git a/Cargo.Bazel.Fuzzing.json.lock b/Cargo.Bazel.Fuzzing.json.lock index 0240985a028c..9e8a598b2336 100644 --- a/Cargo.Bazel.Fuzzing.json.lock +++ b/Cargo.Bazel.Fuzzing.json.lock @@ -1,5 +1,5 @@ { - "checksum": "bff29c17220de7fac8fd4a275c96c1b88ff7b1ecf5f0ef987bacf362e5044cff", + "checksum": "47695d92b61eb83496bb7a917ae6d3dd7938821b587162b6b7166488f2081ab4", "crates": { "abnf 0.12.0": { "name": "abnf", @@ -6830,7 +6830,10 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Zsanitizer=address" + "-Cstrip=none", + "-Zsanitizer=address", + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" ], "selects": {} }, @@ -7620,7 +7623,10 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Zsanitizer=address" + "-Cstrip=none", + "-Zsanitizer=address", + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" ], "selects": {} }, @@ -7737,7 +7743,10 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Zsanitizer=address" + "-Cstrip=none", + "-Zsanitizer=address", + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" ], "selects": {} }, @@ -7884,7 +7893,10 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Zsanitizer=address" + "-Cstrip=none", + "-Zsanitizer=address", + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" ], "selects": {} }, @@ -11412,7 +11424,10 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Zsanitizer=address" + "-Cstrip=none", + "-Zsanitizer=address", + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" ], "selects": {} }, @@ -11762,7 +11777,13 @@ "repository": { "Http": { "url": "https://static.crates.io/crates/cc/1.1.37/download", - "sha256": "40545c26d092346d8a8dab71ee48e7685a7a9cba76e634790c215b41a4a7b4cf" + "sha256": "40545c26d092346d8a8dab71ee48e7685a7a9cba76e634790c215b41a4a7b4cf", + "patch_args": [ + "-p1" + ], + "patches": [ + "@@//bazel:cc_rs.patch" + ] } }, "targets": [ @@ -35020,7 +35041,10 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Zsanitizer=address" + "-Cstrip=none", + "-Zsanitizer=address", + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" ], "selects": {} }, @@ -42705,6 +42729,12 @@ "compile_data_glob": [ "**" ], + "data": { + "common": [ + "@openssl//:gen_dir" + ], + "selects": {} + }, "data_glob": [ "**" ], @@ -42888,7 +42918,8 @@ ], "crate_features": { "common": [ - "libc" + "libc", + "static" ], "selects": {} }, @@ -49911,6 +49942,12 @@ "compile_data_glob": [ "**" ], + "data": { + "common": [ + "@openssl//:gen_dir" + ], + "selects": {} + }, "data_glob": [ "**" ], @@ -49931,6 +49968,15 @@ ], "selects": {} }, + "build_script_env": { + "common": { + "OPENSSL_INCLUDE_DIR": "$(location @openssl//:gen_dir)/include", + "OPENSSL_LIB_DIR": "$(location @openssl//:gen_dir)/lib64", + "OPENSSL_NO_VENDOR": "1", + "OPENSSL_STATIC": "1" + }, + "selects": {} + }, "links": "openssl" }, "license": "MIT", @@ -87037,7 +87083,10 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Zsanitizer=address" + "-Cstrip=none", + "-Zsanitizer=address", + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" ], "selects": {} }, diff --git a/Cargo.Bazel.json.lock b/Cargo.Bazel.json.lock index 00433b6c35cc..2768cfa888d4 100644 --- a/Cargo.Bazel.json.lock +++ b/Cargo.Bazel.json.lock @@ -1,5 +1,5 @@ { - "checksum": "6b5cac5f03c3f73653b9fcc13d0d44e0d958cc37cd309d006ec71b5733d99f13", + "checksum": "cd971a45e5a37d1a9d8d198aceb0e8f3545642c45e517ea8ffd717e3324ae600", "crates": { "abnf 0.12.0": { "name": "abnf", @@ -11658,7 +11658,13 @@ "repository": { "Http": { "url": "https://static.crates.io/crates/cc/1.1.37/download", - "sha256": "40545c26d092346d8a8dab71ee48e7685a7a9cba76e634790c215b41a4a7b4cf" + "sha256": "40545c26d092346d8a8dab71ee48e7685a7a9cba76e634790c215b41a4a7b4cf", + "patch_args": [ + "-p1" + ], + "patches": [ + "@@//bazel:cc_rs.patch" + ] } }, "targets": [ @@ -42539,6 +42545,12 @@ "compile_data_glob": [ "**" ], + "data": { + "common": [ + "@openssl//:gen_dir" + ], + "selects": {} + }, "data_glob": [ "**" ], @@ -42722,7 +42734,8 @@ ], "crate_features": { "common": [ - "libc" + "libc", + "static" ], "selects": {} }, @@ -49751,6 +49764,12 @@ "compile_data_glob": [ "**" ], + "data": { + "common": [ + "@openssl//:gen_dir" + ], + "selects": {} + }, "data_glob": [ "**" ], @@ -49771,6 +49790,15 @@ ], "selects": {} }, + "build_script_env": { + "common": { + "OPENSSL_INCLUDE_DIR": "$(location @openssl//:gen_dir)/include", + "OPENSSL_LIB_DIR": "$(location @openssl//:gen_dir)/lib64", + "OPENSSL_NO_VENDOR": "1", + "OPENSSL_STATIC": "1" + }, + "selects": {} + }, "links": "openssl" }, "license": "MIT", From e594a012f5f84960be7cab65256a485f8502f639 Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Mon, 24 Mar 2025 10:03:18 +0000 Subject: [PATCH 21/28] Buildifier --- MODULE.bazel | 1 + WORKSPACE.bazel | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index 0bcf60ddfd4c..154fddc6a93f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -843,6 +843,7 @@ mainnet_versions( # Set up hermetic cc toolchains for binaries and canisters toolchains = use_extension("@hermetic_cc_toolchain//toolchain:ext.bzl", "toolchains") + # The use_hermetic_cc setting is added to each toolchain to allow us to opt out # in favor of system toolchains, when needed. toolchains.extra_settings(settings = ["//bazel:use_hermetic_cc"]) diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index 1925a455d994..591a7c0002da 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -5,9 +5,9 @@ workspace( load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("//bazel:mainnet-canisters.bzl", "canisters") load("//third_party/jemalloc:repository.bzl", "jemalloc_repository") +load("//third_party/libfuzzer:repository.bzl", "libfuzzer_repository") load("//third_party/lmdb:repository.bzl", "lmdb_repository") load("//third_party/openssl:repository.bzl", "openssl_repository") -load("//third_party/libfuzzer:repository.bzl", "libfuzzer_repository") # We cannot derive the Bazel repository names (e.g. @mainnet_nns_registry_canister) directly # from the canister names because we use inconsistent repo names. Same goes for filenames. From 12ba397778b1fe4aa00fec34263c9cdfd6be904e Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Mon, 24 Mar 2025 13:40:58 +0000 Subject: [PATCH 22/28] Disable hermetic toolchains for macOS --- MODULE.bazel | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 154fddc6a93f..4b143e09c86f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -854,8 +854,10 @@ register_toolchains( "@zig_sdk//toolchain:linux_amd64_gnu.2.28", # macOS toolchains - "@zig_sdk//toolchain:darwin_amd64", - "@zig_sdk//toolchain:darwin_arm64", + # Do not use hermetic toolchains for macOS until we have had a chance to + # wrap the various system libraries. + # "@zig_sdk//toolchain:darwin_amd64", + # "@zig_sdk//toolchain:darwin_arm64", # wasm toolchains "@zig_sdk//toolchain:none_wasm", From 5ab38a716fe25cd734abca8c85ed0ecf3083183f Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Mon, 24 Mar 2025 15:23:51 +0000 Subject: [PATCH 23/28] Disable tmp speedup for zig --- bazel/conf/.bazelrc.build | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bazel/conf/.bazelrc.build b/bazel/conf/.bazelrc.build index f698fc83ba75..b093ccc65280 100644 --- a/bazel/conf/.bazelrc.build +++ b/bazel/conf/.bazelrc.build @@ -150,4 +150,6 @@ common --noexperimental_inmemory_dotd_files test --test_env=CLICOLOR_FORCE=true # Speed up compilation with zig cc -build --sandbox_add_mount_pair=/tmp +# Currently this is disabled as it breaks pocket-ic tests as the port files +# collide across tests. +# build --sandbox_add_mount_pair=/tmp From f2d084717b67e5380ff27d276b6fa7ef7dba1da0 Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Mon, 24 Mar 2025 15:31:45 +0000 Subject: [PATCH 24/28] Wrap rules_haskell defs at a lower level --- hs/spec_compliance/BUILD.bazel | 64 ++-------------------------------- hs/spec_compliance/defs.bzl | 40 +++++++++++++++++++++ 2 files changed, 43 insertions(+), 61 deletions(-) diff --git a/hs/spec_compliance/BUILD.bazel b/hs/spec_compliance/BUILD.bazel index f18b3bf93f1a..d0e65acb6d25 100644 --- a/hs/spec_compliance/BUILD.bazel +++ b/hs/spec_compliance/BUILD.bazel @@ -1,5 +1,4 @@ -load("@rules_haskell//haskell:defs.bzl", "haskell_binary", "haskell_library") -load("defs.bzl", "disable_hermetic_cc_binary") +load("defs.bzl", "haskell_binary", "haskell_library") STACK_DEPS = [ # Keep sorted. @@ -73,14 +72,8 @@ STACK_DEPS = [ "@stackage//:zlib", ] -disable_hermetic_cc_binary( - name = "ic-ref-test", - binary = ":ic-ref-test", - visibility = ["//visibility:public"], -) - haskell_binary( - name = "ic-ref-test-inner", + name = "ic-ref-test", srcs = [ "bin/ic-ref-test.hs", ], @@ -88,8 +81,8 @@ haskell_binary( "-threaded", # use the threaded Run Time System (RTS) "-rtsopts", # allow passing RTS options like: +RTS -N -RTS ], - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], + visibility = ["//visibility:public"], deps = STACK_DEPS + [ ":IC-CBOR-Parser", ":IC-CBOR-Patterns", @@ -149,7 +142,6 @@ haskell_library( name = "IC-Crypto-BLS", srcs = ["src/IC/Crypto/BLS.hsc"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Id-Forms", @@ -163,7 +155,6 @@ haskell_library( name = "IC-Id-Fresh", srcs = ["src/IC/Id/Fresh.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Id-Forms", @@ -177,7 +168,6 @@ haskell_library( name = "IC-Id-Forms", srcs = ["src/IC/Id/Forms.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Hash", @@ -190,7 +180,6 @@ haskell_library( name = "IC-Utils", srcs = ["src/IC/Utils.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Constants", @@ -205,7 +194,6 @@ haskell_library( name = "IC-Certificate", srcs = ["src/IC/Certificate.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-HashTree", @@ -218,7 +206,6 @@ haskell_library( name = "IC-Management", srcs = ["src/IC/Management.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Types", @@ -231,7 +218,6 @@ haskell_library( name = "IC-Purify", srcs = ["src/IC/Purify.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ "@haskell-candid//:candid", @@ -243,7 +229,6 @@ haskell_library( name = "IC-Crypto", srcs = ["src/IC/Crypto.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Crypto-BLS", @@ -262,7 +247,6 @@ haskell_library( name = "IC-HTTP-RequestId", srcs = ["src/IC/HTTP/RequestId.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-HTTP-GenR", @@ -277,7 +261,6 @@ haskell_library( name = "IC-HTTP-CBOR", srcs = ["src/IC/HTTP/CBOR.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-HTTP-GenR", @@ -290,7 +273,6 @@ haskell_library( name = "IC-HTTP-GenR", srcs = ["src/IC/HTTP/GenR.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ "@haskell-candid//:candid", @@ -302,7 +284,6 @@ haskell_library( name = "IC-HTTP-GenR-Parse", srcs = ["src/IC/HTTP/GenR/Parse.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-HTTP-GenR", @@ -315,7 +296,6 @@ haskell_library( name = "IC-Types", srcs = ["src/IC/Types.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ "@haskell-candid//:candid", @@ -327,7 +307,6 @@ haskell_library( name = "IC-HashTree-CBOR", srcs = ["src/IC/HashTree/CBOR.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-CBOR-Patterns", @@ -341,7 +320,6 @@ haskell_library( name = "IC-HashTree", srcs = ["src/IC/HashTree.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ "@haskell-candid//:candid", @@ -353,7 +331,6 @@ haskell_library( name = "IC-Constants", srcs = ["src/IC/Constants.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Types", @@ -366,7 +343,6 @@ haskell_library( name = "IC-Certificate-Value", srcs = ["src/IC/Certificate/Value.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Types", @@ -380,7 +356,6 @@ haskell_library( name = "IC-Certificate-Validate", srcs = ["src/IC/Certificate/Validate.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Certificate", @@ -397,7 +372,6 @@ haskell_library( name = "IC-Certificate-CBOR", srcs = ["src/IC/Certificate/CBOR.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-CBOR-Parser", @@ -414,7 +388,6 @@ haskell_library( name = "IC-Crypto-Secp256k1", srcs = ["src/IC/Crypto/Secp256k1.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ "@haskell-candid//:candid", @@ -426,7 +399,6 @@ haskell_library( name = "IC-Crypto-Ed25519", srcs = ["src/IC/Crypto/Ed25519.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ "@haskell-candid//:candid", @@ -438,7 +410,6 @@ haskell_library( name = "IC-Crypto-DER-Decode", srcs = ["src/IC/Crypto/DER/Decode.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ "@haskell-candid//:candid", @@ -450,7 +421,6 @@ haskell_library( name = "IC-Crypto-DER", srcs = ["src/IC/Crypto/DER.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Crypto-DER-Decode", @@ -463,7 +433,6 @@ haskell_library( name = "IC-Crypto-DER_BLS", srcs = ["src/IC/Crypto/DER_BLS.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Crypto-BLS", @@ -477,7 +446,6 @@ haskell_library( name = "IC-Crypto-WebAuthn", srcs = ["src/IC/Crypto/WebAuthn.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-CBOR-Parser", @@ -495,7 +463,6 @@ haskell_library( name = "IC-Crypto-CanisterSig", srcs = ["src/IC/Crypto/CanisterSig.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-CBOR-Parser", @@ -516,7 +483,6 @@ haskell_library( name = "IC-Crypto-ECDSA", srcs = ["src/IC/Crypto/ECDSA.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ "@haskell-candid//:candid", @@ -528,7 +494,6 @@ haskell_library( name = "IC-Version", srcs = ["src/IC/Version.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":SourceId", @@ -541,7 +506,6 @@ haskell_library( name = "IC-Hash", srcs = ["src/IC/Hash.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ "@haskell-candid//:candid", @@ -553,7 +517,6 @@ haskell_library( name = "IC-CBOR-Utils", srcs = ["src/IC/CBOR/Utils.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-HTTP-CBOR", @@ -568,7 +531,6 @@ haskell_library( name = "IC-CBOR-Parser", srcs = ["src/IC/CBOR/Parser.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-CBOR-Patterns", @@ -581,7 +543,6 @@ haskell_library( name = "IC-CBOR-Patterns", srcs = ["src/IC/CBOR/Patterns.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ "@haskell-candid//:candid", @@ -593,7 +554,6 @@ haskell_library( name = "IC-DRun-Parse", srcs = ["src/IC/DRun/Parse.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ "@haskell-candid//:candid", @@ -605,7 +565,6 @@ haskell_library( name = "IC-Test-Secp256k1", srcs = ["src/IC/Test/Secp256k1.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Crypto-Secp256k1", @@ -618,7 +577,6 @@ haskell_library( name = "IC-Test-Spec", srcs = ["src/IC/Test/Spec.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Certificate", @@ -652,7 +610,6 @@ haskell_library( name = "IC-Test-WebAuthn", srcs = ["src/IC/Test/WebAuthn.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Crypto-WebAuthn", @@ -665,7 +622,6 @@ haskell_library( name = "IC-Test-Spec-Utils", srcs = ["src/IC/Test/Spec/Utils.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Crypto", @@ -687,7 +643,6 @@ haskell_library( name = "IC-Test-Spec-CanisterHistory", srcs = ["src/IC/Test/Spec/CanisterHistory.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Hash", @@ -708,7 +663,6 @@ haskell_library( name = "IC-Test-Spec-CanisterVersion", srcs = ["src/IC/Test/Spec/CanisterVersion.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Test-Agent", @@ -726,7 +680,6 @@ haskell_library( name = "IC-Test-Spec-Timer", srcs = ["src/IC/Test/Spec/Timer.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Management", @@ -743,7 +696,6 @@ haskell_library( name = "IC-Test-BLS", srcs = ["src/IC/Test/BLS.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Crypto-BLS", @@ -756,7 +708,6 @@ haskell_library( name = "IC-Test-ECDSA", srcs = ["src/IC/Test/ECDSA.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Crypto-ECDSA", @@ -769,7 +720,6 @@ haskell_library( name = "IC-Test-Agent", srcs = ["src/IC/Test/Agent.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Certificate", @@ -799,7 +749,6 @@ haskell_library( name = "IC-Test-HashTree", srcs = ["src/IC/Test/HashTree.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-HashTree", @@ -813,7 +762,6 @@ haskell_library( name = "IC-Test-Universal", srcs = ["src/IC/Test/Universal.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ "@haskell-candid//:candid", @@ -825,7 +773,6 @@ haskell_library( name = "IC-Test-Options", srcs = ["src/IC/Test/Options.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Constants", @@ -841,7 +788,6 @@ haskell_library( name = "IC-Test-Agent-UnsafeCalls", srcs = ["src/IC/Test/Agent/UnsafeCalls.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Id-Forms", @@ -859,7 +805,6 @@ haskell_library( name = "IC-Test-Agent-SafeCalls", srcs = ["src/IC/Test/Agent/SafeCalls.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Id-Forms", @@ -877,7 +822,6 @@ haskell_library( name = "IC-Test-Agent-Calls", srcs = ["src/IC/Test/Agent/Calls.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Id-Forms", @@ -892,7 +836,6 @@ haskell_library( name = "IC-Test-Agent-UserCalls", srcs = ["src/IC/Test/Agent/UserCalls.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ ":IC-Id-Forms", @@ -908,7 +851,6 @@ haskell_library( name = "SourceId", srcs = ["src/SourceId.hs"], src_strip_prefix = "src", - tags = ["manual"], target_compatible_with = ["@platforms//os:linux"], deps = STACK_DEPS + [ "@haskell-candid//:candid", diff --git a/hs/spec_compliance/defs.bzl b/hs/spec_compliance/defs.bzl index 2eba1237365f..ad6084660b14 100644 --- a/hs/spec_compliance/defs.bzl +++ b/hs/spec_compliance/defs.bzl @@ -1,5 +1,7 @@ """Create transition to disable hermetic_cc_toolchains for hs targets.""" +load("@rules_haskell//haskell:defs.bzl", haskell_binary_inner = "haskell_binary", haskell_library_inner = "haskell_library") + def _disable_hermetic_cc_transition(_settings, _attr): return { "//bazel:hermetic_cc": False, @@ -34,3 +36,41 @@ disable_hermetic_cc_binary = rule( "binary": attr.label(mandatory = True, cfg = disable_hermetic_cc_transition, allow_single_file = True), }, ) + +def haskell_binary(name, **kwargs): + """ + Wrap the rules_haskell haskell_binary with a transition that disables hermetic toolchains. + + These are not supported together, because ghc will not work with the zig compiler. + + Args: + name: Name for the generated filegroup. + **kwargs: Pass through args to the inner haskell_binary. + """ + + tags = kwargs.pop("tags", []) + tags.append("manual") + new_name = name + "_inner" + label = ":" + new_name + + haskell_binary_inner(new_name, tags = tags, **kwargs) + + disable_hermetic_cc_binary( + name = name, + binary = label, + visibility = ["//visibility:public"], + ) + +def haskell_library(**kwargs): + """ + Wrap the rules_haskell haskell_library with a transition that disables hermetic toolchains. + + These are not supported together, because ghc will not work with the zig compiler. + + Args: + **kwargs: Pass through args to the inner haskell_library. + """ + tags = kwargs.pop("tags", []) + tags.append("manual") + + haskell_library_inner(tags = tags, **kwargs) From 33f03b0b0715a66ef39de4ead17256fd2ed7ebbd Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Wed, 26 Mar 2025 14:34:08 +0000 Subject: [PATCH 25/28] Fix release stripping --- publish/defs.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/publish/defs.bzl b/publish/defs.bzl index bcac2b391b5b..b965e66890b4 100644 --- a/publish/defs.bzl +++ b/publish/defs.bzl @@ -10,9 +10,9 @@ def _release_nostrip_transition(_settings, _attr): "//command_line_option:strip": "never", # opt mode will have rules_rust strip debug symbols, regardless of the # strip setting. Unfortunately zig cc (from hermetic_cc_toolchain) - # strips as "all or nothing", so we lose all symbols, unless we also - # override the strip setting here. - "@rules_rust//:extra_rustc_flags": ["-Cdebug-assertions=off", "-Cstrip=none"], + # strips as "all or nothing", so we lose all symbols in opt mode. + # Instead, we have the compiler strip nothing, then strip debug in linking. + "@rules_rust//:extra_rustc_flags": ["-Cdebug-assertions=off", "-Cstrip=none", "-Clink-args=-Wl,--strip-debug"], } release_nostrip_transition = transition( From 7fec8394aa1424f7254768ae8a1a058665120390 Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Wed, 26 Mar 2025 15:51:17 +0000 Subject: [PATCH 26/28] Fix release stripping for macOS --- publish/defs.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/publish/defs.bzl b/publish/defs.bzl index b965e66890b4..ee71cab669d7 100644 --- a/publish/defs.bzl +++ b/publish/defs.bzl @@ -12,7 +12,7 @@ def _release_nostrip_transition(_settings, _attr): # strip setting. Unfortunately zig cc (from hermetic_cc_toolchain) # strips as "all or nothing", so we lose all symbols in opt mode. # Instead, we have the compiler strip nothing, then strip debug in linking. - "@rules_rust//:extra_rustc_flags": ["-Cdebug-assertions=off", "-Cstrip=none", "-Clink-args=-Wl,--strip-debug"], + "@rules_rust//:extra_rustc_flags": ["-Cdebug-assertions=off", "-Cstrip=none", "-Clink-args=-Wl,-S"], } release_nostrip_transition = transition( From 356cb8f9fe19554d0f2060c10cff4d32f439a3d9 Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Thu, 27 Mar 2025 10:47:06 +0000 Subject: [PATCH 27/28] Drop lockfiles --- Cargo.Bazel.Fuzzing.json.lock | 69 +++++------------------------------ Cargo.Bazel.json.lock | 34 ++--------------- 2 files changed, 13 insertions(+), 90 deletions(-) diff --git a/Cargo.Bazel.Fuzzing.json.lock b/Cargo.Bazel.Fuzzing.json.lock index 9e8a598b2336..0240985a028c 100644 --- a/Cargo.Bazel.Fuzzing.json.lock +++ b/Cargo.Bazel.Fuzzing.json.lock @@ -1,5 +1,5 @@ { - "checksum": "47695d92b61eb83496bb7a917ae6d3dd7938821b587162b6b7166488f2081ab4", + "checksum": "bff29c17220de7fac8fd4a275c96c1b88ff7b1ecf5f0ef987bacf362e5044cff", "crates": { "abnf 0.12.0": { "name": "abnf", @@ -6830,10 +6830,7 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Cstrip=none", - "-Zsanitizer=address", - "-Zexternal-clangrt", - "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" + "-Zsanitizer=address" ], "selects": {} }, @@ -7623,10 +7620,7 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Cstrip=none", - "-Zsanitizer=address", - "-Zexternal-clangrt", - "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" + "-Zsanitizer=address" ], "selects": {} }, @@ -7743,10 +7737,7 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Cstrip=none", - "-Zsanitizer=address", - "-Zexternal-clangrt", - "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" + "-Zsanitizer=address" ], "selects": {} }, @@ -7893,10 +7884,7 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Cstrip=none", - "-Zsanitizer=address", - "-Zexternal-clangrt", - "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" + "-Zsanitizer=address" ], "selects": {} }, @@ -11424,10 +11412,7 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Cstrip=none", - "-Zsanitizer=address", - "-Zexternal-clangrt", - "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" + "-Zsanitizer=address" ], "selects": {} }, @@ -11777,13 +11762,7 @@ "repository": { "Http": { "url": "https://static.crates.io/crates/cc/1.1.37/download", - "sha256": "40545c26d092346d8a8dab71ee48e7685a7a9cba76e634790c215b41a4a7b4cf", - "patch_args": [ - "-p1" - ], - "patches": [ - "@@//bazel:cc_rs.patch" - ] + "sha256": "40545c26d092346d8a8dab71ee48e7685a7a9cba76e634790c215b41a4a7b4cf" } }, "targets": [ @@ -35041,10 +35020,7 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Cstrip=none", - "-Zsanitizer=address", - "-Zexternal-clangrt", - "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" + "-Zsanitizer=address" ], "selects": {} }, @@ -42729,12 +42705,6 @@ "compile_data_glob": [ "**" ], - "data": { - "common": [ - "@openssl//:gen_dir" - ], - "selects": {} - }, "data_glob": [ "**" ], @@ -42918,8 +42888,7 @@ ], "crate_features": { "common": [ - "libc", - "static" + "libc" ], "selects": {} }, @@ -49942,12 +49911,6 @@ "compile_data_glob": [ "**" ], - "data": { - "common": [ - "@openssl//:gen_dir" - ], - "selects": {} - }, "data_glob": [ "**" ], @@ -49968,15 +49931,6 @@ ], "selects": {} }, - "build_script_env": { - "common": { - "OPENSSL_INCLUDE_DIR": "$(location @openssl//:gen_dir)/include", - "OPENSSL_LIB_DIR": "$(location @openssl//:gen_dir)/lib64", - "OPENSSL_NO_VENDOR": "1", - "OPENSSL_STATIC": "1" - }, - "selects": {} - }, "links": "openssl" }, "license": "MIT", @@ -87083,10 +87037,7 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Cstrip=none", - "-Zsanitizer=address", - "-Zexternal-clangrt", - "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" + "-Zsanitizer=address" ], "selects": {} }, diff --git a/Cargo.Bazel.json.lock b/Cargo.Bazel.json.lock index 2768cfa888d4..00433b6c35cc 100644 --- a/Cargo.Bazel.json.lock +++ b/Cargo.Bazel.json.lock @@ -1,5 +1,5 @@ { - "checksum": "cd971a45e5a37d1a9d8d198aceb0e8f3545642c45e517ea8ffd717e3324ae600", + "checksum": "6b5cac5f03c3f73653b9fcc13d0d44e0d958cc37cd309d006ec71b5733d99f13", "crates": { "abnf 0.12.0": { "name": "abnf", @@ -11658,13 +11658,7 @@ "repository": { "Http": { "url": "https://static.crates.io/crates/cc/1.1.37/download", - "sha256": "40545c26d092346d8a8dab71ee48e7685a7a9cba76e634790c215b41a4a7b4cf", - "patch_args": [ - "-p1" - ], - "patches": [ - "@@//bazel:cc_rs.patch" - ] + "sha256": "40545c26d092346d8a8dab71ee48e7685a7a9cba76e634790c215b41a4a7b4cf" } }, "targets": [ @@ -42545,12 +42539,6 @@ "compile_data_glob": [ "**" ], - "data": { - "common": [ - "@openssl//:gen_dir" - ], - "selects": {} - }, "data_glob": [ "**" ], @@ -42734,8 +42722,7 @@ ], "crate_features": { "common": [ - "libc", - "static" + "libc" ], "selects": {} }, @@ -49764,12 +49751,6 @@ "compile_data_glob": [ "**" ], - "data": { - "common": [ - "@openssl//:gen_dir" - ], - "selects": {} - }, "data_glob": [ "**" ], @@ -49790,15 +49771,6 @@ ], "selects": {} }, - "build_script_env": { - "common": { - "OPENSSL_INCLUDE_DIR": "$(location @openssl//:gen_dir)/include", - "OPENSSL_LIB_DIR": "$(location @openssl//:gen_dir)/lib64", - "OPENSSL_NO_VENDOR": "1", - "OPENSSL_STATIC": "1" - }, - "selects": {} - }, "links": "openssl" }, "license": "MIT", From 57fbca4b32c1a09b3f639757cc8fe77224fc2673 Mon Sep 17 00:00:00 2001 From: IDX GitHub Automation Date: Thu, 27 Mar 2025 10:58:09 +0000 Subject: [PATCH 28/28] Automatically updated Cargo*.lock --- Cargo.Bazel.Fuzzing.json.lock | 69 ++++++++++++++++++++++++++++++----- Cargo.Bazel.json.lock | 34 +++++++++++++++-- 2 files changed, 90 insertions(+), 13 deletions(-) diff --git a/Cargo.Bazel.Fuzzing.json.lock b/Cargo.Bazel.Fuzzing.json.lock index 0dd37b3bafa5..3c22df3faa83 100644 --- a/Cargo.Bazel.Fuzzing.json.lock +++ b/Cargo.Bazel.Fuzzing.json.lock @@ -1,5 +1,5 @@ { - "checksum": "68aa001ce9abfda2b276ee96f8a584a5fe878c7eef2f8817c1a4f6301cf09995", + "checksum": "29c7b37a159557860517fe0b7082300eddc8cdf865c029fd23664fec048d4423", "crates": { "abnf 0.12.0": { "name": "abnf", @@ -6830,7 +6830,10 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Zsanitizer=address" + "-Cstrip=none", + "-Zsanitizer=address", + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" ], "selects": {} }, @@ -7620,7 +7623,10 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Zsanitizer=address" + "-Cstrip=none", + "-Zsanitizer=address", + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" ], "selects": {} }, @@ -7737,7 +7743,10 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Zsanitizer=address" + "-Cstrip=none", + "-Zsanitizer=address", + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" ], "selects": {} }, @@ -7884,7 +7893,10 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Zsanitizer=address" + "-Cstrip=none", + "-Zsanitizer=address", + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" ], "selects": {} }, @@ -11412,7 +11424,10 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Zsanitizer=address" + "-Cstrip=none", + "-Zsanitizer=address", + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" ], "selects": {} }, @@ -11762,7 +11777,13 @@ "repository": { "Http": { "url": "https://static.crates.io/crates/cc/1.1.37/download", - "sha256": "40545c26d092346d8a8dab71ee48e7685a7a9cba76e634790c215b41a4a7b4cf" + "sha256": "40545c26d092346d8a8dab71ee48e7685a7a9cba76e634790c215b41a4a7b4cf", + "patch_args": [ + "-p1" + ], + "patches": [ + "@@//bazel:cc_rs.patch" + ] } }, "targets": [ @@ -35019,7 +35040,10 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Zsanitizer=address" + "-Cstrip=none", + "-Zsanitizer=address", + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" ], "selects": {} }, @@ -42704,6 +42728,12 @@ "compile_data_glob": [ "**" ], + "data": { + "common": [ + "@openssl//:gen_dir" + ], + "selects": {} + }, "data_glob": [ "**" ], @@ -42887,7 +42917,8 @@ ], "crate_features": { "common": [ - "libc" + "libc", + "static" ], "selects": {} }, @@ -49931,6 +49962,12 @@ "compile_data_glob": [ "**" ], + "data": { + "common": [ + "@openssl//:gen_dir" + ], + "selects": {} + }, "data_glob": [ "**" ], @@ -49951,6 +49988,15 @@ ], "selects": {} }, + "build_script_env": { + "common": { + "OPENSSL_INCLUDE_DIR": "$(location @openssl//:gen_dir)/include", + "OPENSSL_LIB_DIR": "$(location @openssl//:gen_dir)/lib64", + "OPENSSL_NO_VENDOR": "1", + "OPENSSL_STATIC": "1" + }, + "selects": {} + }, "links": "openssl" }, "license": "MIT", @@ -87057,7 +87103,10 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Zsanitizer=address" + "-Cstrip=none", + "-Zsanitizer=address", + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" ], "selects": {} }, diff --git a/Cargo.Bazel.json.lock b/Cargo.Bazel.json.lock index b59a2e37d231..d0d69d15be01 100644 --- a/Cargo.Bazel.json.lock +++ b/Cargo.Bazel.json.lock @@ -1,5 +1,5 @@ { - "checksum": "0263b9a223c42fc2ecc48a3b8183818e33d79e8eb22e2726049e9d5495906a8c", + "checksum": "20b0704a5807a13511ee3df0a87cca12700e295b4ee17f61396575ec2dfe27e9", "crates": { "abnf 0.12.0": { "name": "abnf", @@ -11658,7 +11658,13 @@ "repository": { "Http": { "url": "https://static.crates.io/crates/cc/1.1.37/download", - "sha256": "40545c26d092346d8a8dab71ee48e7685a7a9cba76e634790c215b41a4a7b4cf" + "sha256": "40545c26d092346d8a8dab71ee48e7685a7a9cba76e634790c215b41a4a7b4cf", + "patch_args": [ + "-p1" + ], + "patches": [ + "@@//bazel:cc_rs.patch" + ] } }, "targets": [ @@ -42538,6 +42544,12 @@ "compile_data_glob": [ "**" ], + "data": { + "common": [ + "@openssl//:gen_dir" + ], + "selects": {} + }, "data_glob": [ "**" ], @@ -42721,7 +42733,8 @@ ], "crate_features": { "common": [ - "libc" + "libc", + "static" ], "selects": {} }, @@ -49771,6 +49784,12 @@ "compile_data_glob": [ "**" ], + "data": { + "common": [ + "@openssl//:gen_dir" + ], + "selects": {} + }, "data_glob": [ "**" ], @@ -49791,6 +49810,15 @@ ], "selects": {} }, + "build_script_env": { + "common": { + "OPENSSL_INCLUDE_DIR": "$(location @openssl//:gen_dir)/include", + "OPENSSL_LIB_DIR": "$(location @openssl//:gen_dir)/lib64", + "OPENSSL_NO_VENDOR": "1", + "OPENSSL_STATIC": "1" + }, + "selects": {} + }, "links": "openssl" }, "license": "MIT",