diff --git a/Cargo.Bazel.Fuzzing.json.lock b/Cargo.Bazel.Fuzzing.json.lock index 0dd37b3bafa5..3c22df3faa83 100644 --- a/Cargo.Bazel.Fuzzing.json.lock +++ b/Cargo.Bazel.Fuzzing.json.lock @@ -1,5 +1,5 @@ { - "checksum": "68aa001ce9abfda2b276ee96f8a584a5fe878c7eef2f8817c1a4f6301cf09995", + "checksum": "29c7b37a159557860517fe0b7082300eddc8cdf865c029fd23664fec048d4423", "crates": { "abnf 0.12.0": { "name": "abnf", @@ -6830,7 +6830,10 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Zsanitizer=address" + "-Cstrip=none", + "-Zsanitizer=address", + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" ], "selects": {} }, @@ -7620,7 +7623,10 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Zsanitizer=address" + "-Cstrip=none", + "-Zsanitizer=address", + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" ], "selects": {} }, @@ -7737,7 +7743,10 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Zsanitizer=address" + "-Cstrip=none", + "-Zsanitizer=address", + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" ], "selects": {} }, @@ -7884,7 +7893,10 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Zsanitizer=address" + "-Cstrip=none", + "-Zsanitizer=address", + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" ], "selects": {} }, @@ -11412,7 +11424,10 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Zsanitizer=address" + "-Cstrip=none", + "-Zsanitizer=address", + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" ], "selects": {} }, @@ -11762,7 +11777,13 @@ "repository": { "Http": { "url": "https://static.crates.io/crates/cc/1.1.37/download", - "sha256": "40545c26d092346d8a8dab71ee48e7685a7a9cba76e634790c215b41a4a7b4cf" + "sha256": "40545c26d092346d8a8dab71ee48e7685a7a9cba76e634790c215b41a4a7b4cf", + "patch_args": [ + "-p1" + ], + "patches": [ + "@@//bazel:cc_rs.patch" + ] } }, "targets": [ @@ -35019,7 +35040,10 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Zsanitizer=address" + "-Cstrip=none", + "-Zsanitizer=address", + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" ], "selects": {} }, @@ -42704,6 +42728,12 @@ "compile_data_glob": [ "**" ], + "data": { + "common": [ + "@openssl//:gen_dir" + ], + "selects": {} + }, "data_glob": [ "**" ], @@ -42887,7 +42917,8 @@ ], "crate_features": { "common": [ - "libc" + "libc", + "static" ], "selects": {} }, @@ -49931,6 +49962,12 @@ "compile_data_glob": [ "**" ], + "data": { + "common": [ + "@openssl//:gen_dir" + ], + "selects": {} + }, "data_glob": [ "**" ], @@ -49951,6 +49988,15 @@ ], "selects": {} }, + "build_script_env": { + "common": { + "OPENSSL_INCLUDE_DIR": "$(location @openssl//:gen_dir)/include", + "OPENSSL_LIB_DIR": "$(location @openssl//:gen_dir)/lib64", + "OPENSSL_NO_VENDOR": "1", + "OPENSSL_STATIC": "1" + }, + "selects": {} + }, "links": "openssl" }, "license": "MIT", @@ -87057,7 +87103,10 @@ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", - "-Zsanitizer=address" + "-Cstrip=none", + "-Zsanitizer=address", + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a" ], "selects": {} }, diff --git a/Cargo.Bazel.json.lock b/Cargo.Bazel.json.lock index b59a2e37d231..d0d69d15be01 100644 --- a/Cargo.Bazel.json.lock +++ b/Cargo.Bazel.json.lock @@ -1,5 +1,5 @@ { - "checksum": "0263b9a223c42fc2ecc48a3b8183818e33d79e8eb22e2726049e9d5495906a8c", + "checksum": "20b0704a5807a13511ee3df0a87cca12700e295b4ee17f61396575ec2dfe27e9", "crates": { "abnf 0.12.0": { "name": "abnf", @@ -11658,7 +11658,13 @@ "repository": { "Http": { "url": "https://static.crates.io/crates/cc/1.1.37/download", - "sha256": "40545c26d092346d8a8dab71ee48e7685a7a9cba76e634790c215b41a4a7b4cf" + "sha256": "40545c26d092346d8a8dab71ee48e7685a7a9cba76e634790c215b41a4a7b4cf", + "patch_args": [ + "-p1" + ], + "patches": [ + "@@//bazel:cc_rs.patch" + ] } }, "targets": [ @@ -42538,6 +42544,12 @@ "compile_data_glob": [ "**" ], + "data": { + "common": [ + "@openssl//:gen_dir" + ], + "selects": {} + }, "data_glob": [ "**" ], @@ -42721,7 +42733,8 @@ ], "crate_features": { "common": [ - "libc" + "libc", + "static" ], "selects": {} }, @@ -49771,6 +49784,12 @@ "compile_data_glob": [ "**" ], + "data": { + "common": [ + "@openssl//:gen_dir" + ], + "selects": {} + }, "data_glob": [ "**" ], @@ -49791,6 +49810,15 @@ ], "selects": {} }, + "build_script_env": { + "common": { + "OPENSSL_INCLUDE_DIR": "$(location @openssl//:gen_dir)/include", + "OPENSSL_LIB_DIR": "$(location @openssl//:gen_dir)/lib64", + "OPENSSL_NO_VENDOR": "1", + "OPENSSL_STATIC": "1" + }, + "selects": {} + }, "links": "openssl" }, "license": "MIT", diff --git a/MODULE.bazel b/MODULE.bazel index 72f3c85a5f0b..74e859ee2328 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -7,13 +7,20 @@ module( ) # General Bazel helpers - bazel_dep(name = "bazel_skylib", version = "1.7.1") bazel_dep(name = "aspect_bazel_lib", version = "2.9.0") # CC dependencies (for C libs like miracl-core, etc) bazel_dep(name = "rules_cc", version = "0.0.13") bazel_dep(name = "platforms", version = "0.0.10") +bazel_dep(name = "hermetic_cc_toolchain", version = "3.1.1") +archive_override( + module_name = "hermetic_cc_toolchain", + integrity = "sha256-kHdFv5FVX3foI0wLlTNx5srFunFdHPEv9kFJbdG86dE=", + patch_strip = 1, + patches = ["//bazel:hermetic_cc_toolchain.patch"], + urls = ["https://github.com/uber/hermetic_cc_toolchain/releases/download/v3.1.1/hermetic_cc_toolchain-v3.1.1.tar.gz"], +) # configure/make dependencies bazel_dep(name = "rules_foreign_cc", version = "0.12.0") @@ -833,3 +840,28 @@ mainnet_icos_versions( name = "mainnet_icos_versions", path = "//:mainnet-icos-revisions.json", ) + +# Set up hermetic cc toolchains for binaries and canisters +toolchains = use_extension("@hermetic_cc_toolchain//toolchain:ext.bzl", "toolchains") + +# The use_hermetic_cc setting is added to each toolchain to allow us to opt out +# in favor of system toolchains, when needed. +toolchains.extra_settings(settings = ["//bazel:use_hermetic_cc"]) +use_repo(toolchains, "zig_sdk") + +register_toolchains( + # Linux toolchains + "@zig_sdk//toolchain:linux_amd64_gnu.2.28", + + # macOS toolchains + # Do not use hermetic toolchains for macOS until we have had a chance to + # wrap the various system libraries. + # "@zig_sdk//toolchain:darwin_amd64", + # "@zig_sdk//toolchain:darwin_arm64", + + # wasm toolchains + "@zig_sdk//toolchain:none_wasm", + + # These toolchains are only registered locally. + dev_dependency = True, +) diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index 36de3c9816c5..591a7c0002da 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -5,7 +5,9 @@ workspace( load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("//bazel:mainnet-canisters.bzl", "canisters") load("//third_party/jemalloc:repository.bzl", "jemalloc_repository") +load("//third_party/libfuzzer:repository.bzl", "libfuzzer_repository") load("//third_party/lmdb:repository.bzl", "lmdb_repository") +load("//third_party/openssl:repository.bzl", "openssl_repository") # We cannot derive the Bazel repository names (e.g. @mainnet_nns_registry_canister) directly # from the canister names because we use inconsistent repo names. Same goes for filenames. @@ -172,6 +174,10 @@ lmdb_repository() jemalloc_repository() +openssl_repository() + +libfuzzer_repository() + http_archive( name = "buildifier_prebuilt", sha256 = "72b5bb0853aac597cce6482ee6c62513318e7f2c0050bc7c319d75d03d8a3875", @@ -266,3 +272,29 @@ sns_quill(name = "sns_quill") load("//bazel:idl2json.bzl", "idl_to_json") idl_to_json(name = "idl2json") + +# Use libsystemd from the host environment +# Ideally this is pulled hermetically, but as this is only used by infogetty +# within IC images, we have less compatibility to worry about, and packaging it +# this way is easier. +new_local_repository( + name = "libsystemd", + build_file_content = """ +cc_import( + name = "libsystemd-internal", + hdrs = glob(["include/systemd/*.h"]), + interface_library = "lib/x86_64-linux-gnu/libsystemd.so", + system_provided = True, + visibility = ["//visibility:private"], +) + +# Use an extra cc_library to hide the depth of the include folder +cc_library( + name = "libsystemd", + includes = ["include"], + deps = ["libsystemd-internal"], + visibility = ["//visibility:public"], +) +""", + path = "/usr", +) diff --git a/bazel/BUILD.bazel b/bazel/BUILD.bazel index 33493d6004ff..5bcc8c58745c 100644 --- a/bazel/BUILD.bazel +++ b/bazel/BUILD.bazel @@ -40,6 +40,20 @@ config_setting( }, ) +# Allow targets to opt out of hermetic toolchains, in favor of the one provided +# by the system +bool_flag( + name = "hermetic_cc", + build_setting_default = True, +) + +config_setting( + name = "use_hermetic_cc", + flag_values = { + ":hermetic_cc": "True", + }, +) + string_flag( name = "timeout_value", build_setting_default = "10m", diff --git a/bazel/cc_rs.patch b/bazel/cc_rs.patch new file mode 100644 index 000000000000..7a5f01a33845 --- /dev/null +++ b/bazel/cc_rs.patch @@ -0,0 +1,23 @@ +# Adjust target naming to match what zig expects. +# +# zig does not plan to change their target naming: https://github.com/ziglang/zig/issues/4911 +# and cc-rs is waiting on a 1.0 release to support zig's scheme: https://github.com/rust-lang/cc-rs/pull/986 +# Other related links: https://github.com/bazelbuild/rules_rust/issues/2529 +# https://github.com/Asana/cc-rs +diff --git a/src/lib.rs b/src/lib.rs +index f75b951..8e770a7 100644 +--- a/src/lib.rs ++++ b/src/lib.rs +@@ -2091,6 +2091,12 @@ impl Build { + target.versioned_llvm_target(None) + }; + ++ let llvm_target = match llvm_target.clone() { ++ std::borrow::Cow::Borrowed("x86_64-unknown-linux-gnu") => "x86_64-linux-gnu", ++ std::borrow::Cow::Borrowed("wasm32-unknown-unknown") => "wasm32-freestanding-musl", ++ _other => &llvm_target, ++ }; ++ + // Pass `--target` with the LLVM target to properly + // configure Clang even when cross-compiling. + cmd.push_cc_arg(format!("--target={llvm_target}").into()); diff --git a/bazel/conf/.bazelrc.build b/bazel/conf/.bazelrc.build index ae02b1ff136e..b093ccc65280 100644 --- a/bazel/conf/.bazelrc.build +++ b/bazel/conf/.bazelrc.build @@ -48,6 +48,7 @@ common --flag_alias=s3_endpoint=//ci/src/artifacts:s3_endpoint common --flag_alias=s3_upload=//ci/src/artifacts:s3_upload common --flag_alias=k8s=//rs/tests:k8s common --flag_alias=timeout_value=//bazel:timeout_value +common --flag_alias=hermetic_cc=//bazel:hermetic_cc common:stamped --workspace_status_command='$(pwd)/bazel/workspace_status.sh --stamp' common:release --config=stamped --s3_upload=True @@ -117,6 +118,11 @@ build:afl --action_env="LD=afl-clang-lto++" build:afl --action_env="LLVM_CONFIG=llvm-config-18" build:afl --action_env="RANLIB=llvm-ranlib-18" build:afl --config=fuzzing +# Note: Instrumenting with AFL is done by overriding the above variables, but +# these are not respected by the hermetic toolchains. Instead, we use +# toolchains from the host system. +# For more context see: https://github.com/dfinity/ic/pull/3508 +build:afl --hermetic_cc=false build:afl --build_tag_filters=afl run:afl --run_under="//bin:afl_wrapper" @@ -130,11 +136,6 @@ run:sandbox_fuzzing --run_under="ASAN_OPTIONS=detect_leaks=0:allow_user_segv_han query --ui_event_filters=-info,-debug --noshow_progress cquery --ui_event_filters=-info,-debug --noshow_progress -# The default value makes rules_rust pick a dummy toolchain that breaks -# our canister that depend on C deps -# https://github.com/bazelbuild/rules_rust/issues/2764 -common --noincompatible_enable_cc_toolchain_resolution - # This is disabled by default on bazel 7+ some of our targets choke # on this (not yet clear why) common --remote_download_all @@ -147,3 +148,8 @@ common --noexperimental_inmemory_dotd_files # convention would also benefit. If the test does not support this, this "almost # certainly" does no harm. test --test_env=CLICOLOR_FORCE=true + +# Speed up compilation with zig cc +# Currently this is disabled as it breaks pocket-ic tests as the port files +# collide across tests. +# build --sandbox_add_mount_pair=/tmp diff --git a/bazel/external_crates.bzl b/bazel/external_crates.bzl index 8f464b90aff4..e72237302447 100644 --- a/bazel/external_crates.bzl +++ b/bazel/external_crates.bzl @@ -24,13 +24,34 @@ BUILD_INFO_REV = "701a696844fba5c87df162fbbc1ccef96f27c9d7" def external_crates_repository(name, cargo_lockfile, lockfile, sanitizers_enabled): CRATE_ANNOTATIONS = { + "openssl-sys": [crate.annotation( + build_script_data = [ + "@openssl//:gen_dir", + ], + build_script_env = { + "OPENSSL_NO_VENDOR": "1", + "OPENSSL_LIB_DIR": "$(location @openssl//:gen_dir)/lib64", + "OPENSSL_INCLUDE_DIR": "$(location @openssl//:gen_dir)/include", + "OPENSSL_STATIC": "1", + }, + )], "canbench": [crate.annotation( gen_binaries = True, )], + "cc": [crate.annotation( + patch_args = ["-p1"], + patches = ["@@//bazel:cc_rs.patch"], + )], "libssh2-sys": [crate.annotation( # Patch for determinism issues patch_args = ["-p1"], patches = ["@@//bazel:libssh2-sys.patch"], + build_script_data = [ + "@openssl//:gen_dir", + ], + )], + "libz-sys": [crate.annotation( + crate_features = ["static"], )], "curve25519-dalek": [crate.annotation( rustc_flags = [ diff --git a/bazel/fuzz_testing.bzl b/bazel/fuzz_testing.bzl index 205dc5a3c65f..cead960e5a29 100644 --- a/bazel/fuzz_testing.bzl +++ b/bazel/fuzz_testing.bzl @@ -21,12 +21,16 @@ DEFAULT_RUSTC_FLAGS = [ "-Ccodegen-units=1", "-Zextra-const-ub-checks", "-Zstrict-init-checks", + "-Cstrip=none", # TODO(PSEC): Add configuration to enable only during profiling # "-Cinstrument-coverage", ] DEFAULT_SANITIZERS = [ "-Zsanitizer=address", + # zig doesn't like how rustc pushes the sanitizers, so do it ourselves. + "-Zexternal-clangrt", + "-Clink-arg=bazel-out/k8-opt/bin/external/rust_linux_x86_64__x86_64-unknown-linux-gnu__nightly_tools/rust_toolchain/lib/rustlib/x86_64-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a", ] # This flag will be used by third party crates and internal rust_libraries during fuzzing @@ -53,18 +57,12 @@ def rust_fuzz_test_binary(name, srcs, rustc_flags = [], sanitizers = [], crate_f # This would only work inside the devcontainer if allow_main: - FUZZER_LIB = [ - "-Clink-arg=/usr/lib/llvm-18/lib/clang/18/lib/linux/libclang_rt.fuzzer_no_main-x86_64.a", - ] TAGS = ["sandbox_libfuzzer"] else: # default - FUZZER_LIB = [ - "-Clink-arg=/usr/lib/llvm-18/lib/clang/18/lib/linux/libclang_rt.fuzzer-x86_64.a", - ] TAGS = [] - RUSTC_FLAGS_LIBFUZZER = DEFAULT_RUSTC_FLAGS + FUZZER_LIB + RUSTC_FLAGS_LIBFUZZER = DEFAULT_RUSTC_FLAGS + ["-Clink-arg=$(location @libfuzzer//:fuzzer)"] kwargs.setdefault("testonly", True) @@ -75,6 +73,7 @@ def rust_fuzz_test_binary(name, srcs, rustc_flags = [], sanitizers = [], crate_f crate_features = crate_features + ["fuzzing"], proc_macro_deps = proc_macro_deps, deps = deps, + compile_data = ["@libfuzzer//:fuzzer"], rustc_flags = rustc_flags + RUSTC_FLAGS_LIBFUZZER + sanitizers, tags = [ # Makes sure this target is not run in normal CI builds. It would fail due to non-nightly Rust toolchain. diff --git a/bazel/hermetic_cc_toolchain.patch b/bazel/hermetic_cc_toolchain.patch new file mode 100644 index 000000000000..b9660d75486b --- /dev/null +++ b/bazel/hermetic_cc_toolchain.patch @@ -0,0 +1,213 @@ +# Extend hermetic_cc_toolchain with a wasm target that does not use wasi +# (https://github.com/uber/hermetic_cc_toolchain/pull/214), and add a path to +# put additional restrictions on the generated toolchains so that we can +# restrict which targets pick up the toolchains +# (https://github.com/uber/hermetic_cc_toolchain/pull/213). +diff --git a/MODULE.bazel b/MODULE.bazel +index 409d626..1265298 100644 +--- a/MODULE.bazel ++++ b/MODULE.bazel +@@ -49,6 +49,7 @@ register_toolchains( + "@zig_sdk//libc_aware/toolchain:linux_arm64_musl", + # wasm/wasi toolchains + "@zig_sdk//toolchain:wasip1_wasm", ++ "@zig_sdk//toolchain:none_wasm", + + # These toolchains are only registered locally. + dev_dependency = True, +diff --git a/toolchain/defs.bzl b/toolchain/defs.bzl +index f6f613e..02d5597 100644 +--- a/toolchain/defs.bzl ++++ b/toolchain/defs.bzl +@@ -60,7 +60,8 @@ def toolchains( + version = VERSION, + url_formats = [], + host_platform_sha256 = HOST_PLATFORM_SHA256, +- host_platform_ext = _HOST_PLATFORM_EXT): ++ host_platform_ext = _HOST_PLATFORM_EXT, ++ extra_settings = []): + """ + Download zig toolchain and declare bazel toolchains. + The platforms are not registered automatically, that should be done by +@@ -83,6 +84,7 @@ def toolchains( + url_formats = url_formats, + host_platform_sha256 = host_platform_sha256, + host_platform_ext = host_platform_ext, ++ extra_settings = extra_settings + ) + + def _quote(s): +@@ -110,6 +112,8 @@ def _zig_repository_impl(repository_ctx): + "host_platform": host_platform, + } + ++ extra_settings = "[" + " ".join([_quote(str(setting)) for setting in repository_ctx.attr.extra_settings]) + "]" ++ + # Fetch Label dependencies before doing download/extract. + # The Bazel docs are not very clear about this behavior but see: + # https://bazel.build/extending/repo#when_is_the_implementation_function_executed +@@ -117,7 +121,6 @@ def _zig_repository_impl(repository_ctx): + # https://github.com/bazelbuild/bazel-gazelle/pull/1206 + for dest, src in { + "platform/BUILD": "//toolchain/platform:BUILD", +- "toolchain/BUILD": "//toolchain/toolchain:BUILD", + "libc/BUILD": "//toolchain/libc:BUILD", + "libc_aware/platform/BUILD": "//toolchain/libc_aware/platform:BUILD", + "libc_aware/toolchain/BUILD": "//toolchain/libc_aware/toolchain:BUILD", +@@ -126,6 +129,7 @@ def _zig_repository_impl(repository_ctx): + + for dest, src in { + "BUILD": "//toolchain:BUILD.sdk.bazel", ++ "toolchain/BUILD": "//toolchain/toolchain:BUILD", + }.items(): + repository_ctx.template( + dest, +@@ -134,6 +138,7 @@ def _zig_repository_impl(repository_ctx): + substitutions = { + "{zig_sdk_path}": _quote("external/zig_sdk"), + "{os}": _quote(os), ++ "{extra_settings}": extra_settings, + }, + ) + +@@ -230,6 +235,7 @@ zig_repository = repository_rule( + "host_platform_sha256": attr.string_dict(), + "url_formats": attr.string_list(allow_empty = False), + "host_platform_ext": attr.string_dict(), ++ "extra_settings": attr.label_list(), + }, + environ = ["HERMETIC_CC_TOOLCHAIN_CACHE_PREFIX"], + implementation = _zig_repository_impl, +diff --git a/toolchain/ext.bzl b/toolchain/ext.bzl +index ebf0ff8..c2e8af3 100644 +--- a/toolchain/ext.bzl ++++ b/toolchain/ext.bzl +@@ -1,6 +1,11 @@ + load("@hermetic_cc_toolchain//toolchain:defs.bzl", zig_toolchains = "toolchains") + + def _toolchains_impl(ctx): +- zig_toolchains() ++ extra_settings = [] ++ for mod in ctx.modules: ++ for tag in mod.tags.extra_settings: ++ extra_settings += tag.settings + +-toolchains = module_extension(implementation = _toolchains_impl) ++ zig_toolchains(extra_settings = extra_settings) ++ ++toolchains = module_extension(implementation = _toolchains_impl, tag_classes = { "extra_settings": tag_class(attrs = { "settings": attr.label_list(doc = "Each setting is added to every toolchain to make them more restrictive.")})}) +diff --git a/toolchain/platform/defs.bzl b/toolchain/platform/defs.bzl +index d4a8344..faafc5b 100644 +--- a/toolchain/platform/defs.bzl ++++ b/toolchain/platform/defs.bzl +@@ -16,6 +16,7 @@ def declare_platforms(): + + # We can support GOARCH=wasm32 after https://github.com/golang/go/issues/63131 + declare_platform("wasm", "wasm32", "wasi", "wasip1") ++ declare_platform("wasm", "wasm32", "none", "none") + + def declare_libc_aware_platforms(): + # create @zig_sdk//{os}_{arch}_platform entries with zig and go conventions +diff --git a/toolchain/private/defs.bzl b/toolchain/private/defs.bzl +index 716a3a3..b88d082 100644 +--- a/toolchain/private/defs.bzl ++++ b/toolchain/private/defs.bzl +@@ -49,6 +49,7 @@ def target_structs(): + for glibc in _GLIBCS: + ret.append(_target_linux_gnu(gocpu, zigcpu, glibc)) + ret.append(_target_wasm()) ++ ret.append(_target_wasm_no_wasi()) + return ret + + def _target_macos(gocpu, zigcpu): +@@ -222,3 +223,22 @@ def _target_wasm(): + ld_zig_subcmd = "wasm-ld", + artifact_name_patterns = [], + ) ++ ++def _target_wasm_no_wasi(): ++ return struct( ++ gotarget = "none_wasm", ++ zigtarget = "wasm32-freestanding-musl", ++ includes = [] + _INCLUDE_TAIL, ++ linkopts = [], ++ dynamic_library_linkopts = [], ++ supports_dynamic_linker = False, ++ copts = [], ++ libc = "musl", ++ bazel_target_cpu = "wasm32", ++ constraint_values = [ ++ "@platforms//os:none", ++ "@platforms//cpu:wasm32", ++ ], ++ ld_zig_subcmd = "wasm-ld", ++ artifact_name_patterns = [], ++ ) +diff --git a/toolchain/zig-wrapper.zig b/toolchain/zig-wrapper.zig +index d1d59f9..5e2984b 100644 +--- a/toolchain/zig-wrapper.zig ++++ b/toolchain/zig-wrapper.zig +@@ -283,7 +283,7 @@ fn getRunMode(self_exe: []const u8, self_base_noexe: []const u8) error{BadParent + return error.BadParent; + + const got_os = it.next() orelse return error.BadParent; +- if (mem.indexOf(u8, "linux,macos,windows,wasi", got_os) == null) ++ if (mem.indexOf(u8, "linux,macos,windows,wasi,freestanding", got_os) == null) + return error.BadParent; + + // ABI triple is too much of a moving target +diff --git a/toolchain/toolchain/BUILD b/toolchain/toolchain/BUILD +index 552fcaa..8f7dba5 100644 +--- a/toolchain/toolchain/BUILD ++++ b/toolchain/toolchain/BUILD +@@ -4,4 +4,4 @@ package( + default_visibility = ["//visibility:public"], + ) + +-declare_toolchains() ++declare_toolchains(extra_settings = {extra_settings}) +diff --git a/toolchain/toolchain/defs.bzl b/toolchain/toolchain/defs.bzl +index 50cc881..0549c26 100644 +--- a/toolchain/toolchain/defs.bzl ++++ b/toolchain/toolchain/defs.bzl +@@ -1,6 +1,6 @@ + load("@hermetic_cc_toolchain//toolchain/private:defs.bzl", "target_structs") + +-def declare_toolchains(): ++def declare_toolchains(extra_settings = []): + for target_config in target_structs(): + gotarget = target_config.gotarget + zigtarget = target_config.zigtarget +@@ -12,7 +12,7 @@ def declare_toolchains(): + if hasattr(target_config, "libc_constraint"): + extra_constraints = ["@zig_sdk//libc:unconstrained"] + +- _declare_toolchain(gotarget, zigtarget, target_config.constraint_values + extra_constraints) ++ _declare_toolchain(gotarget, zigtarget, target_config.constraint_values + extra_constraints, extra_settings) + + def declare_libc_aware_toolchains(): + for target_config in target_structs(): +@@ -25,13 +25,14 @@ def declare_libc_aware_toolchains(): + if hasattr(target_config, "libc_constraint"): + _declare_toolchain(gotarget, zigtarget, target_config.constraint_values + [target_config.libc_constraint]) + +-def _declare_toolchain(gotarget, zigtarget, target_compatible_with): ++def _declare_toolchain(gotarget, zigtarget, target_compatible_with, extra_settings): + # register two kinds of toolchain targets: Go and Zig conventions. + # Go convention: amd64/arm64, linux/darwin + native.toolchain( + name = gotarget, + exec_compatible_with = None, + target_compatible_with = target_compatible_with, ++ target_settings = extra_settings, + toolchain = "@zig_sdk//:%s_cc" % zigtarget, + toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", + ) +@@ -41,6 +42,7 @@ def _declare_toolchain(gotarget, zigtarget, target_compatible_with): + name = zigtarget, + exec_compatible_with = None, + target_compatible_with = target_compatible_with, ++ target_settings = extra_settings, + toolchain = "@zig_sdk//:%s_cc" % zigtarget, + toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", + ) diff --git a/cpp/BUILD.bazel b/cpp/BUILD.bazel index 6f0dad10bd30..e9ede47c7e08 100644 --- a/cpp/BUILD.bazel +++ b/cpp/BUILD.bazel @@ -7,13 +7,14 @@ cc_binary( "infogetty-cpp/network_info.cc", "infogetty-cpp/network_info.h", ], - copts = ["-std=c++17"], - linkopts = [ - "-lsystemd", + copts = [ + "-std=c++17", + "-Wl,--strip-debug", ], target_compatible_with = [ "@platforms//os:linux", ], + deps = ["@libsystemd"], ) genrule( diff --git a/hs/spec_compliance/BUILD.bazel b/hs/spec_compliance/BUILD.bazel index 7dd71a755810..d0e65acb6d25 100644 --- a/hs/spec_compliance/BUILD.bazel +++ b/hs/spec_compliance/BUILD.bazel @@ -1,4 +1,4 @@ -load("@rules_haskell//haskell:defs.bzl", "haskell_binary", "haskell_library") +load("defs.bzl", "haskell_binary", "haskell_library") STACK_DEPS = [ # Keep sorted. diff --git a/hs/spec_compliance/defs.bzl b/hs/spec_compliance/defs.bzl new file mode 100644 index 000000000000..ad6084660b14 --- /dev/null +++ b/hs/spec_compliance/defs.bzl @@ -0,0 +1,76 @@ +"""Create transition to disable hermetic_cc_toolchains for hs targets.""" + +load("@rules_haskell//haskell:defs.bzl", haskell_binary_inner = "haskell_binary", haskell_library_inner = "haskell_library") + +def _disable_hermetic_cc_transition(_settings, _attr): + return { + "//bazel:hermetic_cc": False, + } + +disable_hermetic_cc_transition = transition( + implementation = _disable_hermetic_cc_transition, + inputs = [], + outputs = [ + "//bazel:hermetic_cc", + ], +) + +def _disable_hermetic_cc_impl(ctx): + bin = ctx.attr.binary[0] + info = bin[DefaultInfo] + + executable = ctx.actions.declare_file(ctx.label.name) + ctx.actions.symlink(output = executable, target_file = ctx.file.binary) + + return [ + DefaultInfo(files = info.files, runfiles = info.default_runfiles, executable = executable), + ] + +disable_hermetic_cc_binary = rule( + implementation = _disable_hermetic_cc_impl, + executable = True, + attrs = { + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist", + ), + "binary": attr.label(mandatory = True, cfg = disable_hermetic_cc_transition, allow_single_file = True), + }, +) + +def haskell_binary(name, **kwargs): + """ + Wrap the rules_haskell haskell_binary with a transition that disables hermetic toolchains. + + These are not supported together, because ghc will not work with the zig compiler. + + Args: + name: Name for the generated filegroup. + **kwargs: Pass through args to the inner haskell_binary. + """ + + tags = kwargs.pop("tags", []) + tags.append("manual") + new_name = name + "_inner" + label = ":" + new_name + + haskell_binary_inner(new_name, tags = tags, **kwargs) + + disable_hermetic_cc_binary( + name = name, + binary = label, + visibility = ["//visibility:public"], + ) + +def haskell_library(**kwargs): + """ + Wrap the rules_haskell haskell_library with a transition that disables hermetic toolchains. + + These are not supported together, because ghc will not work with the zig compiler. + + Args: + **kwargs: Pass through args to the inner haskell_library. + """ + tags = kwargs.pop("tags", []) + tags.append("manual") + + haskell_library_inner(tags = tags, **kwargs) diff --git a/publish/defs.bzl b/publish/defs.bzl index 22430897e0d3..ee71cab669d7 100644 --- a/publish/defs.bzl +++ b/publish/defs.bzl @@ -8,7 +8,11 @@ def _release_nostrip_transition(_settings, _attr): return { "//command_line_option:compilation_mode": "opt", "//command_line_option:strip": "never", - "@rules_rust//:extra_rustc_flags": ["-Cdebug-assertions=off"], + # opt mode will have rules_rust strip debug symbols, regardless of the + # strip setting. Unfortunately zig cc (from hermetic_cc_toolchain) + # strips as "all or nothing", so we lose all symbols in opt mode. + # Instead, we have the compiler strip nothing, then strip debug in linking. + "@rules_rust//:extra_rustc_flags": ["-Cdebug-assertions=off", "-Cstrip=none", "-Clink-args=-Wl,-S"], } release_nostrip_transition = transition( diff --git a/third_party/libfuzzer/BUILD.bazel b/third_party/libfuzzer/BUILD.bazel new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/third_party/libfuzzer/repository.bzl b/third_party/libfuzzer/repository.bzl new file mode 100644 index 000000000000..bf9164e27633 --- /dev/null +++ b/third_party/libfuzzer/repository.bzl @@ -0,0 +1,29 @@ +""" +Build libfuzzer to ensure compatibility with our toolchain, and so that it can +be optionally linked by the fuzzers. +""" + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") + +def libfuzzer_repository(): + maybe( + http_archive, + name = "libfuzzer", + build_file_content = """ +load("@rules_cc//cc:defs.bzl", "cc_library") + +cc_library( + name = "fuzzer", + srcs = glob(["*.cpp"]), + hdrs = glob(["*.h"]), + additional_compiler_inputs = glob(["*.def"]), + cxxopts = ["-g", "-O2", "-fno-omit-frame-pointer", "-std=c++17"], + linkstatic = True, + visibility = ["//visibility:public"], +) +""", + integrity = "sha256-CLw4JzN3fdo8liWeNzL/lsHfmNBHDE+FsWMnTq5of08=", + strip_prefix = "llvm-project-llvmorg-20.1.0/compiler-rt/lib/fuzzer", + urls = ["https://github.com/llvm/llvm-project/archive/refs/tags/llvmorg-20.1.0.tar.gz"], + ) diff --git a/third_party/openssl/BUILD.bazel b/third_party/openssl/BUILD.bazel new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/third_party/openssl/repository.bzl b/third_party/openssl/repository.bzl new file mode 100644 index 000000000000..6317b80a71b8 --- /dev/null +++ b/third_party/openssl/repository.bzl @@ -0,0 +1,65 @@ +""" +With hermetic toolchains, we would like to not depend on the openssl from the +build enviornment. Normally the vendored feature would build the library from +source alongside the crate, but this does not work when using rules_rust: +https://github.com/bazelbuild/rules_rust/issues/1519 +Instead, we build it ourselves, here. +""" + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") + +def openssl_repository(): + maybe( + http_archive, + name = "openssl", + build_file_content = """ +load("@rules_foreign_cc//foreign_cc:defs.bzl", "configure_make") + +filegroup( + name = "all_srcs", + srcs = glob( + include = ["**"], + exclude = ["*.bazel"], + ), +) + +# https://wiki.openssl.org/index.php/Compilation_and_Installation +CONFIGURE_OPTIONS = [ + "no-comp", + "no-idea", + "no-weak-ssl-ciphers", + "no-shared", +] + +MAKE_TARGETS = [ + "build_libs", + "install_dev", +] + +configure_make( + name = "openssl", + args = ["-j12"], + configure_command = "config", + configure_in_place = True, + configure_options = CONFIGURE_OPTIONS, + lib_name = "openssl", + lib_source = ":all_srcs", + out_lib_dir = "lib64", + out_shared_libs = [], + out_static_libs = ["libssl.a"], + targets = MAKE_TARGETS, + visibility = ["//visibility:public"], +) + +filegroup( + name = "gen_dir", + srcs = [":openssl"], + output_group = "gen_dir", + visibility = ["//visibility:public"], +) +""", + integrity = "sha256-4V3agv4v6BOdwqwho21MoB1TE8dfmfRsTooncJtylL8=", + strip_prefix = "openssl-3.4.0", + urls = ["https://github.com/openssl/openssl/releases/download/openssl-3.4.0/openssl-3.4.0.tar.gz"], + )