From 341b6d69eabf798c83dd9d3db8feda35fdd6bded Mon Sep 17 00:00:00 2001 From: Ben Rogers Date: Thu, 12 Jun 2025 12:56:43 -0400 Subject: [PATCH 01/11] feat(fbcode_builder): Enable python `pex` archives to allow native library dependencies Summary: Fixes integration test failure due to lack of a functioning python-psutil (on Linux, this requires the inclusion of native code which is not permitted in a zipapp). Test Plan: Run the following: ``` act --container-architecture linux/x86_64 -W ".github/workflows/edenfs_linux.yml" ``` Before this change, integration tests fail with a failure to initialize python-psutil. After this change, integraion tests run cleanly --- .../fbcode_builder/CMake/FBPythonBinary.cmake | 6 ++++ .../fbcode_builder/CMake/make_fbpy_archive.py | 32 +++++++++++++++++++ build/fbcode_builder/getdeps.py | 15 +++++---- 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/build/fbcode_builder/CMake/FBPythonBinary.cmake b/build/fbcode_builder/CMake/FBPythonBinary.cmake index f91ebaf326452..2be86fcb3574f 100644 --- a/build/fbcode_builder/CMake/FBPythonBinary.cmake +++ b/build/fbcode_builder/CMake/FBPythonBinary.cmake @@ -236,6 +236,7 @@ function(add_fb_python_unittest TARGET) set( one_value_args WORKING_DIRECTORY BASE_DIR NAMESPACE TEST_LIST DISCOVERY_TIMEOUT + WORKING_DIRECTORY BASE_DIR NAMESPACE TEST_LIST DISCOVERY_TIMEOUT TYPE ) fb_cmake_parse_args( ARG "" "${one_value_args}" "${multi_value_args}" "${ARGN}" @@ -286,8 +287,13 @@ function(add_fb_python_unittest TARGET) list(APPEND ARG_SOURCES "${FB_PY_TEST_MAIN}=__main__.py") list(APPEND ARG_SOURCES "${test_modules_path}=__test_modules__.py") + if(NOT DEFINED ARG_TYPE) + set(ARG_TYPE "zipapp") + endif() + add_fb_python_executable( "${TARGET}" + TYPE "${ARG_TYPE}" NAMESPACE "${ARG_NAMESPACE}" BASE_DIR "${ARG_BASE_DIR}" SOURCES ${ARG_SOURCES} diff --git a/build/fbcode_builder/CMake/make_fbpy_archive.py b/build/fbcode_builder/CMake/make_fbpy_archive.py index 3724feb2183f7..23aeb8b844a23 100755 --- a/build/fbcode_builder/CMake/make_fbpy_archive.py +++ b/build/fbcode_builder/CMake/make_fbpy_archive.py @@ -7,6 +7,7 @@ import errno import os import shutil +import subprocess import sys import tempfile import zipapp @@ -123,6 +124,36 @@ def install_file(info): pass +def build_pex(args, path_map): + """Create a self executing python binary using the PEX tool + + This type of Python binary is more complex as it requires a third-party tool, + but it does support native language extensions (.so/.dll files). + """ + dest_dir = os.path.dirname(args.output) + with tempfile.TemporaryDirectory(prefix="make_fbpy.", dir=dest_dir) as tmpdir: + inst_dir = os.path.join(tmpdir, "tree") + populate_install_tree(inst_dir, path_map) + + if os.path.exists(os.path.join(inst_dir, "__main__.py")): + os.rename( + os.path.join(inst_dir, "__main__.py"), + os.path.join(inst_dir, "main.py"), + ) + args.main = "main" + + tmp_output = os.path.abspath(os.path.join(tmpdir, "output.exe")) + subprocess.check_call( + ["pex"] + + ["--output-file", tmp_output]+ + ["--python", args.python] + + ["--sources-directory", inst_dir] + + ["-e", args.main] + ) + + os.replace(tmp_output, args.output) + + def build_zipapp(args, path_map): """Create a self executing python binary using Python 3's built-in zipapp module. @@ -262,6 +293,7 @@ def check_main_module(args, path_map): BUILD_TYPES = { + "pex": build_pex, "zipapp": build_zipapp, "dir": build_install_dir, "lib-install": install_library, diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py index cd3e38b8d6ddd..bacc835ded785 100755 --- a/build/fbcode_builder/getdeps.py +++ b/build/fbcode_builder/getdeps.py @@ -432,29 +432,30 @@ def run_project_cmd(self, args, loader, manifest): merged += v all_packages[k] = merged - cmd_args = None + cmd_argss = [] if manager == "rpm": packages = sorted(set(all_packages["rpm"])) if packages: - cmd_args = ["sudo", "dnf", "install", "-y", "--skip-broken"] + packages + cmd_argss.append(["sudo", "dnf", "install", "-y", "--skip-broken"] + packages) elif manager == "deb": packages = sorted(set(all_packages["deb"])) if packages: - cmd_args = [ + cmd_argss.append([ "sudo", "--preserve-env=http_proxy", "apt-get", "install", "-y", - ] + packages + ] + packages) + cmd_argss.append(["pip", "install", "pex"]) elif manager == "homebrew": packages = sorted(set(all_packages["homebrew"])) if packages: - cmd_args = ["brew", "install"] + packages + cmd_argss.append(["brew", "install"] + packages) elif manager == "pacman-package": packages = sorted(list(set(all_packages["pacman-package"]))) if packages: - cmd_args = ["pacman", "-S"] + packages + cmd_argss.append(["pacman", "-S"] + packages) else: host_tuple = loader.build_opts.host_type.as_tuple_string() print( @@ -462,7 +463,7 @@ def run_project_cmd(self, args, loader, manifest): ) return - if cmd_args: + for cmd_args in cmd_argss: if args.dry_run: print(" ".join(cmd_args)) else: From 1b293fc89f39851dd6180a23554f556fcbf07521 Mon Sep 17 00:00:00 2001 From: Ben Rogers Date: Thu, 12 Jun 2025 12:56:43 -0400 Subject: [PATCH 02/11] fix: Fix integration tests by running as a pex executable Summary: Fixes integration test failure due to lack of a functioning python-psutil (on Linux, this requires the inclusion of native code which is not permitted in a zipapp). Test Plan: Run the following: ``` act --container-architecture linux/x86_64 -W ".github/workflows/edenfs_linux.yml" ``` Before this change, integration tests fail with a failure to initialize python-psutil. After this change, integraion tests run cleanly --- eden/integration/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/eden/integration/CMakeLists.txt b/eden/integration/CMakeLists.txt index de6e0ca95812d..c16bd179a60f0 100644 --- a/eden/integration/CMakeLists.txt +++ b/eden/integration/CMakeLists.txt @@ -44,6 +44,7 @@ endif() add_fb_python_unittest( integration_tests + TYPE "pex" SOURCES ${INTEGRATION_SRCS} WORKING_DIRECTORY "${CMAKE_BINARY_DIR}" DEPENDS From 1908ba850bf5f51d422ab89f97a61d653c88a7d8 Mon Sep 17 00:00:00 2001 From: Ben Rogers Date: Thu, 12 Jun 2025 12:56:43 -0400 Subject: [PATCH 03/11] fix: make apt-get commands more safe/reliable Summary: Test Plan: --- build/fbcode_builder/getdeps.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py index bacc835ded785..9a0ed21774eef 100755 --- a/build/fbcode_builder/getdeps.py +++ b/build/fbcode_builder/getdeps.py @@ -440,12 +440,22 @@ def run_project_cmd(self, args, loader, manifest): elif manager == "deb": packages = sorted(set(all_packages["deb"])) if packages: + cmd_argss.append(["sed", "-i", "-e", "s/http:/https:/g", "/etc/apt/sources.list"]) + cmd_argss.append([ + "sudo", + "--preserve-env=http_proxy", + "apt-get", + "update", + ]) cmd_argss.append([ "sudo", "--preserve-env=http_proxy", "apt-get", "install", "-y", + "--no-install-recommends", + "-o", + "Acquire::Retries=3", ] + packages) cmd_argss.append(["pip", "install", "pex"]) elif manager == "homebrew": From 5be6e4ddd603043bfc9680602659ab37f4e84a44 Mon Sep 17 00:00:00 2001 From: Ben Rogers Date: Wed, 25 Jun 2025 14:14:29 -0400 Subject: [PATCH 04/11] fix: Update to c++ 20 to fix build issue in Mountd.cpp --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a6a4c34c7e3e3..f1c1948e0fcb7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -99,7 +99,7 @@ else() endif() if(NOT CMAKE_CXX_STANDARD) - set(CMAKE_CXX_STANDARD 17) + set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) message(STATUS "setting C++ standard to C++${CMAKE_CXX_STANDARD}") endif() From 1ff8e08444d5e4b8b306935ca495ee8fd93bb6f9 Mon Sep 17 00:00:00 2001 From: Ben Rogers Date: Thu, 12 Jun 2025 12:56:43 -0400 Subject: [PATCH 05/11] fix: No module named 'psutil' Summary: Test Plan: --- CMake/EdenConfigChecks.cmake | 1 + build/fbcode_builder/manifests/eden | 1 + build/fbcode_builder/manifests/python-psutil | 10 ++++++++++ eden/integration/CMakeLists.txt | 1 + 4 files changed, 13 insertions(+) create mode 100644 build/fbcode_builder/manifests/python-psutil diff --git a/CMake/EdenConfigChecks.cmake b/CMake/EdenConfigChecks.cmake index 38314ab59da7c..64b47cd17a4c1 100644 --- a/CMake/EdenConfigChecks.cmake +++ b/CMake/EdenConfigChecks.cmake @@ -110,6 +110,7 @@ endif() find_package(python-toml REQUIRED) find_package(python-filelock REQUIRED) +find_package(python-psutil REQUIRED) # pexpect is used by some of the integration tests. # If we don't find it we simply won't run those tests. diff --git a/build/fbcode_builder/manifests/eden b/build/fbcode_builder/manifests/eden index d7bc911612e39..b6019422596d2 100644 --- a/build/fbcode_builder/manifests/eden +++ b/build/fbcode_builder/manifests/eden @@ -27,6 +27,7 @@ rocksdb re2 libgit2 pexpect +python-psutil python-toml python-filelock edencommon diff --git a/build/fbcode_builder/manifests/python-psutil b/build/fbcode_builder/manifests/python-psutil new file mode 100644 index 0000000000000..921781fc09c33 --- /dev/null +++ b/build/fbcode_builder/manifests/python-psutil @@ -0,0 +1,10 @@ +[manifest] +name = python-psutil + +[download] +url = https://files.pythonhosted.org/packages/bf/b9/b0eb3f3cbcb734d930fdf839431606844a825b23eaf9a6ab371edac8162c/psutil-7.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl +sha256 = 4b1388a4f6875d7e2aff5c4ca1cc16c545ed41dd8bb596cefea80111db353a34 + +[build] +builder = python-wheel + diff --git a/eden/integration/CMakeLists.txt b/eden/integration/CMakeLists.txt index c16bd179a60f0..faca26159f6d1 100644 --- a/eden/integration/CMakeLists.txt +++ b/eden/integration/CMakeLists.txt @@ -51,6 +51,7 @@ add_fb_python_unittest( edenfsctl.real.main_lib eden_integration_lib eden_hg_integration_lib + python-psutil::python-psutil ENV "CMAKE_SOURCE_DIR=${CMAKE_SOURCE_DIR}" ) From dc1f31d3543c5d3f7582b452306ede0ee168c9d0 Mon Sep 17 00:00:00 2001 From: Ben Rogers Date: Thu, 12 Jun 2025 12:56:43 -0400 Subject: [PATCH 06/11] fix: `backingstore` did not match any packages Summary: Test Plan: --- eden/scm/lib/backingstore/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eden/scm/lib/backingstore/CMakeLists.txt b/eden/scm/lib/backingstore/CMakeLists.txt index 62bde26384d4a..1635b93417dcd 100644 --- a/eden/scm/lib/backingstore/CMakeLists.txt +++ b/eden/scm/lib/backingstore/CMakeLists.txt @@ -10,7 +10,7 @@ if (IS_FB_BUILD) USE_CXX_INCLUDE FEATURES fb) else() - rust_static_library(rust_backingstore CRATE backingstore USE_CXX_INCLUDE) + rust_static_library(rust_backingstore CRATE sapling-backingstore USE_CXX_INCLUDE) endif() install_rust_static_library( From 1369ca3847ff80fd254cfa48fb5176cef467322a Mon Sep 17 00:00:00 2001 From: Ben Rogers Date: Thu, 12 Jun 2025 12:56:43 -0400 Subject: [PATCH 07/11] fix: compile error in TakeoverData.cpp Summary: TakeoverData is failing to compile in the OSS build Test Plan: Confirm the following gets past the compilation step of TakeoverData.cpp ``` act --container-architecture linux/x86_64 -W ".github/workflows/edenfs_linux.yml" ``` --- eden/fs/takeover/TakeoverData.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eden/fs/takeover/TakeoverData.cpp b/eden/fs/takeover/TakeoverData.cpp index e88623117e988..458ef641a823b 100644 --- a/eden/fs/takeover/TakeoverData.cpp +++ b/eden/fs/takeover/TakeoverData.cpp @@ -279,12 +279,12 @@ void TakeoverData::serializeFd( "Unexpected FileDescriptorType {}", fmt::underlying(type)); } - XLOGF(DBG7, "serializing file type: {} fd: {}", type, fileToSerialize->fd()); + XLOGF(DBG7, "serializing file type: {} fd: {}", fmt::underlying(type), fileToSerialize->fd()); files.push_back(std::move(*fileToSerialize)); } void TakeoverData::deserializeFd(FileDescriptorType type, folly::File& file) { - XLOGF(DBG7, "deserializing file type: {} fd: {}", type, file.fd()); + XLOGF(DBG7, "deserializing file type: {} fd: {}", fmt::underlying(type), file.fd()); switch (type) { case FileDescriptorType::LOCK_FILE: lockFile = std::move(file); From 87fd2068ebf688c7b35be3a9536e4f46a33b4159 Mon Sep 17 00:00:00 2001 From: Ben Rogers Date: Thu, 12 Jun 2025 12:56:43 -0400 Subject: [PATCH 08/11] fix: zstd needs libclang to build Summary: Test Plan: --- build/fbcode_builder/manifests/zstd | 1 + 1 file changed, 1 insertion(+) diff --git a/build/fbcode_builder/manifests/zstd b/build/fbcode_builder/manifests/zstd index 1bd0875388f3f..56bb64f7ec5ec 100644 --- a/build/fbcode_builder/manifests/zstd +++ b/build/fbcode_builder/manifests/zstd @@ -6,6 +6,7 @@ zstd # 18.04 zstd is too old [debs.not(all(distro=ubuntu,distro_vers="18.04"))] +libclang-dev libzstd-dev zstd From 73327f7f544a5a7914b7d3f161b6c196ef4bd9e4 Mon Sep 17 00:00:00 2001 From: Ben Rogers Date: Tue, 17 Jun 2025 14:32:32 -0400 Subject: [PATCH 09/11] Make backingstore's use of iobuf optional (fixing oss build issue) --- eden/scm/lib/backingstore/BUCK | 8 ++++++++ eden/scm/lib/backingstore/src/ffi.rs | 7 +++++++ 2 files changed, 15 insertions(+) diff --git a/eden/scm/lib/backingstore/BUCK b/eden/scm/lib/backingstore/BUCK index 82d3a13ba6535..c82736c373dbd 100644 --- a/eden/scm/lib/backingstore/BUCK +++ b/eden/scm/lib/backingstore/BUCK @@ -52,6 +52,14 @@ rust_library( }}, "features": {"fb": ["sapling-configloader/fb"]}, "lib": {"name": "backingstore"}, + "lints": { + "rust": { + "unexpected_cfgs": { + "check-cfg": ["cfg(fbcode_build)"], + "level": "warn", + }, + }, + }, "package": { "authors": ["Meta Source Control Team "], "homepage": "https://sapling-scm.com/", diff --git a/eden/scm/lib/backingstore/src/ffi.rs b/eden/scm/lib/backingstore/src/ffi.rs index 42efab153f0f9..52daee54ad6e0 100644 --- a/eden/scm/lib/backingstore/src/ffi.rs +++ b/eden/scm/lib/backingstore/src/ffi.rs @@ -15,6 +15,7 @@ use anyhow::Result; use anyhow::anyhow; use cxx::SharedPtr; use cxx::UniquePtr; +#[cfg(fbcode_build)] use iobuf::IOBuf; use storemodel::FileAuxData as ScmStoreFileAuxData; use types::FetchContext; @@ -118,6 +119,7 @@ pub(crate) mod ffi { unsafe extern "C++" { include!("eden/scm/lib/backingstore/include/ffi.h"); + #[cfg(fbcode_build)] #[namespace = "folly"] type IOBuf = iobuf::IOBuf; @@ -140,6 +142,7 @@ pub(crate) mod ffi { tree: SharedPtr, ); + #[cfg(fbcode_build)] unsafe fn sapling_backingstore_get_blob_batch_handler( resolve_state: SharedPtr, index: usize, @@ -193,12 +196,14 @@ pub(crate) mod ffi { resolver: SharedPtr, ); + #[cfg(fbcode_build)] pub fn sapling_backingstore_get_blob( store: &BackingStore, node: &[u8], fetch_mode: FetchMode, ) -> Result>; + #[cfg(fbcode_build)] pub fn sapling_backingstore_get_blob_batch( store: &BackingStore, requests: &[Request], @@ -430,6 +435,7 @@ pub fn sapling_backingstore_get_tree_aux_batch( ); } +#[cfg(fbcode_build)] pub fn sapling_backingstore_get_blob( store: &BackingStore, node: &[u8], @@ -445,6 +451,7 @@ pub fn sapling_backingstore_get_blob( } } +#[cfg(fbcode_build)] pub fn sapling_backingstore_get_blob_batch( store: &BackingStore, requests: &[ffi::Request], From 849cd79b48a584eef04bc26d9afdd95af7a511ae Mon Sep 17 00:00:00 2001 From: Ben Rogers Date: Mon, 30 Jun 2025 11:07:11 -0400 Subject: [PATCH 10/11] fix: Fix OSS compile error introduced by D76936387 --- eden/fs/privhelper/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eden/fs/privhelper/CMakeLists.txt b/eden/fs/privhelper/CMakeLists.txt index 7363021ff6649..da5e5dffbc290 100644 --- a/eden/fs/privhelper/CMakeLists.txt +++ b/eden/fs/privhelper/CMakeLists.txt @@ -3,7 +3,7 @@ # This software may be used and distributed according to the terms of the # GNU General Public License version 2. -file(GLOB PRIVHELPER_SRCS "*.cpp") +file(GLOB PRIVHELPER_SRCS "*.cpp" "priority/*.cpp") add_library( eden_fuse_privhelper STATIC From 2331461c186e297fa971a5eb82f8efb55f7a0910 Mon Sep 17 00:00:00 2001 From: Ben Rogers Date: Tue, 17 Jun 2025 14:32:32 -0400 Subject: [PATCH 11/11] fix: Remove the double-import of sapling-util to fix oss build issue --- eden/fs/cli_rs/edenfs-client/BUCK | 3 --- eden/fs/cli_rs/edenfs-client/Cargo.toml | 1 - 2 files changed, 4 deletions(-) diff --git a/eden/fs/cli_rs/edenfs-client/BUCK b/eden/fs/cli_rs/edenfs-client/BUCK index 6f85ed0e6514e..532c4cc8ab6c6 100644 --- a/eden/fs/cli_rs/edenfs-client/BUCK +++ b/eden/fs/cli_rs/edenfs-client/BUCK @@ -5,9 +5,6 @@ oncall("scm_client_infra") rust_library( name = "edenfs-client", srcs = glob(["src/**/*.rs"]), - named_deps = { - "hg_util": "//eden/scm/lib/util:util", - }, os_deps = [ ( "windows", diff --git a/eden/fs/cli_rs/edenfs-client/Cargo.toml b/eden/fs/cli_rs/edenfs-client/Cargo.toml index 88b72db101b4b..48479cde6ae5b 100644 --- a/eden/fs/cli_rs/edenfs-client/Cargo.toml +++ b/eden/fs/cli_rs/edenfs-client/Cargo.toml @@ -21,7 +21,6 @@ fbinit = { version = "0.2.0", git = "https://github.com/facebookexperimental/rus futures = { version = "0.3.30", features = ["async-await", "compat"] } futures_stats = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" } hex = { version = "0.4.3", features = ["alloc"] } -hg_util = { package = "sapling-util", version = "0.1.0", path = "../../../scm/lib/util" } lazy_static = { version = "1.5", features = ["spin_no_std"], default-features = false } libc = "0.2.139" parking_lot = { version = "0.12.1", features = ["send_guard"] }