From 16ddabb3217e55e98c7f75e6325be178c9272130 Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Tue, 29 Apr 2025 10:28:21 -0700 Subject: [PATCH 01/15] Update to the latest nightly. Update the syntax for naked functions. --- .github/workflows/main.yml | 4 ++-- Cargo.toml | 2 +- src/lib.rs | 2 -- src/naked.rs | 2 +- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 10113f6..cf68a89 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,7 +23,7 @@ jobs: strategy: matrix: build: [ubuntu, i686-linux, aarch64-linux, riscv64-linux] - rust: [1.84, nightly-2025-03-05] + rust: [1.84, nightly-2025-04-28] include: - build: ubuntu os: ubuntu-latest @@ -51,7 +51,7 @@ jobs: qemu: qemu-riscv64 -L /usr/riscv64-linux-gnu qemu_target: riscv64-linux-user host_target: riscv64gc-unknown-linux-gnu - - rust: nightly-2025-03-05 + - rust: nightly-2025-04-28 features: nightly steps: - uses: actions/checkout@v4 diff --git a/Cargo.toml b/Cargo.toml index 474644c..82a084c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,7 +44,7 @@ alloc = { version = "1.0.0", optional = true, package = "rustc-std-workspace-all # Use the unwinding crate if support for unwinding is needed. This depends on # nightly Rust. And it's not supported on ARM yet. [target.'cfg(not(target_arch = "arm"))'.dependencies.unwinding] -version = "0.2.5" +version = "0.2.6" default-features = false features = ["unwinder"] optional = true diff --git a/src/lib.rs b/src/lib.rs index 463e220..9d6fb76 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,8 +3,6 @@ #![no_std] #![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(feature = "experimental-relocate", feature(cfg_relocation_model))] -// On nightly, enable `#[naked]` functions. -#![cfg_attr(feature = "nightly", feature(naked_functions))] // On nightly, enable llvm intrinsics for additional debug asserts. #![cfg_attr(all(debug_assertions, feature = "nightly"), allow(internal_features))] #![cfg_attr( diff --git a/src/naked.rs b/src/naked.rs index ba67aae..9269c7a 100644 --- a/src/naked.rs +++ b/src/naked.rs @@ -40,7 +40,7 @@ macro_rules! naked_fn { $($label:ident = $kind:ident $path:path),* ) => { #[doc = $doc] - #[naked] + #[unsafe(naked)] #[no_mangle] $vis unsafe extern "C" fn $name $args -> $ret { core::arch::naked_asm!( From 94f98f2f74a92e584b658ada313053d2a58c9c48 Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Tue, 29 Apr 2025 10:31:55 -0700 Subject: [PATCH 02/15] Update rust-version to 1.84.1. --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 82a084c..c8cf80a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ edition = "2021" keywords = ["linux"] categories = ["no-std"] include = ["src", "Cargo.toml", "COPYRIGHT", "LICENSE*", "/*.md"] -rust-version = "1.84" +rust-version = "1.84.1" [dependencies] linux-raw-sys = { version = "0.9.2", default-features = false, optional = true, features = ["general", "no_std", "elf"] } From 121e01614a70766d3eb0db2fad9583552a8fda25 Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Tue, 29 Apr 2025 11:15:38 -0700 Subject: [PATCH 03/15] Use Rust 1.84.1. --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cf68a89..856a559 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,7 +23,7 @@ jobs: strategy: matrix: build: [ubuntu, i686-linux, aarch64-linux, riscv64-linux] - rust: [1.84, nightly-2025-04-28] + rust: [1.84.1, nightly-2025-04-28] include: - build: ubuntu os: ubuntu-latest From d5fcee8c8654be678a04e57f59cd9a132f83ea71 Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Tue, 29 Apr 2025 11:26:01 -0700 Subject: [PATCH 04/15] Update to Rust 1.85 for edition2024. --- .github/workflows/main.yml | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 856a559..ee06a4b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,7 +23,7 @@ jobs: strategy: matrix: build: [ubuntu, i686-linux, aarch64-linux, riscv64-linux] - rust: [1.84.1, nightly-2025-04-28] + rust: [1.85, nightly-2025-04-28] include: - build: ubuntu os: ubuntu-latest diff --git a/Cargo.toml b/Cargo.toml index c8cf80a..79ef280 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ edition = "2021" keywords = ["linux"] categories = ["no-std"] include = ["src", "Cargo.toml", "COPYRIGHT", "LICENSE*", "/*.md"] -rust-version = "1.84.1" +rust-version = "1.85" [dependencies] linux-raw-sys = { version = "0.9.2", default-features = false, optional = true, features = ["general", "no_std", "elf"] } From f5e1f479a6df706b6a0eea142deb2502fe5b8c28 Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Tue, 29 Apr 2025 11:31:00 -0700 Subject: [PATCH 05/15] Fix a warning. --- src/mem/impls.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mem/impls.rs b/src/mem/impls.rs index 505ca96..8cf263d 100644 --- a/src/mem/impls.rs +++ b/src/mem/impls.rs @@ -30,7 +30,7 @@ unsafe fn read_usize_unaligned(x: *const usize) -> usize { // Do not use `core::ptr::read_unaligned` here, since it calls `copy_nonoverlapping` which // is translated to memcpy in LLVM. let x_read = (x as *const [u8; core::mem::size_of::()]).read(); - core::mem::transmute(x_read) + usize::from_ne_bytes(x_read) } #[inline(always)] From 90cbc8e27f5119f228407c36f2900e0c93aa5064 Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Tue, 29 Apr 2025 11:40:32 -0700 Subject: [PATCH 06/15] Update MSRV in README.md. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0402667..a292f9b 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ This is used by [Mustang] and [Eyra] in their libc implementations, and in the [Origin Studio] project in its std implementation, which are three different ways to support building Rust programs written entirely in Rust. -It works with both stable (currently Rust >= 1.78) and nightly Rust. If you're +It works with both stable (currently Rust >= 1.85) and nightly Rust. If you're using nightly Rust, enable the feature "nightly" to let origin use nightly-only features, which include proper support for unwinding, better safety checks, and better optimizations. From 5a1ddd2ad21ff173e3efb70fba75641fc1aa2c0c Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Tue, 29 Apr 2025 11:41:37 -0700 Subject: [PATCH 07/15] cargo fix --edition --- example-crates/external-start/src/main.rs | 14 ++--- .../origin-start-dynamic-linker/src/lib.rs | 6 +- example-crates/origin-start-lto/src/main.rs | 6 +- .../origin-start-no-alloc/src/main.rs | 2 +- .../origin-start-panic-abort/src/main.rs | 6 +- .../origin-start-stable/src/main.rs | 6 +- example-crates/origin-start/src/main.rs | 6 +- example-crates/tiny-hello/src/main.rs | 6 +- example-crates/tiny/src/main.rs | 2 +- src/arch/x86_64.rs | 24 ++++---- src/mem/fast.rs | 36 ++++++------ src/mem/small.rs | 36 ++++++------ src/mem/x86_64.rs | 36 ++++++------ src/naked.rs | 4 +- src/program/libc.rs | 8 +-- src/program/linux_raw.rs | 20 +++---- src/relocate.rs | 8 +-- src/signal/libc.rs | 4 +- src/signal/linux_raw.rs | 4 +- src/stubs.rs | 2 +- src/thread/libc.rs | 18 +++--- src/thread/linux_raw.rs | 56 +++++++++---------- src/unwind_unimplemented.rs | 28 +++++----- .../origin-start/src/bin/abort-via-raise.rs | 2 +- test-crates/origin-start/src/bin/canary.rs | 8 +-- test-crates/origin-start/src/bin/detach.rs | 6 +- .../src/bin/main-thread-dtors-adding-dtors.rs | 2 +- .../src/bin/program-dtors-adding-dtors.rs | 2 +- .../src/bin/thread-dtors-adding-dtors.rs | 6 +- test-crates/origin-start/src/bin/thread-id.rs | 6 +- test-crates/origin-start/src/bin/tls.rs | 6 +- test-crates/origin-start/src/bin/trap.rs | 2 +- 32 files changed, 189 insertions(+), 189 deletions(-) diff --git a/example-crates/external-start/src/main.rs b/example-crates/external-start/src/main.rs index f58e79d..fd76d1d 100644 --- a/example-crates/external-start/src/main.rs +++ b/example-crates/external-start/src/main.rs @@ -19,10 +19,10 @@ static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::Glob /// and start running the constructors but to immediately take over. /// /// [here]: https://github.com/rust-lang/rfcs/pull/2735 -#[link_section = ".init_array.00000"] +#[unsafe(link_section = ".init_array.00000")] #[used] static EARLY_INIT_ARRAY: unsafe extern "C" fn(i32, *mut *mut u8) = { - unsafe extern "C" fn function(_argc: i32, argv: *mut *mut u8) { + unsafe extern "C" fn function(_argc: i32, argv: *mut *mut u8) { unsafe { // Libc was calling constructors (we're one of them), but origin will // be doing that now, so just exit when we're called a second time. static FIRST: AtomicBool = AtomicBool::new(false); @@ -37,12 +37,12 @@ static EARLY_INIT_ARRAY: unsafe extern "C" fn(i32, *mut *mut u8) = { let mem = argv.sub(1); origin::program::start(mem as _); - } + }} function }; -#[no_mangle] -unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { +#[unsafe(no_mangle)] +unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { unsafe { eprintln!("Hello from main thread"); program::at_exit(Box::new(|| { @@ -70,11 +70,11 @@ unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> eprintln!("Goodbye from main"); program::exit(0); -} +}} // Libc calls `main` so we need to provide a definition to satisfy the // linker, however origin gains control before libc can call this `main`. -#[no_mangle] +#[unsafe(no_mangle)] unsafe fn main(_argc: i32, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { eprintln!("Main was not supposed to be called!"); program::trap(); diff --git a/example-crates/origin-start-dynamic-linker/src/lib.rs b/example-crates/origin-start-dynamic-linker/src/lib.rs index cae3fd9..6dc8828 100644 --- a/example-crates/origin-start-dynamic-linker/src/lib.rs +++ b/example-crates/origin-start-dynamic-linker/src/lib.rs @@ -12,8 +12,8 @@ use origin::{program, thread}; #[global_allocator] static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::GlobalDlmalloc; -#[no_mangle] -unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { +#[unsafe(no_mangle)] +unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { unsafe { eprintln!("Hello from main thread"); program::at_exit(Box::new(|| { @@ -41,4 +41,4 @@ unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> eprintln!("Goodbye from main"); program::exit(0); -} +}} diff --git a/example-crates/origin-start-lto/src/main.rs b/example-crates/origin-start-lto/src/main.rs index 0ef5ac8..8b2a55b 100644 --- a/example-crates/origin-start-lto/src/main.rs +++ b/example-crates/origin-start-lto/src/main.rs @@ -12,8 +12,8 @@ use origin::{program, thread}; #[global_allocator] static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::GlobalDlmalloc; -#[no_mangle] -unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { +#[unsafe(no_mangle)] +unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { unsafe { eprintln!("Hello from main thread"); program::at_exit(Box::new(|| { @@ -41,4 +41,4 @@ unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> eprintln!("Goodbye from main"); program::exit(0); -} +}} diff --git a/example-crates/origin-start-no-alloc/src/main.rs b/example-crates/origin-start-no-alloc/src/main.rs index 01f6e1a..fc46870 100644 --- a/example-crates/origin-start-no-alloc/src/main.rs +++ b/example-crates/origin-start-no-alloc/src/main.rs @@ -6,7 +6,7 @@ use atomic_dbg::eprintln; use origin::program; -#[no_mangle] +#[unsafe(no_mangle)] unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { eprintln!("Hello!"); diff --git a/example-crates/origin-start-panic-abort/src/main.rs b/example-crates/origin-start-panic-abort/src/main.rs index 0ef5ac8..8b2a55b 100644 --- a/example-crates/origin-start-panic-abort/src/main.rs +++ b/example-crates/origin-start-panic-abort/src/main.rs @@ -12,8 +12,8 @@ use origin::{program, thread}; #[global_allocator] static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::GlobalDlmalloc; -#[no_mangle] -unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { +#[unsafe(no_mangle)] +unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { unsafe { eprintln!("Hello from main thread"); program::at_exit(Box::new(|| { @@ -41,4 +41,4 @@ unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> eprintln!("Goodbye from main"); program::exit(0); -} +}} diff --git a/example-crates/origin-start-stable/src/main.rs b/example-crates/origin-start-stable/src/main.rs index 0ef5ac8..8b2a55b 100644 --- a/example-crates/origin-start-stable/src/main.rs +++ b/example-crates/origin-start-stable/src/main.rs @@ -12,8 +12,8 @@ use origin::{program, thread}; #[global_allocator] static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::GlobalDlmalloc; -#[no_mangle] -unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { +#[unsafe(no_mangle)] +unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { unsafe { eprintln!("Hello from main thread"); program::at_exit(Box::new(|| { @@ -41,4 +41,4 @@ unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> eprintln!("Goodbye from main"); program::exit(0); -} +}} diff --git a/example-crates/origin-start/src/main.rs b/example-crates/origin-start/src/main.rs index 0ef5ac8..8b2a55b 100644 --- a/example-crates/origin-start/src/main.rs +++ b/example-crates/origin-start/src/main.rs @@ -12,8 +12,8 @@ use origin::{program, thread}; #[global_allocator] static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::GlobalDlmalloc; -#[no_mangle] -unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { +#[unsafe(no_mangle)] +unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { unsafe { eprintln!("Hello from main thread"); program::at_exit(Box::new(|| { @@ -41,4 +41,4 @@ unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> eprintln!("Goodbye from main"); program::exit(0); -} +}} diff --git a/example-crates/tiny-hello/src/main.rs b/example-crates/tiny-hello/src/main.rs index 8960527..6e68914 100644 --- a/example-crates/tiny-hello/src/main.rs +++ b/example-crates/tiny-hello/src/main.rs @@ -5,8 +5,8 @@ extern crate origin; -#[no_mangle] -unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { +#[unsafe(no_mangle)] +unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { unsafe { let message = b"Hello, world!\n"; let mut remaining = &message[..]; while !remaining.is_empty() { @@ -17,4 +17,4 @@ unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> } } origin::program::immediate_exit(0); -} +}} diff --git a/example-crates/tiny/src/main.rs b/example-crates/tiny/src/main.rs index a21d6e1..94389d3 100644 --- a/example-crates/tiny/src/main.rs +++ b/example-crates/tiny/src/main.rs @@ -5,7 +5,7 @@ extern crate origin; -#[no_mangle] +#[unsafe(no_mangle)] unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { 42 } diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index 8a6edca..d42d261 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -106,7 +106,7 @@ pub(super) fn ehdr_addr() -> *const Elf_Ehdr { #[cfg(all(feature = "experimental-relocate", feature = "origin-start"))] #[cfg(relocation_model = "pic")] #[inline] -pub(super) unsafe fn relocation_load(ptr: usize) -> usize { +pub(super) unsafe fn relocation_load(ptr: usize) -> usize { unsafe { let r0; // This is read-only but we don't use `readonly` because this memory access @@ -120,7 +120,7 @@ pub(super) unsafe fn relocation_load(ptr: usize) -> usize { ); r0 -} +} } /// Perform a single store operation, outside the Rust memory model. /// @@ -138,14 +138,14 @@ pub(super) unsafe fn relocation_load(ptr: usize) -> usize { #[cfg(all(feature = "experimental-relocate", feature = "origin-start"))] #[cfg(relocation_model = "pic")] #[inline] -pub(super) unsafe fn relocation_store(ptr: usize, value: usize) { +pub(super) unsafe fn relocation_store(ptr: usize, value: usize) { unsafe { asm!( "mov [{}], {}", in(reg) ptr, in(reg) value, options(nostack, preserves_flags), ); -} +} } /// Mark “relro” memory as readonly. /// @@ -166,7 +166,7 @@ pub(super) unsafe fn relocation_store(ptr: usize, value: usize) { #[cfg(all(feature = "experimental-relocate", feature = "origin-start"))] #[cfg(relocation_model = "pic")] #[inline] -pub(super) unsafe fn relocation_mprotect_readonly(ptr: usize, len: usize) { +pub(super) unsafe fn relocation_mprotect_readonly(ptr: usize, len: usize) { unsafe { let r0: usize; // This is read-only but we don't use `readonly` because the side effects @@ -188,7 +188,7 @@ pub(super) unsafe fn relocation_mprotect_readonly(ptr: usize, len: usize) { // yet initialized at this point. trap(); } -} +} } /// The required alignment for the stack pointer. #[cfg(feature = "take-charge")] @@ -211,7 +211,7 @@ pub(super) unsafe fn clone( newtls: *mut c_void, fn_: extern "C" fn(), num_args: usize, -) -> isize { +) -> isize { unsafe { let r0; asm!( "syscall", // Do the `clone` system call. @@ -243,17 +243,17 @@ pub(super) unsafe fn clone( options(nostack) ); r0 -} +} } /// Write a value to the platform thread-pointer register. #[cfg(feature = "take-charge")] #[cfg(feature = "thread")] #[inline] -pub(super) unsafe fn set_thread_pointer(ptr: *mut c_void) { +pub(super) unsafe fn set_thread_pointer(ptr: *mut c_void) { unsafe { rustix::runtime::set_fs(ptr); debug_assert_eq!(*ptr.cast::<*const c_void>(), ptr); debug_assert_eq!(thread_pointer(), ptr); -} +} } /// Read the value of the platform thread-pointer register. #[cfg(feature = "take-charge")] @@ -281,7 +281,7 @@ pub(super) const TLS_OFFSET: usize = 0; #[cfg(feature = "take-charge")] #[cfg(feature = "thread")] #[inline] -pub(super) unsafe fn munmap_and_exit_thread(map_addr: *mut c_void, map_len: usize) -> ! { +pub(super) unsafe fn munmap_and_exit_thread(map_addr: *mut c_void, map_len: usize) -> ! { unsafe { asm!( "syscall", "xor edi, edi", @@ -294,7 +294,7 @@ pub(super) unsafe fn munmap_and_exit_thread(map_addr: *mut c_void, map_len: usiz in("rsi") map_len, options(noreturn, nostack) ); -} +} } #[cfg(feature = "take-charge")] #[cfg(feature = "signal")] diff --git a/src/mem/fast.rs b/src/mem/fast.rs index 74e2590..712d84c 100644 --- a/src/mem/fast.rs +++ b/src/mem/fast.rs @@ -11,14 +11,14 @@ #[cfg_attr(not(target_arch = "x86_64"), path = "impls.rs")] mod impls; -#[no_mangle] -unsafe extern "C" fn memcpy(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 { +#[unsafe(no_mangle)] +unsafe extern "C" fn memcpy(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 { unsafe { impls::copy_forward(dest, src, n); dest -} +} } -#[no_mangle] -unsafe extern "C" fn memmove(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 { +#[unsafe(no_mangle)] +unsafe extern "C" fn memmove(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 { unsafe { let delta = dest.addr().wrapping_sub(src.addr()); if delta >= n { // We can copy forwards because either dest is far enough ahead of src, @@ -28,25 +28,25 @@ unsafe extern "C" fn memmove(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 impls::copy_backward(dest, src, n); } dest -} +} } -#[no_mangle] -unsafe extern "C" fn memset(s: *mut u8, c: core::ffi::c_int, n: usize) -> *mut u8 { +#[unsafe(no_mangle)] +unsafe extern "C" fn memset(s: *mut u8, c: core::ffi::c_int, n: usize) -> *mut u8 { unsafe { impls::set_bytes(s, c as u8, n); s -} +} } -#[no_mangle] -unsafe extern "C" fn memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 { +#[unsafe(no_mangle)] +unsafe extern "C" fn memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 { unsafe { impls::compare_bytes(s1, s2, n) -} +} } -#[no_mangle] -unsafe extern "C" fn bcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 { +#[unsafe(no_mangle)] +unsafe extern "C" fn bcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 { unsafe { memcmp(s1, s2, n) -} +} } -#[no_mangle] -unsafe extern "C" fn strlen(s: *const core::ffi::c_char) -> usize { +#[unsafe(no_mangle)] +unsafe extern "C" fn strlen(s: *const core::ffi::c_char) -> usize { unsafe { impls::c_string_length(s) -} +} } diff --git a/src/mem/small.rs b/src/mem/small.rs index 77dc433..4bde079 100644 --- a/src/mem/small.rs +++ b/src/mem/small.rs @@ -5,8 +5,8 @@ use core::ffi::{c_char, c_int, c_void}; -#[no_mangle] -unsafe extern "C" fn memcpy(dst: *mut c_void, src: *const c_void, len: usize) -> *mut c_void { +#[unsafe(no_mangle)] +unsafe extern "C" fn memcpy(dst: *mut c_void, src: *const c_void, len: usize) -> *mut c_void { unsafe { let start = dst; let mut dst = dst.cast::(); let mut src = src.cast::(); @@ -18,10 +18,10 @@ unsafe extern "C" fn memcpy(dst: *mut c_void, src: *const c_void, len: usize) -> core::arch::asm!(""); } start -} +} } -#[no_mangle] -unsafe extern "C" fn memmove(dst: *mut c_void, src: *const c_void, len: usize) -> *mut c_void { +#[unsafe(no_mangle)] +unsafe extern "C" fn memmove(dst: *mut c_void, src: *const c_void, len: usize) -> *mut c_void { unsafe { let start = dst; let mut dst = dst.cast::(); let mut src = src.cast::(); @@ -46,10 +46,10 @@ unsafe extern "C" fn memmove(dst: *mut c_void, src: *const c_void, len: usize) - } } start -} +} } -#[no_mangle] -unsafe extern "C" fn memset(dst: *mut c_void, fill: c_int, len: usize) -> *mut c_void { +#[unsafe(no_mangle)] +unsafe extern "C" fn memset(dst: *mut c_void, fill: c_int, len: usize) -> *mut c_void { unsafe { let mut s = dst.cast::(); let end = s.add(len); while s < end { @@ -58,10 +58,10 @@ unsafe extern "C" fn memset(dst: *mut c_void, fill: c_int, len: usize) -> *mut c core::arch::asm!(""); } dst -} +} } -#[no_mangle] -unsafe extern "C" fn memcmp(a: *const c_void, b: *const c_void, len: usize) -> c_int { +#[unsafe(no_mangle)] +unsafe extern "C" fn memcmp(a: *const c_void, b: *const c_void, len: usize) -> c_int { unsafe { let a = a.cast::(); let b = b.cast::(); let mut i = 0; @@ -75,16 +75,16 @@ unsafe extern "C" fn memcmp(a: *const c_void, b: *const c_void, len: usize) -> c core::arch::asm!(""); } 0 -} +} } // Obsolescent -#[no_mangle] -unsafe extern "C" fn bcmp(a: *const c_void, b: *const c_void, len: usize) -> c_int { +#[unsafe(no_mangle)] +unsafe extern "C" fn bcmp(a: *const c_void, b: *const c_void, len: usize) -> c_int { unsafe { memcmp(a, b, len) -} +} } -#[no_mangle] -unsafe extern "C" fn strlen(s: *const c_char) -> usize { +#[unsafe(no_mangle)] +unsafe extern "C" fn strlen(s: *const c_char) -> usize { unsafe { let mut s = s; let mut n = 0; while *s != 0 { @@ -93,4 +93,4 @@ unsafe extern "C" fn strlen(s: *const c_char) -> usize { core::arch::asm!(""); } n -} +} } diff --git a/src/mem/x86_64.rs b/src/mem/x86_64.rs index 7a28430..2d6fc91 100644 --- a/src/mem/x86_64.rs +++ b/src/mem/x86_64.rs @@ -27,7 +27,7 @@ use core::mem; #[inline(always)] #[cfg(target_feature = "ermsb")] -pub unsafe fn copy_forward(dest: *mut u8, src: *const u8, count: usize) { +pub unsafe fn copy_forward(dest: *mut u8, src: *const u8, count: usize) { unsafe { // FIXME: Use the Intel syntax once we drop LLVM 9 support on rust-lang/rust. core::arch::asm!( "repe movsb (%rsi), (%rdi)", @@ -36,11 +36,11 @@ pub unsafe fn copy_forward(dest: *mut u8, src: *const u8, count: usize) { inout("rsi") src => _, options(att_syntax, nostack, preserves_flags) ); -} +} } #[inline(always)] #[cfg(not(target_feature = "ermsb"))] -pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, count: usize) { +pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, count: usize) { unsafe { let (pre_byte_count, qword_count, byte_count) = rep_param(dest, count); // Separating the blocks gives the compiler more freedom to reorder instructions. asm!( @@ -64,10 +64,10 @@ pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, count: usize) inout("rsi") src => _, options(att_syntax, nostack, preserves_flags) ); -} +} } #[inline(always)] -pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, count: usize) { +pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, count: usize) { unsafe { let (pre_byte_count, qword_count, byte_count) = rep_param(dest, count); // We can't separate this block due to std/cld asm!( @@ -91,11 +91,11 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, count: usize) { // We modify flags, but we restore it afterwards options(att_syntax, nostack, preserves_flags) ); -} +} } #[inline(always)] #[cfg(target_feature = "ermsb")] -pub unsafe fn set_bytes(dest: *mut u8, c: u8, count: usize) { +pub unsafe fn set_bytes(dest: *mut u8, c: u8, count: usize) { unsafe { // FIXME: Use the Intel syntax once we drop LLVM 9 support on rust-lang/rust. core::arch::asm!( "repe stosb %al, (%rdi)", @@ -104,11 +104,11 @@ pub unsafe fn set_bytes(dest: *mut u8, c: u8, count: usize) { inout("al") c => _, options(att_syntax, nostack, preserves_flags) ) -} +} } #[inline(always)] #[cfg(not(target_feature = "ermsb"))] -pub unsafe fn set_bytes(mut dest: *mut u8, c: u8, count: usize) { +pub unsafe fn set_bytes(mut dest: *mut u8, c: u8, count: usize) { unsafe { let c = c as u64 * 0x0101_0101_0101_0101; let (pre_byte_count, qword_count, byte_count) = rep_param(dest, count); // Separating the blocks gives the compiler more freedom to reorder instructions. @@ -133,17 +133,17 @@ pub unsafe fn set_bytes(mut dest: *mut u8, c: u8, count: usize) { in("rax") c, options(att_syntax, nostack, preserves_flags) ); -} +} } #[inline(always)] -pub unsafe fn compare_bytes(a: *const u8, b: *const u8, n: usize) -> i32 { +pub unsafe fn compare_bytes(a: *const u8, b: *const u8, n: usize) -> i32 { unsafe { #[inline(always)] unsafe fn cmp(mut a: *const T, mut b: *const T, n: usize, f: F) -> i32 where T: Clone + Copy + Eq, U: Clone + Copy + Eq, F: FnOnce(*const U, *const U, usize) -> i32, - { + { unsafe { // Ensure T is not a ZST. assert!(mem::size_of::() != 0); @@ -156,7 +156,7 @@ pub unsafe fn compare_bytes(a: *const u8, b: *const u8, n: usize) -> i32 { b = b.add(1); } f(a.cast(), b.cast(), n % mem::size_of::()) - } + } } let c1 = |mut a: *const u8, mut b: *const u8, n| { for _ in 0..n { if a.read() != b.read() { @@ -172,7 +172,7 @@ pub unsafe fn compare_bytes(a: *const u8, b: *const u8, n: usize) -> i32 { let c8 = |a: *const u64, b, n| cmp(a, b, n, c4); let c16 = |a: *const u128, b, n| cmp(a, b, n, c8); c16(a.cast(), b.cast(), n) -} +} } // In order to process more than on byte simultaneously when executing strlen, // two things must be considered: @@ -186,7 +186,7 @@ pub unsafe fn compare_bytes(a: *const u8, b: *const u8, n: usize) -> i32 { #[cfg(target_feature = "sse2")] #[inline(always)] -pub unsafe fn c_string_length(mut s: *const core::ffi::c_char) -> usize { +pub unsafe fn c_string_length(mut s: *const core::ffi::c_char) -> usize { unsafe { use core::arch::x86_64::{__m128i, _mm_cmpeq_epi8, _mm_movemask_epi8, _mm_set1_epi8}; let mut n = 0; @@ -249,13 +249,13 @@ pub unsafe fn c_string_length(mut s: *const core::ffi::c_char) -> usize { return n + cmp.trailing_zeros() as usize; } } -} +} } // Provided for scenarios like kernel development, where SSE might not // be available. #[cfg(not(target_feature = "sse2"))] #[inline(always)] -pub unsafe fn c_string_length(mut s: *const core::ffi::c_char) -> usize { +pub unsafe fn c_string_length(mut s: *const core::ffi::c_char) -> usize { unsafe { let mut n = 0; // Check bytes in steps of one until @@ -302,7 +302,7 @@ pub unsafe fn c_string_length(mut s: *const core::ffi::c_char) -> usize { s = s.add(1); } } -} +} } /// Determine optimal parameters for a `rep` instruction. fn rep_param(dest: *mut u8, mut count: usize) -> (usize, usize, usize) { diff --git a/src/naked.rs b/src/naked.rs index 9269c7a..a5f285b 100644 --- a/src/naked.rs +++ b/src/naked.rs @@ -41,7 +41,7 @@ macro_rules! naked_fn { ) => { #[doc = $doc] #[unsafe(naked)] - #[no_mangle] + #[unsafe(no_mangle)] $vis unsafe extern "C" fn $name $args -> $ret { core::arch::naked_asm!( $($code),*, @@ -62,7 +62,7 @@ macro_rules! naked_fn { $($code:literal),*; $($label:ident = $kind:ident $path:path),* ) => { - extern "C" { + unsafe extern "C" { #[doc = $doc] $vis fn $name $args -> $ret; } diff --git a/src/program/libc.rs b/src/program/libc.rs index 517a791..42228f1 100644 --- a/src/program/libc.rs +++ b/src/program/libc.rs @@ -8,7 +8,7 @@ //! /// //! /// SAFETY: `argc`, `argv`, and `envp` describe incoming program //! /// command-line arguments and environment variables. -//! #[no_mangle] +//! #[unsafe(no_mangle)] //! unsafe fn origin_main(argc: usize, argv: *mut *mut u8, envp: *mut *mut u8) -> i32 { //! todo!("Run the program and return the program exit status.") //! } @@ -40,7 +40,7 @@ use libc::c_int; pub fn at_exit(func: Box) { use core::ffi::c_void; - extern "C" { + unsafe extern "C" { // fn __cxa_atexit( func: unsafe extern "C" fn(*mut c_void), @@ -50,9 +50,9 @@ pub fn at_exit(func: Box) { } // The function to pass to `__cxa_atexit`. - unsafe extern "C" fn at_exit_func(arg: *mut c_void) { + unsafe extern "C" fn at_exit_func(arg: *mut c_void) { unsafe { Box::from_raw(arg.cast::>())(); - } + }} let at_exit_arg = Box::into_raw(Box::new(func)).cast::(); let r = unsafe { __cxa_atexit(at_exit_func, at_exit_arg, null_mut()) }; diff --git a/src/program/linux_raw.rs b/src/program/linux_raw.rs index 43e3b33..ea1b675 100644 --- a/src/program/linux_raw.rs +++ b/src/program/linux_raw.rs @@ -46,7 +46,7 @@ compile_error!("\"origin-program\" depends on either \"origin-start\" or \"exter /// # Safety /// /// `mem` must point to the stack as provided by the operating system. -pub(super) unsafe extern "C" fn entry(mem: *mut usize) -> ! { +pub(super) unsafe extern "C" fn entry(mem: *mut usize) -> ! { unsafe { // Do some basic precondition checks, to ensure that our assembly code did // what we expect it to do. These are debug-only, to keep the release-mode // startup code small and simple to disassemble and inspect. @@ -60,7 +60,7 @@ pub(super) unsafe extern "C" fn entry(mem: *mut usize) -> ! { // If we have nightly, we can do additional checks. #[cfg(feature = "nightly")] { - extern "C" { + unsafe extern "C" { #[link_name = "llvm.frameaddress"] fn builtin_frame_address(level: i32) -> *const u8; #[link_name = "llvm.returnaddress"] @@ -144,7 +144,7 @@ pub(super) unsafe extern "C" fn entry(mem: *mut usize) -> ! { { // Declare `origin_main` as documented in [`crate::program`]. - extern "Rust" { + unsafe extern "Rust" { fn origin_main(argc: usize, argv: *mut *mut u8, envp: *mut *mut u8) -> i32; } @@ -161,7 +161,7 @@ pub(super) unsafe extern "C" fn entry(mem: *mut usize) -> ! { // `origin_main`'s return value. exit(status) } -} +} } /// A program entry point similar to `_start`, but which is meant to be called /// by something else in the program rather than the OS. @@ -171,16 +171,16 @@ pub(super) unsafe extern "C" fn entry(mem: *mut usize) -> ! { /// `mem` must point to a stack with the contents that the OS would provide /// on the initial stack. #[cfg(feature = "external-start")] -pub unsafe fn start(mem: *mut usize) -> ! { +pub unsafe fn start(mem: *mut usize) -> ! { unsafe { entry(mem) -} +} } /// Compute `argc`, `argv`, and `envp`. /// /// # Safety /// /// `mem` must point to the stack as provided by the operating system. -unsafe fn compute_args(mem: *mut usize) -> (i32, *mut *mut u8, *mut *mut u8) { +unsafe fn compute_args(mem: *mut usize) -> (i32, *mut *mut u8, *mut *mut u8) { unsafe { use linux_raw_sys::ctypes::c_uint; let kernel_argc = *mem; @@ -194,7 +194,7 @@ unsafe fn compute_args(mem: *mut usize) -> (i32, *mut *mut u8, *mut *mut u8) { debug_assert_eq!(*argv.add(argc as usize), core::ptr::null_mut()); (argc, argv, envp) -} +} } /// Initialize `origin` and `rustix` runtime state. /// @@ -207,7 +207,7 @@ unsafe fn init_runtime(mem: *mut usize, envp: *mut *mut u8) { // Explicitly initialize `rustix`. This is needed for things like // `page_size()` to work. #[cfg(feature = "param")] - rustix::param::init(envp); + unsafe { rustix::param::init(envp); } // Read the program headers and extract the TLS info. #[cfg(feature = "thread")] @@ -215,7 +215,7 @@ unsafe fn init_runtime(mem: *mut usize, envp: *mut *mut u8) { // Initialize the main thread. #[cfg(feature = "thread")] - thread::initialize_main(mem.cast()); + unsafe { thread::initialize_main(mem.cast()); } } /// Functions registered with [`at_exit`]. diff --git a/src/relocate.rs b/src/relocate.rs index 288370c..8dcaebc 100644 --- a/src/relocate.rs +++ b/src/relocate.rs @@ -77,7 +77,7 @@ macro_rules! debug_assert_eq { /// /// So yes, there's a reason this code is behind a feature flag. #[cold] -pub(super) unsafe fn relocate(envp: *mut *mut u8) { +pub(super) unsafe fn relocate(envp: *mut *mut u8) { unsafe { // Locate the AUX records we need. let auxp = compute_auxp(envp); @@ -354,10 +354,10 @@ pub(super) unsafe fn relocate(envp: *mut *mut u8) { let mprotect_addr = relro.wrapping_add(offset) & auxv_page_size.wrapping_neg(); relocation_mprotect_readonly(mprotect_addr, relro_size); } -} +} } /// Compute the address of the AUX table. -unsafe fn compute_auxp(envp: *mut *mut u8) -> *const Elf_auxv_t { +unsafe fn compute_auxp(envp: *mut *mut u8) -> *const Elf_auxv_t { unsafe { // Locate the AUX records we need. We don't use rustix to do this because // that would involve calling a function in another crate. let mut auxp = envp; @@ -366,7 +366,7 @@ unsafe fn compute_auxp(envp: *mut *mut u8) -> *const Elf_auxv_t { auxp = auxp.add(1); } auxp.add(1).cast() -} +} } /// Load the address of `_start` from static memory. /// diff --git a/src/signal/libc.rs b/src/signal/libc.rs index 27d8405..2f26b45 100644 --- a/src/signal/libc.rs +++ b/src/signal/libc.rs @@ -52,7 +52,7 @@ bitflags::bitflags! { /// # Safety /// /// yolo. At least this function handles `sa_restorer` automatically though. -pub unsafe fn sigaction(sig: Signal, action: Option) -> io::Result { +pub unsafe fn sigaction(sig: Signal, action: Option) -> io::Result { unsafe { let action: *const Sigaction = match action { Some(action) => &action, None => null(), @@ -64,7 +64,7 @@ pub unsafe fn sigaction(sig: Signal, action: Option) -> io::Result) -> io::Result { +pub unsafe fn sigaction(sig: Signal, action: Option) -> io::Result { unsafe { #[allow(unused_mut)] let mut action = action; @@ -32,7 +32,7 @@ pub unsafe fn sigaction(sig: Signal, action: Option) -> io::Result Option> { +pub unsafe fn join(thread: Thread) -> Option> { unsafe { let thread = thread.0; let mut return_value: *mut c_void = null_mut(); assert_eq!(libc::pthread_join(thread, &mut return_value), 0); NonNull::new(return_value) -} +}} /// Registers a function to call when the current thread exits. #[cfg(feature = "thread-at-exit")] @@ -287,7 +287,7 @@ pub fn current_tls_addr(module: usize, offset: usize) -> *mut c_void { /// `thread` must point to a valid thread record. #[inline] #[must_use] -pub unsafe fn stack(thread: Thread) -> (*mut c_void, usize, usize) { +pub unsafe fn stack(thread: Thread) -> (*mut c_void, usize, usize) { unsafe { let thread = thread.0; let mut attr: libc::pthread_attr_t = zeroed(); @@ -304,7 +304,7 @@ pub unsafe fn stack(thread: Thread) -> (*mut c_void, usize, usize) { assert_eq!(libc::pthread_attr_getguardsize(&attr, &mut guard_size), 0); (stack_addr, stack_size, guard_size) -} +}} /// Return the default stack size for new threads. #[inline] @@ -344,7 +344,7 @@ pub fn yield_current() { /// Return the address of `__dso_handle`, appropriately casted. #[cfg(feature = "thread-at-exit")] -unsafe fn dso_handle() -> *mut c_void { +unsafe fn dso_handle() -> *mut c_void { unsafe { let dso_handle: *const *const c_void = &__dso_handle; dso_handle.cast::().cast_mut() -} +}} diff --git a/src/thread/linux_raw.rs b/src/thread/linux_raw.rs index 24579ae..a51a981 100644 --- a/src/thread/linux_raw.rs +++ b/src/thread/linux_raw.rs @@ -54,9 +54,9 @@ impl Thread { /// /// `raw` must be a valid non-null thread pointer. #[inline] - pub unsafe fn from_raw_unchecked(raw: *mut c_void) -> Self { + pub unsafe fn from_raw_unchecked(raw: *mut c_void) -> Self { unsafe { Self(NonNull::new_unchecked(raw.cast())) - } + } } /// Convert to `Self` from a raw non-null pointer that was returned from /// `Thread::to_raw_non_null`. @@ -301,7 +301,7 @@ pub(super) fn initialize_startup_info() { } } -extern "C" { +unsafe extern "C" { /// Declare the `_DYNAMIC` symbol so that we can compare its address with /// the static address in the `PT_DYNAMIC` header to learn our offset. Use /// a weak symbol because `_DYNAMIC` is not always present. @@ -323,7 +323,7 @@ core::arch::global_asm!(".weak _DYNAMIC"); /// `initialize_startup_info` must be called before this. And `mem` must be the /// initial value of the stack pointer in a new process, pointing to the /// initial contents of the stack. -pub(super) unsafe fn initialize_main(mem: *mut c_void) { +pub(super) unsafe fn initialize_main(mem: *mut c_void) { unsafe { // Determine the top of the stack. Linux puts the `AT_EXECFN` string at // the top, so find the end of that, and then round up to the page size. // See for details. @@ -361,7 +361,7 @@ pub(super) unsafe fn initialize_main(mem: *mut c_void) { .unwrap() .cast::(); - let metadata_align = max(unsafe { STARTUP_TLS_INFO.align }, align_of::()); + let metadata_align = max(STARTUP_TLS_INFO.align, align_of::()); debug_assert_eq!(new.addr() % metadata_align, 0); let tls_data = new.add(tls_data_bottom); @@ -381,7 +381,7 @@ pub(super) unsafe fn initialize_main(mem: *mut c_void) { // Point the platform thread-pointer register at the new thread metadata. set_thread_pointer(newtls); -} +} } fn calculate_tls_size(map_size: &mut usize) -> (usize, usize) { // SAFETY: `STARTUP_TLS_INFO` is initialized at program startup before @@ -435,7 +435,7 @@ unsafe fn initialize_tls( stack_size: usize, guard_size: usize, map_size: usize, -) -> (*mut c_void, *mut i32) { +) -> (*mut c_void, *mut i32) { unsafe { let newtls: *mut c_void = (*metadata).abi.thread_pointee.as_mut_ptr().cast(); // Initialize the thread metadata. @@ -469,7 +469,7 @@ unsafe fn initialize_tls( let thread_id_ptr = (*metadata).thread.thread_id.as_ptr().cast::(); (newtls, thread_id_ptr) -} +} } /// Creates a new thread. /// @@ -628,7 +628,7 @@ pub(super) unsafe extern "C" fn entry( fn_: extern "C" fn(), args: *mut *mut c_void, num_args: usize, -) -> ! { +) -> ! { unsafe { #[cfg(feature = "log")] log::trace!("Thread[{:?}] launched", current_id().as_raw_nonzero()); @@ -641,7 +641,7 @@ pub(super) unsafe extern "C" fn entry( // If we have nightly, we can do additional checks. #[cfg(feature = "nightly")] { - extern "C" { + unsafe extern "C" { #[link_name = "llvm.frameaddress"] fn builtin_frame_address(level: i32) -> *const u8; #[link_name = "llvm.returnaddress"] @@ -678,10 +678,10 @@ pub(super) unsafe extern "C" fn entry( let return_value = fn_(args); exit(return_value) -} +} } /// Call the destructors registered with [`at_exit`] and exit the thread. -unsafe fn exit(return_value: Option>) -> ! { +unsafe fn exit(return_value: Option>) -> ! { unsafe { let current = current(); #[cfg(feature = "log")] @@ -765,7 +765,7 @@ unsafe fn exit(return_value: Option>) -> ! { // Terminate the thread. rustix::runtime::exit_thread(0) -} +} } /// Call the destructors registered with [`at_exit`]. #[cfg(feature = "thread-at-exit")] @@ -800,7 +800,7 @@ pub(crate) fn call_dtors(current: Thread) { /// `thread` must point to a valid thread record that has not yet been detached /// and will not be joined. #[inline] -pub unsafe fn detach(thread: Thread) { +pub unsafe fn detach(thread: Thread) { unsafe { #[cfg(feature = "log")] let thread_id = thread.0.as_ref().thread_id.load(SeqCst); @@ -821,7 +821,7 @@ pub unsafe fn detach(thread: Thread) { free_memory(thread); } -} +} } /// Waits for a thread to finish. /// @@ -832,7 +832,7 @@ pub unsafe fn detach(thread: Thread) { /// /// `thread` must point to a valid thread record that has not already been /// detached or joined. -pub unsafe fn join(thread: Thread) -> Option> { +pub unsafe fn join(thread: Thread) -> Option> { unsafe { let thread_data = thread.0.as_ref(); #[cfg(feature = "log")] @@ -863,13 +863,13 @@ pub unsafe fn join(thread: Thread) -> Option> { // Convert the `*mut c_void` we stored in the `AtomicPtr` back into an // `Option>`. NonNull::new(return_value) -} +} } /// Wait until `thread` has exited. /// /// `thread` must point to a valid thread record that has not already been /// detached or joined. -unsafe fn wait_for_exit(thread: Thread) { +unsafe fn wait_for_exit(thread: Thread) { unsafe { use rustix::thread::futex; // Check whether the thread has exited already; we set the @@ -893,7 +893,7 @@ unsafe fn wait_for_exit(thread: Thread) { Err(e) => debug_assert_eq!(e, io::Errno::AGAIN), } } -} +} } #[cfg(feature = "log")] fn log_thread_to_be_freed(thread_id: i32) { @@ -908,7 +908,7 @@ fn log_thread_to_be_freed(thread_id: i32) { /// /// `thread` must point to a valid thread record for a thread that has /// already exited. -unsafe fn free_memory(thread: Thread) { +unsafe fn free_memory(thread: Thread) { unsafe { use rustix::mm::munmap; // The thread was detached. Prepare to free the memory. First read out @@ -925,7 +925,7 @@ unsafe fn free_memory(thread: Thread) { let map = stack_addr.byte_sub(guard_size); munmap(map, map_size).unwrap(); } -} +} } /// Registers a function to call when the current thread exits. #[cfg(feature = "thread-at-exit")] @@ -986,7 +986,7 @@ pub fn current_id() -> ThreadId { /// return. #[doc(hidden)] #[inline] -pub unsafe fn set_current_id_after_a_fork(tid: ThreadId) { +pub unsafe fn set_current_id_after_a_fork(tid: ThreadId) { unsafe { let current = current(); debug_assert_ne!( tid.as_raw_nonzero().get(), @@ -999,7 +999,7 @@ pub unsafe fn set_current_id_after_a_fork(tid: ThreadId) { .as_ref() .thread_id .store(tid.as_raw_nonzero().get(), SeqCst); -} +} } /// Return the address of the thread-local `errno` state. /// @@ -1048,10 +1048,10 @@ pub fn current_tls_addr(module: usize, offset: usize) -> *mut c_void { /// `thread` must point to a valid thread record. #[inline] #[cfg_attr(docsrs, doc(cfg(feature = "take-charge")))] -pub unsafe fn id(thread: Thread) -> Option { +pub unsafe fn id(thread: Thread) -> Option { unsafe { let raw = thread.0.as_ref().thread_id.load(SeqCst); ThreadId::from_raw(raw) -} +} } /// Return the current thread's stack address (lowest address), size, and guard /// size. @@ -1061,10 +1061,10 @@ pub unsafe fn id(thread: Thread) -> Option { /// `thread` must point to a valid thread record. #[inline] #[must_use] -pub unsafe fn stack(thread: Thread) -> (*mut c_void, usize, usize) { +pub unsafe fn stack(thread: Thread) -> (*mut c_void, usize, usize) { unsafe { let data = thread.0.as_ref(); (data.stack_addr, data.stack_size, data.guard_size) -} +} } /// Return the default stack size for new threads. #[inline] @@ -1099,7 +1099,7 @@ extern "C" fn __aeabi_read_tp() -> *mut c_void { } /// Some targets use this global variable instead of the TLS `canary` field. -#[no_mangle] +#[unsafe(no_mangle)] static mut __stack_chk_guard: usize = 0; const fn round_up(addr: usize, boundary: usize) -> usize { diff --git a/src/unwind_unimplemented.rs b/src/unwind_unimplemented.rs index 19d4ac9..509de39 100644 --- a/src/unwind_unimplemented.rs +++ b/src/unwind_unimplemented.rs @@ -3,72 +3,72 @@ //! //! Entirely `unimplemented!`. -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_Backtrace() { unimplemented!("_Unwind_Backtrace") } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_DeleteException() { unimplemented!("_Unwind_DeleteException") } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_GetDataRelBase() { unimplemented!("_Unwind_GetDataRelBase") } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_GetIP() { unimplemented!("_Unwind_GetIP") } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_GetIPInfo() { unimplemented!("_Unwind_GetIPInfo") } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_GetLanguageSpecificData() { unimplemented!("_Unwind_GetLanguageSpecificData") } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_GetRegionStart() { unimplemented!("_Unwind_GetRegionStart") } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_GetTextRelBase() { unimplemented!("_Unwind_GetTextRelBase") } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_RaiseException() { unimplemented!("_Unwind_RaiseException") } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_Resume() { unimplemented!("_Unwind_Resume") } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_SetGR() { unimplemented!("_Unwind_SetGR") } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_SetIP() { unimplemented!("_Unwind_SetIP") } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_FindEnclosingFunction() { unimplemented!("_Unwind_FindEnclosingFunction") } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_GetCFA() { unimplemented!("_Unwind_GetCFA") } diff --git a/test-crates/origin-start/src/bin/abort-via-raise.rs b/test-crates/origin-start/src/bin/abort-via-raise.rs index 4253840..db46cb3 100644 --- a/test-crates/origin-start/src/bin/abort-via-raise.rs +++ b/test-crates/origin-start/src/bin/abort-via-raise.rs @@ -11,7 +11,7 @@ use origin::{program, signal}; #[global_allocator] static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::GlobalDlmalloc; -#[no_mangle] +#[unsafe(no_mangle)] unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { // Terminate the current process using `Signal::ABORT`. rustix::process::kill_process(rustix::process::getpid(), signal::Signal::ABORT).ok(); diff --git a/test-crates/origin-start/src/bin/canary.rs b/test-crates/origin-start/src/bin/canary.rs index 7afb613..9d768a0 100644 --- a/test-crates/origin-start/src/bin/canary.rs +++ b/test-crates/origin-start/src/bin/canary.rs @@ -13,7 +13,7 @@ use origin::{program, thread}; #[global_allocator] static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::GlobalDlmalloc; -extern "C" { +unsafe extern "C" { static __stack_chk_guard: UnsafeCell; } @@ -40,8 +40,8 @@ fn tls_guard() -> usize { ret } -#[no_mangle] -unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { +#[unsafe(no_mangle)] +unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { unsafe { assert_ne!(*__stack_chk_guard.get(), 0); assert_eq!(*__stack_chk_guard.get(), tls_guard()); @@ -63,4 +63,4 @@ unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> assert_eq!(*__stack_chk_guard.get(), tls_guard()); program::exit(203); -} +}} diff --git a/test-crates/origin-start/src/bin/detach.rs b/test-crates/origin-start/src/bin/detach.rs index c49135d..dfad516 100644 --- a/test-crates/origin-start/src/bin/detach.rs +++ b/test-crates/origin-start/src/bin/detach.rs @@ -10,8 +10,8 @@ use origin::{program, thread}; #[global_allocator] static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::GlobalDlmalloc; -#[no_mangle] -unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { +#[unsafe(no_mangle)] +unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { unsafe { let long_thread = thread::create( |_args| None, &[], @@ -43,4 +43,4 @@ unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> thread::detach(long_thread); program::exit(202); -} +}} diff --git a/test-crates/origin-start/src/bin/main-thread-dtors-adding-dtors.rs b/test-crates/origin-start/src/bin/main-thread-dtors-adding-dtors.rs index ab18ceb..8dc4878 100644 --- a/test-crates/origin-start/src/bin/main-thread-dtors-adding-dtors.rs +++ b/test-crates/origin-start/src/bin/main-thread-dtors-adding-dtors.rs @@ -14,7 +14,7 @@ static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::Glob static FLAG: AtomicBool = AtomicBool::new(false); -#[no_mangle] +#[unsafe(no_mangle)] unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { thread::at_exit(Box::new(|| { assert!(FLAG.load(Ordering::Relaxed)); diff --git a/test-crates/origin-start/src/bin/program-dtors-adding-dtors.rs b/test-crates/origin-start/src/bin/program-dtors-adding-dtors.rs index 0560609..d00490b 100644 --- a/test-crates/origin-start/src/bin/program-dtors-adding-dtors.rs +++ b/test-crates/origin-start/src/bin/program-dtors-adding-dtors.rs @@ -14,7 +14,7 @@ static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::Glob static FLAG: AtomicBool = AtomicBool::new(false); -#[no_mangle] +#[unsafe(no_mangle)] unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { program::at_exit(Box::new(|| { assert!(FLAG.load(Ordering::Relaxed)); diff --git a/test-crates/origin-start/src/bin/thread-dtors-adding-dtors.rs b/test-crates/origin-start/src/bin/thread-dtors-adding-dtors.rs index 0f855ab..800708b 100644 --- a/test-crates/origin-start/src/bin/thread-dtors-adding-dtors.rs +++ b/test-crates/origin-start/src/bin/thread-dtors-adding-dtors.rs @@ -14,8 +14,8 @@ static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::Glob static FLAG: AtomicBool = AtomicBool::new(false); -#[no_mangle] -unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { +#[unsafe(no_mangle)] +unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { unsafe { let thread = thread::create( |_args| { thread::at_exit(Box::new(|| { @@ -45,4 +45,4 @@ unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> thread::join(thread); 0 -} +}} diff --git a/test-crates/origin-start/src/bin/thread-id.rs b/test-crates/origin-start/src/bin/thread-id.rs index 3c06f4d..c2f58c8 100644 --- a/test-crates/origin-start/src/bin/thread-id.rs +++ b/test-crates/origin-start/src/bin/thread-id.rs @@ -21,8 +21,8 @@ enum Id { Child(thread::ThreadId), } -#[no_mangle] -unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { +#[unsafe(no_mangle)] +unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { unsafe { assert_eq!(thread::current_id(), thread::id(thread::current()).unwrap()); program::at_exit(Box::new(|| { assert_eq!(thread::current_id(), thread::id(thread::current()).unwrap()) @@ -109,4 +109,4 @@ unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> assert_eq!(thread::current_id(), thread::id(thread::current()).unwrap()); program::exit(201); -} +}} diff --git a/test-crates/origin-start/src/bin/tls.rs b/test-crates/origin-start/src/bin/tls.rs index 3f2b85b..53560c3 100644 --- a/test-crates/origin-start/src/bin/tls.rs +++ b/test-crates/origin-start/src/bin/tls.rs @@ -16,8 +16,8 @@ use origin::{program, thread}; #[global_allocator] static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::GlobalDlmalloc; -#[no_mangle] -unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { +#[unsafe(no_mangle)] +unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { unsafe { // Assert that the main thread initialized its TLS properly. check_eq(TEST_DATA.0); @@ -79,7 +79,7 @@ unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> check_eq([TEST_DATA.0[0], without_provenance_mut(78), TEST_DATA.0[2]]); program::exit(200); -} +}} struct SyncTestData([*const u32; 3]); unsafe impl Sync for SyncTestData {} diff --git a/test-crates/origin-start/src/bin/trap.rs b/test-crates/origin-start/src/bin/trap.rs index aeadcca..2136f41 100644 --- a/test-crates/origin-start/src/bin/trap.rs +++ b/test-crates/origin-start/src/bin/trap.rs @@ -11,7 +11,7 @@ use origin::program; #[global_allocator] static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::GlobalDlmalloc; -#[no_mangle] +#[unsafe(no_mangle)] unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { program::trap() } From 2d4998ddcc3e706c4f8d2e011c8a08c4a670cf66 Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Tue, 29 Apr 2025 11:45:58 -0700 Subject: [PATCH 08/15] Bump editions --- Cargo.toml | 2 +- example-crates/basic/Cargo.toml | 2 +- example-crates/external-start/Cargo.toml | 2 +- example-crates/origin-start-dynamic-linker/Cargo.toml | 2 +- example-crates/origin-start-lto/Cargo.toml | 2 +- example-crates/origin-start-no-alloc/Cargo.toml | 2 +- example-crates/origin-start-panic-abort/Cargo.toml | 2 +- example-crates/origin-start-stable/Cargo.toml | 2 +- example-crates/origin-start/Cargo.toml | 2 +- example-crates/tiny-hello/Cargo.toml | 2 +- example-crates/tiny/Cargo.toml | 2 +- test-crates/origin-start/Cargo.toml | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 79ef280..578eef4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ description = "Program startup and thread support written in Rust" documentation = "https://docs.rs/origin" license = "Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT" repository = "https://github.com/sunfishcode/origin" -edition = "2021" +edition = "2024" keywords = ["linux"] categories = ["no-std"] include = ["src", "Cargo.toml", "COPYRIGHT", "LICENSE*", "/*.md"] diff --git a/example-crates/basic/Cargo.toml b/example-crates/basic/Cargo.toml index 51e0574..b425a2b 100644 --- a/example-crates/basic/Cargo.toml +++ b/example-crates/basic/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "basic" version = "0.0.0" -edition = "2021" +edition = "2024" publish = false [dependencies] diff --git a/example-crates/external-start/Cargo.toml b/example-crates/external-start/Cargo.toml index 098165e..74d3853 100644 --- a/example-crates/external-start/Cargo.toml +++ b/example-crates/external-start/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "external-start" version = "0.0.0" -edition = "2021" +edition = "2024" publish = false [dependencies] diff --git a/example-crates/origin-start-dynamic-linker/Cargo.toml b/example-crates/origin-start-dynamic-linker/Cargo.toml index be034aa..6430629 100644 --- a/example-crates/origin-start-dynamic-linker/Cargo.toml +++ b/example-crates/origin-start-dynamic-linker/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "origin-start-dynamic-linker" version = "0.0.0" -edition = "2021" +edition = "2024" publish = false [lib] diff --git a/example-crates/origin-start-lto/Cargo.toml b/example-crates/origin-start-lto/Cargo.toml index e384b7e..63c7363 100644 --- a/example-crates/origin-start-lto/Cargo.toml +++ b/example-crates/origin-start-lto/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "origin-start-lto" version = "0.0.0" -edition = "2021" +edition = "2024" publish = false [dependencies] diff --git a/example-crates/origin-start-no-alloc/Cargo.toml b/example-crates/origin-start-no-alloc/Cargo.toml index ced2aa5..cde37b7 100644 --- a/example-crates/origin-start-no-alloc/Cargo.toml +++ b/example-crates/origin-start-no-alloc/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "origin-start-no-alloc" version = "0.0.0" -edition = "2021" +edition = "2024" publish = false [dependencies] diff --git a/example-crates/origin-start-panic-abort/Cargo.toml b/example-crates/origin-start-panic-abort/Cargo.toml index e349344..44f7823 100644 --- a/example-crates/origin-start-panic-abort/Cargo.toml +++ b/example-crates/origin-start-panic-abort/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "origin-start-panic-abort" version = "0.0.0" -edition = "2021" +edition = "2024" publish = false [dependencies] diff --git a/example-crates/origin-start-stable/Cargo.toml b/example-crates/origin-start-stable/Cargo.toml index 959c53b..59c3c3a 100644 --- a/example-crates/origin-start-stable/Cargo.toml +++ b/example-crates/origin-start-stable/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "origin-start-stable" version = "0.0.0" -edition = "2021" +edition = "2024" publish = false [dependencies] diff --git a/example-crates/origin-start/Cargo.toml b/example-crates/origin-start/Cargo.toml index f2548ce..ee6cab7 100644 --- a/example-crates/origin-start/Cargo.toml +++ b/example-crates/origin-start/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "origin-start" version = "0.0.0" -edition = "2021" +edition = "2024" publish = false [dependencies] diff --git a/example-crates/tiny-hello/Cargo.toml b/example-crates/tiny-hello/Cargo.toml index 4bfff34..e043e6b 100644 --- a/example-crates/tiny-hello/Cargo.toml +++ b/example-crates/tiny-hello/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "tiny-hello" version = "0.0.0" -edition = "2021" +edition = "2024" publish = false [dependencies] diff --git a/example-crates/tiny/Cargo.toml b/example-crates/tiny/Cargo.toml index 28b711a..7a612b8 100644 --- a/example-crates/tiny/Cargo.toml +++ b/example-crates/tiny/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "tiny" version = "0.0.0" -edition = "2021" +edition = "2024" publish = false [dependencies] diff --git a/test-crates/origin-start/Cargo.toml b/test-crates/origin-start/Cargo.toml index 03ea686..ba1593a 100644 --- a/test-crates/origin-start/Cargo.toml +++ b/test-crates/origin-start/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "origin-start-tests" version = "0.0.0" -edition = "2021" +edition = "2024" publish = false [dependencies] From eae6707119703eb5cbbae76bb0ed14aac66d3a94 Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Tue, 29 Apr 2025 12:05:40 -0700 Subject: [PATCH 09/15] rustfmt --- example-crates/external-start/src/main.rs | 82 ++- .../origin-start-dynamic-linker/src/lib.rs | 60 +- example-crates/origin-start-lto/src/main.rs | 60 +- .../origin-start-panic-abort/src/main.rs | 60 +- .../origin-start-stable/src/main.rs | 60 +- example-crates/origin-start/src/main.rs | 60 +- example-crates/tiny-hello/src/main.rs | 22 +- src/arch/riscv64.rs | 7 +- src/arch/x86_64.rs | 192 ++--- src/mem/fast.rs | 60 +- src/mem/small.rs | 142 ++-- src/mem/x86_64.rs | 440 ++++++------ src/program/libc.rs | 8 +- src/program/linux_raw.rs | 238 +++--- src/relocate.rs | 526 +++++++------- src/signal/libc.rs | 26 +- src/signal/linux_raw.rs | 32 +- src/thread/libc.rs | 66 +- src/thread/linux_raw.rs | 675 +++++++++--------- test-crates/origin-start/src/bin/canary.rs | 40 +- test-crates/origin-start/src/bin/detach.rs | 68 +- .../src/bin/thread-dtors-adding-dtors.rs | 48 +- test-crates/origin-start/src/bin/thread-id.rs | 154 ++-- test-crates/origin-start/src/bin/tls.rs | 98 +-- 24 files changed, 1673 insertions(+), 1551 deletions(-) diff --git a/example-crates/external-start/src/main.rs b/example-crates/external-start/src/main.rs index fd76d1d..c2b7bec 100644 --- a/example-crates/external-start/src/main.rs +++ b/example-crates/external-start/src/main.rs @@ -22,55 +22,59 @@ static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::Glob #[unsafe(link_section = ".init_array.00000")] #[used] static EARLY_INIT_ARRAY: unsafe extern "C" fn(i32, *mut *mut u8) = { - unsafe extern "C" fn function(_argc: i32, argv: *mut *mut u8) { unsafe { - // Libc was calling constructors (we're one of them), but origin will - // be doing that now, so just exit when we're called a second time. - static FIRST: AtomicBool = AtomicBool::new(false); - if FIRST - .compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed) - .is_err() - { - return; - } + unsafe extern "C" fn function(_argc: i32, argv: *mut *mut u8) { + unsafe { + // Libc was calling constructors (we're one of them), but origin will + // be doing that now, so just exit when we're called a second time. + static FIRST: AtomicBool = AtomicBool::new(false); + if FIRST + .compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed) + .is_err() + { + return; + } - // Compute the initial stack address provided by the kernel. - let mem = argv.sub(1); + // Compute the initial stack address provided by the kernel. + let mem = argv.sub(1); - origin::program::start(mem as _); - }} + origin::program::start(mem as _); + } + } function }; #[unsafe(no_mangle)] -unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { unsafe { - eprintln!("Hello from main thread"); +unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { + unsafe { + eprintln!("Hello from main thread"); - program::at_exit(Box::new(|| { - eprintln!("Hello from a `program::at_exit` handler") - })); - thread::at_exit(Box::new(|| { - eprintln!("Hello from a main-thread `thread::at_exit` handler") - })); + program::at_exit(Box::new(|| { + eprintln!("Hello from a `program::at_exit` handler") + })); + thread::at_exit(Box::new(|| { + eprintln!("Hello from a main-thread `thread::at_exit` handler") + })); - let thread = thread::create( - |_args| { - eprintln!("Hello from child thread"); - thread::at_exit(Box::new(|| { - eprintln!("Hello from child thread's `thread::at_exit` handler") - })); - None - }, - &[], - thread::default_stack_size(), - thread::default_guard_size(), - ) - .unwrap(); + let thread = thread::create( + |_args| { + eprintln!("Hello from child thread"); + thread::at_exit(Box::new(|| { + eprintln!("Hello from child thread's `thread::at_exit` handler") + })); + None + }, + &[], + thread::default_stack_size(), + thread::default_guard_size(), + ) + .unwrap(); - thread::join(thread); + thread::join(thread); - eprintln!("Goodbye from main"); - program::exit(0); -}} + eprintln!("Goodbye from main"); + program::exit(0); + } +} // Libc calls `main` so we need to provide a definition to satisfy the // linker, however origin gains control before libc can call this `main`. diff --git a/example-crates/origin-start-dynamic-linker/src/lib.rs b/example-crates/origin-start-dynamic-linker/src/lib.rs index 6dc8828..e86cb68 100644 --- a/example-crates/origin-start-dynamic-linker/src/lib.rs +++ b/example-crates/origin-start-dynamic-linker/src/lib.rs @@ -13,32 +13,34 @@ use origin::{program, thread}; static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::GlobalDlmalloc; #[unsafe(no_mangle)] -unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { unsafe { - eprintln!("Hello from main thread"); - - program::at_exit(Box::new(|| { - eprintln!("Hello from a `program::at_exit` handler") - })); - thread::at_exit(Box::new(|| { - eprintln!("Hello from a main-thread `thread::at_exit` handler") - })); - - let thread = thread::create( - |_args| { - eprintln!("Hello from child thread"); - thread::at_exit(Box::new(|| { - eprintln!("Hello from child thread's `thread::at_exit` handler") - })); - None - }, - &[], - thread::default_stack_size(), - thread::default_guard_size(), - ) - .unwrap(); - - thread::join(thread); - - eprintln!("Goodbye from main"); - program::exit(0); -}} +unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { + unsafe { + eprintln!("Hello from main thread"); + + program::at_exit(Box::new(|| { + eprintln!("Hello from a `program::at_exit` handler") + })); + thread::at_exit(Box::new(|| { + eprintln!("Hello from a main-thread `thread::at_exit` handler") + })); + + let thread = thread::create( + |_args| { + eprintln!("Hello from child thread"); + thread::at_exit(Box::new(|| { + eprintln!("Hello from child thread's `thread::at_exit` handler") + })); + None + }, + &[], + thread::default_stack_size(), + thread::default_guard_size(), + ) + .unwrap(); + + thread::join(thread); + + eprintln!("Goodbye from main"); + program::exit(0); + } +} diff --git a/example-crates/origin-start-lto/src/main.rs b/example-crates/origin-start-lto/src/main.rs index 8b2a55b..cfe9bae 100644 --- a/example-crates/origin-start-lto/src/main.rs +++ b/example-crates/origin-start-lto/src/main.rs @@ -13,32 +13,34 @@ use origin::{program, thread}; static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::GlobalDlmalloc; #[unsafe(no_mangle)] -unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { unsafe { - eprintln!("Hello from main thread"); - - program::at_exit(Box::new(|| { - eprintln!("Hello from a `program::at_exit` handler") - })); - thread::at_exit(Box::new(|| { - eprintln!("Hello from a main-thread `thread::at_exit` handler") - })); - - let thread = thread::create( - |_args| { - eprintln!("Hello from child thread"); - thread::at_exit(Box::new(|| { - eprintln!("Hello from child thread's `thread::at_exit` handler") - })); - None - }, - &[], - thread::default_stack_size(), - thread::default_guard_size(), - ) - .unwrap(); - - thread::join(thread); - - eprintln!("Goodbye from main"); - program::exit(0); -}} +unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { + unsafe { + eprintln!("Hello from main thread"); + + program::at_exit(Box::new(|| { + eprintln!("Hello from a `program::at_exit` handler") + })); + thread::at_exit(Box::new(|| { + eprintln!("Hello from a main-thread `thread::at_exit` handler") + })); + + let thread = thread::create( + |_args| { + eprintln!("Hello from child thread"); + thread::at_exit(Box::new(|| { + eprintln!("Hello from child thread's `thread::at_exit` handler") + })); + None + }, + &[], + thread::default_stack_size(), + thread::default_guard_size(), + ) + .unwrap(); + + thread::join(thread); + + eprintln!("Goodbye from main"); + program::exit(0); + } +} diff --git a/example-crates/origin-start-panic-abort/src/main.rs b/example-crates/origin-start-panic-abort/src/main.rs index 8b2a55b..cfe9bae 100644 --- a/example-crates/origin-start-panic-abort/src/main.rs +++ b/example-crates/origin-start-panic-abort/src/main.rs @@ -13,32 +13,34 @@ use origin::{program, thread}; static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::GlobalDlmalloc; #[unsafe(no_mangle)] -unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { unsafe { - eprintln!("Hello from main thread"); - - program::at_exit(Box::new(|| { - eprintln!("Hello from a `program::at_exit` handler") - })); - thread::at_exit(Box::new(|| { - eprintln!("Hello from a main-thread `thread::at_exit` handler") - })); - - let thread = thread::create( - |_args| { - eprintln!("Hello from child thread"); - thread::at_exit(Box::new(|| { - eprintln!("Hello from child thread's `thread::at_exit` handler") - })); - None - }, - &[], - thread::default_stack_size(), - thread::default_guard_size(), - ) - .unwrap(); - - thread::join(thread); - - eprintln!("Goodbye from main"); - program::exit(0); -}} +unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { + unsafe { + eprintln!("Hello from main thread"); + + program::at_exit(Box::new(|| { + eprintln!("Hello from a `program::at_exit` handler") + })); + thread::at_exit(Box::new(|| { + eprintln!("Hello from a main-thread `thread::at_exit` handler") + })); + + let thread = thread::create( + |_args| { + eprintln!("Hello from child thread"); + thread::at_exit(Box::new(|| { + eprintln!("Hello from child thread's `thread::at_exit` handler") + })); + None + }, + &[], + thread::default_stack_size(), + thread::default_guard_size(), + ) + .unwrap(); + + thread::join(thread); + + eprintln!("Goodbye from main"); + program::exit(0); + } +} diff --git a/example-crates/origin-start-stable/src/main.rs b/example-crates/origin-start-stable/src/main.rs index 8b2a55b..cfe9bae 100644 --- a/example-crates/origin-start-stable/src/main.rs +++ b/example-crates/origin-start-stable/src/main.rs @@ -13,32 +13,34 @@ use origin::{program, thread}; static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::GlobalDlmalloc; #[unsafe(no_mangle)] -unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { unsafe { - eprintln!("Hello from main thread"); - - program::at_exit(Box::new(|| { - eprintln!("Hello from a `program::at_exit` handler") - })); - thread::at_exit(Box::new(|| { - eprintln!("Hello from a main-thread `thread::at_exit` handler") - })); - - let thread = thread::create( - |_args| { - eprintln!("Hello from child thread"); - thread::at_exit(Box::new(|| { - eprintln!("Hello from child thread's `thread::at_exit` handler") - })); - None - }, - &[], - thread::default_stack_size(), - thread::default_guard_size(), - ) - .unwrap(); - - thread::join(thread); - - eprintln!("Goodbye from main"); - program::exit(0); -}} +unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { + unsafe { + eprintln!("Hello from main thread"); + + program::at_exit(Box::new(|| { + eprintln!("Hello from a `program::at_exit` handler") + })); + thread::at_exit(Box::new(|| { + eprintln!("Hello from a main-thread `thread::at_exit` handler") + })); + + let thread = thread::create( + |_args| { + eprintln!("Hello from child thread"); + thread::at_exit(Box::new(|| { + eprintln!("Hello from child thread's `thread::at_exit` handler") + })); + None + }, + &[], + thread::default_stack_size(), + thread::default_guard_size(), + ) + .unwrap(); + + thread::join(thread); + + eprintln!("Goodbye from main"); + program::exit(0); + } +} diff --git a/example-crates/origin-start/src/main.rs b/example-crates/origin-start/src/main.rs index 8b2a55b..cfe9bae 100644 --- a/example-crates/origin-start/src/main.rs +++ b/example-crates/origin-start/src/main.rs @@ -13,32 +13,34 @@ use origin::{program, thread}; static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::GlobalDlmalloc; #[unsafe(no_mangle)] -unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { unsafe { - eprintln!("Hello from main thread"); - - program::at_exit(Box::new(|| { - eprintln!("Hello from a `program::at_exit` handler") - })); - thread::at_exit(Box::new(|| { - eprintln!("Hello from a main-thread `thread::at_exit` handler") - })); - - let thread = thread::create( - |_args| { - eprintln!("Hello from child thread"); - thread::at_exit(Box::new(|| { - eprintln!("Hello from child thread's `thread::at_exit` handler") - })); - None - }, - &[], - thread::default_stack_size(), - thread::default_guard_size(), - ) - .unwrap(); - - thread::join(thread); - - eprintln!("Goodbye from main"); - program::exit(0); -}} +unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { + unsafe { + eprintln!("Hello from main thread"); + + program::at_exit(Box::new(|| { + eprintln!("Hello from a `program::at_exit` handler") + })); + thread::at_exit(Box::new(|| { + eprintln!("Hello from a main-thread `thread::at_exit` handler") + })); + + let thread = thread::create( + |_args| { + eprintln!("Hello from child thread"); + thread::at_exit(Box::new(|| { + eprintln!("Hello from child thread's `thread::at_exit` handler") + })); + None + }, + &[], + thread::default_stack_size(), + thread::default_guard_size(), + ) + .unwrap(); + + thread::join(thread); + + eprintln!("Goodbye from main"); + program::exit(0); + } +} diff --git a/example-crates/tiny-hello/src/main.rs b/example-crates/tiny-hello/src/main.rs index 6e68914..b66b2bf 100644 --- a/example-crates/tiny-hello/src/main.rs +++ b/example-crates/tiny-hello/src/main.rs @@ -6,15 +6,17 @@ extern crate origin; #[unsafe(no_mangle)] -unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { unsafe { - let message = b"Hello, world!\n"; - let mut remaining = &message[..]; - while !remaining.is_empty() { - match rustix::io::write(rustix::stdio::stdout(), message) { - Ok(n) => remaining = &remaining[n as usize..], - Err(rustix::io::Errno::INTR) => continue, - Err(_) => origin::program::trap(), +unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { + unsafe { + let message = b"Hello, world!\n"; + let mut remaining = &message[..]; + while !remaining.is_empty() { + match rustix::io::write(rustix::stdio::stdout(), message) { + Ok(n) => remaining = &remaining[n as usize..], + Err(rustix::io::Errno::INTR) => continue, + Err(_) => origin::program::trap(), + } } + origin::program::immediate_exit(0); } - origin::program::immediate_exit(0); -}} +} diff --git a/src/arch/riscv64.rs b/src/arch/riscv64.rs index 5e53b36..5a57f1f 100644 --- a/src/arch/riscv64.rs +++ b/src/arch/riscv64.rs @@ -119,14 +119,15 @@ pub(super) unsafe fn relocation_load(ptr: usize) -> usize { r0 } -/// Perform a raw load operation to memory that Rust may consider out of bounds. +/// Perform a raw load operation to memory that Rust may consider out of +/// bounds. /// /// Data loaded from out-of-bounds bytes will have nondeterministic values. /// /// # Safety /// -/// `ptr` must be aligned for loading a `usize` and must point to enough readable -/// memory for loading a `usize`. +/// `ptr` must be aligned for loading a `usize` and must point to enough +/// readable memory for loading a `usize`. #[cfg(all(feature = "take-charge", not(feature = "optimize_for_size")))] #[inline] pub(super) unsafe fn oob_load(ptr: *const usize) -> usize { diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index d42d261..ce0d52f 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -106,21 +106,23 @@ pub(super) fn ehdr_addr() -> *const Elf_Ehdr { #[cfg(all(feature = "experimental-relocate", feature = "origin-start"))] #[cfg(relocation_model = "pic")] #[inline] -pub(super) unsafe fn relocation_load(ptr: usize) -> usize { unsafe { - let r0; +pub(super) unsafe fn relocation_load(ptr: usize) -> usize { + unsafe { + let r0; - // This is read-only but we don't use `readonly` because this memory access - // happens outside the Rust memory model. As far as Rust knows, this is - // just an arbitrary side-effecting opaque operation. - asm!( - "mov {}, [{}]", - out(reg) r0, - in(reg) ptr, - options(nostack, preserves_flags), - ); + // This is read-only but we don't use `readonly` because this memory access + // happens outside the Rust memory model. As far as Rust knows, this is + // just an arbitrary side-effecting opaque operation. + asm!( + "mov {}, [{}]", + out(reg) r0, + in(reg) ptr, + options(nostack, preserves_flags), + ); - r0 -} } + r0 + } +} /// Perform a single store operation, outside the Rust memory model. /// @@ -138,14 +140,16 @@ pub(super) unsafe fn relocation_load(ptr: usize) -> usize { unsafe { #[cfg(all(feature = "experimental-relocate", feature = "origin-start"))] #[cfg(relocation_model = "pic")] #[inline] -pub(super) unsafe fn relocation_store(ptr: usize, value: usize) { unsafe { - asm!( - "mov [{}], {}", - in(reg) ptr, - in(reg) value, - options(nostack, preserves_flags), - ); -} } +pub(super) unsafe fn relocation_store(ptr: usize, value: usize) { + unsafe { + asm!( + "mov [{}], {}", + in(reg) ptr, + in(reg) value, + options(nostack, preserves_flags), + ); + } +} /// Mark “relro” memory as readonly. /// @@ -166,29 +170,31 @@ pub(super) unsafe fn relocation_store(ptr: usize, value: usize) { unsafe { #[cfg(all(feature = "experimental-relocate", feature = "origin-start"))] #[cfg(relocation_model = "pic")] #[inline] -pub(super) unsafe fn relocation_mprotect_readonly(ptr: usize, len: usize) { unsafe { - let r0: usize; +pub(super) unsafe fn relocation_mprotect_readonly(ptr: usize, len: usize) { + unsafe { + let r0: usize; - // This is read-only but we don't use `readonly` because the side effects - // happen outside the Rust memory model. As far as Rust knows, this is - // just an arbitrary side-effecting opaque operation. - asm!( - "syscall", - inlateout("rax") __NR_mprotect as usize => r0, - in("rdi") ptr, - in("rsi") len, - in("rdx") PROT_READ, - lateout("rcx") _, - lateout("r11") _, - options(nostack, preserves_flags), - ); + // This is read-only but we don't use `readonly` because the side effects + // happen outside the Rust memory model. As far as Rust knows, this is + // just an arbitrary side-effecting opaque operation. + asm!( + "syscall", + inlateout("rax") __NR_mprotect as usize => r0, + in("rdi") ptr, + in("rsi") len, + in("rdx") PROT_READ, + lateout("rcx") _, + lateout("r11") _, + options(nostack, preserves_flags), + ); - if r0 != 0 { - // Do not panic here as libstd's panic handler needs TLS, which is not - // yet initialized at this point. - trap(); + if r0 != 0 { + // Do not panic here as libstd's panic handler needs TLS, which is not + // yet initialized at this point. + trap(); + } } -} } +} /// The required alignment for the stack pointer. #[cfg(feature = "take-charge")] @@ -211,49 +217,53 @@ pub(super) unsafe fn clone( newtls: *mut c_void, fn_: extern "C" fn(), num_args: usize, -) -> isize { unsafe { - let r0; - asm!( - "syscall", // Do the `clone` system call. - "test eax, eax", // Branch if we're in the parent thread. - "jnz 2f", +) -> isize { + unsafe { + let r0; + asm!( + "syscall", // Do the `clone` system call. + "test eax, eax", // Branch if we're in the parent thread. + "jnz 2f", - // Child thread. - "mov rdi, r9", // Pass `fn_` as the first argument. - "mov rsi, rsp", // Pass the args pointer as the second argument. - "mov rdx, r12", // Pass `num_args` as the third argument. - "xor ebp, ebp", // Zero the frame address. - "push rax", // Zero the return address. - "jmp {entry}", // Call `entry`. + // Child thread. + "mov rdi, r9", // Pass `fn_` as the first argument. + "mov rsi, rsp", // Pass the args pointer as the second argument. + "mov rdx, r12", // Pass `num_args` as the third argument. + "xor ebp, ebp", // Zero the frame address. + "push rax", // Zero the return address. + "jmp {entry}", // Call `entry`. - // Parent thread. - "2:", + // Parent thread. + "2:", - entry = sym super::thread::entry, - inlateout("rax") __NR_clone as usize => r0, - in("rdi") flags, - in("rsi") child_stack, - in("rdx") parent_tid, - in("r10") child_tid, - in("r8") newtls, - in("r9") fn_, - in("r12") num_args, - lateout("rcx") _, - lateout("r11") _, - options(nostack) - ); - r0 -} } + entry = sym super::thread::entry, + inlateout("rax") __NR_clone as usize => r0, + in("rdi") flags, + in("rsi") child_stack, + in("rdx") parent_tid, + in("r10") child_tid, + in("r8") newtls, + in("r9") fn_, + in("r12") num_args, + lateout("rcx") _, + lateout("r11") _, + options(nostack) + ); + r0 + } +} /// Write a value to the platform thread-pointer register. #[cfg(feature = "take-charge")] #[cfg(feature = "thread")] #[inline] -pub(super) unsafe fn set_thread_pointer(ptr: *mut c_void) { unsafe { - rustix::runtime::set_fs(ptr); - debug_assert_eq!(*ptr.cast::<*const c_void>(), ptr); - debug_assert_eq!(thread_pointer(), ptr); -} } +pub(super) unsafe fn set_thread_pointer(ptr: *mut c_void) { + unsafe { + rustix::runtime::set_fs(ptr); + debug_assert_eq!(*ptr.cast::<*const c_void>(), ptr); + debug_assert_eq!(thread_pointer(), ptr); + } +} /// Read the value of the platform thread-pointer register. #[cfg(feature = "take-charge")] @@ -281,20 +291,22 @@ pub(super) const TLS_OFFSET: usize = 0; #[cfg(feature = "take-charge")] #[cfg(feature = "thread")] #[inline] -pub(super) unsafe fn munmap_and_exit_thread(map_addr: *mut c_void, map_len: usize) -> ! { unsafe { - asm!( - "syscall", - "xor edi, edi", - "mov eax, {__NR_exit}", - "syscall", - "ud2", - __NR_exit = const __NR_exit, - in("rax") __NR_munmap, - in("rdi") map_addr, - in("rsi") map_len, - options(noreturn, nostack) - ); -} } +pub(super) unsafe fn munmap_and_exit_thread(map_addr: *mut c_void, map_len: usize) -> ! { + unsafe { + asm!( + "syscall", + "xor edi, edi", + "mov eax, {__NR_exit}", + "syscall", + "ud2", + __NR_exit = const __NR_exit, + in("rax") __NR_munmap, + in("rdi") map_addr, + in("rsi") map_len, + options(noreturn, nostack) + ); + } +} #[cfg(feature = "take-charge")] #[cfg(feature = "signal")] diff --git a/src/mem/fast.rs b/src/mem/fast.rs index 712d84c..0cfc230 100644 --- a/src/mem/fast.rs +++ b/src/mem/fast.rs @@ -12,41 +12,47 @@ mod impls; #[unsafe(no_mangle)] -unsafe extern "C" fn memcpy(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 { unsafe { - impls::copy_forward(dest, src, n); - dest -} } +unsafe extern "C" fn memcpy(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 { + unsafe { + impls::copy_forward(dest, src, n); + dest + } +} #[unsafe(no_mangle)] -unsafe extern "C" fn memmove(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 { unsafe { - let delta = dest.addr().wrapping_sub(src.addr()); - if delta >= n { - // We can copy forwards because either dest is far enough ahead of src, - // or src is ahead of dest (and delta overflowed). - impls::copy_forward(dest, src, n); - } else { - impls::copy_backward(dest, src, n); +unsafe extern "C" fn memmove(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 { + unsafe { + let delta = dest.addr().wrapping_sub(src.addr()); + if delta >= n { + // We can copy forwards because either dest is far enough ahead of src, + // or src is ahead of dest (and delta overflowed). + impls::copy_forward(dest, src, n); + } else { + impls::copy_backward(dest, src, n); + } + dest } - dest -} } +} #[unsafe(no_mangle)] -unsafe extern "C" fn memset(s: *mut u8, c: core::ffi::c_int, n: usize) -> *mut u8 { unsafe { - impls::set_bytes(s, c as u8, n); - s -} } +unsafe extern "C" fn memset(s: *mut u8, c: core::ffi::c_int, n: usize) -> *mut u8 { + unsafe { + impls::set_bytes(s, c as u8, n); + s + } +} #[unsafe(no_mangle)] -unsafe extern "C" fn memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 { unsafe { - impls::compare_bytes(s1, s2, n) -} } +unsafe extern "C" fn memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 { + unsafe { impls::compare_bytes(s1, s2, n) } +} #[unsafe(no_mangle)] -unsafe extern "C" fn bcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 { unsafe { - memcmp(s1, s2, n) -} } +unsafe extern "C" fn bcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 { + unsafe { memcmp(s1, s2, n) } +} #[unsafe(no_mangle)] -unsafe extern "C" fn strlen(s: *const core::ffi::c_char) -> usize { unsafe { - impls::c_string_length(s) -} } +unsafe extern "C" fn strlen(s: *const core::ffi::c_char) -> usize { + unsafe { impls::c_string_length(s) } +} diff --git a/src/mem/small.rs b/src/mem/small.rs index 4bde079..06b33af 100644 --- a/src/mem/small.rs +++ b/src/mem/small.rs @@ -6,27 +6,11 @@ use core::ffi::{c_char, c_int, c_void}; #[unsafe(no_mangle)] -unsafe extern "C" fn memcpy(dst: *mut c_void, src: *const c_void, len: usize) -> *mut c_void { unsafe { - let start = dst; - let mut dst = dst.cast::(); - let mut src = src.cast::(); - let dst_end = dst.add(len); - while dst < dst_end { - *dst = *src; - dst = dst.add(1); - src = src.add(1); - core::arch::asm!(""); - } - start -} } - -#[unsafe(no_mangle)] -unsafe extern "C" fn memmove(dst: *mut c_void, src: *const c_void, len: usize) -> *mut c_void { unsafe { - let start = dst; - let mut dst = dst.cast::(); - let mut src = src.cast::(); - let delta = (dst.addr()).wrapping_sub(src.addr()); - if delta >= len { +unsafe extern "C" fn memcpy(dst: *mut c_void, src: *const c_void, len: usize) -> *mut c_void { + unsafe { + let start = dst; + let mut dst = dst.cast::(); + let mut src = src.cast::(); let dst_end = dst.add(len); while dst < dst_end { *dst = *src; @@ -34,63 +18,89 @@ unsafe extern "C" fn memmove(dst: *mut c_void, src: *const c_void, len: usize) - src = src.add(1); core::arch::asm!(""); } - } else { - let dst_start = dst; - let mut dst = dst.add(len); - let mut src = src.add(len); - while dst > dst_start { - dst = dst.sub(1); - src = src.sub(1); - *dst = *src; - core::arch::asm!(""); + start + } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn memmove(dst: *mut c_void, src: *const c_void, len: usize) -> *mut c_void { + unsafe { + let start = dst; + let mut dst = dst.cast::(); + let mut src = src.cast::(); + let delta = (dst.addr()).wrapping_sub(src.addr()); + if delta >= len { + let dst_end = dst.add(len); + while dst < dst_end { + *dst = *src; + dst = dst.add(1); + src = src.add(1); + core::arch::asm!(""); + } + } else { + let dst_start = dst; + let mut dst = dst.add(len); + let mut src = src.add(len); + while dst > dst_start { + dst = dst.sub(1); + src = src.sub(1); + *dst = *src; + core::arch::asm!(""); + } } + start } - start -} } +} #[unsafe(no_mangle)] -unsafe extern "C" fn memset(dst: *mut c_void, fill: c_int, len: usize) -> *mut c_void { unsafe { - let mut s = dst.cast::(); - let end = s.add(len); - while s < end { - *s = fill as _; - s = s.add(1); - core::arch::asm!(""); +unsafe extern "C" fn memset(dst: *mut c_void, fill: c_int, len: usize) -> *mut c_void { + unsafe { + let mut s = dst.cast::(); + let end = s.add(len); + while s < end { + *s = fill as _; + s = s.add(1); + core::arch::asm!(""); + } + dst } - dst -} } +} #[unsafe(no_mangle)] -unsafe extern "C" fn memcmp(a: *const c_void, b: *const c_void, len: usize) -> c_int { unsafe { - let a = a.cast::(); - let b = b.cast::(); - let mut i = 0; - while i < len { - let a = *a.add(i); - let b = *b.add(i); - if a != b { - return a as i32 - b as i32; +unsafe extern "C" fn memcmp(a: *const c_void, b: *const c_void, len: usize) -> c_int { + unsafe { + let a = a.cast::(); + let b = b.cast::(); + let mut i = 0; + while i < len { + let a = *a.add(i); + let b = *b.add(i); + if a != b { + return a as i32 - b as i32; + } + i += 1; + core::arch::asm!(""); } - i += 1; - core::arch::asm!(""); + 0 } - 0 -} } +} // Obsolescent #[unsafe(no_mangle)] -unsafe extern "C" fn bcmp(a: *const c_void, b: *const c_void, len: usize) -> c_int { unsafe { - memcmp(a, b, len) -} } +unsafe extern "C" fn bcmp(a: *const c_void, b: *const c_void, len: usize) -> c_int { + unsafe { memcmp(a, b, len) } +} #[unsafe(no_mangle)] -unsafe extern "C" fn strlen(s: *const c_char) -> usize { unsafe { - let mut s = s; - let mut n = 0; - while *s != 0 { - n += 1; - s = s.add(1); - core::arch::asm!(""); +unsafe extern "C" fn strlen(s: *const c_char) -> usize { + unsafe { + let mut s = s; + let mut n = 0; + while *s != 0 { + n += 1; + s = s.add(1); + core::arch::asm!(""); + } + n } - n -} } +} diff --git a/src/mem/x86_64.rs b/src/mem/x86_64.rs index 2d6fc91..6b016ff 100644 --- a/src/mem/x86_64.rs +++ b/src/mem/x86_64.rs @@ -27,152 +27,166 @@ use core::mem; #[inline(always)] #[cfg(target_feature = "ermsb")] -pub unsafe fn copy_forward(dest: *mut u8, src: *const u8, count: usize) { unsafe { - // FIXME: Use the Intel syntax once we drop LLVM 9 support on rust-lang/rust. - core::arch::asm!( - "repe movsb (%rsi), (%rdi)", - inout("rcx") count => _, - inout("rdi") dest => _, - inout("rsi") src => _, - options(att_syntax, nostack, preserves_flags) - ); -} } +pub unsafe fn copy_forward(dest: *mut u8, src: *const u8, count: usize) { + unsafe { + // FIXME: Use the Intel syntax once we drop LLVM 9 support on rust-lang/rust. + core::arch::asm!( + "repe movsb (%rsi), (%rdi)", + inout("rcx") count => _, + inout("rdi") dest => _, + inout("rsi") src => _, + options(att_syntax, nostack, preserves_flags) + ); + } +} #[inline(always)] #[cfg(not(target_feature = "ermsb"))] -pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, count: usize) { unsafe { - let (pre_byte_count, qword_count, byte_count) = rep_param(dest, count); - // Separating the blocks gives the compiler more freedom to reorder instructions. - asm!( - "rep movsb", - inout("ecx") pre_byte_count => _, - inout("rdi") dest => dest, - inout("rsi") src => src, - options(att_syntax, nostack, preserves_flags) - ); - asm!( - "rep movsq", - inout("rcx") qword_count => _, - inout("rdi") dest => dest, - inout("rsi") src => src, - options(att_syntax, nostack, preserves_flags) - ); - asm!( - "rep movsb", - inout("ecx") byte_count => _, - inout("rdi") dest => _, - inout("rsi") src => _, - options(att_syntax, nostack, preserves_flags) - ); -} } +pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, count: usize) { + unsafe { + let (pre_byte_count, qword_count, byte_count) = rep_param(dest, count); + // Separating the blocks gives the compiler more freedom to reorder instructions. + asm!( + "rep movsb", + inout("ecx") pre_byte_count => _, + inout("rdi") dest => dest, + inout("rsi") src => src, + options(att_syntax, nostack, preserves_flags) + ); + asm!( + "rep movsq", + inout("rcx") qword_count => _, + inout("rdi") dest => dest, + inout("rsi") src => src, + options(att_syntax, nostack, preserves_flags) + ); + asm!( + "rep movsb", + inout("ecx") byte_count => _, + inout("rdi") dest => _, + inout("rsi") src => _, + options(att_syntax, nostack, preserves_flags) + ); + } +} #[inline(always)] -pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, count: usize) { unsafe { - let (pre_byte_count, qword_count, byte_count) = rep_param(dest, count); - // We can't separate this block due to std/cld - asm!( - "std", - "rep movsb", - "sub $7, %rsi", - "sub $7, %rdi", - "mov {qword_count}, %rcx", - "rep movsq", - "test {pre_byte_count:e}, {pre_byte_count:e}", - "add $7, %rsi", - "add $7, %rdi", - "mov {pre_byte_count:e}, %ecx", - "rep movsb", - "cld", - pre_byte_count = in(reg) pre_byte_count, - qword_count = in(reg) qword_count, - inout("ecx") byte_count => _, - inout("rdi") dest.add(count - 1) => _, - inout("rsi") src.add(count - 1) => _, - // We modify flags, but we restore it afterwards - options(att_syntax, nostack, preserves_flags) - ); -} } +pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, count: usize) { + unsafe { + let (pre_byte_count, qword_count, byte_count) = rep_param(dest, count); + // We can't separate this block due to std/cld + asm!( + "std", + "rep movsb", + "sub $7, %rsi", + "sub $7, %rdi", + "mov {qword_count}, %rcx", + "rep movsq", + "test {pre_byte_count:e}, {pre_byte_count:e}", + "add $7, %rsi", + "add $7, %rdi", + "mov {pre_byte_count:e}, %ecx", + "rep movsb", + "cld", + pre_byte_count = in(reg) pre_byte_count, + qword_count = in(reg) qword_count, + inout("ecx") byte_count => _, + inout("rdi") dest.add(count - 1) => _, + inout("rsi") src.add(count - 1) => _, + // We modify flags, but we restore it afterwards + options(att_syntax, nostack, preserves_flags) + ); + } +} #[inline(always)] #[cfg(target_feature = "ermsb")] -pub unsafe fn set_bytes(dest: *mut u8, c: u8, count: usize) { unsafe { - // FIXME: Use the Intel syntax once we drop LLVM 9 support on rust-lang/rust. - core::arch::asm!( - "repe stosb %al, (%rdi)", - inout("rcx") count => _, - inout("rdi") dest => _, - inout("al") c => _, - options(att_syntax, nostack, preserves_flags) - ) -} } +pub unsafe fn set_bytes(dest: *mut u8, c: u8, count: usize) { + unsafe { + // FIXME: Use the Intel syntax once we drop LLVM 9 support on rust-lang/rust. + core::arch::asm!( + "repe stosb %al, (%rdi)", + inout("rcx") count => _, + inout("rdi") dest => _, + inout("al") c => _, + options(att_syntax, nostack, preserves_flags) + ) + } +} #[inline(always)] #[cfg(not(target_feature = "ermsb"))] -pub unsafe fn set_bytes(mut dest: *mut u8, c: u8, count: usize) { unsafe { - let c = c as u64 * 0x0101_0101_0101_0101; - let (pre_byte_count, qword_count, byte_count) = rep_param(dest, count); - // Separating the blocks gives the compiler more freedom to reorder instructions. - asm!( - "rep stosb", - inout("ecx") pre_byte_count => _, - inout("rdi") dest => dest, - in("rax") c, - options(att_syntax, nostack, preserves_flags) - ); - asm!( - "rep stosq", - inout("rcx") qword_count => _, - inout("rdi") dest => dest, - in("rax") c, - options(att_syntax, nostack, preserves_flags) - ); - asm!( - "rep stosb", - inout("ecx") byte_count => _, - inout("rdi") dest => _, - in("rax") c, - options(att_syntax, nostack, preserves_flags) - ); -} } +pub unsafe fn set_bytes(mut dest: *mut u8, c: u8, count: usize) { + unsafe { + let c = c as u64 * 0x0101_0101_0101_0101; + let (pre_byte_count, qword_count, byte_count) = rep_param(dest, count); + // Separating the blocks gives the compiler more freedom to reorder instructions. + asm!( + "rep stosb", + inout("ecx") pre_byte_count => _, + inout("rdi") dest => dest, + in("rax") c, + options(att_syntax, nostack, preserves_flags) + ); + asm!( + "rep stosq", + inout("rcx") qword_count => _, + inout("rdi") dest => dest, + in("rax") c, + options(att_syntax, nostack, preserves_flags) + ); + asm!( + "rep stosb", + inout("ecx") byte_count => _, + inout("rdi") dest => _, + in("rax") c, + options(att_syntax, nostack, preserves_flags) + ); + } +} #[inline(always)] -pub unsafe fn compare_bytes(a: *const u8, b: *const u8, n: usize) -> i32 { unsafe { - #[inline(always)] - unsafe fn cmp(mut a: *const T, mut b: *const T, n: usize, f: F) -> i32 - where - T: Clone + Copy + Eq, - U: Clone + Copy + Eq, - F: FnOnce(*const U, *const U, usize) -> i32, - { unsafe { - // Ensure T is not a ZST. - assert!(mem::size_of::() != 0); +pub unsafe fn compare_bytes(a: *const u8, b: *const u8, n: usize) -> i32 { + unsafe { + #[inline(always)] + unsafe fn cmp(mut a: *const T, mut b: *const T, n: usize, f: F) -> i32 + where + T: Clone + Copy + Eq, + U: Clone + Copy + Eq, + F: FnOnce(*const U, *const U, usize) -> i32, + { + unsafe { + // Ensure T is not a ZST. + assert!(mem::size_of::() != 0); - let end = a.add(n / mem::size_of::()); - while a != end { - if a.read_unaligned() != b.read_unaligned() { - return f(a.cast(), b.cast(), mem::size_of::()); + let end = a.add(n / mem::size_of::()); + while a != end { + if a.read_unaligned() != b.read_unaligned() { + return f(a.cast(), b.cast(), mem::size_of::()); + } + a = a.add(1); + b = b.add(1); + } + f(a.cast(), b.cast(), n % mem::size_of::()) } - a = a.add(1); - b = b.add(1); } - f(a.cast(), b.cast(), n % mem::size_of::()) - } } - let c1 = |mut a: *const u8, mut b: *const u8, n| { - for _ in 0..n { - if a.read() != b.read() { - return i32::from(a.read()) - i32::from(b.read()); + let c1 = |mut a: *const u8, mut b: *const u8, n| { + for _ in 0..n { + if a.read() != b.read() { + return i32::from(a.read()) - i32::from(b.read()); + } + a = a.add(1); + b = b.add(1); } - a = a.add(1); - b = b.add(1); - } - 0 - }; - let c2 = |a: *const u16, b, n| cmp(a, b, n, c1); - let c4 = |a: *const u32, b, n| cmp(a, b, n, c2); - let c8 = |a: *const u64, b, n| cmp(a, b, n, c4); - let c16 = |a: *const u128, b, n| cmp(a, b, n, c8); - c16(a.cast(), b.cast(), n) -} } + 0 + }; + let c2 = |a: *const u16, b, n| cmp(a, b, n, c1); + let c4 = |a: *const u32, b, n| cmp(a, b, n, c2); + let c8 = |a: *const u64, b, n| cmp(a, b, n, c4); + let c16 = |a: *const u128, b, n| cmp(a, b, n, c8); + c16(a.cast(), b.cast(), n) + } +} // In order to process more than on byte simultaneously when executing strlen, // two things must be considered: @@ -186,51 +200,32 @@ pub unsafe fn compare_bytes(a: *const u8, b: *const u8, n: usize) -> i32 { unsaf #[cfg(target_feature = "sse2")] #[inline(always)] -pub unsafe fn c_string_length(mut s: *const core::ffi::c_char) -> usize { unsafe { - use core::arch::x86_64::{__m128i, _mm_cmpeq_epi8, _mm_movemask_epi8, _mm_set1_epi8}; +pub unsafe fn c_string_length(mut s: *const core::ffi::c_char) -> usize { + unsafe { + use core::arch::x86_64::{__m128i, _mm_cmpeq_epi8, _mm_movemask_epi8, _mm_set1_epi8}; - let mut n = 0; + let mut n = 0; - // The use of _mm_movemask_epi8 and company allow for speedups, - // but they aren't cheap by themselves. Thus, possibly small strings - // are handled in simple loops. + // The use of _mm_movemask_epi8 and company allow for speedups, + // but they aren't cheap by themselves. Thus, possibly small strings + // are handled in simple loops. - for _ in 0..4 { - if *s == 0 { - return n; - } - - n += 1; - s = s.add(1); - } - - // Shave of the least significand bits to align the address to a 16 - // byte boundary. The shaved of bits are used to correct the first iteration. - - let align = s.addr() & 15; - let mut s = s.with_addr(s.addr() - align) as *const __m128i; - let zero = _mm_set1_epi8(0); + for _ in 0..4 { + if *s == 0 { + return n; + } - let x = { - let r; - asm!( - "movdqa ({addr}), {dest}", - addr = in(reg) s, - dest = out(xmm_reg) r, - options(att_syntax, nostack), - ); - r - }; - let cmp = _mm_movemask_epi8(_mm_cmpeq_epi8(x, zero)) >> align; + n += 1; + s = s.add(1); + } - if cmp != 0 { - return n + cmp.trailing_zeros() as usize; - } + // Shave of the least significand bits to align the address to a 16 + // byte boundary. The shaved of bits are used to correct the first iteration. - n += 16 - align; - s = s.add(1); + let align = s.addr() & 15; + let mut s = s.with_addr(s.addr() - align) as *const __m128i; + let zero = _mm_set1_epi8(0); - loop { let x = { let r; asm!( @@ -241,68 +236,91 @@ pub unsafe fn c_string_length(mut s: *const core::ffi::c_char) -> usize { unsafe ); r }; - let cmp = _mm_movemask_epi8(_mm_cmpeq_epi8(x, zero)) as u32; - if cmp == 0 { - n += 16; - s = s.add(1); - } else { + let cmp = _mm_movemask_epi8(_mm_cmpeq_epi8(x, zero)) >> align; + + if cmp != 0 { return n + cmp.trailing_zeros() as usize; } + + n += 16 - align; + s = s.add(1); + + loop { + let x = { + let r; + asm!( + "movdqa ({addr}), {dest}", + addr = in(reg) s, + dest = out(xmm_reg) r, + options(att_syntax, nostack), + ); + r + }; + let cmp = _mm_movemask_epi8(_mm_cmpeq_epi8(x, zero)) as u32; + if cmp == 0 { + n += 16; + s = s.add(1); + } else { + return n + cmp.trailing_zeros() as usize; + } + } } -} } +} // Provided for scenarios like kernel development, where SSE might not // be available. #[cfg(not(target_feature = "sse2"))] #[inline(always)] -pub unsafe fn c_string_length(mut s: *const core::ffi::c_char) -> usize { unsafe { - let mut n = 0; +pub unsafe fn c_string_length(mut s: *const core::ffi::c_char) -> usize { + unsafe { + let mut n = 0; - // Check bytes in steps of one until - // either a zero byte is discovered or - // pointer is aligned to an eight byte boundary. + // Check bytes in steps of one until + // either a zero byte is discovered or + // pointer is aligned to an eight byte boundary. - while s as usize & 7 != 0 { - if *s == 0 { - return n; + while s as usize & 7 != 0 { + if *s == 0 { + return n; + } + n += 1; + s = s.add(1); } - n += 1; - s = s.add(1); - } - // Check bytes in steps of eight until a zero - // byte is discovered. + // Check bytes in steps of eight until a zero + // byte is discovered. - let mut s = s as *const u64; + let mut s = s as *const u64; - loop { - let mut cs = { - let r: u64; - asm!( - "mov ({addr}), {dest}", - addr = in(reg) s, - dest = out(reg) r, - options(att_syntax, nostack), - ); - r - }; - // Detect if a word has a zero byte, taken from - // https://graphics.stanford.edu/~seander/bithacks.html - if (cs.wrapping_sub(0x0101010101010101) & !cs & 0x8080808080808080) != 0 { - loop { - if cs & 255 == 0 { - return n; - } else { - cs >>= 8; - n += 1; + loop { + let mut cs = { + let r: u64; + asm!( + "mov ({addr}), {dest}", + addr = in(reg) s, + dest = out(reg) r, + options(att_syntax, nostack), + ); + r + }; + // Detect if a word has a zero byte, taken from + // https://graphics.stanford.edu/~seander/bithacks.html + if (cs.wrapping_sub(0x0101010101010101) & !cs & 0x8080808080808080) != 0 { + loop { + if cs & 255 == 0 { + return n; + } else { + cs >>= 8; + n += 1; + } } + } else { + n += 8; + s = s.add(1); } - } else { - n += 8; - s = s.add(1); } } -} } +} /// Determine optimal parameters for a `rep` instruction. fn rep_param(dest: *mut u8, mut count: usize) -> (usize, usize, usize) { diff --git a/src/program/libc.rs b/src/program/libc.rs index 42228f1..6318d42 100644 --- a/src/program/libc.rs +++ b/src/program/libc.rs @@ -50,9 +50,11 @@ pub fn at_exit(func: Box) { } // The function to pass to `__cxa_atexit`. - unsafe extern "C" fn at_exit_func(arg: *mut c_void) { unsafe { - Box::from_raw(arg.cast::>())(); - }} + unsafe extern "C" fn at_exit_func(arg: *mut c_void) { + unsafe { + Box::from_raw(arg.cast::>())(); + } + } let at_exit_arg = Box::into_raw(Box::new(func)).cast::(); let r = unsafe { __cxa_atexit(at_exit_func, at_exit_arg, null_mut()) }; diff --git a/src/program/linux_raw.rs b/src/program/linux_raw.rs index ea1b675..e7158a1 100644 --- a/src/program/linux_raw.rs +++ b/src/program/linux_raw.rs @@ -46,122 +46,124 @@ compile_error!("\"origin-program\" depends on either \"origin-start\" or \"exter /// # Safety /// /// `mem` must point to the stack as provided by the operating system. -pub(super) unsafe extern "C" fn entry(mem: *mut usize) -> ! { unsafe { - // Do some basic precondition checks, to ensure that our assembly code did - // what we expect it to do. These are debug-only, to keep the release-mode - // startup code small and simple to disassemble and inspect. - #[cfg(debug_assertions)] - #[cfg(feature = "origin-start")] - { - // Check that `mem` is where we expect it to be. - debug_assert_ne!(mem, core::ptr::null_mut()); - debug_assert_eq!(mem.addr() & 0xf, 0); - - // If we have nightly, we can do additional checks. - #[cfg(feature = "nightly")] +pub(super) unsafe extern "C" fn entry(mem: *mut usize) -> ! { + unsafe { + // Do some basic precondition checks, to ensure that our assembly code did + // what we expect it to do. These are debug-only, to keep the release-mode + // startup code small and simple to disassemble and inspect. + #[cfg(debug_assertions)] + #[cfg(feature = "origin-start")] { - unsafe extern "C" { - #[link_name = "llvm.frameaddress"] - fn builtin_frame_address(level: i32) -> *const u8; - #[link_name = "llvm.returnaddress"] - fn builtin_return_address(level: i32) -> *const u8; + // Check that `mem` is where we expect it to be. + debug_assert_ne!(mem, core::ptr::null_mut()); + debug_assert_eq!(mem.addr() & 0xf, 0); + + // If we have nightly, we can do additional checks. + #[cfg(feature = "nightly")] + { + unsafe extern "C" { + #[link_name = "llvm.frameaddress"] + fn builtin_frame_address(level: i32) -> *const u8; + #[link_name = "llvm.returnaddress"] + fn builtin_return_address(level: i32) -> *const u8; + #[cfg(target_arch = "aarch64")] + #[link_name = "llvm.sponentry"] + fn builtin_sponentry() -> *const u8; + } + + // Check that `mem` is where we expect it to be. + debug_assert!(builtin_frame_address(0).addr() <= mem.addr()); + + // Check that the incoming stack pointer is where we expect it to be. + debug_assert_eq!(builtin_return_address(0), core::ptr::null()); + debug_assert_ne!(builtin_frame_address(0), core::ptr::null()); + #[cfg(not(any(target_arch = "arm", target_arch = "x86")))] + debug_assert_eq!(builtin_frame_address(0).addr() & 0xf, 0); + #[cfg(target_arch = "arm")] + debug_assert_eq!(builtin_frame_address(0).addr() & 0x7, 0); + #[cfg(target_arch = "x86")] + debug_assert_eq!(builtin_frame_address(0).addr() & 0xf, 8); + debug_assert_eq!(builtin_frame_address(1), core::ptr::null()); + #[cfg(target_arch = "aarch64")] + debug_assert_ne!(builtin_sponentry(), core::ptr::null()); #[cfg(target_arch = "aarch64")] - #[link_name = "llvm.sponentry"] - fn builtin_sponentry() -> *const u8; + debug_assert_eq!(builtin_sponentry().addr() & 0xf, 0); } - - // Check that `mem` is where we expect it to be. - debug_assert!(builtin_frame_address(0).addr() <= mem.addr()); - - // Check that the incoming stack pointer is where we expect it to be. - debug_assert_eq!(builtin_return_address(0), core::ptr::null()); - debug_assert_ne!(builtin_frame_address(0), core::ptr::null()); - #[cfg(not(any(target_arch = "arm", target_arch = "x86")))] - debug_assert_eq!(builtin_frame_address(0).addr() & 0xf, 0); - #[cfg(target_arch = "arm")] - debug_assert_eq!(builtin_frame_address(0).addr() & 0x7, 0); - #[cfg(target_arch = "x86")] - debug_assert_eq!(builtin_frame_address(0).addr() & 0xf, 8); - debug_assert_eq!(builtin_frame_address(1), core::ptr::null()); - #[cfg(target_arch = "aarch64")] - debug_assert_ne!(builtin_sponentry(), core::ptr::null()); - #[cfg(target_arch = "aarch64")] - debug_assert_eq!(builtin_sponentry().addr() & 0xf, 0); } - } - // Compute `argc`, `argv`, and `envp`. - let (argc, argv, envp) = compute_args(mem); + // Compute `argc`, `argv`, and `envp`. + let (argc, argv, envp) = compute_args(mem); - // Before doing anything else, perform dynamic relocations. - #[cfg(all(feature = "experimental-relocate", feature = "origin-start"))] - #[cfg(relocation_model = "pic")] - { - crate::relocate::relocate(envp); - } + // Before doing anything else, perform dynamic relocations. + #[cfg(all(feature = "experimental-relocate", feature = "origin-start"))] + #[cfg(relocation_model = "pic")] + { + crate::relocate::relocate(envp); + } - // Initialize program state before running any user code. - init_runtime(mem, envp); + // Initialize program state before running any user code. + init_runtime(mem, envp); - // Call the functions registered via `.init_array`. - #[cfg(feature = "init-array")] - { - use core::arch::asm; - use core::ffi::c_void; + // Call the functions registered via `.init_array`. + #[cfg(feature = "init-array")] + { + use core::arch::asm; + use core::ffi::c_void; + + // The linker-generated symbols that mark the start and end of the + // `.init_array` section. + extern "C" { + static __init_array_start: c_void; + static __init_array_end: c_void; + } - // The linker-generated symbols that mark the start and end of the - // `.init_array` section. - extern "C" { - static __init_array_start: c_void; - static __init_array_end: c_void; + // Call the `.init_array` functions. As glibc does, pass argc, argv, + // and envp as extra arguments. In addition to glibc ABI compatibility, + // c-scape relies on this. + type InitFn = unsafe extern "C" fn(c_int, *mut *mut u8, *mut *mut u8); + let mut init = core::ptr::addr_of!(__init_array_start).cast::(); + let init_end = core::ptr::addr_of!(__init_array_end).cast::(); + // Prevent the optimizer from optimizing the `!=` comparison to true; + // `init` and `init_start` may have the same address. + asm!("# {}", inout(reg) init, options(pure, nomem, nostack, preserves_flags)); + + while init != init_end { + #[cfg(feature = "log")] + log::trace!( + "Calling `.init_array`-registered function `{:?}({:?}, {:?}, {:?})`", + *init, + argc, + argv, + envp + ); + + (*init)(argc, argv, envp); + + init = init.add(1); + } } - // Call the `.init_array` functions. As glibc does, pass argc, argv, - // and envp as extra arguments. In addition to glibc ABI compatibility, - // c-scape relies on this. - type InitFn = unsafe extern "C" fn(c_int, *mut *mut u8, *mut *mut u8); - let mut init = core::ptr::addr_of!(__init_array_start).cast::(); - let init_end = core::ptr::addr_of!(__init_array_end).cast::(); - // Prevent the optimizer from optimizing the `!=` comparison to true; - // `init` and `init_start` may have the same address. - asm!("# {}", inout(reg) init, options(pure, nomem, nostack, preserves_flags)); + { + // Declare `origin_main` as documented in [`crate::program`]. + unsafe extern "Rust" { + fn origin_main(argc: usize, argv: *mut *mut u8, envp: *mut *mut u8) -> i32; + } - while init != init_end { #[cfg(feature = "log")] - log::trace!( - "Calling `.init_array`-registered function `{:?}({:?}, {:?}, {:?})`", - *init, - argc, - argv, - envp - ); + log::trace!("Calling `origin_main({:?}, {:?}, {:?})`", argc, argv, envp); - (*init)(argc, argv, envp); + // Call `origin_main`. + let status = origin_main(argc as usize, argv, envp); - init = init.add(1); - } - } + #[cfg(feature = "log")] + log::trace!("`origin_main` returned `{:?}`", status); - { - // Declare `origin_main` as documented in [`crate::program`]. - unsafe extern "Rust" { - fn origin_main(argc: usize, argv: *mut *mut u8, envp: *mut *mut u8) -> i32; + // Run functions registered with `at_exit`, and exit with + // `origin_main`'s return value. + exit(status) } - - #[cfg(feature = "log")] - log::trace!("Calling `origin_main({:?}, {:?}, {:?})`", argc, argv, envp); - - // Call `origin_main`. - let status = origin_main(argc as usize, argv, envp); - - #[cfg(feature = "log")] - log::trace!("`origin_main` returned `{:?}`", status); - - // Run functions registered with `at_exit`, and exit with - // `origin_main`'s return value. - exit(status) } -} } +} /// A program entry point similar to `_start`, but which is meant to be called /// by something else in the program rather than the OS. @@ -171,30 +173,32 @@ pub(super) unsafe extern "C" fn entry(mem: *mut usize) -> ! { unsafe { /// `mem` must point to a stack with the contents that the OS would provide /// on the initial stack. #[cfg(feature = "external-start")] -pub unsafe fn start(mem: *mut usize) -> ! { unsafe { - entry(mem) -} } +pub unsafe fn start(mem: *mut usize) -> ! { + unsafe { entry(mem) } +} /// Compute `argc`, `argv`, and `envp`. /// /// # Safety /// /// `mem` must point to the stack as provided by the operating system. -unsafe fn compute_args(mem: *mut usize) -> (i32, *mut *mut u8, *mut *mut u8) { unsafe { - use linux_raw_sys::ctypes::c_uint; +unsafe fn compute_args(mem: *mut usize) -> (i32, *mut *mut u8, *mut *mut u8) { + unsafe { + use linux_raw_sys::ctypes::c_uint; - let kernel_argc = *mem; - let argc = kernel_argc as c_int; - let argv = mem.add(1).cast::<*mut u8>(); - let envp = argv.add(argc as c_uint as usize + 1); + let kernel_argc = *mem; + let argc = kernel_argc as c_int; + let argv = mem.add(1).cast::<*mut u8>(); + let envp = argv.add(argc as c_uint as usize + 1); - // Do a few more precondition checks on `argc` and `argv`. - debug_assert!(argc >= 0); - debug_assert_eq!(kernel_argc, argc as _); - debug_assert_eq!(*argv.add(argc as usize), core::ptr::null_mut()); + // Do a few more precondition checks on `argc` and `argv`. + debug_assert!(argc >= 0); + debug_assert_eq!(kernel_argc, argc as _); + debug_assert_eq!(*argv.add(argc as usize), core::ptr::null_mut()); - (argc, argv, envp) -} } + (argc, argv, envp) + } +} /// Initialize `origin` and `rustix` runtime state. /// @@ -207,7 +211,9 @@ unsafe fn init_runtime(mem: *mut usize, envp: *mut *mut u8) { // Explicitly initialize `rustix`. This is needed for things like // `page_size()` to work. #[cfg(feature = "param")] - unsafe { rustix::param::init(envp); } + unsafe { + rustix::param::init(envp); + } // Read the program headers and extract the TLS info. #[cfg(feature = "thread")] @@ -215,7 +221,9 @@ unsafe fn init_runtime(mem: *mut usize, envp: *mut *mut u8) { // Initialize the main thread. #[cfg(feature = "thread")] - unsafe { thread::initialize_main(mem.cast()); } + unsafe { + thread::initialize_main(mem.cast()); + } } /// Functions registered with [`at_exit`]. diff --git a/src/relocate.rs b/src/relocate.rs index 8dcaebc..383f0ed 100644 --- a/src/relocate.rs +++ b/src/relocate.rs @@ -77,296 +77,300 @@ macro_rules! debug_assert_eq { /// /// So yes, there's a reason this code is behind a feature flag. #[cold] -pub(super) unsafe fn relocate(envp: *mut *mut u8) { unsafe { - // Locate the AUX records we need. - let auxp = compute_auxp(envp); - - // The page size, segment headers info, and runtime entry address. - let mut auxv_base = null_mut(); - let mut auxv_page_size = 0; - let mut auxv_entry = null_mut(); - - // Look through the AUX records to find the segment headers, page size, - // and runtime entry address. - let mut current_aux = auxp; - loop { - let Elf_auxv_t { a_type, a_val } = *current_aux; - current_aux = current_aux.add(1); - - match a_type as _ { - AT_BASE => auxv_base = a_val, - AT_PAGESZ => auxv_page_size = a_val.addr(), - AT_ENTRY => auxv_entry = a_val, - AT_NULL => break, - _ => (), +pub(super) unsafe fn relocate(envp: *mut *mut u8) { + unsafe { + // Locate the AUX records we need. + let auxp = compute_auxp(envp); + + // The page size, segment headers info, and runtime entry address. + let mut auxv_base = null_mut(); + let mut auxv_page_size = 0; + let mut auxv_entry = null_mut(); + + // Look through the AUX records to find the segment headers, page size, + // and runtime entry address. + let mut current_aux = auxp; + loop { + let Elf_auxv_t { a_type, a_val } = *current_aux; + current_aux = current_aux.add(1); + + match a_type as _ { + AT_BASE => auxv_base = a_val, + AT_PAGESZ => auxv_page_size = a_val.addr(), + AT_ENTRY => auxv_entry = a_val, + AT_NULL => break, + _ => (), + } } - } - - // There are four cases to consider here: - // - // 1. Static executable. - // This is the trivial case. No relocations necessary. - // 2. Static pie executable. - // We need to relocate ourself. `AT_PHDR` points to our own program - // headers and `AT_ENTRY` to our own entry point. - // 3. Dynamic pie executable with external dynamic linker. - // We don't need to relocate ourself as the dynamic linker has already - // done this. `AT_PHDR` points to our own program headers and `AT_ENTRY` - // to our own entry point. `AT_BASE` contains the relocation offset of - // the dynamic linker. - // 4. Shared library acting as dynamic linker. - // We do need to relocate ourself. `AT_PHDR` doesn't point to our own - // program headers and `AT_ENTRY` doesn't point to our own entry point. - // `AT_BASE` contains our own relocation offset. - - if load_static_start() == auxv_entry.addr() { - // This is case 1) or case 3). If `AT_BASE` doesn't exist, then we are - // already loaded at our static address despite the lack of any dynamic - // linker. As such it would be case 1). If `AT_BASE` does exist, we have - // already been relocated by the dynamic linker, which is case 3). - // In either case there is no need to do any relocations. - return; - } - let the_ehdr = &*ehdr_addr(); - - let base = if auxv_base == null_mut() { - // Obtain the static address of `_start`, which is recorded in the - // entry field of the ELF header. - let static_start = the_ehdr.e_entry; - - // This is case 2) as without dynamic linker `AT_BASE` doesn't exist - // and we have already excluded case 1) above. - auxv_entry.wrapping_sub(static_start) - } else { - // This is case 4) as `AT_BASE` indicates that there is a dynamic - // linker, yet we are not already relocated by a dynamic linker. - // `AT_BASE` contains the relocation offset of the dynamic linker. - auxv_base - }; - let offset = base.addr(); - - // This is case 2) or 4). We need to do all `R_RELATIVE` relocations. - // There should be no other kind of relocation because we are either a - // static PIE binary or a dynamic linker compiled with `-Bsymbolic`. - - // Compute the dynamic address of `_DYNAMIC`. - let dynv = dynamic_table_addr(); - - // Rela tables contain `Elf_Rela` elements which have an - // `r_addend` field. - let mut rela_ptr: *const Elf_Rela = null(); - let mut rela_total_size = 0; - - // Rel tables contain `Elf_Rel` elements which lack an - // `r_addend` field and store the addend in the memory to be - // relocated. - let mut rel_ptr: *const Elf_Rel = null(); - let mut rel_total_size = 0; - - // Relr tables contain `Elf_Relr` elements which are a compact - // way of representing relative relocations. - let mut relr_ptr: *const Elf_Relr = null(); - let mut relr_total_size = 0; - - // Look through the `Elf_Dyn` entries to find the location and - // size of the relocation table(s). - let mut current_dyn: *const Elf_Dyn = dynv; - loop { - let Elf_Dyn { d_tag, d_un } = &*current_dyn; - current_dyn = current_dyn.add(1); - - match *d_tag { - // We found a rela table. As above, model this as - // `with_exposed_provenance`. - DT_RELA => rela_ptr = base.byte_add(d_un.d_ptr).cast::(), - DT_RELASZ => rela_total_size = d_un.d_val as usize, - #[cfg(debug_assertions)] - DT_RELAENT => debug_assert_eq!(d_un.d_val as usize, size_of::()), - - // We found a rel table. As above, model this as - // `with_exposed_provenance`. - DT_REL => rel_ptr = base.byte_add(d_un.d_ptr).cast::(), - DT_RELSZ => rel_total_size = d_un.d_val as usize, - #[cfg(debug_assertions)] - DT_RELENT => debug_assert_eq!(d_un.d_val as usize, size_of::()), - - // We found a relr table. As above, model this as - // `with_exposed_provenance`. - DT_RELR => relr_ptr = base.byte_add(d_un.d_ptr).cast::(), - DT_RELRSZ => relr_total_size = d_un.d_val as usize, - #[cfg(debug_assertions)] - DT_RELRENT => debug_assert_eq!(d_un.d_val as usize, size_of::()), - - // End of the Dynamic section - DT_NULL => break, - - _ => (), + // There are four cases to consider here: + // + // 1. Static executable. + // This is the trivial case. No relocations necessary. + // 2. Static pie executable. + // We need to relocate ourself. `AT_PHDR` points to our own program + // headers and `AT_ENTRY` to our own entry point. + // 3. Dynamic pie executable with external dynamic linker. + // We don't need to relocate ourself as the dynamic linker has already + // done this. `AT_PHDR` points to our own program headers and `AT_ENTRY` + // to our own entry point. `AT_BASE` contains the relocation offset of + // the dynamic linker. + // 4. Shared library acting as dynamic linker. + // We do need to relocate ourself. `AT_PHDR` doesn't point to our own + // program headers and `AT_ENTRY` doesn't point to our own entry point. + // `AT_BASE` contains our own relocation offset. + + if load_static_start() == auxv_entry.addr() { + // This is case 1) or case 3). If `AT_BASE` doesn't exist, then we are + // already loaded at our static address despite the lack of any dynamic + // linker. As such it would be case 1). If `AT_BASE` does exist, we have + // already been relocated by the dynamic linker, which is case 3). + // In either case there is no need to do any relocations. + return; } - } - // Perform the rela relocations. - let mut current_rela = rela_ptr; - let rela_end = current_rela.byte_add(rela_total_size); - while current_rela != rela_end { - let rela = &*current_rela; - current_rela = current_rela.add(1); + let the_ehdr = &*ehdr_addr(); - // Calculate the location the relocation will apply at. - let reloc_addr = rela.r_offset.wrapping_add(offset); + let base = if auxv_base == null_mut() { + // Obtain the static address of `_start`, which is recorded in the + // entry field of the ELF header. + let static_start = the_ehdr.e_entry; - // Perform the rela relocation. - match rela.type_() { - R_RELATIVE => { - let addend = rela.r_addend; - let reloc_value = addend.wrapping_add(offset); - relocation_store(reloc_addr, reloc_value); + // This is case 2) as without dynamic linker `AT_BASE` doesn't exist + // and we have already excluded case 1) above. + auxv_entry.wrapping_sub(static_start) + } else { + // This is case 4) as `AT_BASE` indicates that there is a dynamic + // linker, yet we are not already relocated by a dynamic linker. + // `AT_BASE` contains the relocation offset of the dynamic linker. + auxv_base + }; + let offset = base.addr(); + + // This is case 2) or 4). We need to do all `R_RELATIVE` relocations. + // There should be no other kind of relocation because we are either a + // static PIE binary or a dynamic linker compiled with `-Bsymbolic`. + + // Compute the dynamic address of `_DYNAMIC`. + let dynv = dynamic_table_addr(); + + // Rela tables contain `Elf_Rela` elements which have an + // `r_addend` field. + let mut rela_ptr: *const Elf_Rela = null(); + let mut rela_total_size = 0; + + // Rel tables contain `Elf_Rel` elements which lack an + // `r_addend` field and store the addend in the memory to be + // relocated. + let mut rel_ptr: *const Elf_Rel = null(); + let mut rel_total_size = 0; + + // Relr tables contain `Elf_Relr` elements which are a compact + // way of representing relative relocations. + let mut relr_ptr: *const Elf_Relr = null(); + let mut relr_total_size = 0; + + // Look through the `Elf_Dyn` entries to find the location and + // size of the relocation table(s). + let mut current_dyn: *const Elf_Dyn = dynv; + loop { + let Elf_Dyn { d_tag, d_un } = &*current_dyn; + current_dyn = current_dyn.add(1); + + match *d_tag { + // We found a rela table. As above, model this as + // `with_exposed_provenance`. + DT_RELA => rela_ptr = base.byte_add(d_un.d_ptr).cast::(), + DT_RELASZ => rela_total_size = d_un.d_val as usize, + #[cfg(debug_assertions)] + DT_RELAENT => debug_assert_eq!(d_un.d_val as usize, size_of::()), + + // We found a rel table. As above, model this as + // `with_exposed_provenance`. + DT_REL => rel_ptr = base.byte_add(d_un.d_ptr).cast::(), + DT_RELSZ => rel_total_size = d_un.d_val as usize, + #[cfg(debug_assertions)] + DT_RELENT => debug_assert_eq!(d_un.d_val as usize, size_of::()), + + // We found a relr table. As above, model this as + // `with_exposed_provenance`. + DT_RELR => relr_ptr = base.byte_add(d_un.d_ptr).cast::(), + DT_RELRSZ => relr_total_size = d_un.d_val as usize, + #[cfg(debug_assertions)] + DT_RELRENT => debug_assert_eq!(d_un.d_val as usize, size_of::()), + + // End of the Dynamic section + DT_NULL => break, + + _ => (), } - // Trap the process without panicking as panicking requires - // relocations to be performed first. - _ => trap(), } - } - // Perform the rel relocations. - let mut current_rel = rel_ptr; - let rel_end = current_rel.byte_add(rel_total_size); - while current_rel != rel_end { - let rel = &*current_rel; - current_rel = current_rel.add(1); + // Perform the rela relocations. + let mut current_rela = rela_ptr; + let rela_end = current_rela.byte_add(rela_total_size); + while current_rela != rela_end { + let rela = &*current_rela; + current_rela = current_rela.add(1); - // Calculate the location the relocation will apply at. - let reloc_addr = rel.r_offset.wrapping_add(offset); + // Calculate the location the relocation will apply at. + let reloc_addr = rela.r_offset.wrapping_add(offset); - // Perform the rel relocation. - match rel.type_() { - R_RELATIVE => { - let addend = relocation_load(reloc_addr); - let reloc_value = addend.wrapping_add(offset); - relocation_store(reloc_addr, reloc_value); + // Perform the rela relocation. + match rela.type_() { + R_RELATIVE => { + let addend = rela.r_addend; + let reloc_value = addend.wrapping_add(offset); + relocation_store(reloc_addr, reloc_value); + } + // Trap the process without panicking as panicking requires + // relocations to be performed first. + _ => trap(), } - // Trap the process without panicking as panicking requires - // relocations to be performed first. - _ => trap(), } - } - // Perform the relr relocations. - let mut current_relr = relr_ptr; - let relr_end = current_relr.byte_add(relr_total_size); - let mut reloc_addr = 0; - while current_relr != relr_end { - let mut entry = *current_relr; - current_relr = current_relr.add(1); - - if entry & 1 == 0 { - // Entry encodes offset to relocate; calculate the location - // the relocation will apply at. - reloc_addr = offset + entry; - - // Perform the relr relocation. - let addend = relocation_load(reloc_addr); - let reloc_value = addend.wrapping_add(offset); - relocation_store(reloc_addr, reloc_value); - - // Advance relocation location. - reloc_addr += mem::size_of::(); - } else { - // Entry encodes bitmask with locations to relocate; apply each entry. - - let mut i = 0; - loop { - // Shift to next item in the bitmask. - entry >>= 1; - if entry == 0 { - // No more entries left in the bitmask; early exit. - break; - } + // Perform the rel relocations. + let mut current_rel = rel_ptr; + let rel_end = current_rel.byte_add(rel_total_size); + while current_rel != rel_end { + let rel = &*current_rel; + current_rel = current_rel.add(1); + + // Calculate the location the relocation will apply at. + let reloc_addr = rel.r_offset.wrapping_add(offset); - if entry & 1 != 0 { - // Perform the relr relocation. - let addend = relocation_load(reloc_addr + i * mem::size_of::()); + // Perform the rel relocation. + match rel.type_() { + R_RELATIVE => { + let addend = relocation_load(reloc_addr); let reloc_value = addend.wrapping_add(offset); - relocation_store(reloc_addr + i * mem::size_of::(), reloc_value); + relocation_store(reloc_addr, reloc_value); } - - i += 1; + // Trap the process without panicking as panicking requires + // relocations to be performed first. + _ => trap(), } - - // Advance relocation location. - reloc_addr += (mem::size_of::() * 8 - 1) * mem::size_of::(); } - } - // FIXME split function into two here with a hint::black_box around the - // function pointer to prevent the compiler from moving code between the - // functions. - - // Check that the page size is a power of two. - debug_assert!(auxv_page_size.is_power_of_two()); - - // This code doesn't rely on the offset being page aligned, but it is - // useful to check to make sure we computed it correctly. - debug_assert_eq!(offset & (auxv_page_size - 1), 0); - - // Check that relocation did its job. Do the same static start - // computation we did earlier; this time it should match the dynamic - // address. - // AT_ENTRY points to the main executable's entry point rather than our - // entry point when AT_BASE is not zero and thus a dynamic linker is in - // use. In this case the assertion would fail. - if auxv_base == null_mut() { - debug_assert_eq!(load_static_start(), auxv_entry.addr()); - } + // Perform the relr relocations. + let mut current_relr = relr_ptr; + let relr_end = current_relr.byte_add(relr_total_size); + let mut reloc_addr = 0; + while current_relr != relr_end { + let mut entry = *current_relr; + current_relr = current_relr.add(1); + + if entry & 1 == 0 { + // Entry encodes offset to relocate; calculate the location + // the relocation will apply at. + reloc_addr = offset + entry; + + // Perform the relr relocation. + let addend = relocation_load(reloc_addr); + let reloc_value = addend.wrapping_add(offset); + relocation_store(reloc_addr, reloc_value); - // Finally, look through the static segment headers (phdrs) to find the - // the relro description if present. Also do a debug assertion that - // the dynv argument matches the PT_DYNAMIC segment. - - // The location and size of the `relro` region. - let mut relro = 0; - let mut relro_size = 0; - - let phentsize = the_ehdr.e_phentsize as usize; - let mut current_phdr = base.byte_add(the_ehdr.e_phoff).cast::(); - let phdrs_end = current_phdr.byte_add(the_ehdr.e_phnum as usize * phentsize); - while current_phdr != phdrs_end { - let phdr = &*current_phdr; - current_phdr = current_phdr.byte_add(phentsize); - - match phdr.p_type { - #[cfg(debug_assertions)] - PT_DYNAMIC => { - assert_eq!(dynv, base.byte_add(phdr.p_vaddr).cast::()); + // Advance relocation location. + reloc_addr += mem::size_of::(); + } else { + // Entry encodes bitmask with locations to relocate; apply each entry. + + let mut i = 0; + loop { + // Shift to next item in the bitmask. + entry >>= 1; + if entry == 0 { + // No more entries left in the bitmask; early exit. + break; + } + + if entry & 1 != 0 { + // Perform the relr relocation. + let addend = relocation_load(reloc_addr + i * mem::size_of::()); + let reloc_value = addend.wrapping_add(offset); + relocation_store(reloc_addr + i * mem::size_of::(), reloc_value); + } + + i += 1; + } + + // Advance relocation location. + reloc_addr += (mem::size_of::() * 8 - 1) * mem::size_of::(); } - PT_GNU_RELRO => { - // A relro description is present. Make a note of it so that we - // can mark the memory readonly after relocations are done. - relro = phdr.p_vaddr; - relro_size = phdr.p_memsz; + } + + // FIXME split function into two here with a hint::black_box around the + // function pointer to prevent the compiler from moving code between the + // functions. + + // Check that the page size is a power of two. + debug_assert!(auxv_page_size.is_power_of_two()); + + // This code doesn't rely on the offset being page aligned, but it is + // useful to check to make sure we computed it correctly. + debug_assert_eq!(offset & (auxv_page_size - 1), 0); + + // Check that relocation did its job. Do the same static start + // computation we did earlier; this time it should match the dynamic + // address. + // AT_ENTRY points to the main executable's entry point rather than our + // entry point when AT_BASE is not zero and thus a dynamic linker is in + // use. In this case the assertion would fail. + if auxv_base == null_mut() { + debug_assert_eq!(load_static_start(), auxv_entry.addr()); + } + + // Finally, look through the static segment headers (phdrs) to find the + // the relro description if present. Also do a debug assertion that + // the dynv argument matches the PT_DYNAMIC segment. + + // The location and size of the `relro` region. + let mut relro = 0; + let mut relro_size = 0; + + let phentsize = the_ehdr.e_phentsize as usize; + let mut current_phdr = base.byte_add(the_ehdr.e_phoff).cast::(); + let phdrs_end = current_phdr.byte_add(the_ehdr.e_phnum as usize * phentsize); + while current_phdr != phdrs_end { + let phdr = &*current_phdr; + current_phdr = current_phdr.byte_add(phentsize); + + match phdr.p_type { + #[cfg(debug_assertions)] + PT_DYNAMIC => { + assert_eq!(dynv, base.byte_add(phdr.p_vaddr).cast::()); + } + PT_GNU_RELRO => { + // A relro description is present. Make a note of it so that we + // can mark the memory readonly after relocations are done. + relro = phdr.p_vaddr; + relro_size = phdr.p_memsz; + } + _ => (), } - _ => (), } - } - // If we saw a relro description, mark the memory readonly. - if relro_size != 0 { - let mprotect_addr = relro.wrapping_add(offset) & auxv_page_size.wrapping_neg(); - relocation_mprotect_readonly(mprotect_addr, relro_size); + // If we saw a relro description, mark the memory readonly. + if relro_size != 0 { + let mprotect_addr = relro.wrapping_add(offset) & auxv_page_size.wrapping_neg(); + relocation_mprotect_readonly(mprotect_addr, relro_size); + } } -} } +} /// Compute the address of the AUX table. -unsafe fn compute_auxp(envp: *mut *mut u8) -> *const Elf_auxv_t { unsafe { - // Locate the AUX records we need. We don't use rustix to do this because - // that would involve calling a function in another crate. - let mut auxp = envp; - // Don't use `is_null` because that's not `inline(always)`. - while *auxp != null_mut() { - auxp = auxp.add(1); +unsafe fn compute_auxp(envp: *mut *mut u8) -> *const Elf_auxv_t { + unsafe { + // Locate the AUX records we need. We don't use rustix to do this because + // that would involve calling a function in another crate. + let mut auxp = envp; + // Don't use `is_null` because that's not `inline(always)`. + while *auxp != null_mut() { + auxp = auxp.add(1); + } + auxp.add(1).cast() } - auxp.add(1).cast() -} } +} /// Load the address of `_start` from static memory. /// diff --git a/src/signal/libc.rs b/src/signal/libc.rs index 2f26b45..d8c7164 100644 --- a/src/signal/libc.rs +++ b/src/signal/libc.rs @@ -52,19 +52,21 @@ bitflags::bitflags! { /// # Safety /// /// yolo. At least this function handles `sa_restorer` automatically though. -pub unsafe fn sigaction(sig: Signal, action: Option) -> io::Result { unsafe { - let action: *const Sigaction = match action { - Some(action) => &action, - None => null(), - }; - let mut old = MaybeUninit::::uninit(); - - if libc::sigaction(sig.as_raw(), action, old.as_mut_ptr()) == 0 { - Ok(old.assume_init()) - } else { - Err(rustix::io::Errno::from_raw_os_error(errno::errno().0)) +pub unsafe fn sigaction(sig: Signal, action: Option) -> io::Result { + unsafe { + let action: *const Sigaction = match action { + Some(action) => &action, + None => null(), + }; + let mut old = MaybeUninit::::uninit(); + + if libc::sigaction(sig.as_raw(), action, old.as_mut_ptr()) == 0 { + Ok(old.assume_init()) + } else { + Err(rustix::io::Errno::from_raw_os_error(errno::errno().0)) + } } -}} +} /// Return a special “ignore” signal handler for ignoring signals. /// diff --git a/src/signal/linux_raw.rs b/src/signal/linux_raw.rs index 8f4259a..84d72b4 100644 --- a/src/signal/linux_raw.rs +++ b/src/signal/linux_raw.rs @@ -16,23 +16,25 @@ pub type Sighandler = rustix::runtime::KernelSighandler; /// # Safety /// /// yolo. At least this function handles `sa_restorer` automatically though. -pub unsafe fn sigaction(sig: Signal, action: Option) -> io::Result { unsafe { - #[allow(unused_mut)] - let mut action = action; - - #[cfg(not(target_arch = "riscv64"))] - if let Some(action) = &mut action { - action.sa_flags |= SigactionFlags::RESTORER; - - if action.sa_flags.contains(SigactionFlags::SIGINFO) { - action.sa_restorer = Some(arch::return_from_signal_handler); - } else { - action.sa_restorer = Some(arch::return_from_signal_handler_noinfo); +pub unsafe fn sigaction(sig: Signal, action: Option) -> io::Result { + unsafe { + #[allow(unused_mut)] + let mut action = action; + + #[cfg(not(target_arch = "riscv64"))] + if let Some(action) = &mut action { + action.sa_flags |= SigactionFlags::RESTORER; + + if action.sa_flags.contains(SigactionFlags::SIGINFO) { + action.sa_restorer = Some(arch::return_from_signal_handler); + } else { + action.sa_restorer = Some(arch::return_from_signal_handler_noinfo); + } } - } - rustix::runtime::kernel_sigaction(sig, action) -} } + rustix::runtime::kernel_sigaction(sig, action) + } +} /// Return a special “ignore” signal handler for ignoring signals. /// diff --git a/src/thread/libc.rs b/src/thread/libc.rs index a98e2ab..12bdd43 100644 --- a/src/thread/libc.rs +++ b/src/thread/libc.rs @@ -11,7 +11,7 @@ use alloc::boxed::Box; use core::ffi::{c_int, c_void}; use core::mem::{size_of, transmute, zeroed}; -use core::ptr::{null_mut, with_exposed_provenance_mut, without_provenance_mut, NonNull}; +use core::ptr::{NonNull, null_mut, with_exposed_provenance_mut, without_provenance_mut}; use core::slice; use rustix::io; @@ -184,11 +184,13 @@ pub unsafe fn create( /// `thread` must point to a valid thread record that has not yet been detached /// and will not be joined. #[inline] -pub unsafe fn detach(thread: Thread) { unsafe { - let thread = thread.0; +pub unsafe fn detach(thread: Thread) { + unsafe { + let thread = thread.0; - assert_eq!(libc::pthread_detach(thread), 0); -}} + assert_eq!(libc::pthread_detach(thread), 0); + } +} /// Waits for a thread to finish. /// @@ -199,14 +201,16 @@ pub unsafe fn detach(thread: Thread) { unsafe { /// /// `thread` must point to a valid thread record that has not already been /// detached or joined. -pub unsafe fn join(thread: Thread) -> Option> { unsafe { - let thread = thread.0; +pub unsafe fn join(thread: Thread) -> Option> { + unsafe { + let thread = thread.0; - let mut return_value: *mut c_void = null_mut(); - assert_eq!(libc::pthread_join(thread, &mut return_value), 0); + let mut return_value: *mut c_void = null_mut(); + assert_eq!(libc::pthread_join(thread, &mut return_value), 0); - NonNull::new(return_value) -}} + NonNull::new(return_value) + } +} /// Registers a function to call when the current thread exits. #[cfg(feature = "thread-at-exit")] @@ -287,24 +291,26 @@ pub fn current_tls_addr(module: usize, offset: usize) -> *mut c_void { /// `thread` must point to a valid thread record. #[inline] #[must_use] -pub unsafe fn stack(thread: Thread) -> (*mut c_void, usize, usize) { unsafe { - let thread = thread.0; +pub unsafe fn stack(thread: Thread) -> (*mut c_void, usize, usize) { + unsafe { + let thread = thread.0; - let mut attr: libc::pthread_attr_t = zeroed(); - assert_eq!(libc::pthread_getattr_np(thread, &mut attr), 0); + let mut attr: libc::pthread_attr_t = zeroed(); + assert_eq!(libc::pthread_getattr_np(thread, &mut attr), 0); - let mut stack_size = 0; - let mut stack_addr = null_mut(); - assert_eq!( - libc::pthread_attr_getstack(&attr, &mut stack_addr, &mut stack_size), - 0 - ); + let mut stack_size = 0; + let mut stack_addr = null_mut(); + assert_eq!( + libc::pthread_attr_getstack(&attr, &mut stack_addr, &mut stack_size), + 0 + ); - let mut guard_size = 0; - assert_eq!(libc::pthread_attr_getguardsize(&attr, &mut guard_size), 0); + let mut guard_size = 0; + assert_eq!(libc::pthread_attr_getguardsize(&attr, &mut guard_size), 0); - (stack_addr, stack_size, guard_size) -}} + (stack_addr, stack_size, guard_size) + } +} /// Return the default stack size for new threads. #[inline] @@ -344,7 +350,9 @@ pub fn yield_current() { /// Return the address of `__dso_handle`, appropriately casted. #[cfg(feature = "thread-at-exit")] -unsafe fn dso_handle() -> *mut c_void { unsafe { - let dso_handle: *const *const c_void = &__dso_handle; - dso_handle.cast::().cast_mut() -}} +unsafe fn dso_handle() -> *mut c_void { + unsafe { + let dso_handle: *const *const c_void = &__dso_handle; + dso_handle.cast::().cast_mut() + } +} diff --git a/src/thread/linux_raw.rs b/src/thread/linux_raw.rs index a51a981..fe19f6f 100644 --- a/src/thread/linux_raw.rs +++ b/src/thread/linux_raw.rs @@ -8,7 +8,7 @@ //! pthreads or `std::thread::Thread`. use crate::arch::{ - clone, munmap_and_exit_thread, set_thread_pointer, thread_pointer, STACK_ALIGNMENT, TLS_OFFSET, + STACK_ALIGNMENT, TLS_OFFSET, clone, munmap_and_exit_thread, set_thread_pointer, thread_pointer, }; #[cfg(feature = "thread-at-exit")] use alloc::boxed::Box; @@ -17,18 +17,18 @@ use core::cell::Cell; use core::cmp::max; use core::ffi::c_void; use core::mem::{align_of, offset_of, size_of}; -use core::ptr::{copy_nonoverlapping, drop_in_place, null, null_mut, NonNull}; +use core::ptr::{NonNull, copy_nonoverlapping, drop_in_place, null, null_mut}; use core::slice; use core::sync::atomic::Ordering::SeqCst; -use core::sync::atomic::{AtomicI32, AtomicPtr, AtomicU32, AtomicU8}; +use core::sync::atomic::{AtomicI32, AtomicPtr, AtomicU8, AtomicU32}; use linux_raw_sys::elf::*; use rustix::io; -use rustix::mm::{mmap_anonymous, mprotect, MapFlags, MprotectFlags, ProtFlags}; +use rustix::mm::{MapFlags, MprotectFlags, ProtFlags, mmap_anonymous, mprotect}; use rustix::param::{linux_execfn, page_size}; -use rustix::process::{getrlimit, Resource}; -use rustix::runtime::{exe_phdrs, set_tid_address}; +use rustix::process::{Resource, getrlimit}; #[cfg(feature = "signal")] -use rustix::runtime::{kernel_sigprocmask, How, KernelSigSet}; +use rustix::runtime::{How, KernelSigSet, kernel_sigprocmask}; +use rustix::runtime::{exe_phdrs, set_tid_address}; use rustix::thread::gettid; pub use rustix::thread::Pid as ThreadId; @@ -54,9 +54,9 @@ impl Thread { /// /// `raw` must be a valid non-null thread pointer. #[inline] - pub unsafe fn from_raw_unchecked(raw: *mut c_void) -> Self { unsafe { - Self(NonNull::new_unchecked(raw.cast())) - } } + pub unsafe fn from_raw_unchecked(raw: *mut c_void) -> Self { + unsafe { Self(NonNull::new_unchecked(raw.cast())) } + } /// Convert to `Self` from a raw non-null pointer that was returned from /// `Thread::to_raw_non_null`. @@ -323,65 +323,67 @@ core::arch::global_asm!(".weak _DYNAMIC"); /// `initialize_startup_info` must be called before this. And `mem` must be the /// initial value of the stack pointer in a new process, pointing to the /// initial contents of the stack. -pub(super) unsafe fn initialize_main(mem: *mut c_void) { unsafe { - // Determine the top of the stack. Linux puts the `AT_EXECFN` string at - // the top, so find the end of that, and then round up to the page size. - // See for details. - let execfn = linux_execfn().to_bytes_with_nul(); - let stack_base = execfn.as_ptr().add(execfn.len()); - let stack_base = stack_base - .map_addr(|ptr| round_up(ptr, page_size())) - .cast_mut(); - - // We're running before any user code, so the startup soft stack limit is - // the effective stack size. Linux sets up inaccessible memory at the end - // of the stack. - let stack_map_size = getrlimit(Resource::Stack).current.unwrap() as usize; - let stack_least = stack_base.sub(stack_map_size); - let stack_size = stack_least.offset_from(mem.cast::()) as usize; - let guard_size = page_size(); - - // Initialize the canary value from the OS-provided random bytes. - let random_ptr = rustix::runtime::random().cast::(); - let canary = random_ptr.read_unaligned(); - __stack_chk_guard = canary; - - let mut alloc_size = 0; - let (tls_data_bottom, header) = calculate_tls_size(&mut alloc_size); - - // Allocate the thread data. Use `mmap_anonymous` rather than `alloc` here - // as the allocator may depend on thread-local data, which is what we're - // initializing here. - let new = mmap_anonymous( - null_mut(), - alloc_size, - ProtFlags::READ | ProtFlags::WRITE, - MapFlags::PRIVATE, - ) - .unwrap() - .cast::(); - - let metadata_align = max(STARTUP_TLS_INFO.align, align_of::()); - debug_assert_eq!(new.addr() % metadata_align, 0); - - let tls_data = new.add(tls_data_bottom); - let metadata: *mut Metadata = new.add(header).cast(); - - let (newtls, thread_id_ptr) = initialize_tls( - tls_data, - metadata, - canary, - stack_least, - stack_size, - guard_size, - 0, - ); - let tid = rustix::runtime::set_tid_address(thread_id_ptr.cast()); - *thread_id_ptr = tid.as_raw_nonzero().get(); - - // Point the platform thread-pointer register at the new thread metadata. - set_thread_pointer(newtls); -} } +pub(super) unsafe fn initialize_main(mem: *mut c_void) { + unsafe { + // Determine the top of the stack. Linux puts the `AT_EXECFN` string at + // the top, so find the end of that, and then round up to the page size. + // See for details. + let execfn = linux_execfn().to_bytes_with_nul(); + let stack_base = execfn.as_ptr().add(execfn.len()); + let stack_base = stack_base + .map_addr(|ptr| round_up(ptr, page_size())) + .cast_mut(); + + // We're running before any user code, so the startup soft stack limit is + // the effective stack size. Linux sets up inaccessible memory at the end + // of the stack. + let stack_map_size = getrlimit(Resource::Stack).current.unwrap() as usize; + let stack_least = stack_base.sub(stack_map_size); + let stack_size = stack_least.offset_from(mem.cast::()) as usize; + let guard_size = page_size(); + + // Initialize the canary value from the OS-provided random bytes. + let random_ptr = rustix::runtime::random().cast::(); + let canary = random_ptr.read_unaligned(); + __stack_chk_guard = canary; + + let mut alloc_size = 0; + let (tls_data_bottom, header) = calculate_tls_size(&mut alloc_size); + + // Allocate the thread data. Use `mmap_anonymous` rather than `alloc` here + // as the allocator may depend on thread-local data, which is what we're + // initializing here. + let new = mmap_anonymous( + null_mut(), + alloc_size, + ProtFlags::READ | ProtFlags::WRITE, + MapFlags::PRIVATE, + ) + .unwrap() + .cast::(); + + let metadata_align = max(STARTUP_TLS_INFO.align, align_of::()); + debug_assert_eq!(new.addr() % metadata_align, 0); + + let tls_data = new.add(tls_data_bottom); + let metadata: *mut Metadata = new.add(header).cast(); + + let (newtls, thread_id_ptr) = initialize_tls( + tls_data, + metadata, + canary, + stack_least, + stack_size, + guard_size, + 0, + ); + let tid = rustix::runtime::set_tid_address(thread_id_ptr.cast()); + *thread_id_ptr = tid.as_raw_nonzero().get(); + + // Point the platform thread-pointer register at the new thread metadata. + set_thread_pointer(newtls); + } +} fn calculate_tls_size(map_size: &mut usize) -> (usize, usize) { // SAFETY: `STARTUP_TLS_INFO` is initialized at program startup before @@ -435,41 +437,43 @@ unsafe fn initialize_tls( stack_size: usize, guard_size: usize, map_size: usize, -) -> (*mut c_void, *mut i32) { unsafe { - let newtls: *mut c_void = (*metadata).abi.thread_pointee.as_mut_ptr().cast(); +) -> (*mut c_void, *mut i32) { + unsafe { + let newtls: *mut c_void = (*metadata).abi.thread_pointee.as_mut_ptr().cast(); + + // Initialize the thread metadata. + metadata.write(Metadata { + abi: Abi { + canary, + dtv: null(), + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + this: newtls, + _pad: Default::default(), + thread_pointee: [], + }, + thread: ThreadData::new(stack_least.cast(), stack_size, guard_size, map_size), + }); + + // Initialize the TLS data with explicit initializer data. + slice::from_raw_parts_mut(tls_data, STARTUP_TLS_INFO.file_size).copy_from_slice( + slice::from_raw_parts( + STARTUP_TLS_INFO.addr.cast::(), + STARTUP_TLS_INFO.file_size, + ), + ); - // Initialize the thread metadata. - metadata.write(Metadata { - abi: Abi { - canary, - dtv: null(), - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - this: newtls, - _pad: Default::default(), - thread_pointee: [], - }, - thread: ThreadData::new(stack_least.cast(), stack_size, guard_size, map_size), - }); - - // Initialize the TLS data with explicit initializer data. - slice::from_raw_parts_mut(tls_data, STARTUP_TLS_INFO.file_size).copy_from_slice( - slice::from_raw_parts( - STARTUP_TLS_INFO.addr.cast::(), - STARTUP_TLS_INFO.file_size, - ), - ); - - // Initialize the TLS data beyond `file_size` which is zero-filled. - slice::from_raw_parts_mut( - tls_data.add(STARTUP_TLS_INFO.file_size), - STARTUP_TLS_INFO.mem_size - STARTUP_TLS_INFO.file_size, - ) - .fill(0); - - let thread_id_ptr = (*metadata).thread.thread_id.as_ptr().cast::(); - - (newtls, thread_id_ptr) -} } + // Initialize the TLS data beyond `file_size` which is zero-filled. + slice::from_raw_parts_mut( + tls_data.add(STARTUP_TLS_INFO.file_size), + STARTUP_TLS_INFO.mem_size - STARTUP_TLS_INFO.file_size, + ) + .fill(0); + + let thread_id_ptr = (*metadata).thread.thread_id.as_ptr().cast::(); + + (newtls, thread_id_ptr) + } +} /// Creates a new thread. /// @@ -628,144 +632,149 @@ pub(super) unsafe extern "C" fn entry( fn_: extern "C" fn(), args: *mut *mut c_void, num_args: usize, -) -> ! { unsafe { - #[cfg(feature = "log")] - log::trace!("Thread[{:?}] launched", current_id().as_raw_nonzero()); - - // Do some basic precondition checks, to ensure that our assembly code did - // what we expect it to do. These are debug-only for now, to keep the - // release-mode startup code simple to disassemble and inspect, while we're - // getting started. - #[cfg(debug_assertions)] - { - // If we have nightly, we can do additional checks. - #[cfg(feature = "nightly")] +) -> ! { + unsafe { + #[cfg(feature = "log")] + log::trace!("Thread[{:?}] launched", current_id().as_raw_nonzero()); + + // Do some basic precondition checks, to ensure that our assembly code did + // what we expect it to do. These are debug-only for now, to keep the + // release-mode startup code simple to disassemble and inspect, while we're + // getting started. + #[cfg(debug_assertions)] { - unsafe extern "C" { - #[link_name = "llvm.frameaddress"] - fn builtin_frame_address(level: i32) -> *const u8; - #[link_name = "llvm.returnaddress"] - fn builtin_return_address(level: i32) -> *const u8; + // If we have nightly, we can do additional checks. + #[cfg(feature = "nightly")] + { + unsafe extern "C" { + #[link_name = "llvm.frameaddress"] + fn builtin_frame_address(level: i32) -> *const u8; + #[link_name = "llvm.returnaddress"] + fn builtin_return_address(level: i32) -> *const u8; + #[cfg(target_arch = "aarch64")] + #[link_name = "llvm.sponentry"] + fn builtin_sponentry() -> *const u8; + } + + // Check that the incoming stack pointer is where we expect it to be. + debug_assert_eq!(builtin_return_address(0), null()); + debug_assert_ne!(builtin_frame_address(0), null()); + #[cfg(not(any(target_arch = "x86", target_arch = "arm")))] + debug_assert_eq!(builtin_frame_address(0).addr() & 0xf, 0); + #[cfg(target_arch = "arm")] + debug_assert_eq!(builtin_frame_address(0).addr() & 0x3, 0); + #[cfg(target_arch = "x86")] + debug_assert_eq!(builtin_frame_address(0).addr() & 0xf, 8); + debug_assert_eq!(builtin_frame_address(1), null()); #[cfg(target_arch = "aarch64")] - #[link_name = "llvm.sponentry"] - fn builtin_sponentry() -> *const u8; + debug_assert_ne!(builtin_sponentry(), null()); + #[cfg(target_arch = "aarch64")] + debug_assert_eq!(builtin_sponentry().addr() & 0xf, 0); } - // Check that the incoming stack pointer is where we expect it to be. - debug_assert_eq!(builtin_return_address(0), null()); - debug_assert_ne!(builtin_frame_address(0), null()); - #[cfg(not(any(target_arch = "x86", target_arch = "arm")))] - debug_assert_eq!(builtin_frame_address(0).addr() & 0xf, 0); - #[cfg(target_arch = "arm")] - debug_assert_eq!(builtin_frame_address(0).addr() & 0x3, 0); - #[cfg(target_arch = "x86")] - debug_assert_eq!(builtin_frame_address(0).addr() & 0xf, 8); - debug_assert_eq!(builtin_frame_address(1), null()); - #[cfg(target_arch = "aarch64")] - debug_assert_ne!(builtin_sponentry(), null()); - #[cfg(target_arch = "aarch64")] - debug_assert_eq!(builtin_sponentry().addr() & 0xf, 0); + // Check that `clone` stored our thread id as we expected. + debug_assert_eq!(current_id(), gettid()); } - // Check that `clone` stored our thread id as we expected. - debug_assert_eq!(current_id(), gettid()); - } - - // Call the user thread function. In `std`, this is `thread_start`. Ignore - // the return value for now, as `std` doesn't need it. - let fn_: unsafe fn(&mut [*mut c_void]) -> Option> = core::mem::transmute(fn_); - let args = slice::from_raw_parts_mut(args, num_args); - let return_value = fn_(args); + // Call the user thread function. In `std`, this is `thread_start`. Ignore + // the return value for now, as `std` doesn't need it. + let fn_: unsafe fn(&mut [*mut c_void]) -> Option> = + core::mem::transmute(fn_); + let args = slice::from_raw_parts_mut(args, num_args); + let return_value = fn_(args); - exit(return_value) -} } - -/// Call the destructors registered with [`at_exit`] and exit the thread. -unsafe fn exit(return_value: Option>) -> ! { unsafe { - let current = current(); - - #[cfg(feature = "log")] - if log::log_enabled!(log::Level::Trace) { - log::trace!( - "Thread[{:?}] returned {:?}", - current.0.as_ref().thread_id.load(SeqCst), - return_value - ); + exit(return_value) } +} - // Call functions registered with `at_exit`. - #[cfg(feature = "thread-at-exit")] - call_dtors(current); - - // Read the thread's state, and set it to `ABANDONED` if it was `INITIAL`, - // which tells `join_thread` to free the memory. Otherwise, it's in the - // `DETACHED` state, and we free the memory immediately. - let state = current - .0 - .as_ref() - .detached - .compare_exchange(INITIAL, ABANDONED, SeqCst, SeqCst); - if let Err(e) = state { - // The thread was detached. Prepare to free the memory. First read out - // all the fields that we'll need before freeing it. - #[cfg(feature = "log")] - let current_thread_id = current.0.as_ref().thread_id.load(SeqCst); - let current_map_size = current.0.as_ref().map_size; - let current_stack_addr = current.0.as_ref().stack_addr; - let current_guard_size = current.0.as_ref().guard_size; +/// Call the destructors registered with [`at_exit`] and exit the thread. +unsafe fn exit(return_value: Option>) -> ! { + unsafe { + let current = current(); #[cfg(feature = "log")] - log::trace!("Thread[{:?}] exiting as detached", current_thread_id); - debug_assert_eq!(e, DETACHED); + if log::log_enabled!(log::Level::Trace) { + log::trace!( + "Thread[{:?}] returned {:?}", + current.0.as_ref().thread_id.load(SeqCst), + return_value + ); + } - // Deallocate the `ThreadData`. - drop_in_place(current.0.as_ptr()); + // Call functions registered with `at_exit`. + #[cfg(feature = "thread-at-exit")] + call_dtors(current); + + // Read the thread's state, and set it to `ABANDONED` if it was `INITIAL`, + // which tells `join_thread` to free the memory. Otherwise, it's in the + // `DETACHED` state, and we free the memory immediately. + let state = current + .0 + .as_ref() + .detached + .compare_exchange(INITIAL, ABANDONED, SeqCst, SeqCst); + if let Err(e) = state { + // The thread was detached. Prepare to free the memory. First read out + // all the fields that we'll need before freeing it. + #[cfg(feature = "log")] + let current_thread_id = current.0.as_ref().thread_id.load(SeqCst); + let current_map_size = current.0.as_ref().map_size; + let current_stack_addr = current.0.as_ref().stack_addr; + let current_guard_size = current.0.as_ref().guard_size; - // Free the thread's `mmap` region, if we allocated it. - let map_size = current_map_size; - if map_size != 0 { - // Null out the tid address so that the kernel doesn't write to - // memory that we've freed trying to clear our tid when we exit. - let _ = set_tid_address(null_mut()); + #[cfg(feature = "log")] + log::trace!("Thread[{:?}] exiting as detached", current_thread_id); + debug_assert_eq!(e, DETACHED); + + // Deallocate the `ThreadData`. + drop_in_place(current.0.as_ptr()); + + // Free the thread's `mmap` region, if we allocated it. + let map_size = current_map_size; + if map_size != 0 { + // Null out the tid address so that the kernel doesn't write to + // memory that we've freed trying to clear our tid when we exit. + let _ = set_tid_address(null_mut()); + + // In preparation for freeing the stack, block all signals, so that + // no signals for the process are delivered to this thread. + #[cfg(feature = "signal")] + { + let all = KernelSigSet::all(); + kernel_sigprocmask(How::BLOCK, Some(&all)).ok(); + } - // In preparation for freeing the stack, block all signals, so that - // no signals for the process are delivered to this thread. - #[cfg(feature = "signal")] - { - let all = KernelSigSet::all(); - kernel_sigprocmask(How::BLOCK, Some(&all)).ok(); + // `munmap` the memory, which also frees the stack we're currently + // on, and do an `exit` carefully without touching the stack. + let map = current_stack_addr.byte_sub(current_guard_size); + munmap_and_exit_thread(map, map_size); + } + } else { + // The thread was not detached, so its memory will be freed when it's + // joined. + #[cfg(feature = "log")] + if log::log_enabled!(log::Level::Trace) { + log::trace!( + "Thread[{:?}] exiting as joinable", + current.0.as_ref().thread_id.load(SeqCst) + ); } - // `munmap` the memory, which also frees the stack we're currently - // on, and do an `exit` carefully without touching the stack. - let map = current_stack_addr.byte_sub(current_guard_size); - munmap_and_exit_thread(map, map_size); - } - } else { - // The thread was not detached, so its memory will be freed when it's - // joined. - #[cfg(feature = "log")] - if log::log_enabled!(log::Level::Trace) { - log::trace!( - "Thread[{:?}] exiting as joinable", - current.0.as_ref().thread_id.load(SeqCst) - ); - } + // Convert `return_value` into a `*mut c_void` so that we can store it + // in an `AtomicPtr`. + let return_value = match return_value { + Some(return_value) => return_value.as_ptr(), + None => null_mut(), + }; - // Convert `return_value` into a `*mut c_void` so that we can store it - // in an `AtomicPtr`. - let return_value = match return_value { - Some(return_value) => return_value.as_ptr(), - None => null_mut(), - }; + // Store the return value in the thread for `join_thread` to read. + current.0.as_ref().return_value.store(return_value, SeqCst); + } - // Store the return value in the thread for `join_thread` to read. - current.0.as_ref().return_value.store(return_value, SeqCst); + // Terminate the thread. + rustix::runtime::exit_thread(0) } - - // Terminate the thread. - rustix::runtime::exit_thread(0) -} } +} /// Call the destructors registered with [`at_exit`]. #[cfg(feature = "thread-at-exit")] @@ -800,28 +809,30 @@ pub(crate) fn call_dtors(current: Thread) { /// `thread` must point to a valid thread record that has not yet been detached /// and will not be joined. #[inline] -pub unsafe fn detach(thread: Thread) { unsafe { - #[cfg(feature = "log")] - let thread_id = thread.0.as_ref().thread_id.load(SeqCst); +pub unsafe fn detach(thread: Thread) { + unsafe { + #[cfg(feature = "log")] + let thread_id = thread.0.as_ref().thread_id.load(SeqCst); - #[cfg(feature = "log")] - if log::log_enabled!(log::Level::Trace) { - log::trace!( - "Thread[{:?}] marked as detached by Thread[{:?}]", - thread_id, - current_id().as_raw_nonzero() - ); - } + #[cfg(feature = "log")] + if log::log_enabled!(log::Level::Trace) { + log::trace!( + "Thread[{:?}] marked as detached by Thread[{:?}]", + thread_id, + current_id().as_raw_nonzero() + ); + } - if thread.0.as_ref().detached.swap(DETACHED, SeqCst) == ABANDONED { - wait_for_exit(thread); + if thread.0.as_ref().detached.swap(DETACHED, SeqCst) == ABANDONED { + wait_for_exit(thread); - #[cfg(feature = "log")] - log_thread_to_be_freed(thread_id); + #[cfg(feature = "log")] + log_thread_to_be_freed(thread_id); - free_memory(thread); + free_memory(thread); + } } -} } +} /// Waits for a thread to finish. /// @@ -832,68 +843,72 @@ pub unsafe fn detach(thread: Thread) { unsafe { /// /// `thread` must point to a valid thread record that has not already been /// detached or joined. -pub unsafe fn join(thread: Thread) -> Option> { unsafe { - let thread_data = thread.0.as_ref(); +pub unsafe fn join(thread: Thread) -> Option> { + unsafe { + let thread_data = thread.0.as_ref(); - #[cfg(feature = "log")] - let thread_id = thread_data.thread_id.load(SeqCst); + #[cfg(feature = "log")] + let thread_id = thread_data.thread_id.load(SeqCst); - #[cfg(feature = "log")] - if log::log_enabled!(log::Level::Trace) { - log::trace!( - "Thread[{:?}] is being joined by Thread[{:?}]", - thread_id, - current_id().as_raw_nonzero() - ); - } + #[cfg(feature = "log")] + if log::log_enabled!(log::Level::Trace) { + log::trace!( + "Thread[{:?}] is being joined by Thread[{:?}]", + thread_id, + current_id().as_raw_nonzero() + ); + } - wait_for_exit(thread); - debug_assert_eq!(thread_data.detached.load(SeqCst), ABANDONED); + wait_for_exit(thread); + debug_assert_eq!(thread_data.detached.load(SeqCst), ABANDONED); - #[cfg(feature = "log")] - log_thread_to_be_freed(thread_id); + #[cfg(feature = "log")] + log_thread_to_be_freed(thread_id); - // Load the return value stored by `exit_thread`, before we free the - // thread's memory. - let return_value = thread_data.return_value.load(SeqCst); + // Load the return value stored by `exit_thread`, before we free the + // thread's memory. + let return_value = thread_data.return_value.load(SeqCst); - // `munmap` the stack and metadata for the thread. - free_memory(thread); + // `munmap` the stack and metadata for the thread. + free_memory(thread); - // Convert the `*mut c_void` we stored in the `AtomicPtr` back into an - // `Option>`. - NonNull::new(return_value) -} } + // Convert the `*mut c_void` we stored in the `AtomicPtr` back into an + // `Option>`. + NonNull::new(return_value) + } +} /// Wait until `thread` has exited. /// /// `thread` must point to a valid thread record that has not already been /// detached or joined. -unsafe fn wait_for_exit(thread: Thread) { unsafe { - use rustix::thread::futex; - - // Check whether the thread has exited already; we set the - // `CloneFlags::CHILD_CLEARTID` flag on the clone syscall, so we can test - // for `NONE` here. - let thread_data = thread.0.as_ref(); - let thread_id = &thread_data.thread_id; - while let Some(id_value) = ThreadId::from_raw(thread_id.load(SeqCst)) { - // This doesn't use any shared memory, but we can't use - // `FutexFlags::PRIVATE` because the wake comes from Linux - // as arranged by the `CloneFlags::CHILD_CLEARTID` flag, - // and Linux doesn't use the private flag for the wake. - match futex::wait( - AtomicU32::from_ptr(thread_id.as_ptr().cast()), - futex::Flags::empty(), - id_value.as_raw_nonzero().get() as u32, - None, - ) { - Ok(_) => break, - Err(io::Errno::INTR) => continue, - Err(e) => debug_assert_eq!(e, io::Errno::AGAIN), +unsafe fn wait_for_exit(thread: Thread) { + unsafe { + use rustix::thread::futex; + + // Check whether the thread has exited already; we set the + // `CloneFlags::CHILD_CLEARTID` flag on the clone syscall, so we can test + // for `NONE` here. + let thread_data = thread.0.as_ref(); + let thread_id = &thread_data.thread_id; + while let Some(id_value) = ThreadId::from_raw(thread_id.load(SeqCst)) { + // This doesn't use any shared memory, but we can't use + // `FutexFlags::PRIVATE` because the wake comes from Linux + // as arranged by the `CloneFlags::CHILD_CLEARTID` flag, + // and Linux doesn't use the private flag for the wake. + match futex::wait( + AtomicU32::from_ptr(thread_id.as_ptr().cast()), + futex::Flags::empty(), + id_value.as_raw_nonzero().get() as u32, + None, + ) { + Ok(_) => break, + Err(io::Errno::INTR) => continue, + Err(e) => debug_assert_eq!(e, io::Errno::AGAIN), + } } } -} } +} #[cfg(feature = "log")] fn log_thread_to_be_freed(thread_id: i32) { @@ -908,24 +923,26 @@ fn log_thread_to_be_freed(thread_id: i32) { /// /// `thread` must point to a valid thread record for a thread that has /// already exited. -unsafe fn free_memory(thread: Thread) { unsafe { - use rustix::mm::munmap; - - // The thread was detached. Prepare to free the memory. First read out - // all the fields that we'll need before freeing it. - let map_size = thread.0.as_ref().map_size; - let stack_addr = thread.0.as_ref().stack_addr; - let guard_size = thread.0.as_ref().guard_size; - - // Deallocate the `ThreadData`. - drop_in_place(thread.0.as_ptr()); - - // Free the thread's `mmap` region, if we allocated it. - if map_size != 0 { - let map = stack_addr.byte_sub(guard_size); - munmap(map, map_size).unwrap(); +unsafe fn free_memory(thread: Thread) { + unsafe { + use rustix::mm::munmap; + + // The thread was detached. Prepare to free the memory. First read out + // all the fields that we'll need before freeing it. + let map_size = thread.0.as_ref().map_size; + let stack_addr = thread.0.as_ref().stack_addr; + let guard_size = thread.0.as_ref().guard_size; + + // Deallocate the `ThreadData`. + drop_in_place(thread.0.as_ptr()); + + // Free the thread's `mmap` region, if we allocated it. + if map_size != 0 { + let map = stack_addr.byte_sub(guard_size); + munmap(map, map_size).unwrap(); + } } -} } +} /// Registers a function to call when the current thread exits. #[cfg(feature = "thread-at-exit")] @@ -986,20 +1003,22 @@ pub fn current_id() -> ThreadId { /// return. #[doc(hidden)] #[inline] -pub unsafe fn set_current_id_after_a_fork(tid: ThreadId) { unsafe { - let current = current(); - debug_assert_ne!( - tid.as_raw_nonzero().get(), - current.0.as_ref().thread_id.load(SeqCst), - "current thread ID already matches new thread ID" - ); - debug_assert_eq!(tid, gettid(), "new thread ID disagrees with `gettid`"); - current - .0 - .as_ref() - .thread_id - .store(tid.as_raw_nonzero().get(), SeqCst); -} } +pub unsafe fn set_current_id_after_a_fork(tid: ThreadId) { + unsafe { + let current = current(); + debug_assert_ne!( + tid.as_raw_nonzero().get(), + current.0.as_ref().thread_id.load(SeqCst), + "current thread ID already matches new thread ID" + ); + debug_assert_eq!(tid, gettid(), "new thread ID disagrees with `gettid`"); + current + .0 + .as_ref() + .thread_id + .store(tid.as_raw_nonzero().get(), SeqCst); + } +} /// Return the address of the thread-local `errno` state. /// @@ -1048,10 +1067,12 @@ pub fn current_tls_addr(module: usize, offset: usize) -> *mut c_void { /// `thread` must point to a valid thread record. #[inline] #[cfg_attr(docsrs, doc(cfg(feature = "take-charge")))] -pub unsafe fn id(thread: Thread) -> Option { unsafe { - let raw = thread.0.as_ref().thread_id.load(SeqCst); - ThreadId::from_raw(raw) -} } +pub unsafe fn id(thread: Thread) -> Option { + unsafe { + let raw = thread.0.as_ref().thread_id.load(SeqCst); + ThreadId::from_raw(raw) + } +} /// Return the current thread's stack address (lowest address), size, and guard /// size. @@ -1061,10 +1082,12 @@ pub unsafe fn id(thread: Thread) -> Option { unsafe { /// `thread` must point to a valid thread record. #[inline] #[must_use] -pub unsafe fn stack(thread: Thread) -> (*mut c_void, usize, usize) { unsafe { - let data = thread.0.as_ref(); - (data.stack_addr, data.stack_size, data.guard_size) -} } +pub unsafe fn stack(thread: Thread) -> (*mut c_void, usize, usize) { + unsafe { + let data = thread.0.as_ref(); + (data.stack_addr, data.stack_size, data.guard_size) + } +} /// Return the default stack size for new threads. #[inline] diff --git a/test-crates/origin-start/src/bin/canary.rs b/test-crates/origin-start/src/bin/canary.rs index 9d768a0..83d7b2e 100644 --- a/test-crates/origin-start/src/bin/canary.rs +++ b/test-crates/origin-start/src/bin/canary.rs @@ -41,26 +41,28 @@ fn tls_guard() -> usize { } #[unsafe(no_mangle)] -unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { unsafe { - assert_ne!(*__stack_chk_guard.get(), 0); - assert_eq!(*__stack_chk_guard.get(), tls_guard()); +unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { + unsafe { + assert_ne!(*__stack_chk_guard.get(), 0); + assert_eq!(*__stack_chk_guard.get(), tls_guard()); - let thread = thread::create( - |_args| { - assert_ne!(*__stack_chk_guard.get(), 0); - assert_eq!(*__stack_chk_guard.get(), tls_guard()); - None - }, - &[], - thread::default_stack_size(), - thread::default_guard_size(), - ) - .unwrap(); + let thread = thread::create( + |_args| { + assert_ne!(*__stack_chk_guard.get(), 0); + assert_eq!(*__stack_chk_guard.get(), tls_guard()); + None + }, + &[], + thread::default_stack_size(), + thread::default_guard_size(), + ) + .unwrap(); - thread::join(thread); + thread::join(thread); - assert_ne!(*__stack_chk_guard.get(), 0); - assert_eq!(*__stack_chk_guard.get(), tls_guard()); + assert_ne!(*__stack_chk_guard.get(), 0); + assert_eq!(*__stack_chk_guard.get(), tls_guard()); - program::exit(203); -}} + program::exit(203); + } +} diff --git a/test-crates/origin-start/src/bin/detach.rs b/test-crates/origin-start/src/bin/detach.rs index dfad516..5899e06 100644 --- a/test-crates/origin-start/src/bin/detach.rs +++ b/test-crates/origin-start/src/bin/detach.rs @@ -11,36 +11,38 @@ use origin::{program, thread}; static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::GlobalDlmalloc; #[unsafe(no_mangle)] -unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { unsafe { - let long_thread = thread::create( - |_args| None, - &[], - thread::default_stack_size(), - thread::default_guard_size(), - ) - .unwrap(); - - let short_thread = thread::create( - |_args| None, - &[], - thread::default_stack_size(), - thread::default_guard_size(), - ) - .unwrap(); - thread::detach(short_thread); - - let _thread = thread::create( - |_args| { - thread::detach(thread::current()); - None - }, - &[], - thread::default_stack_size(), - thread::default_guard_size(), - ) - .unwrap(); - - thread::detach(long_thread); - - program::exit(202); -}} +unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { + unsafe { + let long_thread = thread::create( + |_args| None, + &[], + thread::default_stack_size(), + thread::default_guard_size(), + ) + .unwrap(); + + let short_thread = thread::create( + |_args| None, + &[], + thread::default_stack_size(), + thread::default_guard_size(), + ) + .unwrap(); + thread::detach(short_thread); + + let _thread = thread::create( + |_args| { + thread::detach(thread::current()); + None + }, + &[], + thread::default_stack_size(), + thread::default_guard_size(), + ) + .unwrap(); + + thread::detach(long_thread); + + program::exit(202); + } +} diff --git a/test-crates/origin-start/src/bin/thread-dtors-adding-dtors.rs b/test-crates/origin-start/src/bin/thread-dtors-adding-dtors.rs index 800708b..846e34a 100644 --- a/test-crates/origin-start/src/bin/thread-dtors-adding-dtors.rs +++ b/test-crates/origin-start/src/bin/thread-dtors-adding-dtors.rs @@ -15,34 +15,36 @@ static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::Glob static FLAG: AtomicBool = AtomicBool::new(false); #[unsafe(no_mangle)] -unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { unsafe { - let thread = thread::create( - |_args| { - thread::at_exit(Box::new(|| { - assert!(FLAG.load(Ordering::Relaxed)); - })); - - thread::at_exit(Box::new(|| { +unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { + unsafe { + let thread = thread::create( + |_args| { + thread::at_exit(Box::new(|| { + assert!(FLAG.load(Ordering::Relaxed)); + })); + thread::at_exit(Box::new(|| { thread::at_exit(Box::new(|| { - FLAG.store(true, Ordering::Relaxed); + thread::at_exit(Box::new(|| { + FLAG.store(true, Ordering::Relaxed); + })); })); })); - })); - thread::at_exit(Box::new(|| { - assert!(!FLAG.load(Ordering::Relaxed)); - })); + thread::at_exit(Box::new(|| { + assert!(!FLAG.load(Ordering::Relaxed)); + })); - None - }, - &[], - thread::default_stack_size(), - thread::default_guard_size(), - ) - .unwrap(); + None + }, + &[], + thread::default_stack_size(), + thread::default_guard_size(), + ) + .unwrap(); - thread::join(thread); + thread::join(thread); - 0 -}} + 0 + } +} diff --git a/test-crates/origin-start/src/bin/thread-id.rs b/test-crates/origin-start/src/bin/thread-id.rs index c2f58c8..471d0bc 100644 --- a/test-crates/origin-start/src/bin/thread-id.rs +++ b/test-crates/origin-start/src/bin/thread-id.rs @@ -8,7 +8,7 @@ extern crate alloc; use alloc::boxed::Box; use alloc::sync::Arc; use core::ffi::c_void; -use core::ptr::{null_mut, NonNull}; +use core::ptr::{NonNull, null_mut}; use origin::{program, thread}; use rustix_futex_sync::{Condvar, Mutex}; @@ -22,91 +22,93 @@ enum Id { } #[unsafe(no_mangle)] -unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { unsafe { - assert_eq!(thread::current_id(), thread::id(thread::current()).unwrap()); - program::at_exit(Box::new(|| { - assert_eq!(thread::current_id(), thread::id(thread::current()).unwrap()) - })); - thread::at_exit(Box::new(|| { +unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { + unsafe { assert_eq!(thread::current_id(), thread::id(thread::current()).unwrap()); - })); + program::at_exit(Box::new(|| { + assert_eq!(thread::current_id(), thread::id(thread::current()).unwrap()) + })); + thread::at_exit(Box::new(|| { + assert_eq!(thread::current_id(), thread::id(thread::current()).unwrap()); + })); - let cond = Arc::new((Mutex::new(None), Condvar::new())); - let cond_clone = NonNull::new(Box::into_raw(Box::new(Arc::clone(&cond))).cast()); + let cond = Arc::new((Mutex::new(None), Condvar::new())); + let cond_clone = NonNull::new(Box::into_raw(Box::new(Arc::clone(&cond))).cast()); - let thread = thread::create( - move |args| { - assert_eq!(thread::current_id(), thread::id(thread::current()).unwrap()); - thread::at_exit(Box::new(|| { + let thread = thread::create( + move |args| { assert_eq!(thread::current_id(), thread::id(thread::current()).unwrap()); - })); - - let unpack = |x: Option>| match x { - Some(p) => p.as_ptr().cast::>, Condvar)>>(), - None => null_mut(), - }; - let cond = Box::from_raw(unpack(args[0])); - - // Wait for the parent to tell the child its id, and check it. - let (lock, cvar) = &**cond; - { - let mut id = lock.lock(); - while (*id).is_none() { - id = cvar.wait(id); + thread::at_exit(Box::new(|| { + assert_eq!(thread::current_id(), thread::id(thread::current()).unwrap()); + })); + + let unpack = |x: Option>| match x { + Some(p) => p.as_ptr().cast::>, Condvar)>>(), + None => null_mut(), + }; + let cond = Box::from_raw(unpack(args[0])); + + // Wait for the parent to tell the child its id, and check it. + let (lock, cvar) = &**cond; + { + let mut id = lock.lock(); + while (*id).is_none() { + id = cvar.wait(id); + } + assert_ne!(Id::Parent(thread::current_id()), *id.as_ref().unwrap()); } - assert_ne!(Id::Parent(thread::current_id()), *id.as_ref().unwrap()); - } - // Tell the parent the child's id. - { - let mut id = lock.lock(); - *id = Some(Id::Child(thread::current_id())); - cvar.notify_one(); - } + // Tell the parent the child's id. + { + let mut id = lock.lock(); + *id = Some(Id::Child(thread::current_id())); + cvar.notify_one(); + } - assert_eq!(thread::current_id(), thread::id(thread::current()).unwrap()); - None - }, - &[cond_clone], - thread::default_stack_size(), - thread::default_guard_size(), - ) - .unwrap(); - - // While the child is still running, examine its id. - let child_thread_id_from_main = thread::id(thread).unwrap(); - - // Tell the child the main thread id. - let (lock, cvar) = &*cond; - { - let mut id = lock.lock(); - *id = Some(Id::Parent(thread::current_id())); - cvar.notify_one(); - } + assert_eq!(thread::current_id(), thread::id(thread::current()).unwrap()); + None + }, + &[cond_clone], + thread::default_stack_size(), + thread::default_guard_size(), + ) + .unwrap(); + + // While the child is still running, examine its id. + let child_thread_id_from_main = thread::id(thread).unwrap(); + + // Tell the child the main thread id. + let (lock, cvar) = &*cond; + { + let mut id = lock.lock(); + *id = Some(Id::Parent(thread::current_id())); + cvar.notify_one(); + } - // Wait for the child to tell the main thread its id, and check it. - { - let mut id = lock.lock(); - loop { - if let Some(Id::Child(id)) = *id { - assert_eq!(child_thread_id_from_main, id); - break; + // Wait for the child to tell the main thread its id, and check it. + { + let mut id = lock.lock(); + loop { + if let Some(Id::Child(id)) = *id { + assert_eq!(child_thread_id_from_main, id); + break; + } + id = cvar.wait(id); } - id = cvar.wait(id); } - } - // At some point the child thread should exit, at which point `thread::id` - // will return `None`. - while thread::id(thread).is_some() { - let _ = rustix::thread::nanosleep(&rustix::thread::Timespec { - tv_sec: 0, - tv_nsec: 100_000_000, - }); - } + // At some point the child thread should exit, at which point `thread::id` + // will return `None`. + while thread::id(thread).is_some() { + let _ = rustix::thread::nanosleep(&rustix::thread::Timespec { + tv_sec: 0, + tv_nsec: 100_000_000, + }); + } - thread::join(thread); + thread::join(thread); - assert_eq!(thread::current_id(), thread::id(thread::current()).unwrap()); - program::exit(201); -}} + assert_eq!(thread::current_id(), thread::id(thread::current()).unwrap()); + program::exit(201); + } +} diff --git a/test-crates/origin-start/src/bin/tls.rs b/test-crates/origin-start/src/bin/tls.rs index 53560c3..c5b5a8c 100644 --- a/test-crates/origin-start/src/bin/tls.rs +++ b/test-crates/origin-start/src/bin/tls.rs @@ -17,69 +17,71 @@ use origin::{program, thread}; static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::GlobalDlmalloc; #[unsafe(no_mangle)] -unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { unsafe { - // Assert that the main thread initialized its TLS properly. - check_eq(TEST_DATA.0); - - // Mutate one of the TLS fields. - (*THREAD_LOCAL.get())[1] = without_provenance_mut(77); - - // Assert that the mutation happened properly. - check_eq([TEST_DATA.0[0], without_provenance_mut(77), TEST_DATA.0[2]]); - - program::at_exit(Box::new(|| { - // This is the last thing to run. Assert that we see the value stored - // by the `at_thread_exit` callback. - check_eq([TEST_DATA.0[0], without_provenance_mut(79), TEST_DATA.0[2]]); +unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { + unsafe { + // Assert that the main thread initialized its TLS properly. + check_eq(TEST_DATA.0); // Mutate one of the TLS fields. - (*THREAD_LOCAL.get())[1] = without_provenance_mut(80); - })); - thread::at_exit(Box::new(|| { - // Assert that we see the value stored at the end of `main`. - check_eq([TEST_DATA.0[0], without_provenance_mut(78), TEST_DATA.0[2]]); + (*THREAD_LOCAL.get())[1] = without_provenance_mut(77); - // Mutate one of the TLS fields. - (*THREAD_LOCAL.get())[1] = without_provenance_mut(79); - })); + // Assert that the mutation happened properly. + check_eq([TEST_DATA.0[0], without_provenance_mut(77), TEST_DATA.0[2]]); + + program::at_exit(Box::new(|| { + // This is the last thing to run. Assert that we see the value stored + // by the `at_thread_exit` callback. + check_eq([TEST_DATA.0[0], without_provenance_mut(79), TEST_DATA.0[2]]); - let thread = thread::create( - |_args| { - // Assert that the new thread initialized its TLS properly. - check_eq(TEST_DATA.0); + // Mutate one of the TLS fields. + (*THREAD_LOCAL.get())[1] = without_provenance_mut(80); + })); + thread::at_exit(Box::new(|| { + // Assert that we see the value stored at the end of `main`. + check_eq([TEST_DATA.0[0], without_provenance_mut(78), TEST_DATA.0[2]]); // Mutate one of the TLS fields. - (*THREAD_LOCAL.get())[1] = without_provenance_mut(175); + (*THREAD_LOCAL.get())[1] = without_provenance_mut(79); + })); + + let thread = thread::create( + |_args| { + // Assert that the new thread initialized its TLS properly. + check_eq(TEST_DATA.0); - // Assert that the mutation happened properly. - check_eq([TEST_DATA.0[0], without_provenance_mut(175), TEST_DATA.0[2]]); + // Mutate one of the TLS fields. + (*THREAD_LOCAL.get())[1] = without_provenance_mut(175); - thread::at_exit(Box::new(|| { - // Assert that we still see the value stored in the thread. + // Assert that the mutation happened properly. check_eq([TEST_DATA.0[0], without_provenance_mut(175), TEST_DATA.0[2]]); - })); - None - }, - &[], - thread::default_stack_size(), - thread::default_guard_size(), - ) - .unwrap(); + thread::at_exit(Box::new(|| { + // Assert that we still see the value stored in the thread. + check_eq([TEST_DATA.0[0], without_provenance_mut(175), TEST_DATA.0[2]]); + })); - thread::join(thread); + None + }, + &[], + thread::default_stack_size(), + thread::default_guard_size(), + ) + .unwrap(); - // Assert that the main thread's TLS is still in place. - check_eq([TEST_DATA.0[0], without_provenance_mut(77), TEST_DATA.0[2]]); + thread::join(thread); - // Mutate one of the TLS fields. - (*THREAD_LOCAL.get())[1] = without_provenance_mut(78); + // Assert that the main thread's TLS is still in place. + check_eq([TEST_DATA.0[0], without_provenance_mut(77), TEST_DATA.0[2]]); - // Assert that the mutation happened properly. - check_eq([TEST_DATA.0[0], without_provenance_mut(78), TEST_DATA.0[2]]); + // Mutate one of the TLS fields. + (*THREAD_LOCAL.get())[1] = without_provenance_mut(78); - program::exit(200); -}} + // Assert that the mutation happened properly. + check_eq([TEST_DATA.0[0], without_provenance_mut(78), TEST_DATA.0[2]]); + + program::exit(200); + } +} struct SyncTestData([*const u32; 3]); unsafe impl Sync for SyncTestData {} From 6346750854f05fbc79fa44ee9c302a07f7099fcb Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Tue, 29 Apr 2025 12:18:51 -0700 Subject: [PATCH 10/15] Port more arches to edition2024. --- src/arch/aarch64.rs | 156 ++++++++++++----------- src/arch/arm.rs | 174 +++++++++++++------------ src/arch/riscv64.rs | 174 +++++++++++++------------ src/arch/x86.rs | 304 +++++++++++++++++++++++--------------------- 4 files changed, 429 insertions(+), 379 deletions(-) diff --git a/src/arch/aarch64.rs b/src/arch/aarch64.rs index e08d0fc..ea1da39 100644 --- a/src/arch/aarch64.rs +++ b/src/arch/aarch64.rs @@ -109,19 +109,21 @@ pub(super) fn ehdr_addr() -> *const Elf_Ehdr { #[cfg(relocation_model = "pic")] #[inline] pub(super) unsafe fn relocation_load(ptr: usize) -> usize { - let r0; + unsafe { + let r0; - // This is read-only but we don't use `readonly` because this memory access - // happens outside the Rust memory model. As far as Rust knows, this is - // just an arbitrary side-effecting opaque operation. - asm!( - "ldr {}, [{}]", - out(reg) r0, - in(reg) ptr, - options(nostack, preserves_flags), - ); + // This is read-only but we don't use `readonly` because this memory access + // happens outside the Rust memory model. As far as Rust knows, this is + // just an arbitrary side-effecting opaque operation. + asm!( + "ldr {}, [{}]", + out(reg) r0, + in(reg) ptr, + options(nostack, preserves_flags), + ); - r0 + r0 + } } /// Perform a single store operation, outside the Rust memory model. @@ -140,12 +142,14 @@ pub(super) unsafe fn relocation_load(ptr: usize) -> usize { #[cfg(relocation_model = "pic")] #[inline] pub(super) unsafe fn relocation_store(ptr: usize, value: usize) { - asm!( - "str {}, [{}]", - in(reg) value, - in(reg) ptr, - options(nostack, preserves_flags), - ); + unsafe { + asm!( + "str {}, [{}]", + in(reg) value, + in(reg) ptr, + options(nostack, preserves_flags), + ); + } } /// Mark “relro” memory as readonly. @@ -168,24 +172,26 @@ pub(super) unsafe fn relocation_store(ptr: usize, value: usize) { #[cfg(relocation_model = "pic")] #[inline] pub(super) unsafe fn relocation_mprotect_readonly(ptr: usize, len: usize) { - let r0: usize; + unsafe { + let r0: usize; - // This is read-only but we don't use `readonly` because the side effects - // happen outside the Rust memory model. As far as Rust knows, this is - // just an arbitrary side-effecting opaque operation. - asm!( - "svc 0", - in("x8") __NR_mprotect, - inlateout("x0") ptr as usize => r0, - in("x1") len, - in("x2") PROT_READ, - options(nostack, preserves_flags), - ); + // This is read-only but we don't use `readonly` because the side effects + // happen outside the Rust memory model. As far as Rust knows, this is + // just an arbitrary side-effecting opaque operation. + asm!( + "svc 0", + in("x8") __NR_mprotect, + inlateout("x0") ptr as usize => r0, + in("x1") len, + in("x2") PROT_READ, + options(nostack, preserves_flags), + ); - if r0 != 0 { - // Do not panic here as libstd's panic handler needs TLS, which is not - // yet initialized at this point. - trap(); + if r0 != 0 { + // Do not panic here as libstd's panic handler needs TLS, which is not + // yet initialized at this point. + trap(); + } } } @@ -211,34 +217,36 @@ pub(super) unsafe fn clone( fn_: extern "C" fn(), num_args: usize, ) -> isize { - let r0; - asm!( - "svc 0", // Do the `clone` system call. - "cbnz x0, 0f", // Branch if we're in the parent thread. + unsafe { + let r0; + asm!( + "svc 0", // Do the `clone` system call. + "cbnz x0, 0f", // Branch if we're in the parent thread. - // Child thread. - "mov x0, {fn_}", // Pass `fn_` as the first argument. - "mov x1, sp", // Pass the args pointer as the second argument. - "mov x2, {num_args}", // Pass `num_args` as the third argument. - "mov x29, xzr", // Zero the frame address. - "mov x30, xzr", // Zero the return address. - "b {entry}", // Call `entry`. + // Child thread. + "mov x0, {fn_}", // Pass `fn_` as the first argument. + "mov x1, sp", // Pass the args pointer as the second argument. + "mov x2, {num_args}", // Pass `num_args` as the third argument. + "mov x29, xzr", // Zero the frame address. + "mov x30, xzr", // Zero the return address. + "b {entry}", // Call `entry`. - // Parent thread. - "0:", + // Parent thread. + "0:", - entry = sym super::thread::entry, - fn_ = in(reg) fn_, - num_args = in(reg) num_args, - in("x8") __NR_clone, - inlateout("x0") flags as usize => r0, - in("x1") child_stack, - in("x2") parent_tid, - in("x3") newtls, - in("x4") child_tid, - options(nostack) - ); - r0 + entry = sym super::thread::entry, + fn_ = in(reg) fn_, + num_args = in(reg) num_args, + in("x8") __NR_clone, + inlateout("x0") flags as usize => r0, + in("x1") child_stack, + in("x2") parent_tid, + in("x3") newtls, + in("x4") child_tid, + options(nostack) + ); + r0 + } } /// Write a value to the platform thread-pointer register. @@ -246,8 +254,10 @@ pub(super) unsafe fn clone( #[cfg(feature = "thread")] #[inline] pub(super) unsafe fn set_thread_pointer(ptr: *mut c_void) { - asm!("msr tpidr_el0, {}", in(reg) ptr); - debug_assert_eq!(thread_pointer(), ptr); + unsafe { + asm!("msr tpidr_el0, {}", in(reg) ptr); + debug_assert_eq!(thread_pointer(), ptr); + } } /// Read the value of the platform thread-pointer register. @@ -274,18 +284,20 @@ pub(super) const TLS_OFFSET: usize = 0; #[cfg(feature = "thread")] #[inline] pub(super) unsafe fn munmap_and_exit_thread(map_addr: *mut c_void, map_len: usize) -> ! { - asm!( - "svc 0", - "mov x0, xzr", - "mov x8, {__NR_exit}", - "svc 0", - "udf #16", - __NR_exit = const __NR_exit, - in("x8") __NR_munmap, - in("x0") map_addr, - in("x1") map_len, - options(noreturn, nostack) - ); + unsafe { + asm!( + "svc 0", + "mov x0, xzr", + "mov x8, {__NR_exit}", + "svc 0", + "udf #16", + __NR_exit = const __NR_exit, + in("x8") __NR_munmap, + in("x0") map_addr, + in("x1") map_len, + options(noreturn, nostack) + ); + } } #[cfg(feature = "take-charge")] diff --git a/src/arch/arm.rs b/src/arch/arm.rs index 231c899..a627a2d 100644 --- a/src/arch/arm.rs +++ b/src/arch/arm.rs @@ -121,19 +121,21 @@ pub(super) fn ehdr_addr() -> *const Elf_Ehdr { #[cfg(relocation_model = "pic")] #[inline] pub(super) unsafe fn relocation_load(ptr: usize) -> usize { - let r0; - - // This is read-only but we don't use `readonly` because this memory access - // happens outside the Rust memory model. As far as Rust knows, this is - // just an arbitrary side-effecting opaque operation. - asm!( - "ldr {}, [{}]", - out(reg) r0, - in(reg) ptr, - options(nostack, preserves_flags), - ); - - r0 + { + let r0; + + // This is read-only but we don't use `readonly` because this memory access + // happens outside the Rust memory model. As far as Rust knows, this is + // just an arbitrary side-effecting opaque operation. + asm!( + "ldr {}, [{}]", + out(reg) r0, + in(reg) ptr, + options(nostack, preserves_flags), + ); + + r0 + } } /// Perform a single store operation, outside the Rust memory model. @@ -152,12 +154,14 @@ pub(super) unsafe fn relocation_load(ptr: usize) -> usize { #[cfg(relocation_model = "pic")] #[inline] pub(super) unsafe fn relocation_store(ptr: usize, value: usize) { - asm!( - "str {}, [{}]", - in(reg) value, - in(reg) ptr, - options(nostack, preserves_flags), - ); + unsafe { + asm!( + "str {}, [{}]", + in(reg) value, + in(reg) ptr, + options(nostack, preserves_flags), + ); + } } /// Mark “relro” memory as readonly. @@ -180,24 +184,26 @@ pub(super) unsafe fn relocation_store(ptr: usize, value: usize) { #[cfg(relocation_model = "pic")] #[inline] pub(super) unsafe fn relocation_mprotect_readonly(ptr: usize, len: usize) { - let r0: usize; - - // This is read-only but we don't use `readonly` because the side effects - // happen outside the Rust memory model. As far as Rust knows, this is - // just an arbitrary side-effecting opaque operation. - asm!( - "svc 0", - in("r7") __NR_mprotect, - inlateout("r0") ptr as usize => r0, - in("r1") len, - in("r2") PROT_READ, - options(nostack, preserves_flags), - ); - - if r0 != 0 { - // Do not panic here as libstd's panic handler needs TLS, which is not - // yet initialized at this point. - trap(); + unsafe { + let r0: usize; + + // This is read-only but we don't use `readonly` because the side effects + // happen outside the Rust memory model. As far as Rust knows, this is + // just an arbitrary side-effecting opaque operation. + asm!( + "svc 0", + in("r7") __NR_mprotect, + inlateout("r0") ptr as usize => r0, + in("r1") len, + in("r2") PROT_READ, + options(nostack, preserves_flags), + ); + + if r0 != 0 { + // Do not panic here as libstd's panic handler needs TLS, which is not + // yet initialized at this point. + trap(); + } } } @@ -223,35 +229,37 @@ pub(super) unsafe fn clone( fn_: extern "C" fn(), num_args: usize, ) -> isize { - let r0; - asm!( - "svc 0", // Do the `clone` system call. - "tst r0, r0", // Branch if we're in the parent thread. - "bne 0f", - - // Child thread. - "mov r0, {fn_}", // Pass `fn_` as the first argument. - "mov r1, sp", // Pass the args pointer as the second argument. - "mov r2, {num_args}", // Pass `num_args` as the third argument. - "mov fp, #0", // Zero the frame address. - "mov lr, #0", // Zero the return address. - "b {entry}", // Call `entry`. - - // Parent thread. - "0:", - - entry = sym super::thread::entry, - fn_ = in(reg) fn_, - num_args = in(reg) num_args, - in("r7") __NR_clone, - inlateout("r0") flags as usize => r0, - in("r1") child_stack, - in("r2") parent_tid, - in("r3") newtls, - in("r4") child_tid, - options(nostack) - ); - r0 + unsafe { + let r0; + asm!( + "svc 0", // Do the `clone` system call. + "tst r0, r0", // Branch if we're in the parent thread. + "bne 0f", + + // Child thread. + "mov r0, {fn_}", // Pass `fn_` as the first argument. + "mov r1, sp", // Pass the args pointer as the second argument. + "mov r2, {num_args}", // Pass `num_args` as the third argument. + "mov fp, #0", // Zero the frame address. + "mov lr, #0", // Zero the return address. + "b {entry}", // Call `entry`. + + // Parent thread. + "0:", + + entry = sym super::thread::entry, + fn_ = in(reg) fn_, + num_args = in(reg) num_args, + in("r7") __NR_clone, + inlateout("r0") flags as usize => r0, + in("r1") child_stack, + in("r2") parent_tid, + in("r3") newtls, + in("r4") child_tid, + options(nostack) + ); + r0 + } } /// Write a value to the platform thread-pointer register. @@ -259,9 +267,11 @@ pub(super) unsafe fn clone( #[cfg(feature = "thread")] #[inline] pub(super) unsafe fn set_thread_pointer(ptr: *mut c_void) { - let res = rustix::runtime::arm_set_tls(ptr); - debug_assert_eq!(res, Ok(())); - debug_assert_eq!(thread_pointer(), ptr); + unsafe { + let res = rustix::runtime::arm_set_tls(ptr); + debug_assert_eq!(res, Ok(())); + debug_assert_eq!(thread_pointer(), ptr); + } } /// Read the value of the platform thread-pointer register. @@ -288,18 +298,20 @@ pub(super) const TLS_OFFSET: usize = 0; #[cfg(feature = "thread")] #[inline] pub(super) unsafe fn munmap_and_exit_thread(map_addr: *mut c_void, map_len: usize) -> ! { - asm!( - "svc 0", - "mov r0, #0", - "mov r7, {__NR_exit}", - "svc 0", - "udf #16", - __NR_exit = const __NR_exit, - in("r7") __NR_munmap, - in("r0") map_addr, - in("r1") map_len, - options(noreturn, nostack) - ); + unsafe { + asm!( + "svc 0", + "mov r0, #0", + "mov r7, {__NR_exit}", + "svc 0", + "udf #16", + __NR_exit = const __NR_exit, + in("r7") __NR_munmap, + in("r0") map_addr, + in("r1") map_len, + options(noreturn, nostack) + ); + } } #[cfg(feature = "take-charge")] diff --git a/src/arch/riscv64.rs b/src/arch/riscv64.rs index 5a57f1f..f74d1a8 100644 --- a/src/arch/riscv64.rs +++ b/src/arch/riscv64.rs @@ -104,19 +104,21 @@ pub(super) fn ehdr_addr() -> *const Elf_Ehdr { #[cfg(relocation_model = "pic")] #[inline] pub(super) unsafe fn relocation_load(ptr: usize) -> usize { - let r0; + unsafe { + let r0; - // This is read-only but we don't use `readonly` because this memory access - // happens outside the Rust memory model. As far as Rust knows, this is - // just an arbitrary side-effecting opaque operation. - asm!( - "ld {}, 0({})", - out(reg) r0, - in(reg) ptr, - options(nostack, preserves_flags), - ); + // This is read-only but we don't use `readonly` because this memory access + // happens outside the Rust memory model. As far as Rust knows, this is + // just an arbitrary side-effecting opaque operation. + asm!( + "ld {}, 0({})", + out(reg) r0, + in(reg) ptr, + options(nostack, preserves_flags), + ); - r0 + r0 + } } /// Perform a raw load operation to memory that Rust may consider out of @@ -131,16 +133,18 @@ pub(super) unsafe fn relocation_load(ptr: usize) -> usize { #[cfg(all(feature = "take-charge", not(feature = "optimize_for_size")))] #[inline] pub(super) unsafe fn oob_load(ptr: *const usize) -> usize { - let r0; + unsafe { + let r0; - asm!( - "ld {}, 0({})", - out(reg) r0, - in(reg) ptr, - options(nostack, preserves_flags, readonly), - ); + asm!( + "ld {}, 0({})", + out(reg) r0, + in(reg) ptr, + options(nostack, preserves_flags, readonly), + ); - r0 + r0 + } } /// Perform a single store operation, outside the Rust memory model. @@ -159,12 +163,14 @@ pub(super) unsafe fn oob_load(ptr: *const usize) -> usize { #[cfg(relocation_model = "pic")] #[inline] pub(super) unsafe fn relocation_store(ptr: usize, value: usize) { - asm!( - "sd {}, 0({})", - in(reg) value, - in(reg) ptr, - options(nostack, preserves_flags), - ); + unsafe { + asm!( + "sd {}, 0({})", + in(reg) value, + in(reg) ptr, + options(nostack, preserves_flags), + ); + } } /// Mark “relro” memory as readonly. @@ -187,24 +193,26 @@ pub(super) unsafe fn relocation_store(ptr: usize, value: usize) { #[cfg(relocation_model = "pic")] #[inline] pub(super) unsafe fn relocation_mprotect_readonly(ptr: usize, len: usize) { - let r0: usize; + unsafe { + let r0: usize; - // This is read-only but we don't use `readonly` because the side effects - // happen outside the Rust memory model. As far as Rust knows, this is - // just an arbitrary side-effecting opaque operation. - asm!( - "ecall", - in("a7") __NR_mprotect, - inlateout("a0") ptr as usize => r0, - in("a1") len, - in("a2") PROT_READ, - options(nostack, preserves_flags), - ); + // This is read-only but we don't use `readonly` because the side effects + // happen outside the Rust memory model. As far as Rust knows, this is + // just an arbitrary side-effecting opaque operation. + asm!( + "ecall", + in("a7") __NR_mprotect, + inlateout("a0") ptr as usize => r0, + in("a1") len, + in("a2") PROT_READ, + options(nostack, preserves_flags), + ); - if r0 != 0 { - // Do not panic here as libstd's panic handler needs TLS, which is not - // yet initialized at this point. - trap(); + if r0 != 0 { + // Do not panic here as libstd's panic handler needs TLS, which is not + // yet initialized at this point. + trap(); + } } } @@ -230,34 +238,36 @@ pub(super) unsafe fn clone( fn_: extern "C" fn(), num_args: usize, ) -> isize { - let r0; - asm!( - "ecall", // Do the `clone` system call. - "bnez a0, 0f", // Branch if we're in the parent thread. + unsafe { + let r0; + asm!( + "ecall", // Do the `clone` system call. + "bnez a0, 0f", // Branch if we're in the parent thread. - // Child thread. - "mv a0, {fn_}", // Pass `fn_` as the first argument. - "mv a1, sp", // Pass the args pointer as the second argument. - "mv a2, {num_args}", // Pass `num_args` as the third argument. - "mv fp, zero", // Zero the frame address. - "mv ra, zero", // Zero the return address. - "tail {entry}", // Call `entry`. + // Child thread. + "mv a0, {fn_}", // Pass `fn_` as the first argument. + "mv a1, sp", // Pass the args pointer as the second argument. + "mv a2, {num_args}", // Pass `num_args` as the third argument. + "mv fp, zero", // Zero the frame address. + "mv ra, zero", // Zero the return address. + "tail {entry}", // Call `entry`. - // Parent thread. - "0:", + // Parent thread. + "0:", - entry = sym super::thread::entry, - fn_ = in(reg) fn_, - num_args = in(reg) num_args, - in("a7") __NR_clone, - inlateout("a0") flags as usize => r0, - in("a1") child_stack, - in("a2") parent_tid, - in("a3") newtls, - in("a4") child_tid, - options(nostack) - ); - r0 + entry = sym super::thread::entry, + fn_ = in(reg) fn_, + num_args = in(reg) num_args, + in("a7") __NR_clone, + inlateout("a0") flags as usize => r0, + in("a1") child_stack, + in("a2") parent_tid, + in("a3") newtls, + in("a4") child_tid, + options(nostack) + ); + r0 + } } /// Write a value to the platform thread-pointer register. @@ -265,8 +275,10 @@ pub(super) unsafe fn clone( #[cfg(feature = "thread")] #[inline] pub(super) unsafe fn set_thread_pointer(ptr: *mut c_void) { - asm!("mv tp, {}", in(reg) ptr); - debug_assert_eq!(thread_pointer(), ptr); + unsafe { + asm!("mv tp, {}", in(reg) ptr); + debug_assert_eq!(thread_pointer(), ptr); + } } /// Read the value of the platform thread-pointer register. @@ -294,18 +306,20 @@ pub(super) const TLS_OFFSET: usize = 0x800; #[cfg(feature = "thread")] #[inline] pub(super) unsafe fn munmap_and_exit_thread(map_addr: *mut c_void, map_len: usize) -> ! { - asm!( - "ecall", - "mv a0, zero", - "li a7, {__NR_exit}", - "ecall", - "unimp", - __NR_exit = const __NR_exit, - in("a7") __NR_munmap, - in("a0") map_addr, - in("a1") map_len, - options(noreturn, nostack) - ); + unsafe { + asm!( + "ecall", + "mv a0, zero", + "li a7, {__NR_exit}", + "ecall", + "unimp", + __NR_exit = const __NR_exit, + in("a7") __NR_munmap, + in("a0") map_addr, + in("a1") map_len, + options(noreturn, nostack) + ); + } } // RISC-V doesn't use `__NR_rt_sigreturn` diff --git a/src/arch/x86.rs b/src/arch/x86.rs index ba96e6f..7c1fea1 100644 --- a/src/arch/x86.rs +++ b/src/arch/x86.rs @@ -137,19 +137,21 @@ pub(super) fn ehdr_addr() -> *const Elf_Ehdr { #[cfg(relocation_model = "pic")] #[inline] pub(super) unsafe fn relocation_load(ptr: usize) -> usize { - let r0; - - // This is read-only but we don't use `readonly` because this memory access - // happens outside the Rust memory model. As far as Rust knows, this is - // just an arbitrary side-effecting opaque operation. - asm!( - "mov {}, [{}]", - out(reg) r0, - in(reg) ptr, - options(nostack, preserves_flags), - ); - - r0 + unsafe { + let r0; + + // This is read-only but we don't use `readonly` because this memory access + // happens outside the Rust memory model. As far as Rust knows, this is + // just an arbitrary side-effecting opaque operation. + asm!( + "mov {}, [{}]", + out(reg) r0, + in(reg) ptr, + options(nostack, preserves_flags), + ); + + r0 + } } /// Perform a single store operation, outside the Rust memory model. @@ -168,12 +170,14 @@ pub(super) unsafe fn relocation_load(ptr: usize) -> usize { #[cfg(relocation_model = "pic")] #[inline] pub(super) unsafe fn relocation_store(ptr: usize, value: usize) { - asm!( - "mov [{}], {}", - in(reg) ptr, - in(reg) value, - options(nostack, preserves_flags), - ); + unsafe { + asm!( + "mov [{}], {}", + in(reg) ptr, + in(reg) value, + options(nostack, preserves_flags), + ); + } } /// Mark “relro” memory as readonly. @@ -196,24 +200,26 @@ pub(super) unsafe fn relocation_store(ptr: usize, value: usize) { #[cfg(relocation_model = "pic")] #[inline] pub(super) unsafe fn relocation_mprotect_readonly(ptr: usize, len: usize) { - let r0: usize; - - // This is read-only but we don't use `readonly` because the side effects - // happen outside the Rust memory model. As far as Rust knows, this is - // just an arbitrary side-effecting opaque operation. - asm!( - "int 0x80", - inlateout("eax") __NR_mprotect as usize => r0, - in("ebx") ptr, - in("ecx") len, - in("edx") PROT_READ, - options(nostack, preserves_flags), - ); - - if r0 != 0 { - // Do not panic here as libstd's panic handler needs TLS, which is not - // yet initialized at this point. - trap(); + unsafe { + let r0: usize; + + // This is read-only but we don't use `readonly` because the side effects + // happen outside the Rust memory model. As far as Rust knows, this is + // just an arbitrary side-effecting opaque operation. + asm!( + "int 0x80", + inlateout("eax") __NR_mprotect as usize => r0, + in("ebx") ptr, + in("ecx") len, + in("edx") PROT_READ, + options(nostack, preserves_flags), + ); + + if r0 != 0 { + // Do not panic here as libstd's panic handler needs TLS, which is not + // yet initialized at this point. + trap(); + } } } @@ -239,80 +245,82 @@ pub(super) unsafe fn clone( fn_: extern "C" fn(), num_args: usize, ) -> isize { - let mut gs: u32 = 0; - asm!("mov {0:x}, gs", inout(reg) gs); - let entry_number = gs >> 3; - - let mut user_desc = rustix::runtime::UserDesc { - entry_number, - base_addr: newtls.addr() as u32, - limit: 0xfffff, - _bitfield_align_1: [], - _bitfield_1: Default::default(), - __bindgen_padding_0: [0_u8; 3_usize], - }; - user_desc.set_seg_32bit(1); - user_desc.set_contents(0); - user_desc.set_read_exec_only(0); - user_desc.set_limit_in_pages(1); - user_desc.set_seg_not_present(0); - user_desc.set_useable(1); - let newtls: *const _ = &user_desc; - - // Push `fn_` to the child stack, since after all the `clone` args, and - // `num_args` in `ebp`, there are no more free registers. - let child_stack = child_stack.cast::<*mut c_void>().sub(1); - child_stack.write(fn_ as _); - - // See the comments for x86's `syscall6` in `rustix`. Inline asm isn't - // allowed to name ebp or esi as operands, so we have to jump through - // extra hoops here. - let r0; - asm!( - "push esi", // Save incoming register value. - "push ebp", // Save incoming register value. - - // Pass `num_args` to the child in `ebp`. - "mov ebp, [eax+8]", - - "mov esi, [eax+0]", // Pass `newtls` to the `int 0x80`. - "mov eax, [eax+4]", // Pass `__NR_clone` to the `int 0x80`. - - // Use `int 0x80` instead of vsyscall, following `clone`'s - // documentation; vsyscall would attempt to return to the parent stack - // in the child. - "int 0x80", // Do the `clone` system call. - "test eax, eax", // Branch if we're in the parent. - "jnz 2f", - - // Child thread. - "pop edi", // Load `fn_` from the child stack. - "mov esi, esp", // Snapshot the args pointer. - "push eax", // Pad for stack pointer alignment. - "push ebp", // Pass `num_args` as the third argument. - "push esi", // Pass the args pointer as the second argument. - "push edi", // Pass `fn_` as the first argument. - "xor ebp, ebp", // Zero the frame address. - "push eax", // Zero the return address. - "jmp {entry}", // Call `entry`. - - // Parent thread. - "2:", - "pop ebp", // Restore incoming register value. - "pop esi", // Restore incoming register value. - - entry = sym super::thread::entry, - inout("eax") &[ - newtls.cast::().cast_mut(), - without_provenance_mut(__NR_clone as usize), - without_provenance_mut(num_args) - ] => r0, - in("ebx") flags, - in("ecx") child_stack, - in("edx") parent_tid, - in("edi") child_tid, - ); - r0 + unsafe { + let mut gs: u32 = 0; + asm!("mov {0:x}, gs", inout(reg) gs); + let entry_number = gs >> 3; + + let mut user_desc = rustix::runtime::UserDesc { + entry_number, + base_addr: newtls.addr() as u32, + limit: 0xfffff, + _bitfield_align_1: [], + _bitfield_1: Default::default(), + __bindgen_padding_0: [0_u8; 3_usize], + }; + user_desc.set_seg_32bit(1); + user_desc.set_contents(0); + user_desc.set_read_exec_only(0); + user_desc.set_limit_in_pages(1); + user_desc.set_seg_not_present(0); + user_desc.set_useable(1); + let newtls: *const _ = &user_desc; + + // Push `fn_` to the child stack, since after all the `clone` args, and + // `num_args` in `ebp`, there are no more free registers. + let child_stack = child_stack.cast::<*mut c_void>().sub(1); + child_stack.write(fn_ as _); + + // See the comments for x86's `syscall6` in `rustix`. Inline asm isn't + // allowed to name ebp or esi as operands, so we have to jump through + // extra hoops here. + let r0; + asm!( + "push esi", // Save incoming register value. + "push ebp", // Save incoming register value. + + // Pass `num_args` to the child in `ebp`. + "mov ebp, [eax+8]", + + "mov esi, [eax+0]", // Pass `newtls` to the `int 0x80`. + "mov eax, [eax+4]", // Pass `__NR_clone` to the `int 0x80`. + + // Use `int 0x80` instead of vsyscall, following `clone`'s + // documentation; vsyscall would attempt to return to the parent stack + // in the child. + "int 0x80", // Do the `clone` system call. + "test eax, eax", // Branch if we're in the parent. + "jnz 2f", + + // Child thread. + "pop edi", // Load `fn_` from the child stack. + "mov esi, esp", // Snapshot the args pointer. + "push eax", // Pad for stack pointer alignment. + "push ebp", // Pass `num_args` as the third argument. + "push esi", // Pass the args pointer as the second argument. + "push edi", // Pass `fn_` as the first argument. + "xor ebp, ebp", // Zero the frame address. + "push eax", // Zero the return address. + "jmp {entry}", // Call `entry`. + + // Parent thread. + "2:", + "pop ebp", // Restore incoming register value. + "pop esi", // Restore incoming register value. + + entry = sym super::thread::entry, + inout("eax") &[ + newtls.cast::().cast_mut(), + without_provenance_mut(__NR_clone as usize), + without_provenance_mut(num_args) + ] => r0, + in("ebx") flags, + in("ecx") child_stack, + in("edx") parent_tid, + in("edi") child_tid, + ); + r0 + } } /// Write a value to the platform thread-pointer register. @@ -320,27 +328,29 @@ pub(super) unsafe fn clone( #[cfg(feature = "thread")] #[inline] pub(super) unsafe fn set_thread_pointer(ptr: *mut c_void) { - let mut user_desc = rustix::runtime::UserDesc { - entry_number: !0u32, - base_addr: ptr.addr() as u32, - limit: 0xfffff, - _bitfield_align_1: [], - _bitfield_1: Default::default(), - __bindgen_padding_0: [0_u8; 3_usize], - }; - user_desc.set_seg_32bit(1); - user_desc.set_contents(0); - user_desc.set_read_exec_only(0); - user_desc.set_limit_in_pages(1); - user_desc.set_seg_not_present(0); - user_desc.set_useable(1); - let res = rustix::runtime::set_thread_area(&mut user_desc); - debug_assert_eq!(res, Ok(())); - debug_assert_ne!(user_desc.entry_number, !0); - - asm!("mov gs, {0:x}", in(reg) ((user_desc.entry_number << 3) | 3) as u16); - debug_assert_eq!(*ptr.cast::<*const c_void>(), ptr); - debug_assert_eq!(thread_pointer(), ptr); + unsafe { + let mut user_desc = rustix::runtime::UserDesc { + entry_number: !0u32, + base_addr: ptr.addr() as u32, + limit: 0xfffff, + _bitfield_align_1: [], + _bitfield_1: Default::default(), + __bindgen_padding_0: [0_u8; 3_usize], + }; + user_desc.set_seg_32bit(1); + user_desc.set_contents(0); + user_desc.set_read_exec_only(0); + user_desc.set_limit_in_pages(1); + user_desc.set_seg_not_present(0); + user_desc.set_useable(1); + let res = rustix::runtime::set_thread_area(&mut user_desc); + debug_assert_eq!(res, Ok(())); + debug_assert_ne!(user_desc.entry_number, !0); + + asm!("mov gs, {0:x}", in(reg) ((user_desc.entry_number << 3) | 3) as u16); + debug_assert_eq!(*ptr.cast::<*const c_void>(), ptr); + debug_assert_eq!(thread_pointer(), ptr); + } } /// Read the value of the platform thread-pointer register. @@ -370,20 +380,22 @@ pub(super) const TLS_OFFSET: usize = 0; #[cfg(feature = "thread")] #[inline] pub(super) unsafe fn munmap_and_exit_thread(map_addr: *mut c_void, map_len: usize) -> ! { - asm!( - // Use `int 0x80` instead of vsyscall, since vsyscall would attempt to - // touch the stack after we `munmap` it. - "int 0x80", - "xor ebx, ebx", - "mov eax, {__NR_exit}", - "int 0x80", - "ud2", - __NR_exit = const __NR_exit, - in("eax") __NR_munmap, - in("ebx") map_addr, - in("ecx") map_len, - options(noreturn, nostack) - ); + unsafe { + asm!( + // Use `int 0x80` instead of vsyscall, since vsyscall would attempt to + // touch the stack after we `munmap` it. + "int 0x80", + "xor ebx, ebx", + "mov eax, {__NR_exit}", + "int 0x80", + "ud2", + __NR_exit = const __NR_exit, + in("eax") __NR_munmap, + in("ebx") map_addr, + in("ecx") map_len, + options(noreturn, nostack) + ); + } } #[cfg(feature = "take-charge")] From 04be16af44d2f978caf0c4bcd6f12c840191e722 Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Tue, 29 Apr 2025 12:20:49 -0700 Subject: [PATCH 11/15] More. --- src/mem/impls.rs | 474 ++++++++++++++++++++++++----------------------- 1 file changed, 243 insertions(+), 231 deletions(-) diff --git a/src/mem/impls.rs b/src/mem/impls.rs index 8cf263d..9d9dd9b 100644 --- a/src/mem/impls.rs +++ b/src/mem/impls.rs @@ -27,284 +27,296 @@ const WORD_COPY_THRESHOLD: usize = if 2 * WORD_SIZE > 16 { target_arch = "bpf" ))] // These targets have hardware unaligned access support. unsafe fn read_usize_unaligned(x: *const usize) -> usize { - // Do not use `core::ptr::read_unaligned` here, since it calls `copy_nonoverlapping` which - // is translated to memcpy in LLVM. - let x_read = (x as *const [u8; core::mem::size_of::()]).read(); - usize::from_ne_bytes(x_read) + unsafe { + // Do not use `core::ptr::read_unaligned` here, since it calls `copy_nonoverlapping` which + // is translated to memcpy in LLVM. + let x_read = (x as *const [u8; core::mem::size_of::()]).read(); + usize::from_ne_bytes(x_read) + } } #[inline(always)] pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize) { - #[inline(always)] - unsafe fn copy_forward_bytes(mut dest: *mut u8, mut src: *const u8, n: usize) { - let dest_end = dest.add(n); - while dest < dest_end { - *dest = *src; - dest = dest.add(1); - src = src.add(1); + unsafe { + #[inline(always)] + unsafe fn copy_forward_bytes(mut dest: *mut u8, mut src: *const u8, n: usize) { + let dest_end = dest.add(n); + while dest < dest_end { + *dest = *src; + dest = dest.add(1); + src = src.add(1); + } } - } - - #[inline(always)] - unsafe fn copy_forward_aligned_words(dest: *mut u8, src: *const u8, n: usize) { - let mut dest_usize = dest as *mut usize; - let mut src_usize = src as *mut usize; - let dest_end = dest.add(n) as *mut usize; - while dest_usize < dest_end { - *dest_usize = *src_usize; - dest_usize = dest_usize.add(1); - src_usize = src_usize.add(1); + #[inline(always)] + unsafe fn copy_forward_aligned_words(dest: *mut u8, src: *const u8, n: usize) { + let mut dest_usize = dest as *mut usize; + let mut src_usize = src as *mut usize; + let dest_end = dest.add(n) as *mut usize; + + while dest_usize < dest_end { + *dest_usize = *src_usize; + dest_usize = dest_usize.add(1); + src_usize = src_usize.add(1); + } } - } - #[cfg(not(any( - target_arch = "x86_64", - target_arch = "x86", - target_arch = "aarch64", - target_arch = "bpf" - )))] - #[inline(always)] - unsafe fn copy_forward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) { - let mut dest_usize = dest as *mut usize; - let dest_end = dest.add(n) as *mut usize; - - // Calculate the misalignment offset and shift needed to reassemble value. - let offset = src.addr() & WORD_MASK; - let shift = offset * 8; - - // Realign src - let mut src_aligned = src.with_addr(src.addr() & !WORD_MASK).cast::(); - // This will read (but won't use) bytes out of bound. - let mut prev_word = crate::arch::oob_load(src_aligned); - - while dest_usize < dest_end { - src_aligned = src_aligned.add(1); - let cur_word = *src_aligned; - #[cfg(target_endian = "little")] - let resembled = prev_word >> shift | cur_word << (WORD_SIZE * 8 - shift); - #[cfg(target_endian = "big")] - let resembled = prev_word << shift | cur_word >> (WORD_SIZE * 8 - shift); - prev_word = cur_word; - - *dest_usize = resembled; - dest_usize = dest_usize.add(1); + #[cfg(not(any( + target_arch = "x86_64", + target_arch = "x86", + target_arch = "aarch64", + target_arch = "bpf" + )))] + #[inline(always)] + unsafe fn copy_forward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) { + let mut dest_usize = dest as *mut usize; + let dest_end = dest.add(n) as *mut usize; + + // Calculate the misalignment offset and shift needed to reassemble value. + let offset = src.addr() & WORD_MASK; + let shift = offset * 8; + + // Realign src + let mut src_aligned = src.with_addr(src.addr() & !WORD_MASK).cast::(); + // This will read (but won't use) bytes out of bound. + let mut prev_word = crate::arch::oob_load(src_aligned); + + while dest_usize < dest_end { + src_aligned = src_aligned.add(1); + let cur_word = *src_aligned; + #[cfg(target_endian = "little")] + let resembled = prev_word >> shift | cur_word << (WORD_SIZE * 8 - shift); + #[cfg(target_endian = "big")] + let resembled = prev_word << shift | cur_word >> (WORD_SIZE * 8 - shift); + prev_word = cur_word; + + *dest_usize = resembled; + dest_usize = dest_usize.add(1); + } } - } - #[cfg(any( - target_arch = "x86_64", - target_arch = "x86", - target_arch = "aarch64", - target_arch = "bpf" - ))] - #[inline(always)] - unsafe fn copy_forward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) { - let mut dest_usize = dest as *mut usize; - let mut src_usize = src as *mut usize; - let dest_end = dest.add(n) as *mut usize; - - while dest_usize < dest_end { - *dest_usize = read_usize_unaligned(src_usize); - dest_usize = dest_usize.add(1); - src_usize = src_usize.add(1); + #[cfg(any( + target_arch = "x86_64", + target_arch = "x86", + target_arch = "aarch64", + target_arch = "bpf" + ))] + #[inline(always)] + unsafe fn copy_forward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) { + let mut dest_usize = dest as *mut usize; + let mut src_usize = src as *mut usize; + let dest_end = dest.add(n) as *mut usize; + + while dest_usize < dest_end { + *dest_usize = read_usize_unaligned(src_usize); + dest_usize = dest_usize.add(1); + src_usize = src_usize.add(1); + } } - } - if n >= WORD_COPY_THRESHOLD { - // Align dest - // Because of n >= 2 * WORD_SIZE, dst_misalignment < n - let dest_misalignment = (dest.addr()).wrapping_neg() & WORD_MASK; - copy_forward_bytes(dest, src, dest_misalignment); - dest = dest.add(dest_misalignment); - src = src.add(dest_misalignment); - n -= dest_misalignment; - - let n_words = n & !WORD_MASK; - let src_misalignment = src.addr() & WORD_MASK; - if src_misalignment == 0 { - copy_forward_aligned_words(dest, src, n_words); - } else { - copy_forward_misaligned_words(dest, src, n_words); + if n >= WORD_COPY_THRESHOLD { + // Align dest + // Because of n >= 2 * WORD_SIZE, dst_misalignment < n + let dest_misalignment = (dest.addr()).wrapping_neg() & WORD_MASK; + copy_forward_bytes(dest, src, dest_misalignment); + dest = dest.add(dest_misalignment); + src = src.add(dest_misalignment); + n -= dest_misalignment; + + let n_words = n & !WORD_MASK; + let src_misalignment = src.addr() & WORD_MASK; + if src_misalignment == 0 { + copy_forward_aligned_words(dest, src, n_words); + } else { + copy_forward_misaligned_words(dest, src, n_words); + } + dest = dest.add(n_words); + src = src.add(n_words); + n -= n_words; } - dest = dest.add(n_words); - src = src.add(n_words); - n -= n_words; + copy_forward_bytes(dest, src, n); } - copy_forward_bytes(dest, src, n); } #[inline(always)] pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) { - // The following backward copy helper functions uses the pointers past the end - // as their inputs instead of pointers to the start! - #[inline(always)] - unsafe fn copy_backward_bytes(mut dest: *mut u8, mut src: *const u8, n: usize) { - let dest_start = dest.sub(n); - while dest_start < dest { - dest = dest.sub(1); - src = src.sub(1); - *dest = *src; + unsafe { + // The following backward copy helper functions uses the pointers past the end + // as their inputs instead of pointers to the start! + #[inline(always)] + unsafe fn copy_backward_bytes(mut dest: *mut u8, mut src: *const u8, n: usize) { + let dest_start = dest.sub(n); + while dest_start < dest { + dest = dest.sub(1); + src = src.sub(1); + *dest = *src; + } } - } - #[inline(always)] - unsafe fn copy_backward_aligned_words(dest: *mut u8, src: *const u8, n: usize) { - let mut dest_usize = dest as *mut usize; - let mut src_usize = src as *mut usize; - let dest_start = dest.sub(n) as *mut usize; - - while dest_start < dest_usize { - dest_usize = dest_usize.sub(1); - src_usize = src_usize.sub(1); - *dest_usize = *src_usize; + #[inline(always)] + unsafe fn copy_backward_aligned_words(dest: *mut u8, src: *const u8, n: usize) { + let mut dest_usize = dest as *mut usize; + let mut src_usize = src as *mut usize; + let dest_start = dest.sub(n) as *mut usize; + + while dest_start < dest_usize { + dest_usize = dest_usize.sub(1); + src_usize = src_usize.sub(1); + *dest_usize = *src_usize; + } } - } - #[cfg(not(any( - target_arch = "x86_64", - target_arch = "x86", - target_arch = "aarch64", - target_arch = "bpf" - )))] - #[inline(always)] - unsafe fn copy_backward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) { - let mut dest_usize = dest as *mut usize; - let dest_start = dest.sub(n) as *mut usize; - - // Calculate the misalignment offset and shift needed to reassemble value. - let offset = src.addr() & WORD_MASK; - let shift = offset * 8; - - // Realign src_aligned - let mut src_aligned = src.with_addr(src.addr() & !WORD_MASK).cast::(); - // This will read (but won't use) bytes out of bound. - let mut prev_word = crate::arch::oob_load(src_aligned); - - while dest_start < dest_usize { - src_aligned = src_aligned.sub(1); - let cur_word = *src_aligned; - #[cfg(target_endian = "little")] - let resembled = prev_word << (WORD_SIZE * 8 - shift) | cur_word >> shift; - #[cfg(target_endian = "big")] - let resembled = prev_word >> (WORD_SIZE * 8 - shift) | cur_word << shift; - prev_word = cur_word; - - dest_usize = dest_usize.sub(1); - *dest_usize = resembled; + #[cfg(not(any( + target_arch = "x86_64", + target_arch = "x86", + target_arch = "aarch64", + target_arch = "bpf" + )))] + #[inline(always)] + unsafe fn copy_backward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) { + let mut dest_usize = dest as *mut usize; + let dest_start = dest.sub(n) as *mut usize; + + // Calculate the misalignment offset and shift needed to reassemble value. + let offset = src.addr() & WORD_MASK; + let shift = offset * 8; + + // Realign src_aligned + let mut src_aligned = src.with_addr(src.addr() & !WORD_MASK).cast::(); + // This will read (but won't use) bytes out of bound. + let mut prev_word = crate::arch::oob_load(src_aligned); + + while dest_start < dest_usize { + src_aligned = src_aligned.sub(1); + let cur_word = *src_aligned; + #[cfg(target_endian = "little")] + let resembled = prev_word << (WORD_SIZE * 8 - shift) | cur_word >> shift; + #[cfg(target_endian = "big")] + let resembled = prev_word >> (WORD_SIZE * 8 - shift) | cur_word << shift; + prev_word = cur_word; + + dest_usize = dest_usize.sub(1); + *dest_usize = resembled; + } } - } - #[cfg(any( - target_arch = "x86_64", - target_arch = "x86", - target_arch = "aarch64", - target_arch = "bpf" - ))] - #[inline(always)] - unsafe fn copy_backward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) { - let mut dest_usize = dest as *mut usize; - let mut src_usize = src as *mut usize; - let dest_start = dest.sub(n) as *mut usize; - - while dest_start < dest_usize { - dest_usize = dest_usize.sub(1); - src_usize = src_usize.sub(1); - *dest_usize = read_usize_unaligned(src_usize); + #[cfg(any( + target_arch = "x86_64", + target_arch = "x86", + target_arch = "aarch64", + target_arch = "bpf" + ))] + #[inline(always)] + unsafe fn copy_backward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) { + let mut dest_usize = dest as *mut usize; + let mut src_usize = src as *mut usize; + let dest_start = dest.sub(n) as *mut usize; + + while dest_start < dest_usize { + dest_usize = dest_usize.sub(1); + src_usize = src_usize.sub(1); + *dest_usize = read_usize_unaligned(src_usize); + } } - } - let mut dest = dest.add(n); - let mut src = src.add(n); - - if n >= WORD_COPY_THRESHOLD { - // Align dest - // Because of n >= 2 * WORD_SIZE, dst_misalignment < n - let dest_misalignment = dest.addr() & WORD_MASK; - copy_backward_bytes(dest, src, dest_misalignment); - dest = dest.sub(dest_misalignment); - src = src.sub(dest_misalignment); - n -= dest_misalignment; - - let n_words = n & !WORD_MASK; - let src_misalignment = src.addr() & WORD_MASK; - if src_misalignment == 0 { - copy_backward_aligned_words(dest, src, n_words); - } else { - copy_backward_misaligned_words(dest, src, n_words); + let mut dest = dest.add(n); + let mut src = src.add(n); + + if n >= WORD_COPY_THRESHOLD { + // Align dest + // Because of n >= 2 * WORD_SIZE, dst_misalignment < n + let dest_misalignment = dest.addr() & WORD_MASK; + copy_backward_bytes(dest, src, dest_misalignment); + dest = dest.sub(dest_misalignment); + src = src.sub(dest_misalignment); + n -= dest_misalignment; + + let n_words = n & !WORD_MASK; + let src_misalignment = src.addr() & WORD_MASK; + if src_misalignment == 0 { + copy_backward_aligned_words(dest, src, n_words); + } else { + copy_backward_misaligned_words(dest, src, n_words); + } + dest = dest.sub(n_words); + src = src.sub(n_words); + n -= n_words; } - dest = dest.sub(n_words); - src = src.sub(n_words); - n -= n_words; + copy_backward_bytes(dest, src, n); } - copy_backward_bytes(dest, src, n); } #[inline(always)] pub unsafe fn set_bytes(mut s: *mut u8, c: u8, mut n: usize) { - #[inline(always)] - pub unsafe fn set_bytes_bytes(mut s: *mut u8, c: u8, n: usize) { - let end = s.add(n); - while s < end { - *s = c; - s = s.add(1); + unsafe { + #[inline(always)] + pub unsafe fn set_bytes_bytes(mut s: *mut u8, c: u8, n: usize) { + let end = s.add(n); + while s < end { + *s = c; + s = s.add(1); + } } - } - #[inline(always)] - pub unsafe fn set_bytes_words(s: *mut u8, c: u8, n: usize) { - let mut broadcast = c as usize; - let mut bits = 8; - while bits < WORD_SIZE * 8 { - broadcast |= broadcast << bits; - bits *= 2; + #[inline(always)] + pub unsafe fn set_bytes_words(s: *mut u8, c: u8, n: usize) { + let mut broadcast = c as usize; + let mut bits = 8; + while bits < WORD_SIZE * 8 { + broadcast |= broadcast << bits; + bits *= 2; + } + + let mut s_usize = s as *mut usize; + let end = s.add(n) as *mut usize; + + while s_usize < end { + *s_usize = broadcast; + s_usize = s_usize.add(1); + } } - let mut s_usize = s as *mut usize; - let end = s.add(n) as *mut usize; - - while s_usize < end { - *s_usize = broadcast; - s_usize = s_usize.add(1); + if n >= WORD_COPY_THRESHOLD { + // Align s + // Because of n >= 2 * WORD_SIZE, dst_misalignment < n + let misalignment = (s.addr()).wrapping_neg() & WORD_MASK; + set_bytes_bytes(s, c, misalignment); + s = s.add(misalignment); + n -= misalignment; + + let n_words = n & !WORD_MASK; + set_bytes_words(s, c, n_words); + s = s.add(n_words); + n -= n_words; } + set_bytes_bytes(s, c, n); } - - if n >= WORD_COPY_THRESHOLD { - // Align s - // Because of n >= 2 * WORD_SIZE, dst_misalignment < n - let misalignment = (s.addr()).wrapping_neg() & WORD_MASK; - set_bytes_bytes(s, c, misalignment); - s = s.add(misalignment); - n -= misalignment; - - let n_words = n & !WORD_MASK; - set_bytes_words(s, c, n_words); - s = s.add(n_words); - n -= n_words; - } - set_bytes_bytes(s, c, n); } #[inline(always)] pub unsafe fn compare_bytes(s1: *const u8, s2: *const u8, n: usize) -> i32 { - let mut i = 0; - while i < n { - let a = *s1.add(i); - let b = *s2.add(i); - if a != b { - return a as i32 - b as i32; + unsafe { + let mut i = 0; + while i < n { + let a = *s1.add(i); + let b = *s2.add(i); + if a != b { + return a as i32 - b as i32; + } + i += 1; } - i += 1; + 0 } - 0 } #[inline(always)] pub unsafe fn c_string_length(mut s: *const core::ffi::c_char) -> usize { - let mut n = 0; - while *s != 0 { - n += 1; - s = s.add(1); + unsafe { + let mut n = 0; + while *s != 0 { + n += 1; + s = s.add(1); + } + n } - n } From 4a428148bdf967e299c0f6d5a1be8778e1127d21 Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Tue, 29 Apr 2025 12:24:19 -0700 Subject: [PATCH 12/15] More. --- src/mem/impls.rs | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/src/mem/impls.rs b/src/mem/impls.rs index 9d9dd9b..f49ba18 100644 --- a/src/mem/impls.rs +++ b/src/mem/impls.rs @@ -251,28 +251,32 @@ pub unsafe fn set_bytes(mut s: *mut u8, c: u8, mut n: usize) { unsafe { #[inline(always)] pub unsafe fn set_bytes_bytes(mut s: *mut u8, c: u8, n: usize) { - let end = s.add(n); - while s < end { - *s = c; - s = s.add(1); + unsafe { + let end = s.add(n); + while s < end { + *s = c; + s = s.add(1); + } } } #[inline(always)] pub unsafe fn set_bytes_words(s: *mut u8, c: u8, n: usize) { - let mut broadcast = c as usize; - let mut bits = 8; - while bits < WORD_SIZE * 8 { - broadcast |= broadcast << bits; - bits *= 2; - } - - let mut s_usize = s as *mut usize; - let end = s.add(n) as *mut usize; - - while s_usize < end { - *s_usize = broadcast; - s_usize = s_usize.add(1); + unsafe { + let mut broadcast = c as usize; + let mut bits = 8; + while bits < WORD_SIZE * 8 { + broadcast |= broadcast << bits; + bits *= 2; + } + + let mut s_usize = s as *mut usize; + let end = s.add(n) as *mut usize; + + while s_usize < end { + *s_usize = broadcast; + s_usize = s_usize.add(1); + } } } From e767b7d541285f723ab4fc7a17ea88d9fdcaffdd Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Tue, 29 Apr 2025 12:32:19 -0700 Subject: [PATCH 13/15] More. --- src/mem/impls.rs | 192 +++++++++++++++++++++++++---------------------- 1 file changed, 104 insertions(+), 88 deletions(-) diff --git a/src/mem/impls.rs b/src/mem/impls.rs index f49ba18..72847cc 100644 --- a/src/mem/impls.rs +++ b/src/mem/impls.rs @@ -40,24 +40,28 @@ pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize) unsafe { #[inline(always)] unsafe fn copy_forward_bytes(mut dest: *mut u8, mut src: *const u8, n: usize) { - let dest_end = dest.add(n); - while dest < dest_end { - *dest = *src; - dest = dest.add(1); - src = src.add(1); + unsafe { + let dest_end = dest.add(n); + while dest < dest_end { + *dest = *src; + dest = dest.add(1); + src = src.add(1); + } } } #[inline(always)] unsafe fn copy_forward_aligned_words(dest: *mut u8, src: *const u8, n: usize) { - let mut dest_usize = dest as *mut usize; - let mut src_usize = src as *mut usize; - let dest_end = dest.add(n) as *mut usize; - - while dest_usize < dest_end { - *dest_usize = *src_usize; - dest_usize = dest_usize.add(1); - src_usize = src_usize.add(1); + unsafe { + let mut dest_usize = dest as *mut usize; + let mut src_usize = src as *mut usize; + let dest_end = dest.add(n) as *mut usize; + + while dest_usize < dest_end { + *dest_usize = *src_usize; + dest_usize = dest_usize.add(1); + src_usize = src_usize.add(1); + } } } @@ -69,29 +73,31 @@ pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize) )))] #[inline(always)] unsafe fn copy_forward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) { - let mut dest_usize = dest as *mut usize; - let dest_end = dest.add(n) as *mut usize; - - // Calculate the misalignment offset and shift needed to reassemble value. - let offset = src.addr() & WORD_MASK; - let shift = offset * 8; - - // Realign src - let mut src_aligned = src.with_addr(src.addr() & !WORD_MASK).cast::(); - // This will read (but won't use) bytes out of bound. - let mut prev_word = crate::arch::oob_load(src_aligned); - - while dest_usize < dest_end { - src_aligned = src_aligned.add(1); - let cur_word = *src_aligned; - #[cfg(target_endian = "little")] - let resembled = prev_word >> shift | cur_word << (WORD_SIZE * 8 - shift); - #[cfg(target_endian = "big")] - let resembled = prev_word << shift | cur_word >> (WORD_SIZE * 8 - shift); - prev_word = cur_word; - - *dest_usize = resembled; - dest_usize = dest_usize.add(1); + unsafe { + let mut dest_usize = dest as *mut usize; + let dest_end = dest.add(n) as *mut usize; + + // Calculate the misalignment offset and shift needed to reassemble value. + let offset = src.addr() & WORD_MASK; + let shift = offset * 8; + + // Realign src + let mut src_aligned = src.with_addr(src.addr() & !WORD_MASK).cast::(); + // This will read (but won't use) bytes out of bound. + let mut prev_word = crate::arch::oob_load(src_aligned); + + while dest_usize < dest_end { + src_aligned = src_aligned.add(1); + let cur_word = *src_aligned; + #[cfg(target_endian = "little")] + let resembled = prev_word >> shift | cur_word << (WORD_SIZE * 8 - shift); + #[cfg(target_endian = "big")] + let resembled = prev_word << shift | cur_word >> (WORD_SIZE * 8 - shift); + prev_word = cur_word; + + *dest_usize = resembled; + dest_usize = dest_usize.add(1); + } } } @@ -103,14 +109,16 @@ pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize) ))] #[inline(always)] unsafe fn copy_forward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) { - let mut dest_usize = dest as *mut usize; - let mut src_usize = src as *mut usize; - let dest_end = dest.add(n) as *mut usize; - - while dest_usize < dest_end { - *dest_usize = read_usize_unaligned(src_usize); - dest_usize = dest_usize.add(1); - src_usize = src_usize.add(1); + unsafe { + let mut dest_usize = dest as *mut usize; + let mut src_usize = src as *mut usize; + let dest_end = dest.add(n) as *mut usize; + + while dest_usize < dest_end { + *dest_usize = read_usize_unaligned(src_usize); + dest_usize = dest_usize.add(1); + src_usize = src_usize.add(1); + } } } @@ -145,24 +153,28 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) { // as their inputs instead of pointers to the start! #[inline(always)] unsafe fn copy_backward_bytes(mut dest: *mut u8, mut src: *const u8, n: usize) { - let dest_start = dest.sub(n); - while dest_start < dest { - dest = dest.sub(1); - src = src.sub(1); - *dest = *src; + unsafe { + let dest_start = dest.sub(n); + while dest_start < dest { + dest = dest.sub(1); + src = src.sub(1); + *dest = *src; + } } } #[inline(always)] unsafe fn copy_backward_aligned_words(dest: *mut u8, src: *const u8, n: usize) { - let mut dest_usize = dest as *mut usize; - let mut src_usize = src as *mut usize; - let dest_start = dest.sub(n) as *mut usize; - - while dest_start < dest_usize { - dest_usize = dest_usize.sub(1); - src_usize = src_usize.sub(1); - *dest_usize = *src_usize; + unsafe { + let mut dest_usize = dest as *mut usize; + let mut src_usize = src as *mut usize; + let dest_start = dest.sub(n) as *mut usize; + + while dest_start < dest_usize { + dest_usize = dest_usize.sub(1); + src_usize = src_usize.sub(1); + *dest_usize = *src_usize; + } } } @@ -174,29 +186,31 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) { )))] #[inline(always)] unsafe fn copy_backward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) { - let mut dest_usize = dest as *mut usize; - let dest_start = dest.sub(n) as *mut usize; - - // Calculate the misalignment offset and shift needed to reassemble value. - let offset = src.addr() & WORD_MASK; - let shift = offset * 8; - - // Realign src_aligned - let mut src_aligned = src.with_addr(src.addr() & !WORD_MASK).cast::(); - // This will read (but won't use) bytes out of bound. - let mut prev_word = crate::arch::oob_load(src_aligned); - - while dest_start < dest_usize { - src_aligned = src_aligned.sub(1); - let cur_word = *src_aligned; - #[cfg(target_endian = "little")] - let resembled = prev_word << (WORD_SIZE * 8 - shift) | cur_word >> shift; - #[cfg(target_endian = "big")] - let resembled = prev_word >> (WORD_SIZE * 8 - shift) | cur_word << shift; - prev_word = cur_word; - - dest_usize = dest_usize.sub(1); - *dest_usize = resembled; + unsafe { + let mut dest_usize = dest as *mut usize; + let dest_start = dest.sub(n) as *mut usize; + + // Calculate the misalignment offset and shift needed to reassemble value. + let offset = src.addr() & WORD_MASK; + let shift = offset * 8; + + // Realign src_aligned + let mut src_aligned = src.with_addr(src.addr() & !WORD_MASK).cast::(); + // This will read (but won't use) bytes out of bound. + let mut prev_word = crate::arch::oob_load(src_aligned); + + while dest_start < dest_usize { + src_aligned = src_aligned.sub(1); + let cur_word = *src_aligned; + #[cfg(target_endian = "little")] + let resembled = prev_word << (WORD_SIZE * 8 - shift) | cur_word >> shift; + #[cfg(target_endian = "big")] + let resembled = prev_word >> (WORD_SIZE * 8 - shift) | cur_word << shift; + prev_word = cur_word; + + dest_usize = dest_usize.sub(1); + *dest_usize = resembled; + } } } @@ -208,14 +222,16 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) { ))] #[inline(always)] unsafe fn copy_backward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) { - let mut dest_usize = dest as *mut usize; - let mut src_usize = src as *mut usize; - let dest_start = dest.sub(n) as *mut usize; - - while dest_start < dest_usize { - dest_usize = dest_usize.sub(1); - src_usize = src_usize.sub(1); - *dest_usize = read_usize_unaligned(src_usize); + unsafe { + let mut dest_usize = dest as *mut usize; + let mut src_usize = src as *mut usize; + let dest_start = dest.sub(n) as *mut usize; + + while dest_start < dest_usize { + dest_usize = dest_usize.sub(1); + src_usize = src_usize.sub(1); + *dest_usize = read_usize_unaligned(src_usize); + } } } From 76163f4226fdbfdf2998e2d429ef2926afee2846 Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Tue, 29 Apr 2025 12:37:23 -0700 Subject: [PATCH 14/15] More. --- src/getauxval.rs | 8 ++++---- src/program/linux_raw.rs | 2 +- src/thread/linux_raw.rs | 2 +- src/unwind_unimplemented.rs | 10 +++++----- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/getauxval.rs b/src/getauxval.rs index f5784dd..32e7d72 100644 --- a/src/getauxval.rs +++ b/src/getauxval.rs @@ -7,15 +7,15 @@ use core::ptr::without_provenance_mut; // `getauxval` usually returns `unsigned long`, but we make it a pointer type // so that it preserves provenance. -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn getauxval(type_: c_ulong) -> *mut c_void { - _getauxval(type_) + unsafe { _getauxval(type_) } } #[cfg(target_arch = "aarch64")] -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn __getauxval(type_: c_ulong) -> *mut c_void { - _getauxval(type_) + unsafe { _getauxval(type_) } } fn _getauxval(type_: c_ulong) -> *mut c_void { diff --git a/src/program/linux_raw.rs b/src/program/linux_raw.rs index e7158a1..9266b80 100644 --- a/src/program/linux_raw.rs +++ b/src/program/linux_raw.rs @@ -8,7 +8,7 @@ //! /// //! /// SAFETY: `argc`, `argv`, and `envp` describe incoming program //! /// command-line arguments and environment variables. -//! #[no_mangle] +//! #[unsafe(no_mangle)] //! unsafe fn origin_main(argc: usize, argv: *mut *mut u8, envp: *mut *mut u8) -> i32 { //! todo!("Run the program and return the program exit status.") //! } diff --git a/src/thread/linux_raw.rs b/src/thread/linux_raw.rs index fe19f6f..0870a92 100644 --- a/src/thread/linux_raw.rs +++ b/src/thread/linux_raw.rs @@ -1116,7 +1116,7 @@ pub fn yield_current() { /// The ARM ABI expects this to be defined. #[cfg(target_arch = "arm")] -#[no_mangle] +#[unsafe(no_mangle)] extern "C" fn __aeabi_read_tp() -> *mut c_void { thread_pointer() } diff --git a/src/unwind_unimplemented.rs b/src/unwind_unimplemented.rs index 509de39..65c9cbb 100644 --- a/src/unwind_unimplemented.rs +++ b/src/unwind_unimplemented.rs @@ -74,31 +74,31 @@ unsafe extern "C" fn _Unwind_GetCFA() { } #[cfg(target_arch = "arm")] -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_VRS_Get() { unimplemented!("_Unwind_VRS_Get") } #[cfg(target_arch = "arm")] -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_VRS_Set() { unimplemented!("_Unwind_VRS_Set") } #[cfg(target_arch = "arm")] -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn __aeabi_unwind_cpp_pr0() { unimplemented!("__aeabi_unwind_cpp_pr0") } #[cfg(target_arch = "arm")] -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn __aeabi_unwind_cpp_pr1() { unimplemented!("__aeabi_unwind_cpp_pr1") } #[cfg(target_arch = "arm")] -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn __gnu_unwind_frame() { unimplemented!("__gnu_unwind_frame") } From b3bf84968e0c8a1eadc295ab55b484ab6e7bfdea Mon Sep 17 00:00:00 2001 From: Dan Gohman Date: Tue, 29 Apr 2025 12:42:33 -0700 Subject: [PATCH 15/15] Fix warnings. --- src/getauxval.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/getauxval.rs b/src/getauxval.rs index 32e7d72..923da40 100644 --- a/src/getauxval.rs +++ b/src/getauxval.rs @@ -9,13 +9,13 @@ use core::ptr::without_provenance_mut; // so that it preserves provenance. #[unsafe(no_mangle)] unsafe extern "C" fn getauxval(type_: c_ulong) -> *mut c_void { - unsafe { _getauxval(type_) } + _getauxval(type_) } #[cfg(target_arch = "aarch64")] #[unsafe(no_mangle)] unsafe extern "C" fn __getauxval(type_: c_ulong) -> *mut c_void { - unsafe { _getauxval(type_) } + _getauxval(type_) } fn _getauxval(type_: c_ulong) -> *mut c_void {