diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 10113f6..ee06a4b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,7 +23,7 @@ jobs: strategy: matrix: build: [ubuntu, i686-linux, aarch64-linux, riscv64-linux] - rust: [1.84, nightly-2025-03-05] + rust: [1.85, nightly-2025-04-28] include: - build: ubuntu os: ubuntu-latest @@ -51,7 +51,7 @@ jobs: qemu: qemu-riscv64 -L /usr/riscv64-linux-gnu qemu_target: riscv64-linux-user host_target: riscv64gc-unknown-linux-gnu - - rust: nightly-2025-03-05 + - rust: nightly-2025-04-28 features: nightly steps: - uses: actions/checkout@v4 diff --git a/Cargo.toml b/Cargo.toml index 474644c..578eef4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,11 +8,11 @@ description = "Program startup and thread support written in Rust" documentation = "https://docs.rs/origin" license = "Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT" repository = "https://github.com/sunfishcode/origin" -edition = "2021" +edition = "2024" keywords = ["linux"] categories = ["no-std"] include = ["src", "Cargo.toml", "COPYRIGHT", "LICENSE*", "/*.md"] -rust-version = "1.84" +rust-version = "1.85" [dependencies] linux-raw-sys = { version = "0.9.2", default-features = false, optional = true, features = ["general", "no_std", "elf"] } @@ -44,7 +44,7 @@ alloc = { version = "1.0.0", optional = true, package = "rustc-std-workspace-all # Use the unwinding crate if support for unwinding is needed. This depends on # nightly Rust. And it's not supported on ARM yet. [target.'cfg(not(target_arch = "arm"))'.dependencies.unwinding] -version = "0.2.5" +version = "0.2.6" default-features = false features = ["unwinder"] optional = true diff --git a/README.md b/README.md index 0402667..a292f9b 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ This is used by [Mustang] and [Eyra] in their libc implementations, and in the [Origin Studio] project in its std implementation, which are three different ways to support building Rust programs written entirely in Rust. -It works with both stable (currently Rust >= 1.78) and nightly Rust. If you're +It works with both stable (currently Rust >= 1.85) and nightly Rust. If you're using nightly Rust, enable the feature "nightly" to let origin use nightly-only features, which include proper support for unwinding, better safety checks, and better optimizations. diff --git a/example-crates/basic/Cargo.toml b/example-crates/basic/Cargo.toml index 51e0574..b425a2b 100644 --- a/example-crates/basic/Cargo.toml +++ b/example-crates/basic/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "basic" version = "0.0.0" -edition = "2021" +edition = "2024" publish = false [dependencies] diff --git a/example-crates/external-start/Cargo.toml b/example-crates/external-start/Cargo.toml index 098165e..74d3853 100644 --- a/example-crates/external-start/Cargo.toml +++ b/example-crates/external-start/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "external-start" version = "0.0.0" -edition = "2021" +edition = "2024" publish = false [dependencies] diff --git a/example-crates/external-start/src/main.rs b/example-crates/external-start/src/main.rs index f58e79d..c2b7bec 100644 --- a/example-crates/external-start/src/main.rs +++ b/example-crates/external-start/src/main.rs @@ -19,62 +19,66 @@ static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::Glob /// and start running the constructors but to immediately take over. /// /// [here]: https://github.com/rust-lang/rfcs/pull/2735 -#[link_section = ".init_array.00000"] +#[unsafe(link_section = ".init_array.00000")] #[used] static EARLY_INIT_ARRAY: unsafe extern "C" fn(i32, *mut *mut u8) = { unsafe extern "C" fn function(_argc: i32, argv: *mut *mut u8) { - // Libc was calling constructors (we're one of them), but origin will - // be doing that now, so just exit when we're called a second time. - static FIRST: AtomicBool = AtomicBool::new(false); - if FIRST - .compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed) - .is_err() - { - return; - } + unsafe { + // Libc was calling constructors (we're one of them), but origin will + // be doing that now, so just exit when we're called a second time. + static FIRST: AtomicBool = AtomicBool::new(false); + if FIRST + .compare_exchange(false, true, Ordering::Relaxed, Ordering::Relaxed) + .is_err() + { + return; + } - // Compute the initial stack address provided by the kernel. - let mem = argv.sub(1); + // Compute the initial stack address provided by the kernel. + let mem = argv.sub(1); - origin::program::start(mem as _); + origin::program::start(mem as _); + } } function }; -#[no_mangle] +#[unsafe(no_mangle)] unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { - eprintln!("Hello from main thread"); + unsafe { + eprintln!("Hello from main thread"); - program::at_exit(Box::new(|| { - eprintln!("Hello from a `program::at_exit` handler") - })); - thread::at_exit(Box::new(|| { - eprintln!("Hello from a main-thread `thread::at_exit` handler") - })); + program::at_exit(Box::new(|| { + eprintln!("Hello from a `program::at_exit` handler") + })); + thread::at_exit(Box::new(|| { + eprintln!("Hello from a main-thread `thread::at_exit` handler") + })); - let thread = thread::create( - |_args| { - eprintln!("Hello from child thread"); - thread::at_exit(Box::new(|| { - eprintln!("Hello from child thread's `thread::at_exit` handler") - })); - None - }, - &[], - thread::default_stack_size(), - thread::default_guard_size(), - ) - .unwrap(); + let thread = thread::create( + |_args| { + eprintln!("Hello from child thread"); + thread::at_exit(Box::new(|| { + eprintln!("Hello from child thread's `thread::at_exit` handler") + })); + None + }, + &[], + thread::default_stack_size(), + thread::default_guard_size(), + ) + .unwrap(); - thread::join(thread); + thread::join(thread); - eprintln!("Goodbye from main"); - program::exit(0); + eprintln!("Goodbye from main"); + program::exit(0); + } } // Libc calls `main` so we need to provide a definition to satisfy the // linker, however origin gains control before libc can call this `main`. -#[no_mangle] +#[unsafe(no_mangle)] unsafe fn main(_argc: i32, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { eprintln!("Main was not supposed to be called!"); program::trap(); diff --git a/example-crates/origin-start-dynamic-linker/Cargo.toml b/example-crates/origin-start-dynamic-linker/Cargo.toml index be034aa..6430629 100644 --- a/example-crates/origin-start-dynamic-linker/Cargo.toml +++ b/example-crates/origin-start-dynamic-linker/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "origin-start-dynamic-linker" version = "0.0.0" -edition = "2021" +edition = "2024" publish = false [lib] diff --git a/example-crates/origin-start-dynamic-linker/src/lib.rs b/example-crates/origin-start-dynamic-linker/src/lib.rs index cae3fd9..e86cb68 100644 --- a/example-crates/origin-start-dynamic-linker/src/lib.rs +++ b/example-crates/origin-start-dynamic-linker/src/lib.rs @@ -12,33 +12,35 @@ use origin::{program, thread}; #[global_allocator] static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::GlobalDlmalloc; -#[no_mangle] +#[unsafe(no_mangle)] unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { - eprintln!("Hello from main thread"); - - program::at_exit(Box::new(|| { - eprintln!("Hello from a `program::at_exit` handler") - })); - thread::at_exit(Box::new(|| { - eprintln!("Hello from a main-thread `thread::at_exit` handler") - })); - - let thread = thread::create( - |_args| { - eprintln!("Hello from child thread"); - thread::at_exit(Box::new(|| { - eprintln!("Hello from child thread's `thread::at_exit` handler") - })); - None - }, - &[], - thread::default_stack_size(), - thread::default_guard_size(), - ) - .unwrap(); - - thread::join(thread); - - eprintln!("Goodbye from main"); - program::exit(0); + unsafe { + eprintln!("Hello from main thread"); + + program::at_exit(Box::new(|| { + eprintln!("Hello from a `program::at_exit` handler") + })); + thread::at_exit(Box::new(|| { + eprintln!("Hello from a main-thread `thread::at_exit` handler") + })); + + let thread = thread::create( + |_args| { + eprintln!("Hello from child thread"); + thread::at_exit(Box::new(|| { + eprintln!("Hello from child thread's `thread::at_exit` handler") + })); + None + }, + &[], + thread::default_stack_size(), + thread::default_guard_size(), + ) + .unwrap(); + + thread::join(thread); + + eprintln!("Goodbye from main"); + program::exit(0); + } } diff --git a/example-crates/origin-start-lto/Cargo.toml b/example-crates/origin-start-lto/Cargo.toml index e384b7e..63c7363 100644 --- a/example-crates/origin-start-lto/Cargo.toml +++ b/example-crates/origin-start-lto/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "origin-start-lto" version = "0.0.0" -edition = "2021" +edition = "2024" publish = false [dependencies] diff --git a/example-crates/origin-start-lto/src/main.rs b/example-crates/origin-start-lto/src/main.rs index 0ef5ac8..cfe9bae 100644 --- a/example-crates/origin-start-lto/src/main.rs +++ b/example-crates/origin-start-lto/src/main.rs @@ -12,33 +12,35 @@ use origin::{program, thread}; #[global_allocator] static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::GlobalDlmalloc; -#[no_mangle] +#[unsafe(no_mangle)] unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { - eprintln!("Hello from main thread"); - - program::at_exit(Box::new(|| { - eprintln!("Hello from a `program::at_exit` handler") - })); - thread::at_exit(Box::new(|| { - eprintln!("Hello from a main-thread `thread::at_exit` handler") - })); - - let thread = thread::create( - |_args| { - eprintln!("Hello from child thread"); - thread::at_exit(Box::new(|| { - eprintln!("Hello from child thread's `thread::at_exit` handler") - })); - None - }, - &[], - thread::default_stack_size(), - thread::default_guard_size(), - ) - .unwrap(); - - thread::join(thread); - - eprintln!("Goodbye from main"); - program::exit(0); + unsafe { + eprintln!("Hello from main thread"); + + program::at_exit(Box::new(|| { + eprintln!("Hello from a `program::at_exit` handler") + })); + thread::at_exit(Box::new(|| { + eprintln!("Hello from a main-thread `thread::at_exit` handler") + })); + + let thread = thread::create( + |_args| { + eprintln!("Hello from child thread"); + thread::at_exit(Box::new(|| { + eprintln!("Hello from child thread's `thread::at_exit` handler") + })); + None + }, + &[], + thread::default_stack_size(), + thread::default_guard_size(), + ) + .unwrap(); + + thread::join(thread); + + eprintln!("Goodbye from main"); + program::exit(0); + } } diff --git a/example-crates/origin-start-no-alloc/Cargo.toml b/example-crates/origin-start-no-alloc/Cargo.toml index ced2aa5..cde37b7 100644 --- a/example-crates/origin-start-no-alloc/Cargo.toml +++ b/example-crates/origin-start-no-alloc/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "origin-start-no-alloc" version = "0.0.0" -edition = "2021" +edition = "2024" publish = false [dependencies] diff --git a/example-crates/origin-start-no-alloc/src/main.rs b/example-crates/origin-start-no-alloc/src/main.rs index 01f6e1a..fc46870 100644 --- a/example-crates/origin-start-no-alloc/src/main.rs +++ b/example-crates/origin-start-no-alloc/src/main.rs @@ -6,7 +6,7 @@ use atomic_dbg::eprintln; use origin::program; -#[no_mangle] +#[unsafe(no_mangle)] unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { eprintln!("Hello!"); diff --git a/example-crates/origin-start-panic-abort/Cargo.toml b/example-crates/origin-start-panic-abort/Cargo.toml index e349344..44f7823 100644 --- a/example-crates/origin-start-panic-abort/Cargo.toml +++ b/example-crates/origin-start-panic-abort/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "origin-start-panic-abort" version = "0.0.0" -edition = "2021" +edition = "2024" publish = false [dependencies] diff --git a/example-crates/origin-start-panic-abort/src/main.rs b/example-crates/origin-start-panic-abort/src/main.rs index 0ef5ac8..cfe9bae 100644 --- a/example-crates/origin-start-panic-abort/src/main.rs +++ b/example-crates/origin-start-panic-abort/src/main.rs @@ -12,33 +12,35 @@ use origin::{program, thread}; #[global_allocator] static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::GlobalDlmalloc; -#[no_mangle] +#[unsafe(no_mangle)] unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { - eprintln!("Hello from main thread"); - - program::at_exit(Box::new(|| { - eprintln!("Hello from a `program::at_exit` handler") - })); - thread::at_exit(Box::new(|| { - eprintln!("Hello from a main-thread `thread::at_exit` handler") - })); - - let thread = thread::create( - |_args| { - eprintln!("Hello from child thread"); - thread::at_exit(Box::new(|| { - eprintln!("Hello from child thread's `thread::at_exit` handler") - })); - None - }, - &[], - thread::default_stack_size(), - thread::default_guard_size(), - ) - .unwrap(); - - thread::join(thread); - - eprintln!("Goodbye from main"); - program::exit(0); + unsafe { + eprintln!("Hello from main thread"); + + program::at_exit(Box::new(|| { + eprintln!("Hello from a `program::at_exit` handler") + })); + thread::at_exit(Box::new(|| { + eprintln!("Hello from a main-thread `thread::at_exit` handler") + })); + + let thread = thread::create( + |_args| { + eprintln!("Hello from child thread"); + thread::at_exit(Box::new(|| { + eprintln!("Hello from child thread's `thread::at_exit` handler") + })); + None + }, + &[], + thread::default_stack_size(), + thread::default_guard_size(), + ) + .unwrap(); + + thread::join(thread); + + eprintln!("Goodbye from main"); + program::exit(0); + } } diff --git a/example-crates/origin-start-stable/Cargo.toml b/example-crates/origin-start-stable/Cargo.toml index 959c53b..59c3c3a 100644 --- a/example-crates/origin-start-stable/Cargo.toml +++ b/example-crates/origin-start-stable/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "origin-start-stable" version = "0.0.0" -edition = "2021" +edition = "2024" publish = false [dependencies] diff --git a/example-crates/origin-start-stable/src/main.rs b/example-crates/origin-start-stable/src/main.rs index 0ef5ac8..cfe9bae 100644 --- a/example-crates/origin-start-stable/src/main.rs +++ b/example-crates/origin-start-stable/src/main.rs @@ -12,33 +12,35 @@ use origin::{program, thread}; #[global_allocator] static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::GlobalDlmalloc; -#[no_mangle] +#[unsafe(no_mangle)] unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { - eprintln!("Hello from main thread"); - - program::at_exit(Box::new(|| { - eprintln!("Hello from a `program::at_exit` handler") - })); - thread::at_exit(Box::new(|| { - eprintln!("Hello from a main-thread `thread::at_exit` handler") - })); - - let thread = thread::create( - |_args| { - eprintln!("Hello from child thread"); - thread::at_exit(Box::new(|| { - eprintln!("Hello from child thread's `thread::at_exit` handler") - })); - None - }, - &[], - thread::default_stack_size(), - thread::default_guard_size(), - ) - .unwrap(); - - thread::join(thread); - - eprintln!("Goodbye from main"); - program::exit(0); + unsafe { + eprintln!("Hello from main thread"); + + program::at_exit(Box::new(|| { + eprintln!("Hello from a `program::at_exit` handler") + })); + thread::at_exit(Box::new(|| { + eprintln!("Hello from a main-thread `thread::at_exit` handler") + })); + + let thread = thread::create( + |_args| { + eprintln!("Hello from child thread"); + thread::at_exit(Box::new(|| { + eprintln!("Hello from child thread's `thread::at_exit` handler") + })); + None + }, + &[], + thread::default_stack_size(), + thread::default_guard_size(), + ) + .unwrap(); + + thread::join(thread); + + eprintln!("Goodbye from main"); + program::exit(0); + } } diff --git a/example-crates/origin-start/Cargo.toml b/example-crates/origin-start/Cargo.toml index f2548ce..ee6cab7 100644 --- a/example-crates/origin-start/Cargo.toml +++ b/example-crates/origin-start/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "origin-start" version = "0.0.0" -edition = "2021" +edition = "2024" publish = false [dependencies] diff --git a/example-crates/origin-start/src/main.rs b/example-crates/origin-start/src/main.rs index 0ef5ac8..cfe9bae 100644 --- a/example-crates/origin-start/src/main.rs +++ b/example-crates/origin-start/src/main.rs @@ -12,33 +12,35 @@ use origin::{program, thread}; #[global_allocator] static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::GlobalDlmalloc; -#[no_mangle] +#[unsafe(no_mangle)] unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { - eprintln!("Hello from main thread"); - - program::at_exit(Box::new(|| { - eprintln!("Hello from a `program::at_exit` handler") - })); - thread::at_exit(Box::new(|| { - eprintln!("Hello from a main-thread `thread::at_exit` handler") - })); - - let thread = thread::create( - |_args| { - eprintln!("Hello from child thread"); - thread::at_exit(Box::new(|| { - eprintln!("Hello from child thread's `thread::at_exit` handler") - })); - None - }, - &[], - thread::default_stack_size(), - thread::default_guard_size(), - ) - .unwrap(); - - thread::join(thread); - - eprintln!("Goodbye from main"); - program::exit(0); + unsafe { + eprintln!("Hello from main thread"); + + program::at_exit(Box::new(|| { + eprintln!("Hello from a `program::at_exit` handler") + })); + thread::at_exit(Box::new(|| { + eprintln!("Hello from a main-thread `thread::at_exit` handler") + })); + + let thread = thread::create( + |_args| { + eprintln!("Hello from child thread"); + thread::at_exit(Box::new(|| { + eprintln!("Hello from child thread's `thread::at_exit` handler") + })); + None + }, + &[], + thread::default_stack_size(), + thread::default_guard_size(), + ) + .unwrap(); + + thread::join(thread); + + eprintln!("Goodbye from main"); + program::exit(0); + } } diff --git a/example-crates/tiny-hello/Cargo.toml b/example-crates/tiny-hello/Cargo.toml index 4bfff34..e043e6b 100644 --- a/example-crates/tiny-hello/Cargo.toml +++ b/example-crates/tiny-hello/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "tiny-hello" version = "0.0.0" -edition = "2021" +edition = "2024" publish = false [dependencies] diff --git a/example-crates/tiny-hello/src/main.rs b/example-crates/tiny-hello/src/main.rs index 8960527..b66b2bf 100644 --- a/example-crates/tiny-hello/src/main.rs +++ b/example-crates/tiny-hello/src/main.rs @@ -5,16 +5,18 @@ extern crate origin; -#[no_mangle] +#[unsafe(no_mangle)] unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { - let message = b"Hello, world!\n"; - let mut remaining = &message[..]; - while !remaining.is_empty() { - match rustix::io::write(rustix::stdio::stdout(), message) { - Ok(n) => remaining = &remaining[n as usize..], - Err(rustix::io::Errno::INTR) => continue, - Err(_) => origin::program::trap(), + unsafe { + let message = b"Hello, world!\n"; + let mut remaining = &message[..]; + while !remaining.is_empty() { + match rustix::io::write(rustix::stdio::stdout(), message) { + Ok(n) => remaining = &remaining[n as usize..], + Err(rustix::io::Errno::INTR) => continue, + Err(_) => origin::program::trap(), + } } + origin::program::immediate_exit(0); } - origin::program::immediate_exit(0); } diff --git a/example-crates/tiny/Cargo.toml b/example-crates/tiny/Cargo.toml index 28b711a..7a612b8 100644 --- a/example-crates/tiny/Cargo.toml +++ b/example-crates/tiny/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "tiny" version = "0.0.0" -edition = "2021" +edition = "2024" publish = false [dependencies] diff --git a/example-crates/tiny/src/main.rs b/example-crates/tiny/src/main.rs index a21d6e1..94389d3 100644 --- a/example-crates/tiny/src/main.rs +++ b/example-crates/tiny/src/main.rs @@ -5,7 +5,7 @@ extern crate origin; -#[no_mangle] +#[unsafe(no_mangle)] unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { 42 } diff --git a/src/arch/aarch64.rs b/src/arch/aarch64.rs index e08d0fc..ea1da39 100644 --- a/src/arch/aarch64.rs +++ b/src/arch/aarch64.rs @@ -109,19 +109,21 @@ pub(super) fn ehdr_addr() -> *const Elf_Ehdr { #[cfg(relocation_model = "pic")] #[inline] pub(super) unsafe fn relocation_load(ptr: usize) -> usize { - let r0; + unsafe { + let r0; - // This is read-only but we don't use `readonly` because this memory access - // happens outside the Rust memory model. As far as Rust knows, this is - // just an arbitrary side-effecting opaque operation. - asm!( - "ldr {}, [{}]", - out(reg) r0, - in(reg) ptr, - options(nostack, preserves_flags), - ); + // This is read-only but we don't use `readonly` because this memory access + // happens outside the Rust memory model. As far as Rust knows, this is + // just an arbitrary side-effecting opaque operation. + asm!( + "ldr {}, [{}]", + out(reg) r0, + in(reg) ptr, + options(nostack, preserves_flags), + ); - r0 + r0 + } } /// Perform a single store operation, outside the Rust memory model. @@ -140,12 +142,14 @@ pub(super) unsafe fn relocation_load(ptr: usize) -> usize { #[cfg(relocation_model = "pic")] #[inline] pub(super) unsafe fn relocation_store(ptr: usize, value: usize) { - asm!( - "str {}, [{}]", - in(reg) value, - in(reg) ptr, - options(nostack, preserves_flags), - ); + unsafe { + asm!( + "str {}, [{}]", + in(reg) value, + in(reg) ptr, + options(nostack, preserves_flags), + ); + } } /// Mark “relro” memory as readonly. @@ -168,24 +172,26 @@ pub(super) unsafe fn relocation_store(ptr: usize, value: usize) { #[cfg(relocation_model = "pic")] #[inline] pub(super) unsafe fn relocation_mprotect_readonly(ptr: usize, len: usize) { - let r0: usize; + unsafe { + let r0: usize; - // This is read-only but we don't use `readonly` because the side effects - // happen outside the Rust memory model. As far as Rust knows, this is - // just an arbitrary side-effecting opaque operation. - asm!( - "svc 0", - in("x8") __NR_mprotect, - inlateout("x0") ptr as usize => r0, - in("x1") len, - in("x2") PROT_READ, - options(nostack, preserves_flags), - ); + // This is read-only but we don't use `readonly` because the side effects + // happen outside the Rust memory model. As far as Rust knows, this is + // just an arbitrary side-effecting opaque operation. + asm!( + "svc 0", + in("x8") __NR_mprotect, + inlateout("x0") ptr as usize => r0, + in("x1") len, + in("x2") PROT_READ, + options(nostack, preserves_flags), + ); - if r0 != 0 { - // Do not panic here as libstd's panic handler needs TLS, which is not - // yet initialized at this point. - trap(); + if r0 != 0 { + // Do not panic here as libstd's panic handler needs TLS, which is not + // yet initialized at this point. + trap(); + } } } @@ -211,34 +217,36 @@ pub(super) unsafe fn clone( fn_: extern "C" fn(), num_args: usize, ) -> isize { - let r0; - asm!( - "svc 0", // Do the `clone` system call. - "cbnz x0, 0f", // Branch if we're in the parent thread. + unsafe { + let r0; + asm!( + "svc 0", // Do the `clone` system call. + "cbnz x0, 0f", // Branch if we're in the parent thread. - // Child thread. - "mov x0, {fn_}", // Pass `fn_` as the first argument. - "mov x1, sp", // Pass the args pointer as the second argument. - "mov x2, {num_args}", // Pass `num_args` as the third argument. - "mov x29, xzr", // Zero the frame address. - "mov x30, xzr", // Zero the return address. - "b {entry}", // Call `entry`. + // Child thread. + "mov x0, {fn_}", // Pass `fn_` as the first argument. + "mov x1, sp", // Pass the args pointer as the second argument. + "mov x2, {num_args}", // Pass `num_args` as the third argument. + "mov x29, xzr", // Zero the frame address. + "mov x30, xzr", // Zero the return address. + "b {entry}", // Call `entry`. - // Parent thread. - "0:", + // Parent thread. + "0:", - entry = sym super::thread::entry, - fn_ = in(reg) fn_, - num_args = in(reg) num_args, - in("x8") __NR_clone, - inlateout("x0") flags as usize => r0, - in("x1") child_stack, - in("x2") parent_tid, - in("x3") newtls, - in("x4") child_tid, - options(nostack) - ); - r0 + entry = sym super::thread::entry, + fn_ = in(reg) fn_, + num_args = in(reg) num_args, + in("x8") __NR_clone, + inlateout("x0") flags as usize => r0, + in("x1") child_stack, + in("x2") parent_tid, + in("x3") newtls, + in("x4") child_tid, + options(nostack) + ); + r0 + } } /// Write a value to the platform thread-pointer register. @@ -246,8 +254,10 @@ pub(super) unsafe fn clone( #[cfg(feature = "thread")] #[inline] pub(super) unsafe fn set_thread_pointer(ptr: *mut c_void) { - asm!("msr tpidr_el0, {}", in(reg) ptr); - debug_assert_eq!(thread_pointer(), ptr); + unsafe { + asm!("msr tpidr_el0, {}", in(reg) ptr); + debug_assert_eq!(thread_pointer(), ptr); + } } /// Read the value of the platform thread-pointer register. @@ -274,18 +284,20 @@ pub(super) const TLS_OFFSET: usize = 0; #[cfg(feature = "thread")] #[inline] pub(super) unsafe fn munmap_and_exit_thread(map_addr: *mut c_void, map_len: usize) -> ! { - asm!( - "svc 0", - "mov x0, xzr", - "mov x8, {__NR_exit}", - "svc 0", - "udf #16", - __NR_exit = const __NR_exit, - in("x8") __NR_munmap, - in("x0") map_addr, - in("x1") map_len, - options(noreturn, nostack) - ); + unsafe { + asm!( + "svc 0", + "mov x0, xzr", + "mov x8, {__NR_exit}", + "svc 0", + "udf #16", + __NR_exit = const __NR_exit, + in("x8") __NR_munmap, + in("x0") map_addr, + in("x1") map_len, + options(noreturn, nostack) + ); + } } #[cfg(feature = "take-charge")] diff --git a/src/arch/arm.rs b/src/arch/arm.rs index 231c899..a627a2d 100644 --- a/src/arch/arm.rs +++ b/src/arch/arm.rs @@ -121,19 +121,21 @@ pub(super) fn ehdr_addr() -> *const Elf_Ehdr { #[cfg(relocation_model = "pic")] #[inline] pub(super) unsafe fn relocation_load(ptr: usize) -> usize { - let r0; - - // This is read-only but we don't use `readonly` because this memory access - // happens outside the Rust memory model. As far as Rust knows, this is - // just an arbitrary side-effecting opaque operation. - asm!( - "ldr {}, [{}]", - out(reg) r0, - in(reg) ptr, - options(nostack, preserves_flags), - ); - - r0 + { + let r0; + + // This is read-only but we don't use `readonly` because this memory access + // happens outside the Rust memory model. As far as Rust knows, this is + // just an arbitrary side-effecting opaque operation. + asm!( + "ldr {}, [{}]", + out(reg) r0, + in(reg) ptr, + options(nostack, preserves_flags), + ); + + r0 + } } /// Perform a single store operation, outside the Rust memory model. @@ -152,12 +154,14 @@ pub(super) unsafe fn relocation_load(ptr: usize) -> usize { #[cfg(relocation_model = "pic")] #[inline] pub(super) unsafe fn relocation_store(ptr: usize, value: usize) { - asm!( - "str {}, [{}]", - in(reg) value, - in(reg) ptr, - options(nostack, preserves_flags), - ); + unsafe { + asm!( + "str {}, [{}]", + in(reg) value, + in(reg) ptr, + options(nostack, preserves_flags), + ); + } } /// Mark “relro” memory as readonly. @@ -180,24 +184,26 @@ pub(super) unsafe fn relocation_store(ptr: usize, value: usize) { #[cfg(relocation_model = "pic")] #[inline] pub(super) unsafe fn relocation_mprotect_readonly(ptr: usize, len: usize) { - let r0: usize; - - // This is read-only but we don't use `readonly` because the side effects - // happen outside the Rust memory model. As far as Rust knows, this is - // just an arbitrary side-effecting opaque operation. - asm!( - "svc 0", - in("r7") __NR_mprotect, - inlateout("r0") ptr as usize => r0, - in("r1") len, - in("r2") PROT_READ, - options(nostack, preserves_flags), - ); - - if r0 != 0 { - // Do not panic here as libstd's panic handler needs TLS, which is not - // yet initialized at this point. - trap(); + unsafe { + let r0: usize; + + // This is read-only but we don't use `readonly` because the side effects + // happen outside the Rust memory model. As far as Rust knows, this is + // just an arbitrary side-effecting opaque operation. + asm!( + "svc 0", + in("r7") __NR_mprotect, + inlateout("r0") ptr as usize => r0, + in("r1") len, + in("r2") PROT_READ, + options(nostack, preserves_flags), + ); + + if r0 != 0 { + // Do not panic here as libstd's panic handler needs TLS, which is not + // yet initialized at this point. + trap(); + } } } @@ -223,35 +229,37 @@ pub(super) unsafe fn clone( fn_: extern "C" fn(), num_args: usize, ) -> isize { - let r0; - asm!( - "svc 0", // Do the `clone` system call. - "tst r0, r0", // Branch if we're in the parent thread. - "bne 0f", - - // Child thread. - "mov r0, {fn_}", // Pass `fn_` as the first argument. - "mov r1, sp", // Pass the args pointer as the second argument. - "mov r2, {num_args}", // Pass `num_args` as the third argument. - "mov fp, #0", // Zero the frame address. - "mov lr, #0", // Zero the return address. - "b {entry}", // Call `entry`. - - // Parent thread. - "0:", - - entry = sym super::thread::entry, - fn_ = in(reg) fn_, - num_args = in(reg) num_args, - in("r7") __NR_clone, - inlateout("r0") flags as usize => r0, - in("r1") child_stack, - in("r2") parent_tid, - in("r3") newtls, - in("r4") child_tid, - options(nostack) - ); - r0 + unsafe { + let r0; + asm!( + "svc 0", // Do the `clone` system call. + "tst r0, r0", // Branch if we're in the parent thread. + "bne 0f", + + // Child thread. + "mov r0, {fn_}", // Pass `fn_` as the first argument. + "mov r1, sp", // Pass the args pointer as the second argument. + "mov r2, {num_args}", // Pass `num_args` as the third argument. + "mov fp, #0", // Zero the frame address. + "mov lr, #0", // Zero the return address. + "b {entry}", // Call `entry`. + + // Parent thread. + "0:", + + entry = sym super::thread::entry, + fn_ = in(reg) fn_, + num_args = in(reg) num_args, + in("r7") __NR_clone, + inlateout("r0") flags as usize => r0, + in("r1") child_stack, + in("r2") parent_tid, + in("r3") newtls, + in("r4") child_tid, + options(nostack) + ); + r0 + } } /// Write a value to the platform thread-pointer register. @@ -259,9 +267,11 @@ pub(super) unsafe fn clone( #[cfg(feature = "thread")] #[inline] pub(super) unsafe fn set_thread_pointer(ptr: *mut c_void) { - let res = rustix::runtime::arm_set_tls(ptr); - debug_assert_eq!(res, Ok(())); - debug_assert_eq!(thread_pointer(), ptr); + unsafe { + let res = rustix::runtime::arm_set_tls(ptr); + debug_assert_eq!(res, Ok(())); + debug_assert_eq!(thread_pointer(), ptr); + } } /// Read the value of the platform thread-pointer register. @@ -288,18 +298,20 @@ pub(super) const TLS_OFFSET: usize = 0; #[cfg(feature = "thread")] #[inline] pub(super) unsafe fn munmap_and_exit_thread(map_addr: *mut c_void, map_len: usize) -> ! { - asm!( - "svc 0", - "mov r0, #0", - "mov r7, {__NR_exit}", - "svc 0", - "udf #16", - __NR_exit = const __NR_exit, - in("r7") __NR_munmap, - in("r0") map_addr, - in("r1") map_len, - options(noreturn, nostack) - ); + unsafe { + asm!( + "svc 0", + "mov r0, #0", + "mov r7, {__NR_exit}", + "svc 0", + "udf #16", + __NR_exit = const __NR_exit, + in("r7") __NR_munmap, + in("r0") map_addr, + in("r1") map_len, + options(noreturn, nostack) + ); + } } #[cfg(feature = "take-charge")] diff --git a/src/arch/riscv64.rs b/src/arch/riscv64.rs index 5e53b36..f74d1a8 100644 --- a/src/arch/riscv64.rs +++ b/src/arch/riscv64.rs @@ -104,42 +104,47 @@ pub(super) fn ehdr_addr() -> *const Elf_Ehdr { #[cfg(relocation_model = "pic")] #[inline] pub(super) unsafe fn relocation_load(ptr: usize) -> usize { - let r0; + unsafe { + let r0; - // This is read-only but we don't use `readonly` because this memory access - // happens outside the Rust memory model. As far as Rust knows, this is - // just an arbitrary side-effecting opaque operation. - asm!( - "ld {}, 0({})", - out(reg) r0, - in(reg) ptr, - options(nostack, preserves_flags), - ); + // This is read-only but we don't use `readonly` because this memory access + // happens outside the Rust memory model. As far as Rust knows, this is + // just an arbitrary side-effecting opaque operation. + asm!( + "ld {}, 0({})", + out(reg) r0, + in(reg) ptr, + options(nostack, preserves_flags), + ); - r0 + r0 + } } -/// Perform a raw load operation to memory that Rust may consider out of bounds. +/// Perform a raw load operation to memory that Rust may consider out of +/// bounds. /// /// Data loaded from out-of-bounds bytes will have nondeterministic values. /// /// # Safety /// -/// `ptr` must be aligned for loading a `usize` and must point to enough readable -/// memory for loading a `usize`. +/// `ptr` must be aligned for loading a `usize` and must point to enough +/// readable memory for loading a `usize`. #[cfg(all(feature = "take-charge", not(feature = "optimize_for_size")))] #[inline] pub(super) unsafe fn oob_load(ptr: *const usize) -> usize { - let r0; + unsafe { + let r0; - asm!( - "ld {}, 0({})", - out(reg) r0, - in(reg) ptr, - options(nostack, preserves_flags, readonly), - ); + asm!( + "ld {}, 0({})", + out(reg) r0, + in(reg) ptr, + options(nostack, preserves_flags, readonly), + ); - r0 + r0 + } } /// Perform a single store operation, outside the Rust memory model. @@ -158,12 +163,14 @@ pub(super) unsafe fn oob_load(ptr: *const usize) -> usize { #[cfg(relocation_model = "pic")] #[inline] pub(super) unsafe fn relocation_store(ptr: usize, value: usize) { - asm!( - "sd {}, 0({})", - in(reg) value, - in(reg) ptr, - options(nostack, preserves_flags), - ); + unsafe { + asm!( + "sd {}, 0({})", + in(reg) value, + in(reg) ptr, + options(nostack, preserves_flags), + ); + } } /// Mark “relro” memory as readonly. @@ -186,24 +193,26 @@ pub(super) unsafe fn relocation_store(ptr: usize, value: usize) { #[cfg(relocation_model = "pic")] #[inline] pub(super) unsafe fn relocation_mprotect_readonly(ptr: usize, len: usize) { - let r0: usize; + unsafe { + let r0: usize; - // This is read-only but we don't use `readonly` because the side effects - // happen outside the Rust memory model. As far as Rust knows, this is - // just an arbitrary side-effecting opaque operation. - asm!( - "ecall", - in("a7") __NR_mprotect, - inlateout("a0") ptr as usize => r0, - in("a1") len, - in("a2") PROT_READ, - options(nostack, preserves_flags), - ); + // This is read-only but we don't use `readonly` because the side effects + // happen outside the Rust memory model. As far as Rust knows, this is + // just an arbitrary side-effecting opaque operation. + asm!( + "ecall", + in("a7") __NR_mprotect, + inlateout("a0") ptr as usize => r0, + in("a1") len, + in("a2") PROT_READ, + options(nostack, preserves_flags), + ); - if r0 != 0 { - // Do not panic here as libstd's panic handler needs TLS, which is not - // yet initialized at this point. - trap(); + if r0 != 0 { + // Do not panic here as libstd's panic handler needs TLS, which is not + // yet initialized at this point. + trap(); + } } } @@ -229,34 +238,36 @@ pub(super) unsafe fn clone( fn_: extern "C" fn(), num_args: usize, ) -> isize { - let r0; - asm!( - "ecall", // Do the `clone` system call. - "bnez a0, 0f", // Branch if we're in the parent thread. + unsafe { + let r0; + asm!( + "ecall", // Do the `clone` system call. + "bnez a0, 0f", // Branch if we're in the parent thread. - // Child thread. - "mv a0, {fn_}", // Pass `fn_` as the first argument. - "mv a1, sp", // Pass the args pointer as the second argument. - "mv a2, {num_args}", // Pass `num_args` as the third argument. - "mv fp, zero", // Zero the frame address. - "mv ra, zero", // Zero the return address. - "tail {entry}", // Call `entry`. + // Child thread. + "mv a0, {fn_}", // Pass `fn_` as the first argument. + "mv a1, sp", // Pass the args pointer as the second argument. + "mv a2, {num_args}", // Pass `num_args` as the third argument. + "mv fp, zero", // Zero the frame address. + "mv ra, zero", // Zero the return address. + "tail {entry}", // Call `entry`. - // Parent thread. - "0:", + // Parent thread. + "0:", - entry = sym super::thread::entry, - fn_ = in(reg) fn_, - num_args = in(reg) num_args, - in("a7") __NR_clone, - inlateout("a0") flags as usize => r0, - in("a1") child_stack, - in("a2") parent_tid, - in("a3") newtls, - in("a4") child_tid, - options(nostack) - ); - r0 + entry = sym super::thread::entry, + fn_ = in(reg) fn_, + num_args = in(reg) num_args, + in("a7") __NR_clone, + inlateout("a0") flags as usize => r0, + in("a1") child_stack, + in("a2") parent_tid, + in("a3") newtls, + in("a4") child_tid, + options(nostack) + ); + r0 + } } /// Write a value to the platform thread-pointer register. @@ -264,8 +275,10 @@ pub(super) unsafe fn clone( #[cfg(feature = "thread")] #[inline] pub(super) unsafe fn set_thread_pointer(ptr: *mut c_void) { - asm!("mv tp, {}", in(reg) ptr); - debug_assert_eq!(thread_pointer(), ptr); + unsafe { + asm!("mv tp, {}", in(reg) ptr); + debug_assert_eq!(thread_pointer(), ptr); + } } /// Read the value of the platform thread-pointer register. @@ -293,18 +306,20 @@ pub(super) const TLS_OFFSET: usize = 0x800; #[cfg(feature = "thread")] #[inline] pub(super) unsafe fn munmap_and_exit_thread(map_addr: *mut c_void, map_len: usize) -> ! { - asm!( - "ecall", - "mv a0, zero", - "li a7, {__NR_exit}", - "ecall", - "unimp", - __NR_exit = const __NR_exit, - in("a7") __NR_munmap, - in("a0") map_addr, - in("a1") map_len, - options(noreturn, nostack) - ); + unsafe { + asm!( + "ecall", + "mv a0, zero", + "li a7, {__NR_exit}", + "ecall", + "unimp", + __NR_exit = const __NR_exit, + in("a7") __NR_munmap, + in("a0") map_addr, + in("a1") map_len, + options(noreturn, nostack) + ); + } } // RISC-V doesn't use `__NR_rt_sigreturn` diff --git a/src/arch/x86.rs b/src/arch/x86.rs index ba96e6f..7c1fea1 100644 --- a/src/arch/x86.rs +++ b/src/arch/x86.rs @@ -137,19 +137,21 @@ pub(super) fn ehdr_addr() -> *const Elf_Ehdr { #[cfg(relocation_model = "pic")] #[inline] pub(super) unsafe fn relocation_load(ptr: usize) -> usize { - let r0; - - // This is read-only but we don't use `readonly` because this memory access - // happens outside the Rust memory model. As far as Rust knows, this is - // just an arbitrary side-effecting opaque operation. - asm!( - "mov {}, [{}]", - out(reg) r0, - in(reg) ptr, - options(nostack, preserves_flags), - ); - - r0 + unsafe { + let r0; + + // This is read-only but we don't use `readonly` because this memory access + // happens outside the Rust memory model. As far as Rust knows, this is + // just an arbitrary side-effecting opaque operation. + asm!( + "mov {}, [{}]", + out(reg) r0, + in(reg) ptr, + options(nostack, preserves_flags), + ); + + r0 + } } /// Perform a single store operation, outside the Rust memory model. @@ -168,12 +170,14 @@ pub(super) unsafe fn relocation_load(ptr: usize) -> usize { #[cfg(relocation_model = "pic")] #[inline] pub(super) unsafe fn relocation_store(ptr: usize, value: usize) { - asm!( - "mov [{}], {}", - in(reg) ptr, - in(reg) value, - options(nostack, preserves_flags), - ); + unsafe { + asm!( + "mov [{}], {}", + in(reg) ptr, + in(reg) value, + options(nostack, preserves_flags), + ); + } } /// Mark “relro” memory as readonly. @@ -196,24 +200,26 @@ pub(super) unsafe fn relocation_store(ptr: usize, value: usize) { #[cfg(relocation_model = "pic")] #[inline] pub(super) unsafe fn relocation_mprotect_readonly(ptr: usize, len: usize) { - let r0: usize; - - // This is read-only but we don't use `readonly` because the side effects - // happen outside the Rust memory model. As far as Rust knows, this is - // just an arbitrary side-effecting opaque operation. - asm!( - "int 0x80", - inlateout("eax") __NR_mprotect as usize => r0, - in("ebx") ptr, - in("ecx") len, - in("edx") PROT_READ, - options(nostack, preserves_flags), - ); - - if r0 != 0 { - // Do not panic here as libstd's panic handler needs TLS, which is not - // yet initialized at this point. - trap(); + unsafe { + let r0: usize; + + // This is read-only but we don't use `readonly` because the side effects + // happen outside the Rust memory model. As far as Rust knows, this is + // just an arbitrary side-effecting opaque operation. + asm!( + "int 0x80", + inlateout("eax") __NR_mprotect as usize => r0, + in("ebx") ptr, + in("ecx") len, + in("edx") PROT_READ, + options(nostack, preserves_flags), + ); + + if r0 != 0 { + // Do not panic here as libstd's panic handler needs TLS, which is not + // yet initialized at this point. + trap(); + } } } @@ -239,80 +245,82 @@ pub(super) unsafe fn clone( fn_: extern "C" fn(), num_args: usize, ) -> isize { - let mut gs: u32 = 0; - asm!("mov {0:x}, gs", inout(reg) gs); - let entry_number = gs >> 3; - - let mut user_desc = rustix::runtime::UserDesc { - entry_number, - base_addr: newtls.addr() as u32, - limit: 0xfffff, - _bitfield_align_1: [], - _bitfield_1: Default::default(), - __bindgen_padding_0: [0_u8; 3_usize], - }; - user_desc.set_seg_32bit(1); - user_desc.set_contents(0); - user_desc.set_read_exec_only(0); - user_desc.set_limit_in_pages(1); - user_desc.set_seg_not_present(0); - user_desc.set_useable(1); - let newtls: *const _ = &user_desc; - - // Push `fn_` to the child stack, since after all the `clone` args, and - // `num_args` in `ebp`, there are no more free registers. - let child_stack = child_stack.cast::<*mut c_void>().sub(1); - child_stack.write(fn_ as _); - - // See the comments for x86's `syscall6` in `rustix`. Inline asm isn't - // allowed to name ebp or esi as operands, so we have to jump through - // extra hoops here. - let r0; - asm!( - "push esi", // Save incoming register value. - "push ebp", // Save incoming register value. - - // Pass `num_args` to the child in `ebp`. - "mov ebp, [eax+8]", - - "mov esi, [eax+0]", // Pass `newtls` to the `int 0x80`. - "mov eax, [eax+4]", // Pass `__NR_clone` to the `int 0x80`. - - // Use `int 0x80` instead of vsyscall, following `clone`'s - // documentation; vsyscall would attempt to return to the parent stack - // in the child. - "int 0x80", // Do the `clone` system call. - "test eax, eax", // Branch if we're in the parent. - "jnz 2f", - - // Child thread. - "pop edi", // Load `fn_` from the child stack. - "mov esi, esp", // Snapshot the args pointer. - "push eax", // Pad for stack pointer alignment. - "push ebp", // Pass `num_args` as the third argument. - "push esi", // Pass the args pointer as the second argument. - "push edi", // Pass `fn_` as the first argument. - "xor ebp, ebp", // Zero the frame address. - "push eax", // Zero the return address. - "jmp {entry}", // Call `entry`. - - // Parent thread. - "2:", - "pop ebp", // Restore incoming register value. - "pop esi", // Restore incoming register value. - - entry = sym super::thread::entry, - inout("eax") &[ - newtls.cast::().cast_mut(), - without_provenance_mut(__NR_clone as usize), - without_provenance_mut(num_args) - ] => r0, - in("ebx") flags, - in("ecx") child_stack, - in("edx") parent_tid, - in("edi") child_tid, - ); - r0 + unsafe { + let mut gs: u32 = 0; + asm!("mov {0:x}, gs", inout(reg) gs); + let entry_number = gs >> 3; + + let mut user_desc = rustix::runtime::UserDesc { + entry_number, + base_addr: newtls.addr() as u32, + limit: 0xfffff, + _bitfield_align_1: [], + _bitfield_1: Default::default(), + __bindgen_padding_0: [0_u8; 3_usize], + }; + user_desc.set_seg_32bit(1); + user_desc.set_contents(0); + user_desc.set_read_exec_only(0); + user_desc.set_limit_in_pages(1); + user_desc.set_seg_not_present(0); + user_desc.set_useable(1); + let newtls: *const _ = &user_desc; + + // Push `fn_` to the child stack, since after all the `clone` args, and + // `num_args` in `ebp`, there are no more free registers. + let child_stack = child_stack.cast::<*mut c_void>().sub(1); + child_stack.write(fn_ as _); + + // See the comments for x86's `syscall6` in `rustix`. Inline asm isn't + // allowed to name ebp or esi as operands, so we have to jump through + // extra hoops here. + let r0; + asm!( + "push esi", // Save incoming register value. + "push ebp", // Save incoming register value. + + // Pass `num_args` to the child in `ebp`. + "mov ebp, [eax+8]", + + "mov esi, [eax+0]", // Pass `newtls` to the `int 0x80`. + "mov eax, [eax+4]", // Pass `__NR_clone` to the `int 0x80`. + + // Use `int 0x80` instead of vsyscall, following `clone`'s + // documentation; vsyscall would attempt to return to the parent stack + // in the child. + "int 0x80", // Do the `clone` system call. + "test eax, eax", // Branch if we're in the parent. + "jnz 2f", + + // Child thread. + "pop edi", // Load `fn_` from the child stack. + "mov esi, esp", // Snapshot the args pointer. + "push eax", // Pad for stack pointer alignment. + "push ebp", // Pass `num_args` as the third argument. + "push esi", // Pass the args pointer as the second argument. + "push edi", // Pass `fn_` as the first argument. + "xor ebp, ebp", // Zero the frame address. + "push eax", // Zero the return address. + "jmp {entry}", // Call `entry`. + + // Parent thread. + "2:", + "pop ebp", // Restore incoming register value. + "pop esi", // Restore incoming register value. + + entry = sym super::thread::entry, + inout("eax") &[ + newtls.cast::().cast_mut(), + without_provenance_mut(__NR_clone as usize), + without_provenance_mut(num_args) + ] => r0, + in("ebx") flags, + in("ecx") child_stack, + in("edx") parent_tid, + in("edi") child_tid, + ); + r0 + } } /// Write a value to the platform thread-pointer register. @@ -320,27 +328,29 @@ pub(super) unsafe fn clone( #[cfg(feature = "thread")] #[inline] pub(super) unsafe fn set_thread_pointer(ptr: *mut c_void) { - let mut user_desc = rustix::runtime::UserDesc { - entry_number: !0u32, - base_addr: ptr.addr() as u32, - limit: 0xfffff, - _bitfield_align_1: [], - _bitfield_1: Default::default(), - __bindgen_padding_0: [0_u8; 3_usize], - }; - user_desc.set_seg_32bit(1); - user_desc.set_contents(0); - user_desc.set_read_exec_only(0); - user_desc.set_limit_in_pages(1); - user_desc.set_seg_not_present(0); - user_desc.set_useable(1); - let res = rustix::runtime::set_thread_area(&mut user_desc); - debug_assert_eq!(res, Ok(())); - debug_assert_ne!(user_desc.entry_number, !0); - - asm!("mov gs, {0:x}", in(reg) ((user_desc.entry_number << 3) | 3) as u16); - debug_assert_eq!(*ptr.cast::<*const c_void>(), ptr); - debug_assert_eq!(thread_pointer(), ptr); + unsafe { + let mut user_desc = rustix::runtime::UserDesc { + entry_number: !0u32, + base_addr: ptr.addr() as u32, + limit: 0xfffff, + _bitfield_align_1: [], + _bitfield_1: Default::default(), + __bindgen_padding_0: [0_u8; 3_usize], + }; + user_desc.set_seg_32bit(1); + user_desc.set_contents(0); + user_desc.set_read_exec_only(0); + user_desc.set_limit_in_pages(1); + user_desc.set_seg_not_present(0); + user_desc.set_useable(1); + let res = rustix::runtime::set_thread_area(&mut user_desc); + debug_assert_eq!(res, Ok(())); + debug_assert_ne!(user_desc.entry_number, !0); + + asm!("mov gs, {0:x}", in(reg) ((user_desc.entry_number << 3) | 3) as u16); + debug_assert_eq!(*ptr.cast::<*const c_void>(), ptr); + debug_assert_eq!(thread_pointer(), ptr); + } } /// Read the value of the platform thread-pointer register. @@ -370,20 +380,22 @@ pub(super) const TLS_OFFSET: usize = 0; #[cfg(feature = "thread")] #[inline] pub(super) unsafe fn munmap_and_exit_thread(map_addr: *mut c_void, map_len: usize) -> ! { - asm!( - // Use `int 0x80` instead of vsyscall, since vsyscall would attempt to - // touch the stack after we `munmap` it. - "int 0x80", - "xor ebx, ebx", - "mov eax, {__NR_exit}", - "int 0x80", - "ud2", - __NR_exit = const __NR_exit, - in("eax") __NR_munmap, - in("ebx") map_addr, - in("ecx") map_len, - options(noreturn, nostack) - ); + unsafe { + asm!( + // Use `int 0x80` instead of vsyscall, since vsyscall would attempt to + // touch the stack after we `munmap` it. + "int 0x80", + "xor ebx, ebx", + "mov eax, {__NR_exit}", + "int 0x80", + "ud2", + __NR_exit = const __NR_exit, + in("eax") __NR_munmap, + in("ebx") map_addr, + in("ecx") map_len, + options(noreturn, nostack) + ); + } } #[cfg(feature = "take-charge")] diff --git a/src/arch/x86_64.rs b/src/arch/x86_64.rs index 8a6edca..ce0d52f 100644 --- a/src/arch/x86_64.rs +++ b/src/arch/x86_64.rs @@ -107,19 +107,21 @@ pub(super) fn ehdr_addr() -> *const Elf_Ehdr { #[cfg(relocation_model = "pic")] #[inline] pub(super) unsafe fn relocation_load(ptr: usize) -> usize { - let r0; + unsafe { + let r0; - // This is read-only but we don't use `readonly` because this memory access - // happens outside the Rust memory model. As far as Rust knows, this is - // just an arbitrary side-effecting opaque operation. - asm!( - "mov {}, [{}]", - out(reg) r0, - in(reg) ptr, - options(nostack, preserves_flags), - ); + // This is read-only but we don't use `readonly` because this memory access + // happens outside the Rust memory model. As far as Rust knows, this is + // just an arbitrary side-effecting opaque operation. + asm!( + "mov {}, [{}]", + out(reg) r0, + in(reg) ptr, + options(nostack, preserves_flags), + ); - r0 + r0 + } } /// Perform a single store operation, outside the Rust memory model. @@ -139,12 +141,14 @@ pub(super) unsafe fn relocation_load(ptr: usize) -> usize { #[cfg(relocation_model = "pic")] #[inline] pub(super) unsafe fn relocation_store(ptr: usize, value: usize) { - asm!( - "mov [{}], {}", - in(reg) ptr, - in(reg) value, - options(nostack, preserves_flags), - ); + unsafe { + asm!( + "mov [{}], {}", + in(reg) ptr, + in(reg) value, + options(nostack, preserves_flags), + ); + } } /// Mark “relro” memory as readonly. @@ -167,26 +171,28 @@ pub(super) unsafe fn relocation_store(ptr: usize, value: usize) { #[cfg(relocation_model = "pic")] #[inline] pub(super) unsafe fn relocation_mprotect_readonly(ptr: usize, len: usize) { - let r0: usize; + unsafe { + let r0: usize; - // This is read-only but we don't use `readonly` because the side effects - // happen outside the Rust memory model. As far as Rust knows, this is - // just an arbitrary side-effecting opaque operation. - asm!( - "syscall", - inlateout("rax") __NR_mprotect as usize => r0, - in("rdi") ptr, - in("rsi") len, - in("rdx") PROT_READ, - lateout("rcx") _, - lateout("r11") _, - options(nostack, preserves_flags), - ); + // This is read-only but we don't use `readonly` because the side effects + // happen outside the Rust memory model. As far as Rust knows, this is + // just an arbitrary side-effecting opaque operation. + asm!( + "syscall", + inlateout("rax") __NR_mprotect as usize => r0, + in("rdi") ptr, + in("rsi") len, + in("rdx") PROT_READ, + lateout("rcx") _, + lateout("r11") _, + options(nostack, preserves_flags), + ); - if r0 != 0 { - // Do not panic here as libstd's panic handler needs TLS, which is not - // yet initialized at this point. - trap(); + if r0 != 0 { + // Do not panic here as libstd's panic handler needs TLS, which is not + // yet initialized at this point. + trap(); + } } } @@ -212,37 +218,39 @@ pub(super) unsafe fn clone( fn_: extern "C" fn(), num_args: usize, ) -> isize { - let r0; - asm!( - "syscall", // Do the `clone` system call. - "test eax, eax", // Branch if we're in the parent thread. - "jnz 2f", + unsafe { + let r0; + asm!( + "syscall", // Do the `clone` system call. + "test eax, eax", // Branch if we're in the parent thread. + "jnz 2f", - // Child thread. - "mov rdi, r9", // Pass `fn_` as the first argument. - "mov rsi, rsp", // Pass the args pointer as the second argument. - "mov rdx, r12", // Pass `num_args` as the third argument. - "xor ebp, ebp", // Zero the frame address. - "push rax", // Zero the return address. - "jmp {entry}", // Call `entry`. + // Child thread. + "mov rdi, r9", // Pass `fn_` as the first argument. + "mov rsi, rsp", // Pass the args pointer as the second argument. + "mov rdx, r12", // Pass `num_args` as the third argument. + "xor ebp, ebp", // Zero the frame address. + "push rax", // Zero the return address. + "jmp {entry}", // Call `entry`. - // Parent thread. - "2:", + // Parent thread. + "2:", - entry = sym super::thread::entry, - inlateout("rax") __NR_clone as usize => r0, - in("rdi") flags, - in("rsi") child_stack, - in("rdx") parent_tid, - in("r10") child_tid, - in("r8") newtls, - in("r9") fn_, - in("r12") num_args, - lateout("rcx") _, - lateout("r11") _, - options(nostack) - ); - r0 + entry = sym super::thread::entry, + inlateout("rax") __NR_clone as usize => r0, + in("rdi") flags, + in("rsi") child_stack, + in("rdx") parent_tid, + in("r10") child_tid, + in("r8") newtls, + in("r9") fn_, + in("r12") num_args, + lateout("rcx") _, + lateout("r11") _, + options(nostack) + ); + r0 + } } /// Write a value to the platform thread-pointer register. @@ -250,9 +258,11 @@ pub(super) unsafe fn clone( #[cfg(feature = "thread")] #[inline] pub(super) unsafe fn set_thread_pointer(ptr: *mut c_void) { - rustix::runtime::set_fs(ptr); - debug_assert_eq!(*ptr.cast::<*const c_void>(), ptr); - debug_assert_eq!(thread_pointer(), ptr); + unsafe { + rustix::runtime::set_fs(ptr); + debug_assert_eq!(*ptr.cast::<*const c_void>(), ptr); + debug_assert_eq!(thread_pointer(), ptr); + } } /// Read the value of the platform thread-pointer register. @@ -282,18 +292,20 @@ pub(super) const TLS_OFFSET: usize = 0; #[cfg(feature = "thread")] #[inline] pub(super) unsafe fn munmap_and_exit_thread(map_addr: *mut c_void, map_len: usize) -> ! { - asm!( - "syscall", - "xor edi, edi", - "mov eax, {__NR_exit}", - "syscall", - "ud2", - __NR_exit = const __NR_exit, - in("rax") __NR_munmap, - in("rdi") map_addr, - in("rsi") map_len, - options(noreturn, nostack) - ); + unsafe { + asm!( + "syscall", + "xor edi, edi", + "mov eax, {__NR_exit}", + "syscall", + "ud2", + __NR_exit = const __NR_exit, + in("rax") __NR_munmap, + in("rdi") map_addr, + in("rsi") map_len, + options(noreturn, nostack) + ); + } } #[cfg(feature = "take-charge")] diff --git a/src/getauxval.rs b/src/getauxval.rs index f5784dd..923da40 100644 --- a/src/getauxval.rs +++ b/src/getauxval.rs @@ -7,13 +7,13 @@ use core::ptr::without_provenance_mut; // `getauxval` usually returns `unsigned long`, but we make it a pointer type // so that it preserves provenance. -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn getauxval(type_: c_ulong) -> *mut c_void { _getauxval(type_) } #[cfg(target_arch = "aarch64")] -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn __getauxval(type_: c_ulong) -> *mut c_void { _getauxval(type_) } diff --git a/src/lib.rs b/src/lib.rs index 463e220..9d6fb76 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,8 +3,6 @@ #![no_std] #![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(feature = "experimental-relocate", feature(cfg_relocation_model))] -// On nightly, enable `#[naked]` functions. -#![cfg_attr(feature = "nightly", feature(naked_functions))] // On nightly, enable llvm intrinsics for additional debug asserts. #![cfg_attr(all(debug_assertions, feature = "nightly"), allow(internal_features))] #![cfg_attr( diff --git a/src/mem/fast.rs b/src/mem/fast.rs index 74e2590..0cfc230 100644 --- a/src/mem/fast.rs +++ b/src/mem/fast.rs @@ -11,42 +11,48 @@ #[cfg_attr(not(target_arch = "x86_64"), path = "impls.rs")] mod impls; -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn memcpy(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 { - impls::copy_forward(dest, src, n); - dest + unsafe { + impls::copy_forward(dest, src, n); + dest + } } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn memmove(dest: *mut u8, src: *const u8, n: usize) -> *mut u8 { - let delta = dest.addr().wrapping_sub(src.addr()); - if delta >= n { - // We can copy forwards because either dest is far enough ahead of src, - // or src is ahead of dest (and delta overflowed). - impls::copy_forward(dest, src, n); - } else { - impls::copy_backward(dest, src, n); + unsafe { + let delta = dest.addr().wrapping_sub(src.addr()); + if delta >= n { + // We can copy forwards because either dest is far enough ahead of src, + // or src is ahead of dest (and delta overflowed). + impls::copy_forward(dest, src, n); + } else { + impls::copy_backward(dest, src, n); + } + dest } - dest } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn memset(s: *mut u8, c: core::ffi::c_int, n: usize) -> *mut u8 { - impls::set_bytes(s, c as u8, n); - s + unsafe { + impls::set_bytes(s, c as u8, n); + s + } } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn memcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 { - impls::compare_bytes(s1, s2, n) + unsafe { impls::compare_bytes(s1, s2, n) } } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn bcmp(s1: *const u8, s2: *const u8, n: usize) -> i32 { - memcmp(s1, s2, n) + unsafe { memcmp(s1, s2, n) } } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn strlen(s: *const core::ffi::c_char) -> usize { - impls::c_string_length(s) + unsafe { impls::c_string_length(s) } } diff --git a/src/mem/impls.rs b/src/mem/impls.rs index 505ca96..72847cc 100644 --- a/src/mem/impls.rs +++ b/src/mem/impls.rs @@ -27,284 +27,316 @@ const WORD_COPY_THRESHOLD: usize = if 2 * WORD_SIZE > 16 { target_arch = "bpf" ))] // These targets have hardware unaligned access support. unsafe fn read_usize_unaligned(x: *const usize) -> usize { - // Do not use `core::ptr::read_unaligned` here, since it calls `copy_nonoverlapping` which - // is translated to memcpy in LLVM. - let x_read = (x as *const [u8; core::mem::size_of::()]).read(); - core::mem::transmute(x_read) + unsafe { + // Do not use `core::ptr::read_unaligned` here, since it calls `copy_nonoverlapping` which + // is translated to memcpy in LLVM. + let x_read = (x as *const [u8; core::mem::size_of::()]).read(); + usize::from_ne_bytes(x_read) + } } #[inline(always)] pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize) { - #[inline(always)] - unsafe fn copy_forward_bytes(mut dest: *mut u8, mut src: *const u8, n: usize) { - let dest_end = dest.add(n); - while dest < dest_end { - *dest = *src; - dest = dest.add(1); - src = src.add(1); + unsafe { + #[inline(always)] + unsafe fn copy_forward_bytes(mut dest: *mut u8, mut src: *const u8, n: usize) { + unsafe { + let dest_end = dest.add(n); + while dest < dest_end { + *dest = *src; + dest = dest.add(1); + src = src.add(1); + } + } } - } - - #[inline(always)] - unsafe fn copy_forward_aligned_words(dest: *mut u8, src: *const u8, n: usize) { - let mut dest_usize = dest as *mut usize; - let mut src_usize = src as *mut usize; - let dest_end = dest.add(n) as *mut usize; - while dest_usize < dest_end { - *dest_usize = *src_usize; - dest_usize = dest_usize.add(1); - src_usize = src_usize.add(1); + #[inline(always)] + unsafe fn copy_forward_aligned_words(dest: *mut u8, src: *const u8, n: usize) { + unsafe { + let mut dest_usize = dest as *mut usize; + let mut src_usize = src as *mut usize; + let dest_end = dest.add(n) as *mut usize; + + while dest_usize < dest_end { + *dest_usize = *src_usize; + dest_usize = dest_usize.add(1); + src_usize = src_usize.add(1); + } + } } - } - #[cfg(not(any( - target_arch = "x86_64", - target_arch = "x86", - target_arch = "aarch64", - target_arch = "bpf" - )))] - #[inline(always)] - unsafe fn copy_forward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) { - let mut dest_usize = dest as *mut usize; - let dest_end = dest.add(n) as *mut usize; - - // Calculate the misalignment offset and shift needed to reassemble value. - let offset = src.addr() & WORD_MASK; - let shift = offset * 8; - - // Realign src - let mut src_aligned = src.with_addr(src.addr() & !WORD_MASK).cast::(); - // This will read (but won't use) bytes out of bound. - let mut prev_word = crate::arch::oob_load(src_aligned); - - while dest_usize < dest_end { - src_aligned = src_aligned.add(1); - let cur_word = *src_aligned; - #[cfg(target_endian = "little")] - let resembled = prev_word >> shift | cur_word << (WORD_SIZE * 8 - shift); - #[cfg(target_endian = "big")] - let resembled = prev_word << shift | cur_word >> (WORD_SIZE * 8 - shift); - prev_word = cur_word; - - *dest_usize = resembled; - dest_usize = dest_usize.add(1); + #[cfg(not(any( + target_arch = "x86_64", + target_arch = "x86", + target_arch = "aarch64", + target_arch = "bpf" + )))] + #[inline(always)] + unsafe fn copy_forward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) { + unsafe { + let mut dest_usize = dest as *mut usize; + let dest_end = dest.add(n) as *mut usize; + + // Calculate the misalignment offset and shift needed to reassemble value. + let offset = src.addr() & WORD_MASK; + let shift = offset * 8; + + // Realign src + let mut src_aligned = src.with_addr(src.addr() & !WORD_MASK).cast::(); + // This will read (but won't use) bytes out of bound. + let mut prev_word = crate::arch::oob_load(src_aligned); + + while dest_usize < dest_end { + src_aligned = src_aligned.add(1); + let cur_word = *src_aligned; + #[cfg(target_endian = "little")] + let resembled = prev_word >> shift | cur_word << (WORD_SIZE * 8 - shift); + #[cfg(target_endian = "big")] + let resembled = prev_word << shift | cur_word >> (WORD_SIZE * 8 - shift); + prev_word = cur_word; + + *dest_usize = resembled; + dest_usize = dest_usize.add(1); + } + } } - } - #[cfg(any( - target_arch = "x86_64", - target_arch = "x86", - target_arch = "aarch64", - target_arch = "bpf" - ))] - #[inline(always)] - unsafe fn copy_forward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) { - let mut dest_usize = dest as *mut usize; - let mut src_usize = src as *mut usize; - let dest_end = dest.add(n) as *mut usize; - - while dest_usize < dest_end { - *dest_usize = read_usize_unaligned(src_usize); - dest_usize = dest_usize.add(1); - src_usize = src_usize.add(1); + #[cfg(any( + target_arch = "x86_64", + target_arch = "x86", + target_arch = "aarch64", + target_arch = "bpf" + ))] + #[inline(always)] + unsafe fn copy_forward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) { + unsafe { + let mut dest_usize = dest as *mut usize; + let mut src_usize = src as *mut usize; + let dest_end = dest.add(n) as *mut usize; + + while dest_usize < dest_end { + *dest_usize = read_usize_unaligned(src_usize); + dest_usize = dest_usize.add(1); + src_usize = src_usize.add(1); + } + } } - } - if n >= WORD_COPY_THRESHOLD { - // Align dest - // Because of n >= 2 * WORD_SIZE, dst_misalignment < n - let dest_misalignment = (dest.addr()).wrapping_neg() & WORD_MASK; - copy_forward_bytes(dest, src, dest_misalignment); - dest = dest.add(dest_misalignment); - src = src.add(dest_misalignment); - n -= dest_misalignment; - - let n_words = n & !WORD_MASK; - let src_misalignment = src.addr() & WORD_MASK; - if src_misalignment == 0 { - copy_forward_aligned_words(dest, src, n_words); - } else { - copy_forward_misaligned_words(dest, src, n_words); + if n >= WORD_COPY_THRESHOLD { + // Align dest + // Because of n >= 2 * WORD_SIZE, dst_misalignment < n + let dest_misalignment = (dest.addr()).wrapping_neg() & WORD_MASK; + copy_forward_bytes(dest, src, dest_misalignment); + dest = dest.add(dest_misalignment); + src = src.add(dest_misalignment); + n -= dest_misalignment; + + let n_words = n & !WORD_MASK; + let src_misalignment = src.addr() & WORD_MASK; + if src_misalignment == 0 { + copy_forward_aligned_words(dest, src, n_words); + } else { + copy_forward_misaligned_words(dest, src, n_words); + } + dest = dest.add(n_words); + src = src.add(n_words); + n -= n_words; } - dest = dest.add(n_words); - src = src.add(n_words); - n -= n_words; + copy_forward_bytes(dest, src, n); } - copy_forward_bytes(dest, src, n); } #[inline(always)] pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) { - // The following backward copy helper functions uses the pointers past the end - // as their inputs instead of pointers to the start! - #[inline(always)] - unsafe fn copy_backward_bytes(mut dest: *mut u8, mut src: *const u8, n: usize) { - let dest_start = dest.sub(n); - while dest_start < dest { - dest = dest.sub(1); - src = src.sub(1); - *dest = *src; + unsafe { + // The following backward copy helper functions uses the pointers past the end + // as their inputs instead of pointers to the start! + #[inline(always)] + unsafe fn copy_backward_bytes(mut dest: *mut u8, mut src: *const u8, n: usize) { + unsafe { + let dest_start = dest.sub(n); + while dest_start < dest { + dest = dest.sub(1); + src = src.sub(1); + *dest = *src; + } + } } - } - #[inline(always)] - unsafe fn copy_backward_aligned_words(dest: *mut u8, src: *const u8, n: usize) { - let mut dest_usize = dest as *mut usize; - let mut src_usize = src as *mut usize; - let dest_start = dest.sub(n) as *mut usize; - - while dest_start < dest_usize { - dest_usize = dest_usize.sub(1); - src_usize = src_usize.sub(1); - *dest_usize = *src_usize; + #[inline(always)] + unsafe fn copy_backward_aligned_words(dest: *mut u8, src: *const u8, n: usize) { + unsafe { + let mut dest_usize = dest as *mut usize; + let mut src_usize = src as *mut usize; + let dest_start = dest.sub(n) as *mut usize; + + while dest_start < dest_usize { + dest_usize = dest_usize.sub(1); + src_usize = src_usize.sub(1); + *dest_usize = *src_usize; + } + } } - } - #[cfg(not(any( - target_arch = "x86_64", - target_arch = "x86", - target_arch = "aarch64", - target_arch = "bpf" - )))] - #[inline(always)] - unsafe fn copy_backward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) { - let mut dest_usize = dest as *mut usize; - let dest_start = dest.sub(n) as *mut usize; - - // Calculate the misalignment offset and shift needed to reassemble value. - let offset = src.addr() & WORD_MASK; - let shift = offset * 8; - - // Realign src_aligned - let mut src_aligned = src.with_addr(src.addr() & !WORD_MASK).cast::(); - // This will read (but won't use) bytes out of bound. - let mut prev_word = crate::arch::oob_load(src_aligned); - - while dest_start < dest_usize { - src_aligned = src_aligned.sub(1); - let cur_word = *src_aligned; - #[cfg(target_endian = "little")] - let resembled = prev_word << (WORD_SIZE * 8 - shift) | cur_word >> shift; - #[cfg(target_endian = "big")] - let resembled = prev_word >> (WORD_SIZE * 8 - shift) | cur_word << shift; - prev_word = cur_word; - - dest_usize = dest_usize.sub(1); - *dest_usize = resembled; + #[cfg(not(any( + target_arch = "x86_64", + target_arch = "x86", + target_arch = "aarch64", + target_arch = "bpf" + )))] + #[inline(always)] + unsafe fn copy_backward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) { + unsafe { + let mut dest_usize = dest as *mut usize; + let dest_start = dest.sub(n) as *mut usize; + + // Calculate the misalignment offset and shift needed to reassemble value. + let offset = src.addr() & WORD_MASK; + let shift = offset * 8; + + // Realign src_aligned + let mut src_aligned = src.with_addr(src.addr() & !WORD_MASK).cast::(); + // This will read (but won't use) bytes out of bound. + let mut prev_word = crate::arch::oob_load(src_aligned); + + while dest_start < dest_usize { + src_aligned = src_aligned.sub(1); + let cur_word = *src_aligned; + #[cfg(target_endian = "little")] + let resembled = prev_word << (WORD_SIZE * 8 - shift) | cur_word >> shift; + #[cfg(target_endian = "big")] + let resembled = prev_word >> (WORD_SIZE * 8 - shift) | cur_word << shift; + prev_word = cur_word; + + dest_usize = dest_usize.sub(1); + *dest_usize = resembled; + } + } } - } - #[cfg(any( - target_arch = "x86_64", - target_arch = "x86", - target_arch = "aarch64", - target_arch = "bpf" - ))] - #[inline(always)] - unsafe fn copy_backward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) { - let mut dest_usize = dest as *mut usize; - let mut src_usize = src as *mut usize; - let dest_start = dest.sub(n) as *mut usize; - - while dest_start < dest_usize { - dest_usize = dest_usize.sub(1); - src_usize = src_usize.sub(1); - *dest_usize = read_usize_unaligned(src_usize); + #[cfg(any( + target_arch = "x86_64", + target_arch = "x86", + target_arch = "aarch64", + target_arch = "bpf" + ))] + #[inline(always)] + unsafe fn copy_backward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) { + unsafe { + let mut dest_usize = dest as *mut usize; + let mut src_usize = src as *mut usize; + let dest_start = dest.sub(n) as *mut usize; + + while dest_start < dest_usize { + dest_usize = dest_usize.sub(1); + src_usize = src_usize.sub(1); + *dest_usize = read_usize_unaligned(src_usize); + } + } } - } - let mut dest = dest.add(n); - let mut src = src.add(n); - - if n >= WORD_COPY_THRESHOLD { - // Align dest - // Because of n >= 2 * WORD_SIZE, dst_misalignment < n - let dest_misalignment = dest.addr() & WORD_MASK; - copy_backward_bytes(dest, src, dest_misalignment); - dest = dest.sub(dest_misalignment); - src = src.sub(dest_misalignment); - n -= dest_misalignment; - - let n_words = n & !WORD_MASK; - let src_misalignment = src.addr() & WORD_MASK; - if src_misalignment == 0 { - copy_backward_aligned_words(dest, src, n_words); - } else { - copy_backward_misaligned_words(dest, src, n_words); + let mut dest = dest.add(n); + let mut src = src.add(n); + + if n >= WORD_COPY_THRESHOLD { + // Align dest + // Because of n >= 2 * WORD_SIZE, dst_misalignment < n + let dest_misalignment = dest.addr() & WORD_MASK; + copy_backward_bytes(dest, src, dest_misalignment); + dest = dest.sub(dest_misalignment); + src = src.sub(dest_misalignment); + n -= dest_misalignment; + + let n_words = n & !WORD_MASK; + let src_misalignment = src.addr() & WORD_MASK; + if src_misalignment == 0 { + copy_backward_aligned_words(dest, src, n_words); + } else { + copy_backward_misaligned_words(dest, src, n_words); + } + dest = dest.sub(n_words); + src = src.sub(n_words); + n -= n_words; } - dest = dest.sub(n_words); - src = src.sub(n_words); - n -= n_words; + copy_backward_bytes(dest, src, n); } - copy_backward_bytes(dest, src, n); } #[inline(always)] pub unsafe fn set_bytes(mut s: *mut u8, c: u8, mut n: usize) { - #[inline(always)] - pub unsafe fn set_bytes_bytes(mut s: *mut u8, c: u8, n: usize) { - let end = s.add(n); - while s < end { - *s = c; - s = s.add(1); + unsafe { + #[inline(always)] + pub unsafe fn set_bytes_bytes(mut s: *mut u8, c: u8, n: usize) { + unsafe { + let end = s.add(n); + while s < end { + *s = c; + s = s.add(1); + } + } } - } - #[inline(always)] - pub unsafe fn set_bytes_words(s: *mut u8, c: u8, n: usize) { - let mut broadcast = c as usize; - let mut bits = 8; - while bits < WORD_SIZE * 8 { - broadcast |= broadcast << bits; - bits *= 2; + #[inline(always)] + pub unsafe fn set_bytes_words(s: *mut u8, c: u8, n: usize) { + unsafe { + let mut broadcast = c as usize; + let mut bits = 8; + while bits < WORD_SIZE * 8 { + broadcast |= broadcast << bits; + bits *= 2; + } + + let mut s_usize = s as *mut usize; + let end = s.add(n) as *mut usize; + + while s_usize < end { + *s_usize = broadcast; + s_usize = s_usize.add(1); + } + } } - let mut s_usize = s as *mut usize; - let end = s.add(n) as *mut usize; - - while s_usize < end { - *s_usize = broadcast; - s_usize = s_usize.add(1); + if n >= WORD_COPY_THRESHOLD { + // Align s + // Because of n >= 2 * WORD_SIZE, dst_misalignment < n + let misalignment = (s.addr()).wrapping_neg() & WORD_MASK; + set_bytes_bytes(s, c, misalignment); + s = s.add(misalignment); + n -= misalignment; + + let n_words = n & !WORD_MASK; + set_bytes_words(s, c, n_words); + s = s.add(n_words); + n -= n_words; } + set_bytes_bytes(s, c, n); } - - if n >= WORD_COPY_THRESHOLD { - // Align s - // Because of n >= 2 * WORD_SIZE, dst_misalignment < n - let misalignment = (s.addr()).wrapping_neg() & WORD_MASK; - set_bytes_bytes(s, c, misalignment); - s = s.add(misalignment); - n -= misalignment; - - let n_words = n & !WORD_MASK; - set_bytes_words(s, c, n_words); - s = s.add(n_words); - n -= n_words; - } - set_bytes_bytes(s, c, n); } #[inline(always)] pub unsafe fn compare_bytes(s1: *const u8, s2: *const u8, n: usize) -> i32 { - let mut i = 0; - while i < n { - let a = *s1.add(i); - let b = *s2.add(i); - if a != b { - return a as i32 - b as i32; + unsafe { + let mut i = 0; + while i < n { + let a = *s1.add(i); + let b = *s2.add(i); + if a != b { + return a as i32 - b as i32; + } + i += 1; } - i += 1; + 0 } - 0 } #[inline(always)] pub unsafe fn c_string_length(mut s: *const core::ffi::c_char) -> usize { - let mut n = 0; - while *s != 0 { - n += 1; - s = s.add(1); + unsafe { + let mut n = 0; + while *s != 0 { + n += 1; + s = s.add(1); + } + n } - n } diff --git a/src/mem/small.rs b/src/mem/small.rs index 77dc433..06b33af 100644 --- a/src/mem/small.rs +++ b/src/mem/small.rs @@ -5,28 +5,12 @@ use core::ffi::{c_char, c_int, c_void}; -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn memcpy(dst: *mut c_void, src: *const c_void, len: usize) -> *mut c_void { - let start = dst; - let mut dst = dst.cast::(); - let mut src = src.cast::(); - let dst_end = dst.add(len); - while dst < dst_end { - *dst = *src; - dst = dst.add(1); - src = src.add(1); - core::arch::asm!(""); - } - start -} - -#[no_mangle] -unsafe extern "C" fn memmove(dst: *mut c_void, src: *const c_void, len: usize) -> *mut c_void { - let start = dst; - let mut dst = dst.cast::(); - let mut src = src.cast::(); - let delta = (dst.addr()).wrapping_sub(src.addr()); - if delta >= len { + unsafe { + let start = dst; + let mut dst = dst.cast::(); + let mut src = src.cast::(); let dst_end = dst.add(len); while dst < dst_end { *dst = *src; @@ -34,63 +18,89 @@ unsafe extern "C" fn memmove(dst: *mut c_void, src: *const c_void, len: usize) - src = src.add(1); core::arch::asm!(""); } - } else { - let dst_start = dst; - let mut dst = dst.add(len); - let mut src = src.add(len); - while dst > dst_start { - dst = dst.sub(1); - src = src.sub(1); - *dst = *src; - core::arch::asm!(""); + start + } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn memmove(dst: *mut c_void, src: *const c_void, len: usize) -> *mut c_void { + unsafe { + let start = dst; + let mut dst = dst.cast::(); + let mut src = src.cast::(); + let delta = (dst.addr()).wrapping_sub(src.addr()); + if delta >= len { + let dst_end = dst.add(len); + while dst < dst_end { + *dst = *src; + dst = dst.add(1); + src = src.add(1); + core::arch::asm!(""); + } + } else { + let dst_start = dst; + let mut dst = dst.add(len); + let mut src = src.add(len); + while dst > dst_start { + dst = dst.sub(1); + src = src.sub(1); + *dst = *src; + core::arch::asm!(""); + } } + start } - start } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn memset(dst: *mut c_void, fill: c_int, len: usize) -> *mut c_void { - let mut s = dst.cast::(); - let end = s.add(len); - while s < end { - *s = fill as _; - s = s.add(1); - core::arch::asm!(""); + unsafe { + let mut s = dst.cast::(); + let end = s.add(len); + while s < end { + *s = fill as _; + s = s.add(1); + core::arch::asm!(""); + } + dst } - dst } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn memcmp(a: *const c_void, b: *const c_void, len: usize) -> c_int { - let a = a.cast::(); - let b = b.cast::(); - let mut i = 0; - while i < len { - let a = *a.add(i); - let b = *b.add(i); - if a != b { - return a as i32 - b as i32; + unsafe { + let a = a.cast::(); + let b = b.cast::(); + let mut i = 0; + while i < len { + let a = *a.add(i); + let b = *b.add(i); + if a != b { + return a as i32 - b as i32; + } + i += 1; + core::arch::asm!(""); } - i += 1; - core::arch::asm!(""); + 0 } - 0 } // Obsolescent -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn bcmp(a: *const c_void, b: *const c_void, len: usize) -> c_int { - memcmp(a, b, len) + unsafe { memcmp(a, b, len) } } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn strlen(s: *const c_char) -> usize { - let mut s = s; - let mut n = 0; - while *s != 0 { - n += 1; - s = s.add(1); - core::arch::asm!(""); + unsafe { + let mut s = s; + let mut n = 0; + while *s != 0 { + n += 1; + s = s.add(1); + core::arch::asm!(""); + } + n } - n } diff --git a/src/mem/x86_64.rs b/src/mem/x86_64.rs index 7a28430..6b016ff 100644 --- a/src/mem/x86_64.rs +++ b/src/mem/x86_64.rs @@ -28,150 +28,164 @@ use core::mem; #[inline(always)] #[cfg(target_feature = "ermsb")] pub unsafe fn copy_forward(dest: *mut u8, src: *const u8, count: usize) { - // FIXME: Use the Intel syntax once we drop LLVM 9 support on rust-lang/rust. - core::arch::asm!( - "repe movsb (%rsi), (%rdi)", - inout("rcx") count => _, - inout("rdi") dest => _, - inout("rsi") src => _, - options(att_syntax, nostack, preserves_flags) - ); + unsafe { + // FIXME: Use the Intel syntax once we drop LLVM 9 support on rust-lang/rust. + core::arch::asm!( + "repe movsb (%rsi), (%rdi)", + inout("rcx") count => _, + inout("rdi") dest => _, + inout("rsi") src => _, + options(att_syntax, nostack, preserves_flags) + ); + } } #[inline(always)] #[cfg(not(target_feature = "ermsb"))] pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, count: usize) { - let (pre_byte_count, qword_count, byte_count) = rep_param(dest, count); - // Separating the blocks gives the compiler more freedom to reorder instructions. - asm!( - "rep movsb", - inout("ecx") pre_byte_count => _, - inout("rdi") dest => dest, - inout("rsi") src => src, - options(att_syntax, nostack, preserves_flags) - ); - asm!( - "rep movsq", - inout("rcx") qword_count => _, - inout("rdi") dest => dest, - inout("rsi") src => src, - options(att_syntax, nostack, preserves_flags) - ); - asm!( - "rep movsb", - inout("ecx") byte_count => _, - inout("rdi") dest => _, - inout("rsi") src => _, - options(att_syntax, nostack, preserves_flags) - ); + unsafe { + let (pre_byte_count, qword_count, byte_count) = rep_param(dest, count); + // Separating the blocks gives the compiler more freedom to reorder instructions. + asm!( + "rep movsb", + inout("ecx") pre_byte_count => _, + inout("rdi") dest => dest, + inout("rsi") src => src, + options(att_syntax, nostack, preserves_flags) + ); + asm!( + "rep movsq", + inout("rcx") qword_count => _, + inout("rdi") dest => dest, + inout("rsi") src => src, + options(att_syntax, nostack, preserves_flags) + ); + asm!( + "rep movsb", + inout("ecx") byte_count => _, + inout("rdi") dest => _, + inout("rsi") src => _, + options(att_syntax, nostack, preserves_flags) + ); + } } #[inline(always)] pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, count: usize) { - let (pre_byte_count, qword_count, byte_count) = rep_param(dest, count); - // We can't separate this block due to std/cld - asm!( - "std", - "rep movsb", - "sub $7, %rsi", - "sub $7, %rdi", - "mov {qword_count}, %rcx", - "rep movsq", - "test {pre_byte_count:e}, {pre_byte_count:e}", - "add $7, %rsi", - "add $7, %rdi", - "mov {pre_byte_count:e}, %ecx", - "rep movsb", - "cld", - pre_byte_count = in(reg) pre_byte_count, - qword_count = in(reg) qword_count, - inout("ecx") byte_count => _, - inout("rdi") dest.add(count - 1) => _, - inout("rsi") src.add(count - 1) => _, - // We modify flags, but we restore it afterwards - options(att_syntax, nostack, preserves_flags) - ); + unsafe { + let (pre_byte_count, qword_count, byte_count) = rep_param(dest, count); + // We can't separate this block due to std/cld + asm!( + "std", + "rep movsb", + "sub $7, %rsi", + "sub $7, %rdi", + "mov {qword_count}, %rcx", + "rep movsq", + "test {pre_byte_count:e}, {pre_byte_count:e}", + "add $7, %rsi", + "add $7, %rdi", + "mov {pre_byte_count:e}, %ecx", + "rep movsb", + "cld", + pre_byte_count = in(reg) pre_byte_count, + qword_count = in(reg) qword_count, + inout("ecx") byte_count => _, + inout("rdi") dest.add(count - 1) => _, + inout("rsi") src.add(count - 1) => _, + // We modify flags, but we restore it afterwards + options(att_syntax, nostack, preserves_flags) + ); + } } #[inline(always)] #[cfg(target_feature = "ermsb")] pub unsafe fn set_bytes(dest: *mut u8, c: u8, count: usize) { - // FIXME: Use the Intel syntax once we drop LLVM 9 support on rust-lang/rust. - core::arch::asm!( - "repe stosb %al, (%rdi)", - inout("rcx") count => _, - inout("rdi") dest => _, - inout("al") c => _, - options(att_syntax, nostack, preserves_flags) - ) + unsafe { + // FIXME: Use the Intel syntax once we drop LLVM 9 support on rust-lang/rust. + core::arch::asm!( + "repe stosb %al, (%rdi)", + inout("rcx") count => _, + inout("rdi") dest => _, + inout("al") c => _, + options(att_syntax, nostack, preserves_flags) + ) + } } #[inline(always)] #[cfg(not(target_feature = "ermsb"))] pub unsafe fn set_bytes(mut dest: *mut u8, c: u8, count: usize) { - let c = c as u64 * 0x0101_0101_0101_0101; - let (pre_byte_count, qword_count, byte_count) = rep_param(dest, count); - // Separating the blocks gives the compiler more freedom to reorder instructions. - asm!( - "rep stosb", - inout("ecx") pre_byte_count => _, - inout("rdi") dest => dest, - in("rax") c, - options(att_syntax, nostack, preserves_flags) - ); - asm!( - "rep stosq", - inout("rcx") qword_count => _, - inout("rdi") dest => dest, - in("rax") c, - options(att_syntax, nostack, preserves_flags) - ); - asm!( - "rep stosb", - inout("ecx") byte_count => _, - inout("rdi") dest => _, - in("rax") c, - options(att_syntax, nostack, preserves_flags) - ); + unsafe { + let c = c as u64 * 0x0101_0101_0101_0101; + let (pre_byte_count, qword_count, byte_count) = rep_param(dest, count); + // Separating the blocks gives the compiler more freedom to reorder instructions. + asm!( + "rep stosb", + inout("ecx") pre_byte_count => _, + inout("rdi") dest => dest, + in("rax") c, + options(att_syntax, nostack, preserves_flags) + ); + asm!( + "rep stosq", + inout("rcx") qword_count => _, + inout("rdi") dest => dest, + in("rax") c, + options(att_syntax, nostack, preserves_flags) + ); + asm!( + "rep stosb", + inout("ecx") byte_count => _, + inout("rdi") dest => _, + in("rax") c, + options(att_syntax, nostack, preserves_flags) + ); + } } #[inline(always)] pub unsafe fn compare_bytes(a: *const u8, b: *const u8, n: usize) -> i32 { - #[inline(always)] - unsafe fn cmp(mut a: *const T, mut b: *const T, n: usize, f: F) -> i32 - where - T: Clone + Copy + Eq, - U: Clone + Copy + Eq, - F: FnOnce(*const U, *const U, usize) -> i32, - { - // Ensure T is not a ZST. - assert!(mem::size_of::() != 0); + unsafe { + #[inline(always)] + unsafe fn cmp(mut a: *const T, mut b: *const T, n: usize, f: F) -> i32 + where + T: Clone + Copy + Eq, + U: Clone + Copy + Eq, + F: FnOnce(*const U, *const U, usize) -> i32, + { + unsafe { + // Ensure T is not a ZST. + assert!(mem::size_of::() != 0); - let end = a.add(n / mem::size_of::()); - while a != end { - if a.read_unaligned() != b.read_unaligned() { - return f(a.cast(), b.cast(), mem::size_of::()); + let end = a.add(n / mem::size_of::()); + while a != end { + if a.read_unaligned() != b.read_unaligned() { + return f(a.cast(), b.cast(), mem::size_of::()); + } + a = a.add(1); + b = b.add(1); + } + f(a.cast(), b.cast(), n % mem::size_of::()) } - a = a.add(1); - b = b.add(1); } - f(a.cast(), b.cast(), n % mem::size_of::()) - } - let c1 = |mut a: *const u8, mut b: *const u8, n| { - for _ in 0..n { - if a.read() != b.read() { - return i32::from(a.read()) - i32::from(b.read()); + let c1 = |mut a: *const u8, mut b: *const u8, n| { + for _ in 0..n { + if a.read() != b.read() { + return i32::from(a.read()) - i32::from(b.read()); + } + a = a.add(1); + b = b.add(1); } - a = a.add(1); - b = b.add(1); - } - 0 - }; - let c2 = |a: *const u16, b, n| cmp(a, b, n, c1); - let c4 = |a: *const u32, b, n| cmp(a, b, n, c2); - let c8 = |a: *const u64, b, n| cmp(a, b, n, c4); - let c16 = |a: *const u128, b, n| cmp(a, b, n, c8); - c16(a.cast(), b.cast(), n) + 0 + }; + let c2 = |a: *const u16, b, n| cmp(a, b, n, c1); + let c4 = |a: *const u32, b, n| cmp(a, b, n, c2); + let c8 = |a: *const u64, b, n| cmp(a, b, n, c4); + let c16 = |a: *const u128, b, n| cmp(a, b, n, c8); + c16(a.cast(), b.cast(), n) + } } // In order to process more than on byte simultaneously when executing strlen, @@ -187,50 +201,31 @@ pub unsafe fn compare_bytes(a: *const u8, b: *const u8, n: usize) -> i32 { #[cfg(target_feature = "sse2")] #[inline(always)] pub unsafe fn c_string_length(mut s: *const core::ffi::c_char) -> usize { - use core::arch::x86_64::{__m128i, _mm_cmpeq_epi8, _mm_movemask_epi8, _mm_set1_epi8}; - - let mut n = 0; - - // The use of _mm_movemask_epi8 and company allow for speedups, - // but they aren't cheap by themselves. Thus, possibly small strings - // are handled in simple loops. + unsafe { + use core::arch::x86_64::{__m128i, _mm_cmpeq_epi8, _mm_movemask_epi8, _mm_set1_epi8}; - for _ in 0..4 { - if *s == 0 { - return n; - } - - n += 1; - s = s.add(1); - } + let mut n = 0; - // Shave of the least significand bits to align the address to a 16 - // byte boundary. The shaved of bits are used to correct the first iteration. + // The use of _mm_movemask_epi8 and company allow for speedups, + // but they aren't cheap by themselves. Thus, possibly small strings + // are handled in simple loops. - let align = s.addr() & 15; - let mut s = s.with_addr(s.addr() - align) as *const __m128i; - let zero = _mm_set1_epi8(0); + for _ in 0..4 { + if *s == 0 { + return n; + } - let x = { - let r; - asm!( - "movdqa ({addr}), {dest}", - addr = in(reg) s, - dest = out(xmm_reg) r, - options(att_syntax, nostack), - ); - r - }; - let cmp = _mm_movemask_epi8(_mm_cmpeq_epi8(x, zero)) >> align; + n += 1; + s = s.add(1); + } - if cmp != 0 { - return n + cmp.trailing_zeros() as usize; - } + // Shave of the least significand bits to align the address to a 16 + // byte boundary. The shaved of bits are used to correct the first iteration. - n += 16 - align; - s = s.add(1); + let align = s.addr() & 15; + let mut s = s.with_addr(s.addr() - align) as *const __m128i; + let zero = _mm_set1_epi8(0); - loop { let x = { let r; asm!( @@ -241,13 +236,34 @@ pub unsafe fn c_string_length(mut s: *const core::ffi::c_char) -> usize { ); r }; - let cmp = _mm_movemask_epi8(_mm_cmpeq_epi8(x, zero)) as u32; - if cmp == 0 { - n += 16; - s = s.add(1); - } else { + let cmp = _mm_movemask_epi8(_mm_cmpeq_epi8(x, zero)) >> align; + + if cmp != 0 { return n + cmp.trailing_zeros() as usize; } + + n += 16 - align; + s = s.add(1); + + loop { + let x = { + let r; + asm!( + "movdqa ({addr}), {dest}", + addr = in(reg) s, + dest = out(xmm_reg) r, + options(att_syntax, nostack), + ); + r + }; + let cmp = _mm_movemask_epi8(_mm_cmpeq_epi8(x, zero)) as u32; + if cmp == 0 { + n += 16; + s = s.add(1); + } else { + return n + cmp.trailing_zeros() as usize; + } + } } } @@ -256,50 +272,52 @@ pub unsafe fn c_string_length(mut s: *const core::ffi::c_char) -> usize { #[cfg(not(target_feature = "sse2"))] #[inline(always)] pub unsafe fn c_string_length(mut s: *const core::ffi::c_char) -> usize { - let mut n = 0; + unsafe { + let mut n = 0; - // Check bytes in steps of one until - // either a zero byte is discovered or - // pointer is aligned to an eight byte boundary. + // Check bytes in steps of one until + // either a zero byte is discovered or + // pointer is aligned to an eight byte boundary. - while s as usize & 7 != 0 { - if *s == 0 { - return n; + while s as usize & 7 != 0 { + if *s == 0 { + return n; + } + n += 1; + s = s.add(1); } - n += 1; - s = s.add(1); - } - // Check bytes in steps of eight until a zero - // byte is discovered. + // Check bytes in steps of eight until a zero + // byte is discovered. - let mut s = s as *const u64; + let mut s = s as *const u64; - loop { - let mut cs = { - let r: u64; - asm!( - "mov ({addr}), {dest}", - addr = in(reg) s, - dest = out(reg) r, - options(att_syntax, nostack), - ); - r - }; - // Detect if a word has a zero byte, taken from - // https://graphics.stanford.edu/~seander/bithacks.html - if (cs.wrapping_sub(0x0101010101010101) & !cs & 0x8080808080808080) != 0 { - loop { - if cs & 255 == 0 { - return n; - } else { - cs >>= 8; - n += 1; + loop { + let mut cs = { + let r: u64; + asm!( + "mov ({addr}), {dest}", + addr = in(reg) s, + dest = out(reg) r, + options(att_syntax, nostack), + ); + r + }; + // Detect if a word has a zero byte, taken from + // https://graphics.stanford.edu/~seander/bithacks.html + if (cs.wrapping_sub(0x0101010101010101) & !cs & 0x8080808080808080) != 0 { + loop { + if cs & 255 == 0 { + return n; + } else { + cs >>= 8; + n += 1; + } } + } else { + n += 8; + s = s.add(1); } - } else { - n += 8; - s = s.add(1); } } } diff --git a/src/naked.rs b/src/naked.rs index ba67aae..a5f285b 100644 --- a/src/naked.rs +++ b/src/naked.rs @@ -40,8 +40,8 @@ macro_rules! naked_fn { $($label:ident = $kind:ident $path:path),* ) => { #[doc = $doc] - #[naked] - #[no_mangle] + #[unsafe(naked)] + #[unsafe(no_mangle)] $vis unsafe extern "C" fn $name $args -> $ret { core::arch::naked_asm!( $($code),*, @@ -62,7 +62,7 @@ macro_rules! naked_fn { $($code:literal),*; $($label:ident = $kind:ident $path:path),* ) => { - extern "C" { + unsafe extern "C" { #[doc = $doc] $vis fn $name $args -> $ret; } diff --git a/src/program/libc.rs b/src/program/libc.rs index 517a791..6318d42 100644 --- a/src/program/libc.rs +++ b/src/program/libc.rs @@ -8,7 +8,7 @@ //! /// //! /// SAFETY: `argc`, `argv`, and `envp` describe incoming program //! /// command-line arguments and environment variables. -//! #[no_mangle] +//! #[unsafe(no_mangle)] //! unsafe fn origin_main(argc: usize, argv: *mut *mut u8, envp: *mut *mut u8) -> i32 { //! todo!("Run the program and return the program exit status.") //! } @@ -40,7 +40,7 @@ use libc::c_int; pub fn at_exit(func: Box) { use core::ffi::c_void; - extern "C" { + unsafe extern "C" { // fn __cxa_atexit( func: unsafe extern "C" fn(*mut c_void), @@ -51,7 +51,9 @@ pub fn at_exit(func: Box) { // The function to pass to `__cxa_atexit`. unsafe extern "C" fn at_exit_func(arg: *mut c_void) { - Box::from_raw(arg.cast::>())(); + unsafe { + Box::from_raw(arg.cast::>())(); + } } let at_exit_arg = Box::into_raw(Box::new(func)).cast::(); diff --git a/src/program/linux_raw.rs b/src/program/linux_raw.rs index 43e3b33..9266b80 100644 --- a/src/program/linux_raw.rs +++ b/src/program/linux_raw.rs @@ -8,7 +8,7 @@ //! /// //! /// SAFETY: `argc`, `argv`, and `envp` describe incoming program //! /// command-line arguments and environment variables. -//! #[no_mangle] +//! #[unsafe(no_mangle)] //! unsafe fn origin_main(argc: usize, argv: *mut *mut u8, envp: *mut *mut u8) -> i32 { //! todo!("Run the program and return the program exit status.") //! } @@ -47,119 +47,121 @@ compile_error!("\"origin-program\" depends on either \"origin-start\" or \"exter /// /// `mem` must point to the stack as provided by the operating system. pub(super) unsafe extern "C" fn entry(mem: *mut usize) -> ! { - // Do some basic precondition checks, to ensure that our assembly code did - // what we expect it to do. These are debug-only, to keep the release-mode - // startup code small and simple to disassemble and inspect. - #[cfg(debug_assertions)] - #[cfg(feature = "origin-start")] - { - // Check that `mem` is where we expect it to be. - debug_assert_ne!(mem, core::ptr::null_mut()); - debug_assert_eq!(mem.addr() & 0xf, 0); - - // If we have nightly, we can do additional checks. - #[cfg(feature = "nightly")] + unsafe { + // Do some basic precondition checks, to ensure that our assembly code did + // what we expect it to do. These are debug-only, to keep the release-mode + // startup code small and simple to disassemble and inspect. + #[cfg(debug_assertions)] + #[cfg(feature = "origin-start")] { - extern "C" { - #[link_name = "llvm.frameaddress"] - fn builtin_frame_address(level: i32) -> *const u8; - #[link_name = "llvm.returnaddress"] - fn builtin_return_address(level: i32) -> *const u8; + // Check that `mem` is where we expect it to be. + debug_assert_ne!(mem, core::ptr::null_mut()); + debug_assert_eq!(mem.addr() & 0xf, 0); + + // If we have nightly, we can do additional checks. + #[cfg(feature = "nightly")] + { + unsafe extern "C" { + #[link_name = "llvm.frameaddress"] + fn builtin_frame_address(level: i32) -> *const u8; + #[link_name = "llvm.returnaddress"] + fn builtin_return_address(level: i32) -> *const u8; + #[cfg(target_arch = "aarch64")] + #[link_name = "llvm.sponentry"] + fn builtin_sponentry() -> *const u8; + } + + // Check that `mem` is where we expect it to be. + debug_assert!(builtin_frame_address(0).addr() <= mem.addr()); + + // Check that the incoming stack pointer is where we expect it to be. + debug_assert_eq!(builtin_return_address(0), core::ptr::null()); + debug_assert_ne!(builtin_frame_address(0), core::ptr::null()); + #[cfg(not(any(target_arch = "arm", target_arch = "x86")))] + debug_assert_eq!(builtin_frame_address(0).addr() & 0xf, 0); + #[cfg(target_arch = "arm")] + debug_assert_eq!(builtin_frame_address(0).addr() & 0x7, 0); + #[cfg(target_arch = "x86")] + debug_assert_eq!(builtin_frame_address(0).addr() & 0xf, 8); + debug_assert_eq!(builtin_frame_address(1), core::ptr::null()); #[cfg(target_arch = "aarch64")] - #[link_name = "llvm.sponentry"] - fn builtin_sponentry() -> *const u8; + debug_assert_ne!(builtin_sponentry(), core::ptr::null()); + #[cfg(target_arch = "aarch64")] + debug_assert_eq!(builtin_sponentry().addr() & 0xf, 0); } - - // Check that `mem` is where we expect it to be. - debug_assert!(builtin_frame_address(0).addr() <= mem.addr()); - - // Check that the incoming stack pointer is where we expect it to be. - debug_assert_eq!(builtin_return_address(0), core::ptr::null()); - debug_assert_ne!(builtin_frame_address(0), core::ptr::null()); - #[cfg(not(any(target_arch = "arm", target_arch = "x86")))] - debug_assert_eq!(builtin_frame_address(0).addr() & 0xf, 0); - #[cfg(target_arch = "arm")] - debug_assert_eq!(builtin_frame_address(0).addr() & 0x7, 0); - #[cfg(target_arch = "x86")] - debug_assert_eq!(builtin_frame_address(0).addr() & 0xf, 8); - debug_assert_eq!(builtin_frame_address(1), core::ptr::null()); - #[cfg(target_arch = "aarch64")] - debug_assert_ne!(builtin_sponentry(), core::ptr::null()); - #[cfg(target_arch = "aarch64")] - debug_assert_eq!(builtin_sponentry().addr() & 0xf, 0); } - } - - // Compute `argc`, `argv`, and `envp`. - let (argc, argv, envp) = compute_args(mem); - - // Before doing anything else, perform dynamic relocations. - #[cfg(all(feature = "experimental-relocate", feature = "origin-start"))] - #[cfg(relocation_model = "pic")] - { - crate::relocate::relocate(envp); - } - // Initialize program state before running any user code. - init_runtime(mem, envp); + // Compute `argc`, `argv`, and `envp`. + let (argc, argv, envp) = compute_args(mem); - // Call the functions registered via `.init_array`. - #[cfg(feature = "init-array")] - { - use core::arch::asm; - use core::ffi::c_void; - - // The linker-generated symbols that mark the start and end of the - // `.init_array` section. - extern "C" { - static __init_array_start: c_void; - static __init_array_end: c_void; + // Before doing anything else, perform dynamic relocations. + #[cfg(all(feature = "experimental-relocate", feature = "origin-start"))] + #[cfg(relocation_model = "pic")] + { + crate::relocate::relocate(envp); } - // Call the `.init_array` functions. As glibc does, pass argc, argv, - // and envp as extra arguments. In addition to glibc ABI compatibility, - // c-scape relies on this. - type InitFn = unsafe extern "C" fn(c_int, *mut *mut u8, *mut *mut u8); - let mut init = core::ptr::addr_of!(__init_array_start).cast::(); - let init_end = core::ptr::addr_of!(__init_array_end).cast::(); - // Prevent the optimizer from optimizing the `!=` comparison to true; - // `init` and `init_start` may have the same address. - asm!("# {}", inout(reg) init, options(pure, nomem, nostack, preserves_flags)); + // Initialize program state before running any user code. + init_runtime(mem, envp); - while init != init_end { - #[cfg(feature = "log")] - log::trace!( - "Calling `.init_array`-registered function `{:?}({:?}, {:?}, {:?})`", - *init, - argc, - argv, - envp - ); + // Call the functions registered via `.init_array`. + #[cfg(feature = "init-array")] + { + use core::arch::asm; + use core::ffi::c_void; - (*init)(argc, argv, envp); + // The linker-generated symbols that mark the start and end of the + // `.init_array` section. + extern "C" { + static __init_array_start: c_void; + static __init_array_end: c_void; + } - init = init.add(1); + // Call the `.init_array` functions. As glibc does, pass argc, argv, + // and envp as extra arguments. In addition to glibc ABI compatibility, + // c-scape relies on this. + type InitFn = unsafe extern "C" fn(c_int, *mut *mut u8, *mut *mut u8); + let mut init = core::ptr::addr_of!(__init_array_start).cast::(); + let init_end = core::ptr::addr_of!(__init_array_end).cast::(); + // Prevent the optimizer from optimizing the `!=` comparison to true; + // `init` and `init_start` may have the same address. + asm!("# {}", inout(reg) init, options(pure, nomem, nostack, preserves_flags)); + + while init != init_end { + #[cfg(feature = "log")] + log::trace!( + "Calling `.init_array`-registered function `{:?}({:?}, {:?}, {:?})`", + *init, + argc, + argv, + envp + ); + + (*init)(argc, argv, envp); + + init = init.add(1); + } } - } - { - // Declare `origin_main` as documented in [`crate::program`]. - extern "Rust" { - fn origin_main(argc: usize, argv: *mut *mut u8, envp: *mut *mut u8) -> i32; - } + { + // Declare `origin_main` as documented in [`crate::program`]. + unsafe extern "Rust" { + fn origin_main(argc: usize, argv: *mut *mut u8, envp: *mut *mut u8) -> i32; + } - #[cfg(feature = "log")] - log::trace!("Calling `origin_main({:?}, {:?}, {:?})`", argc, argv, envp); + #[cfg(feature = "log")] + log::trace!("Calling `origin_main({:?}, {:?}, {:?})`", argc, argv, envp); - // Call `origin_main`. - let status = origin_main(argc as usize, argv, envp); + // Call `origin_main`. + let status = origin_main(argc as usize, argv, envp); - #[cfg(feature = "log")] - log::trace!("`origin_main` returned `{:?}`", status); + #[cfg(feature = "log")] + log::trace!("`origin_main` returned `{:?}`", status); - // Run functions registered with `at_exit`, and exit with - // `origin_main`'s return value. - exit(status) + // Run functions registered with `at_exit`, and exit with + // `origin_main`'s return value. + exit(status) + } } } @@ -172,7 +174,7 @@ pub(super) unsafe extern "C" fn entry(mem: *mut usize) -> ! { /// on the initial stack. #[cfg(feature = "external-start")] pub unsafe fn start(mem: *mut usize) -> ! { - entry(mem) + unsafe { entry(mem) } } /// Compute `argc`, `argv`, and `envp`. @@ -181,19 +183,21 @@ pub unsafe fn start(mem: *mut usize) -> ! { /// /// `mem` must point to the stack as provided by the operating system. unsafe fn compute_args(mem: *mut usize) -> (i32, *mut *mut u8, *mut *mut u8) { - use linux_raw_sys::ctypes::c_uint; + unsafe { + use linux_raw_sys::ctypes::c_uint; - let kernel_argc = *mem; - let argc = kernel_argc as c_int; - let argv = mem.add(1).cast::<*mut u8>(); - let envp = argv.add(argc as c_uint as usize + 1); + let kernel_argc = *mem; + let argc = kernel_argc as c_int; + let argv = mem.add(1).cast::<*mut u8>(); + let envp = argv.add(argc as c_uint as usize + 1); - // Do a few more precondition checks on `argc` and `argv`. - debug_assert!(argc >= 0); - debug_assert_eq!(kernel_argc, argc as _); - debug_assert_eq!(*argv.add(argc as usize), core::ptr::null_mut()); + // Do a few more precondition checks on `argc` and `argv`. + debug_assert!(argc >= 0); + debug_assert_eq!(kernel_argc, argc as _); + debug_assert_eq!(*argv.add(argc as usize), core::ptr::null_mut()); - (argc, argv, envp) + (argc, argv, envp) + } } /// Initialize `origin` and `rustix` runtime state. @@ -207,7 +211,9 @@ unsafe fn init_runtime(mem: *mut usize, envp: *mut *mut u8) { // Explicitly initialize `rustix`. This is needed for things like // `page_size()` to work. #[cfg(feature = "param")] - rustix::param::init(envp); + unsafe { + rustix::param::init(envp); + } // Read the program headers and extract the TLS info. #[cfg(feature = "thread")] @@ -215,7 +221,9 @@ unsafe fn init_runtime(mem: *mut usize, envp: *mut *mut u8) { // Initialize the main thread. #[cfg(feature = "thread")] - thread::initialize_main(mem.cast()); + unsafe { + thread::initialize_main(mem.cast()); + } } /// Functions registered with [`at_exit`]. diff --git a/src/relocate.rs b/src/relocate.rs index 288370c..383f0ed 100644 --- a/src/relocate.rs +++ b/src/relocate.rs @@ -78,294 +78,298 @@ macro_rules! debug_assert_eq { /// So yes, there's a reason this code is behind a feature flag. #[cold] pub(super) unsafe fn relocate(envp: *mut *mut u8) { - // Locate the AUX records we need. - let auxp = compute_auxp(envp); - - // The page size, segment headers info, and runtime entry address. - let mut auxv_base = null_mut(); - let mut auxv_page_size = 0; - let mut auxv_entry = null_mut(); - - // Look through the AUX records to find the segment headers, page size, - // and runtime entry address. - let mut current_aux = auxp; - loop { - let Elf_auxv_t { a_type, a_val } = *current_aux; - current_aux = current_aux.add(1); - - match a_type as _ { - AT_BASE => auxv_base = a_val, - AT_PAGESZ => auxv_page_size = a_val.addr(), - AT_ENTRY => auxv_entry = a_val, - AT_NULL => break, - _ => (), + unsafe { + // Locate the AUX records we need. + let auxp = compute_auxp(envp); + + // The page size, segment headers info, and runtime entry address. + let mut auxv_base = null_mut(); + let mut auxv_page_size = 0; + let mut auxv_entry = null_mut(); + + // Look through the AUX records to find the segment headers, page size, + // and runtime entry address. + let mut current_aux = auxp; + loop { + let Elf_auxv_t { a_type, a_val } = *current_aux; + current_aux = current_aux.add(1); + + match a_type as _ { + AT_BASE => auxv_base = a_val, + AT_PAGESZ => auxv_page_size = a_val.addr(), + AT_ENTRY => auxv_entry = a_val, + AT_NULL => break, + _ => (), + } } - } - // There are four cases to consider here: - // - // 1. Static executable. - // This is the trivial case. No relocations necessary. - // 2. Static pie executable. - // We need to relocate ourself. `AT_PHDR` points to our own program - // headers and `AT_ENTRY` to our own entry point. - // 3. Dynamic pie executable with external dynamic linker. - // We don't need to relocate ourself as the dynamic linker has already - // done this. `AT_PHDR` points to our own program headers and `AT_ENTRY` - // to our own entry point. `AT_BASE` contains the relocation offset of - // the dynamic linker. - // 4. Shared library acting as dynamic linker. - // We do need to relocate ourself. `AT_PHDR` doesn't point to our own - // program headers and `AT_ENTRY` doesn't point to our own entry point. - // `AT_BASE` contains our own relocation offset. - - if load_static_start() == auxv_entry.addr() { - // This is case 1) or case 3). If `AT_BASE` doesn't exist, then we are - // already loaded at our static address despite the lack of any dynamic - // linker. As such it would be case 1). If `AT_BASE` does exist, we have - // already been relocated by the dynamic linker, which is case 3). - // In either case there is no need to do any relocations. - return; - } - - let the_ehdr = &*ehdr_addr(); - - let base = if auxv_base == null_mut() { - // Obtain the static address of `_start`, which is recorded in the - // entry field of the ELF header. - let static_start = the_ehdr.e_entry; - - // This is case 2) as without dynamic linker `AT_BASE` doesn't exist - // and we have already excluded case 1) above. - auxv_entry.wrapping_sub(static_start) - } else { - // This is case 4) as `AT_BASE` indicates that there is a dynamic - // linker, yet we are not already relocated by a dynamic linker. - // `AT_BASE` contains the relocation offset of the dynamic linker. - auxv_base - }; - let offset = base.addr(); - - // This is case 2) or 4). We need to do all `R_RELATIVE` relocations. - // There should be no other kind of relocation because we are either a - // static PIE binary or a dynamic linker compiled with `-Bsymbolic`. - - // Compute the dynamic address of `_DYNAMIC`. - let dynv = dynamic_table_addr(); - - // Rela tables contain `Elf_Rela` elements which have an - // `r_addend` field. - let mut rela_ptr: *const Elf_Rela = null(); - let mut rela_total_size = 0; - - // Rel tables contain `Elf_Rel` elements which lack an - // `r_addend` field and store the addend in the memory to be - // relocated. - let mut rel_ptr: *const Elf_Rel = null(); - let mut rel_total_size = 0; - - // Relr tables contain `Elf_Relr` elements which are a compact - // way of representing relative relocations. - let mut relr_ptr: *const Elf_Relr = null(); - let mut relr_total_size = 0; - - // Look through the `Elf_Dyn` entries to find the location and - // size of the relocation table(s). - let mut current_dyn: *const Elf_Dyn = dynv; - loop { - let Elf_Dyn { d_tag, d_un } = &*current_dyn; - current_dyn = current_dyn.add(1); - - match *d_tag { - // We found a rela table. As above, model this as - // `with_exposed_provenance`. - DT_RELA => rela_ptr = base.byte_add(d_un.d_ptr).cast::(), - DT_RELASZ => rela_total_size = d_un.d_val as usize, - #[cfg(debug_assertions)] - DT_RELAENT => debug_assert_eq!(d_un.d_val as usize, size_of::()), - - // We found a rel table. As above, model this as - // `with_exposed_provenance`. - DT_REL => rel_ptr = base.byte_add(d_un.d_ptr).cast::(), - DT_RELSZ => rel_total_size = d_un.d_val as usize, - #[cfg(debug_assertions)] - DT_RELENT => debug_assert_eq!(d_un.d_val as usize, size_of::()), - - // We found a relr table. As above, model this as - // `with_exposed_provenance`. - DT_RELR => relr_ptr = base.byte_add(d_un.d_ptr).cast::(), - DT_RELRSZ => relr_total_size = d_un.d_val as usize, - #[cfg(debug_assertions)] - DT_RELRENT => debug_assert_eq!(d_un.d_val as usize, size_of::()), - - // End of the Dynamic section - DT_NULL => break, - - _ => (), + // There are four cases to consider here: + // + // 1. Static executable. + // This is the trivial case. No relocations necessary. + // 2. Static pie executable. + // We need to relocate ourself. `AT_PHDR` points to our own program + // headers and `AT_ENTRY` to our own entry point. + // 3. Dynamic pie executable with external dynamic linker. + // We don't need to relocate ourself as the dynamic linker has already + // done this. `AT_PHDR` points to our own program headers and `AT_ENTRY` + // to our own entry point. `AT_BASE` contains the relocation offset of + // the dynamic linker. + // 4. Shared library acting as dynamic linker. + // We do need to relocate ourself. `AT_PHDR` doesn't point to our own + // program headers and `AT_ENTRY` doesn't point to our own entry point. + // `AT_BASE` contains our own relocation offset. + + if load_static_start() == auxv_entry.addr() { + // This is case 1) or case 3). If `AT_BASE` doesn't exist, then we are + // already loaded at our static address despite the lack of any dynamic + // linker. As such it would be case 1). If `AT_BASE` does exist, we have + // already been relocated by the dynamic linker, which is case 3). + // In either case there is no need to do any relocations. + return; } - } - // Perform the rela relocations. - let mut current_rela = rela_ptr; - let rela_end = current_rela.byte_add(rela_total_size); - while current_rela != rela_end { - let rela = &*current_rela; - current_rela = current_rela.add(1); + let the_ehdr = &*ehdr_addr(); - // Calculate the location the relocation will apply at. - let reloc_addr = rela.r_offset.wrapping_add(offset); + let base = if auxv_base == null_mut() { + // Obtain the static address of `_start`, which is recorded in the + // entry field of the ELF header. + let static_start = the_ehdr.e_entry; - // Perform the rela relocation. - match rela.type_() { - R_RELATIVE => { - let addend = rela.r_addend; - let reloc_value = addend.wrapping_add(offset); - relocation_store(reloc_addr, reloc_value); + // This is case 2) as without dynamic linker `AT_BASE` doesn't exist + // and we have already excluded case 1) above. + auxv_entry.wrapping_sub(static_start) + } else { + // This is case 4) as `AT_BASE` indicates that there is a dynamic + // linker, yet we are not already relocated by a dynamic linker. + // `AT_BASE` contains the relocation offset of the dynamic linker. + auxv_base + }; + let offset = base.addr(); + + // This is case 2) or 4). We need to do all `R_RELATIVE` relocations. + // There should be no other kind of relocation because we are either a + // static PIE binary or a dynamic linker compiled with `-Bsymbolic`. + + // Compute the dynamic address of `_DYNAMIC`. + let dynv = dynamic_table_addr(); + + // Rela tables contain `Elf_Rela` elements which have an + // `r_addend` field. + let mut rela_ptr: *const Elf_Rela = null(); + let mut rela_total_size = 0; + + // Rel tables contain `Elf_Rel` elements which lack an + // `r_addend` field and store the addend in the memory to be + // relocated. + let mut rel_ptr: *const Elf_Rel = null(); + let mut rel_total_size = 0; + + // Relr tables contain `Elf_Relr` elements which are a compact + // way of representing relative relocations. + let mut relr_ptr: *const Elf_Relr = null(); + let mut relr_total_size = 0; + + // Look through the `Elf_Dyn` entries to find the location and + // size of the relocation table(s). + let mut current_dyn: *const Elf_Dyn = dynv; + loop { + let Elf_Dyn { d_tag, d_un } = &*current_dyn; + current_dyn = current_dyn.add(1); + + match *d_tag { + // We found a rela table. As above, model this as + // `with_exposed_provenance`. + DT_RELA => rela_ptr = base.byte_add(d_un.d_ptr).cast::(), + DT_RELASZ => rela_total_size = d_un.d_val as usize, + #[cfg(debug_assertions)] + DT_RELAENT => debug_assert_eq!(d_un.d_val as usize, size_of::()), + + // We found a rel table. As above, model this as + // `with_exposed_provenance`. + DT_REL => rel_ptr = base.byte_add(d_un.d_ptr).cast::(), + DT_RELSZ => rel_total_size = d_un.d_val as usize, + #[cfg(debug_assertions)] + DT_RELENT => debug_assert_eq!(d_un.d_val as usize, size_of::()), + + // We found a relr table. As above, model this as + // `with_exposed_provenance`. + DT_RELR => relr_ptr = base.byte_add(d_un.d_ptr).cast::(), + DT_RELRSZ => relr_total_size = d_un.d_val as usize, + #[cfg(debug_assertions)] + DT_RELRENT => debug_assert_eq!(d_un.d_val as usize, size_of::()), + + // End of the Dynamic section + DT_NULL => break, + + _ => (), } - // Trap the process without panicking as panicking requires - // relocations to be performed first. - _ => trap(), } - } - // Perform the rel relocations. - let mut current_rel = rel_ptr; - let rel_end = current_rel.byte_add(rel_total_size); - while current_rel != rel_end { - let rel = &*current_rel; - current_rel = current_rel.add(1); + // Perform the rela relocations. + let mut current_rela = rela_ptr; + let rela_end = current_rela.byte_add(rela_total_size); + while current_rela != rela_end { + let rela = &*current_rela; + current_rela = current_rela.add(1); - // Calculate the location the relocation will apply at. - let reloc_addr = rel.r_offset.wrapping_add(offset); + // Calculate the location the relocation will apply at. + let reloc_addr = rela.r_offset.wrapping_add(offset); - // Perform the rel relocation. - match rel.type_() { - R_RELATIVE => { - let addend = relocation_load(reloc_addr); - let reloc_value = addend.wrapping_add(offset); - relocation_store(reloc_addr, reloc_value); + // Perform the rela relocation. + match rela.type_() { + R_RELATIVE => { + let addend = rela.r_addend; + let reloc_value = addend.wrapping_add(offset); + relocation_store(reloc_addr, reloc_value); + } + // Trap the process without panicking as panicking requires + // relocations to be performed first. + _ => trap(), } - // Trap the process without panicking as panicking requires - // relocations to be performed first. - _ => trap(), } - } - // Perform the relr relocations. - let mut current_relr = relr_ptr; - let relr_end = current_relr.byte_add(relr_total_size); - let mut reloc_addr = 0; - while current_relr != relr_end { - let mut entry = *current_relr; - current_relr = current_relr.add(1); - - if entry & 1 == 0 { - // Entry encodes offset to relocate; calculate the location - // the relocation will apply at. - reloc_addr = offset + entry; - - // Perform the relr relocation. - let addend = relocation_load(reloc_addr); - let reloc_value = addend.wrapping_add(offset); - relocation_store(reloc_addr, reloc_value); - - // Advance relocation location. - reloc_addr += mem::size_of::(); - } else { - // Entry encodes bitmask with locations to relocate; apply each entry. - - let mut i = 0; - loop { - // Shift to next item in the bitmask. - entry >>= 1; - if entry == 0 { - // No more entries left in the bitmask; early exit. - break; - } + // Perform the rel relocations. + let mut current_rel = rel_ptr; + let rel_end = current_rel.byte_add(rel_total_size); + while current_rel != rel_end { + let rel = &*current_rel; + current_rel = current_rel.add(1); - if entry & 1 != 0 { - // Perform the relr relocation. - let addend = relocation_load(reloc_addr + i * mem::size_of::()); + // Calculate the location the relocation will apply at. + let reloc_addr = rel.r_offset.wrapping_add(offset); + + // Perform the rel relocation. + match rel.type_() { + R_RELATIVE => { + let addend = relocation_load(reloc_addr); let reloc_value = addend.wrapping_add(offset); - relocation_store(reloc_addr + i * mem::size_of::(), reloc_value); + relocation_store(reloc_addr, reloc_value); } - - i += 1; + // Trap the process without panicking as panicking requires + // relocations to be performed first. + _ => trap(), } - - // Advance relocation location. - reloc_addr += (mem::size_of::() * 8 - 1) * mem::size_of::(); } - } - // FIXME split function into two here with a hint::black_box around the - // function pointer to prevent the compiler from moving code between the - // functions. - - // Check that the page size is a power of two. - debug_assert!(auxv_page_size.is_power_of_two()); - - // This code doesn't rely on the offset being page aligned, but it is - // useful to check to make sure we computed it correctly. - debug_assert_eq!(offset & (auxv_page_size - 1), 0); - - // Check that relocation did its job. Do the same static start - // computation we did earlier; this time it should match the dynamic - // address. - // AT_ENTRY points to the main executable's entry point rather than our - // entry point when AT_BASE is not zero and thus a dynamic linker is in - // use. In this case the assertion would fail. - if auxv_base == null_mut() { - debug_assert_eq!(load_static_start(), auxv_entry.addr()); - } + // Perform the relr relocations. + let mut current_relr = relr_ptr; + let relr_end = current_relr.byte_add(relr_total_size); + let mut reloc_addr = 0; + while current_relr != relr_end { + let mut entry = *current_relr; + current_relr = current_relr.add(1); + + if entry & 1 == 0 { + // Entry encodes offset to relocate; calculate the location + // the relocation will apply at. + reloc_addr = offset + entry; + + // Perform the relr relocation. + let addend = relocation_load(reloc_addr); + let reloc_value = addend.wrapping_add(offset); + relocation_store(reloc_addr, reloc_value); + + // Advance relocation location. + reloc_addr += mem::size_of::(); + } else { + // Entry encodes bitmask with locations to relocate; apply each entry. + + let mut i = 0; + loop { + // Shift to next item in the bitmask. + entry >>= 1; + if entry == 0 { + // No more entries left in the bitmask; early exit. + break; + } + + if entry & 1 != 0 { + // Perform the relr relocation. + let addend = relocation_load(reloc_addr + i * mem::size_of::()); + let reloc_value = addend.wrapping_add(offset); + relocation_store(reloc_addr + i * mem::size_of::(), reloc_value); + } + + i += 1; + } - // Finally, look through the static segment headers (phdrs) to find the - // the relro description if present. Also do a debug assertion that - // the dynv argument matches the PT_DYNAMIC segment. - - // The location and size of the `relro` region. - let mut relro = 0; - let mut relro_size = 0; - - let phentsize = the_ehdr.e_phentsize as usize; - let mut current_phdr = base.byte_add(the_ehdr.e_phoff).cast::(); - let phdrs_end = current_phdr.byte_add(the_ehdr.e_phnum as usize * phentsize); - while current_phdr != phdrs_end { - let phdr = &*current_phdr; - current_phdr = current_phdr.byte_add(phentsize); - - match phdr.p_type { - #[cfg(debug_assertions)] - PT_DYNAMIC => { - assert_eq!(dynv, base.byte_add(phdr.p_vaddr).cast::()); + // Advance relocation location. + reloc_addr += (mem::size_of::() * 8 - 1) * mem::size_of::(); } - PT_GNU_RELRO => { - // A relro description is present. Make a note of it so that we - // can mark the memory readonly after relocations are done. - relro = phdr.p_vaddr; - relro_size = phdr.p_memsz; + } + + // FIXME split function into two here with a hint::black_box around the + // function pointer to prevent the compiler from moving code between the + // functions. + + // Check that the page size is a power of two. + debug_assert!(auxv_page_size.is_power_of_two()); + + // This code doesn't rely on the offset being page aligned, but it is + // useful to check to make sure we computed it correctly. + debug_assert_eq!(offset & (auxv_page_size - 1), 0); + + // Check that relocation did its job. Do the same static start + // computation we did earlier; this time it should match the dynamic + // address. + // AT_ENTRY points to the main executable's entry point rather than our + // entry point when AT_BASE is not zero and thus a dynamic linker is in + // use. In this case the assertion would fail. + if auxv_base == null_mut() { + debug_assert_eq!(load_static_start(), auxv_entry.addr()); + } + + // Finally, look through the static segment headers (phdrs) to find the + // the relro description if present. Also do a debug assertion that + // the dynv argument matches the PT_DYNAMIC segment. + + // The location and size of the `relro` region. + let mut relro = 0; + let mut relro_size = 0; + + let phentsize = the_ehdr.e_phentsize as usize; + let mut current_phdr = base.byte_add(the_ehdr.e_phoff).cast::(); + let phdrs_end = current_phdr.byte_add(the_ehdr.e_phnum as usize * phentsize); + while current_phdr != phdrs_end { + let phdr = &*current_phdr; + current_phdr = current_phdr.byte_add(phentsize); + + match phdr.p_type { + #[cfg(debug_assertions)] + PT_DYNAMIC => { + assert_eq!(dynv, base.byte_add(phdr.p_vaddr).cast::()); + } + PT_GNU_RELRO => { + // A relro description is present. Make a note of it so that we + // can mark the memory readonly after relocations are done. + relro = phdr.p_vaddr; + relro_size = phdr.p_memsz; + } + _ => (), } - _ => (), } - } - // If we saw a relro description, mark the memory readonly. - if relro_size != 0 { - let mprotect_addr = relro.wrapping_add(offset) & auxv_page_size.wrapping_neg(); - relocation_mprotect_readonly(mprotect_addr, relro_size); + // If we saw a relro description, mark the memory readonly. + if relro_size != 0 { + let mprotect_addr = relro.wrapping_add(offset) & auxv_page_size.wrapping_neg(); + relocation_mprotect_readonly(mprotect_addr, relro_size); + } } } /// Compute the address of the AUX table. unsafe fn compute_auxp(envp: *mut *mut u8) -> *const Elf_auxv_t { - // Locate the AUX records we need. We don't use rustix to do this because - // that would involve calling a function in another crate. - let mut auxp = envp; - // Don't use `is_null` because that's not `inline(always)`. - while *auxp != null_mut() { - auxp = auxp.add(1); + unsafe { + // Locate the AUX records we need. We don't use rustix to do this because + // that would involve calling a function in another crate. + let mut auxp = envp; + // Don't use `is_null` because that's not `inline(always)`. + while *auxp != null_mut() { + auxp = auxp.add(1); + } + auxp.add(1).cast() } - auxp.add(1).cast() } /// Load the address of `_start` from static memory. diff --git a/src/signal/libc.rs b/src/signal/libc.rs index 27d8405..d8c7164 100644 --- a/src/signal/libc.rs +++ b/src/signal/libc.rs @@ -53,16 +53,18 @@ bitflags::bitflags! { /// /// yolo. At least this function handles `sa_restorer` automatically though. pub unsafe fn sigaction(sig: Signal, action: Option) -> io::Result { - let action: *const Sigaction = match action { - Some(action) => &action, - None => null(), - }; - let mut old = MaybeUninit::::uninit(); - - if libc::sigaction(sig.as_raw(), action, old.as_mut_ptr()) == 0 { - Ok(old.assume_init()) - } else { - Err(rustix::io::Errno::from_raw_os_error(errno::errno().0)) + unsafe { + let action: *const Sigaction = match action { + Some(action) => &action, + None => null(), + }; + let mut old = MaybeUninit::::uninit(); + + if libc::sigaction(sig.as_raw(), action, old.as_mut_ptr()) == 0 { + Ok(old.assume_init()) + } else { + Err(rustix::io::Errno::from_raw_os_error(errno::errno().0)) + } } } diff --git a/src/signal/linux_raw.rs b/src/signal/linux_raw.rs index 23b5258..84d72b4 100644 --- a/src/signal/linux_raw.rs +++ b/src/signal/linux_raw.rs @@ -17,21 +17,23 @@ pub type Sighandler = rustix::runtime::KernelSighandler; /// /// yolo. At least this function handles `sa_restorer` automatically though. pub unsafe fn sigaction(sig: Signal, action: Option) -> io::Result { - #[allow(unused_mut)] - let mut action = action; - - #[cfg(not(target_arch = "riscv64"))] - if let Some(action) = &mut action { - action.sa_flags |= SigactionFlags::RESTORER; - - if action.sa_flags.contains(SigactionFlags::SIGINFO) { - action.sa_restorer = Some(arch::return_from_signal_handler); - } else { - action.sa_restorer = Some(arch::return_from_signal_handler_noinfo); + unsafe { + #[allow(unused_mut)] + let mut action = action; + + #[cfg(not(target_arch = "riscv64"))] + if let Some(action) = &mut action { + action.sa_flags |= SigactionFlags::RESTORER; + + if action.sa_flags.contains(SigactionFlags::SIGINFO) { + action.sa_restorer = Some(arch::return_from_signal_handler); + } else { + action.sa_restorer = Some(arch::return_from_signal_handler_noinfo); + } } - } - rustix::runtime::kernel_sigaction(sig, action) + rustix::runtime::kernel_sigaction(sig, action) + } } /// Return a special “ignore” signal handler for ignoring signals. diff --git a/src/stubs.rs b/src/stubs.rs index b47200a..8d28dbc 100644 --- a/src/stubs.rs +++ b/src/stubs.rs @@ -3,7 +3,7 @@ // If requested, provide a version of "eh-personality-continue" ourselves. #[cfg(feature = "eh-personality-continue")] -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn rust_eh_personality( version: core::ffi::c_int, _actions: core::ffi::c_int, diff --git a/src/thread/libc.rs b/src/thread/libc.rs index e4fe83d..12bdd43 100644 --- a/src/thread/libc.rs +++ b/src/thread/libc.rs @@ -11,14 +11,14 @@ use alloc::boxed::Box; use core::ffi::{c_int, c_void}; use core::mem::{size_of, transmute, zeroed}; -use core::ptr::{null_mut, with_exposed_provenance_mut, without_provenance_mut, NonNull}; +use core::ptr::{NonNull, null_mut, with_exposed_provenance_mut, without_provenance_mut}; use core::slice; use rustix::io; pub use rustix::thread::Pid as ThreadId; // Symbols defined in libc but not declared in the libc crate. -extern "C" { +unsafe extern "C" { fn __cxa_thread_atexit_impl( func: unsafe extern "C" fn(*mut c_void), obj: *mut c_void, @@ -185,9 +185,11 @@ pub unsafe fn create( /// and will not be joined. #[inline] pub unsafe fn detach(thread: Thread) { - let thread = thread.0; + unsafe { + let thread = thread.0; - assert_eq!(libc::pthread_detach(thread), 0); + assert_eq!(libc::pthread_detach(thread), 0); + } } /// Waits for a thread to finish. @@ -200,12 +202,14 @@ pub unsafe fn detach(thread: Thread) { /// `thread` must point to a valid thread record that has not already been /// detached or joined. pub unsafe fn join(thread: Thread) -> Option> { - let thread = thread.0; + unsafe { + let thread = thread.0; - let mut return_value: *mut c_void = null_mut(); - assert_eq!(libc::pthread_join(thread, &mut return_value), 0); + let mut return_value: *mut c_void = null_mut(); + assert_eq!(libc::pthread_join(thread, &mut return_value), 0); - NonNull::new(return_value) + NonNull::new(return_value) + } } /// Registers a function to call when the current thread exits. @@ -288,22 +292,24 @@ pub fn current_tls_addr(module: usize, offset: usize) -> *mut c_void { #[inline] #[must_use] pub unsafe fn stack(thread: Thread) -> (*mut c_void, usize, usize) { - let thread = thread.0; + unsafe { + let thread = thread.0; - let mut attr: libc::pthread_attr_t = zeroed(); - assert_eq!(libc::pthread_getattr_np(thread, &mut attr), 0); + let mut attr: libc::pthread_attr_t = zeroed(); + assert_eq!(libc::pthread_getattr_np(thread, &mut attr), 0); - let mut stack_size = 0; - let mut stack_addr = null_mut(); - assert_eq!( - libc::pthread_attr_getstack(&attr, &mut stack_addr, &mut stack_size), - 0 - ); + let mut stack_size = 0; + let mut stack_addr = null_mut(); + assert_eq!( + libc::pthread_attr_getstack(&attr, &mut stack_addr, &mut stack_size), + 0 + ); - let mut guard_size = 0; - assert_eq!(libc::pthread_attr_getguardsize(&attr, &mut guard_size), 0); + let mut guard_size = 0; + assert_eq!(libc::pthread_attr_getguardsize(&attr, &mut guard_size), 0); - (stack_addr, stack_size, guard_size) + (stack_addr, stack_size, guard_size) + } } /// Return the default stack size for new threads. @@ -345,6 +351,8 @@ pub fn yield_current() { /// Return the address of `__dso_handle`, appropriately casted. #[cfg(feature = "thread-at-exit")] unsafe fn dso_handle() -> *mut c_void { - let dso_handle: *const *const c_void = &__dso_handle; - dso_handle.cast::().cast_mut() + unsafe { + let dso_handle: *const *const c_void = &__dso_handle; + dso_handle.cast::().cast_mut() + } } diff --git a/src/thread/linux_raw.rs b/src/thread/linux_raw.rs index 24579ae..0870a92 100644 --- a/src/thread/linux_raw.rs +++ b/src/thread/linux_raw.rs @@ -8,7 +8,7 @@ //! pthreads or `std::thread::Thread`. use crate::arch::{ - clone, munmap_and_exit_thread, set_thread_pointer, thread_pointer, STACK_ALIGNMENT, TLS_OFFSET, + STACK_ALIGNMENT, TLS_OFFSET, clone, munmap_and_exit_thread, set_thread_pointer, thread_pointer, }; #[cfg(feature = "thread-at-exit")] use alloc::boxed::Box; @@ -17,18 +17,18 @@ use core::cell::Cell; use core::cmp::max; use core::ffi::c_void; use core::mem::{align_of, offset_of, size_of}; -use core::ptr::{copy_nonoverlapping, drop_in_place, null, null_mut, NonNull}; +use core::ptr::{NonNull, copy_nonoverlapping, drop_in_place, null, null_mut}; use core::slice; use core::sync::atomic::Ordering::SeqCst; -use core::sync::atomic::{AtomicI32, AtomicPtr, AtomicU32, AtomicU8}; +use core::sync::atomic::{AtomicI32, AtomicPtr, AtomicU8, AtomicU32}; use linux_raw_sys::elf::*; use rustix::io; -use rustix::mm::{mmap_anonymous, mprotect, MapFlags, MprotectFlags, ProtFlags}; +use rustix::mm::{MapFlags, MprotectFlags, ProtFlags, mmap_anonymous, mprotect}; use rustix::param::{linux_execfn, page_size}; -use rustix::process::{getrlimit, Resource}; -use rustix::runtime::{exe_phdrs, set_tid_address}; +use rustix::process::{Resource, getrlimit}; #[cfg(feature = "signal")] -use rustix::runtime::{kernel_sigprocmask, How, KernelSigSet}; +use rustix::runtime::{How, KernelSigSet, kernel_sigprocmask}; +use rustix::runtime::{exe_phdrs, set_tid_address}; use rustix::thread::gettid; pub use rustix::thread::Pid as ThreadId; @@ -55,7 +55,7 @@ impl Thread { /// `raw` must be a valid non-null thread pointer. #[inline] pub unsafe fn from_raw_unchecked(raw: *mut c_void) -> Self { - Self(NonNull::new_unchecked(raw.cast())) + unsafe { Self(NonNull::new_unchecked(raw.cast())) } } /// Convert to `Self` from a raw non-null pointer that was returned from @@ -301,7 +301,7 @@ pub(super) fn initialize_startup_info() { } } -extern "C" { +unsafe extern "C" { /// Declare the `_DYNAMIC` symbol so that we can compare its address with /// the static address in the `PT_DYNAMIC` header to learn our offset. Use /// a weak symbol because `_DYNAMIC` is not always present. @@ -324,63 +324,65 @@ core::arch::global_asm!(".weak _DYNAMIC"); /// initial value of the stack pointer in a new process, pointing to the /// initial contents of the stack. pub(super) unsafe fn initialize_main(mem: *mut c_void) { - // Determine the top of the stack. Linux puts the `AT_EXECFN` string at - // the top, so find the end of that, and then round up to the page size. - // See for details. - let execfn = linux_execfn().to_bytes_with_nul(); - let stack_base = execfn.as_ptr().add(execfn.len()); - let stack_base = stack_base - .map_addr(|ptr| round_up(ptr, page_size())) - .cast_mut(); - - // We're running before any user code, so the startup soft stack limit is - // the effective stack size. Linux sets up inaccessible memory at the end - // of the stack. - let stack_map_size = getrlimit(Resource::Stack).current.unwrap() as usize; - let stack_least = stack_base.sub(stack_map_size); - let stack_size = stack_least.offset_from(mem.cast::()) as usize; - let guard_size = page_size(); - - // Initialize the canary value from the OS-provided random bytes. - let random_ptr = rustix::runtime::random().cast::(); - let canary = random_ptr.read_unaligned(); - __stack_chk_guard = canary; - - let mut alloc_size = 0; - let (tls_data_bottom, header) = calculate_tls_size(&mut alloc_size); - - // Allocate the thread data. Use `mmap_anonymous` rather than `alloc` here - // as the allocator may depend on thread-local data, which is what we're - // initializing here. - let new = mmap_anonymous( - null_mut(), - alloc_size, - ProtFlags::READ | ProtFlags::WRITE, - MapFlags::PRIVATE, - ) - .unwrap() - .cast::(); - - let metadata_align = max(unsafe { STARTUP_TLS_INFO.align }, align_of::()); - debug_assert_eq!(new.addr() % metadata_align, 0); - - let tls_data = new.add(tls_data_bottom); - let metadata: *mut Metadata = new.add(header).cast(); - - let (newtls, thread_id_ptr) = initialize_tls( - tls_data, - metadata, - canary, - stack_least, - stack_size, - guard_size, - 0, - ); - let tid = rustix::runtime::set_tid_address(thread_id_ptr.cast()); - *thread_id_ptr = tid.as_raw_nonzero().get(); - - // Point the platform thread-pointer register at the new thread metadata. - set_thread_pointer(newtls); + unsafe { + // Determine the top of the stack. Linux puts the `AT_EXECFN` string at + // the top, so find the end of that, and then round up to the page size. + // See for details. + let execfn = linux_execfn().to_bytes_with_nul(); + let stack_base = execfn.as_ptr().add(execfn.len()); + let stack_base = stack_base + .map_addr(|ptr| round_up(ptr, page_size())) + .cast_mut(); + + // We're running before any user code, so the startup soft stack limit is + // the effective stack size. Linux sets up inaccessible memory at the end + // of the stack. + let stack_map_size = getrlimit(Resource::Stack).current.unwrap() as usize; + let stack_least = stack_base.sub(stack_map_size); + let stack_size = stack_least.offset_from(mem.cast::()) as usize; + let guard_size = page_size(); + + // Initialize the canary value from the OS-provided random bytes. + let random_ptr = rustix::runtime::random().cast::(); + let canary = random_ptr.read_unaligned(); + __stack_chk_guard = canary; + + let mut alloc_size = 0; + let (tls_data_bottom, header) = calculate_tls_size(&mut alloc_size); + + // Allocate the thread data. Use `mmap_anonymous` rather than `alloc` here + // as the allocator may depend on thread-local data, which is what we're + // initializing here. + let new = mmap_anonymous( + null_mut(), + alloc_size, + ProtFlags::READ | ProtFlags::WRITE, + MapFlags::PRIVATE, + ) + .unwrap() + .cast::(); + + let metadata_align = max(STARTUP_TLS_INFO.align, align_of::()); + debug_assert_eq!(new.addr() % metadata_align, 0); + + let tls_data = new.add(tls_data_bottom); + let metadata: *mut Metadata = new.add(header).cast(); + + let (newtls, thread_id_ptr) = initialize_tls( + tls_data, + metadata, + canary, + stack_least, + stack_size, + guard_size, + 0, + ); + let tid = rustix::runtime::set_tid_address(thread_id_ptr.cast()); + *thread_id_ptr = tid.as_raw_nonzero().get(); + + // Point the platform thread-pointer register at the new thread metadata. + set_thread_pointer(newtls); + } } fn calculate_tls_size(map_size: &mut usize) -> (usize, usize) { @@ -436,39 +438,41 @@ unsafe fn initialize_tls( guard_size: usize, map_size: usize, ) -> (*mut c_void, *mut i32) { - let newtls: *mut c_void = (*metadata).abi.thread_pointee.as_mut_ptr().cast(); + unsafe { + let newtls: *mut c_void = (*metadata).abi.thread_pointee.as_mut_ptr().cast(); + + // Initialize the thread metadata. + metadata.write(Metadata { + abi: Abi { + canary, + dtv: null(), + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + this: newtls, + _pad: Default::default(), + thread_pointee: [], + }, + thread: ThreadData::new(stack_least.cast(), stack_size, guard_size, map_size), + }); + + // Initialize the TLS data with explicit initializer data. + slice::from_raw_parts_mut(tls_data, STARTUP_TLS_INFO.file_size).copy_from_slice( + slice::from_raw_parts( + STARTUP_TLS_INFO.addr.cast::(), + STARTUP_TLS_INFO.file_size, + ), + ); - // Initialize the thread metadata. - metadata.write(Metadata { - abi: Abi { - canary, - dtv: null(), - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - this: newtls, - _pad: Default::default(), - thread_pointee: [], - }, - thread: ThreadData::new(stack_least.cast(), stack_size, guard_size, map_size), - }); - - // Initialize the TLS data with explicit initializer data. - slice::from_raw_parts_mut(tls_data, STARTUP_TLS_INFO.file_size).copy_from_slice( - slice::from_raw_parts( - STARTUP_TLS_INFO.addr.cast::(), - STARTUP_TLS_INFO.file_size, - ), - ); - - // Initialize the TLS data beyond `file_size` which is zero-filled. - slice::from_raw_parts_mut( - tls_data.add(STARTUP_TLS_INFO.file_size), - STARTUP_TLS_INFO.mem_size - STARTUP_TLS_INFO.file_size, - ) - .fill(0); - - let thread_id_ptr = (*metadata).thread.thread_id.as_ptr().cast::(); - - (newtls, thread_id_ptr) + // Initialize the TLS data beyond `file_size` which is zero-filled. + slice::from_raw_parts_mut( + tls_data.add(STARTUP_TLS_INFO.file_size), + STARTUP_TLS_INFO.mem_size - STARTUP_TLS_INFO.file_size, + ) + .fill(0); + + let thread_id_ptr = (*metadata).thread.thread_id.as_ptr().cast::(); + + (newtls, thread_id_ptr) + } } /// Creates a new thread. @@ -629,142 +633,147 @@ pub(super) unsafe extern "C" fn entry( args: *mut *mut c_void, num_args: usize, ) -> ! { - #[cfg(feature = "log")] - log::trace!("Thread[{:?}] launched", current_id().as_raw_nonzero()); - - // Do some basic precondition checks, to ensure that our assembly code did - // what we expect it to do. These are debug-only for now, to keep the - // release-mode startup code simple to disassemble and inspect, while we're - // getting started. - #[cfg(debug_assertions)] - { - // If we have nightly, we can do additional checks. - #[cfg(feature = "nightly")] + unsafe { + #[cfg(feature = "log")] + log::trace!("Thread[{:?}] launched", current_id().as_raw_nonzero()); + + // Do some basic precondition checks, to ensure that our assembly code did + // what we expect it to do. These are debug-only for now, to keep the + // release-mode startup code simple to disassemble and inspect, while we're + // getting started. + #[cfg(debug_assertions)] { - extern "C" { - #[link_name = "llvm.frameaddress"] - fn builtin_frame_address(level: i32) -> *const u8; - #[link_name = "llvm.returnaddress"] - fn builtin_return_address(level: i32) -> *const u8; + // If we have nightly, we can do additional checks. + #[cfg(feature = "nightly")] + { + unsafe extern "C" { + #[link_name = "llvm.frameaddress"] + fn builtin_frame_address(level: i32) -> *const u8; + #[link_name = "llvm.returnaddress"] + fn builtin_return_address(level: i32) -> *const u8; + #[cfg(target_arch = "aarch64")] + #[link_name = "llvm.sponentry"] + fn builtin_sponentry() -> *const u8; + } + + // Check that the incoming stack pointer is where we expect it to be. + debug_assert_eq!(builtin_return_address(0), null()); + debug_assert_ne!(builtin_frame_address(0), null()); + #[cfg(not(any(target_arch = "x86", target_arch = "arm")))] + debug_assert_eq!(builtin_frame_address(0).addr() & 0xf, 0); + #[cfg(target_arch = "arm")] + debug_assert_eq!(builtin_frame_address(0).addr() & 0x3, 0); + #[cfg(target_arch = "x86")] + debug_assert_eq!(builtin_frame_address(0).addr() & 0xf, 8); + debug_assert_eq!(builtin_frame_address(1), null()); + #[cfg(target_arch = "aarch64")] + debug_assert_ne!(builtin_sponentry(), null()); #[cfg(target_arch = "aarch64")] - #[link_name = "llvm.sponentry"] - fn builtin_sponentry() -> *const u8; + debug_assert_eq!(builtin_sponentry().addr() & 0xf, 0); } - // Check that the incoming stack pointer is where we expect it to be. - debug_assert_eq!(builtin_return_address(0), null()); - debug_assert_ne!(builtin_frame_address(0), null()); - #[cfg(not(any(target_arch = "x86", target_arch = "arm")))] - debug_assert_eq!(builtin_frame_address(0).addr() & 0xf, 0); - #[cfg(target_arch = "arm")] - debug_assert_eq!(builtin_frame_address(0).addr() & 0x3, 0); - #[cfg(target_arch = "x86")] - debug_assert_eq!(builtin_frame_address(0).addr() & 0xf, 8); - debug_assert_eq!(builtin_frame_address(1), null()); - #[cfg(target_arch = "aarch64")] - debug_assert_ne!(builtin_sponentry(), null()); - #[cfg(target_arch = "aarch64")] - debug_assert_eq!(builtin_sponentry().addr() & 0xf, 0); + // Check that `clone` stored our thread id as we expected. + debug_assert_eq!(current_id(), gettid()); } - // Check that `clone` stored our thread id as we expected. - debug_assert_eq!(current_id(), gettid()); - } - - // Call the user thread function. In `std`, this is `thread_start`. Ignore - // the return value for now, as `std` doesn't need it. - let fn_: unsafe fn(&mut [*mut c_void]) -> Option> = core::mem::transmute(fn_); - let args = slice::from_raw_parts_mut(args, num_args); - let return_value = fn_(args); + // Call the user thread function. In `std`, this is `thread_start`. Ignore + // the return value for now, as `std` doesn't need it. + let fn_: unsafe fn(&mut [*mut c_void]) -> Option> = + core::mem::transmute(fn_); + let args = slice::from_raw_parts_mut(args, num_args); + let return_value = fn_(args); - exit(return_value) + exit(return_value) + } } /// Call the destructors registered with [`at_exit`] and exit the thread. unsafe fn exit(return_value: Option>) -> ! { - let current = current(); - - #[cfg(feature = "log")] - if log::log_enabled!(log::Level::Trace) { - log::trace!( - "Thread[{:?}] returned {:?}", - current.0.as_ref().thread_id.load(SeqCst), - return_value - ); - } - - // Call functions registered with `at_exit`. - #[cfg(feature = "thread-at-exit")] - call_dtors(current); - - // Read the thread's state, and set it to `ABANDONED` if it was `INITIAL`, - // which tells `join_thread` to free the memory. Otherwise, it's in the - // `DETACHED` state, and we free the memory immediately. - let state = current - .0 - .as_ref() - .detached - .compare_exchange(INITIAL, ABANDONED, SeqCst, SeqCst); - if let Err(e) = state { - // The thread was detached. Prepare to free the memory. First read out - // all the fields that we'll need before freeing it. - #[cfg(feature = "log")] - let current_thread_id = current.0.as_ref().thread_id.load(SeqCst); - let current_map_size = current.0.as_ref().map_size; - let current_stack_addr = current.0.as_ref().stack_addr; - let current_guard_size = current.0.as_ref().guard_size; + unsafe { + let current = current(); #[cfg(feature = "log")] - log::trace!("Thread[{:?}] exiting as detached", current_thread_id); - debug_assert_eq!(e, DETACHED); + if log::log_enabled!(log::Level::Trace) { + log::trace!( + "Thread[{:?}] returned {:?}", + current.0.as_ref().thread_id.load(SeqCst), + return_value + ); + } - // Deallocate the `ThreadData`. - drop_in_place(current.0.as_ptr()); + // Call functions registered with `at_exit`. + #[cfg(feature = "thread-at-exit")] + call_dtors(current); + + // Read the thread's state, and set it to `ABANDONED` if it was `INITIAL`, + // which tells `join_thread` to free the memory. Otherwise, it's in the + // `DETACHED` state, and we free the memory immediately. + let state = current + .0 + .as_ref() + .detached + .compare_exchange(INITIAL, ABANDONED, SeqCst, SeqCst); + if let Err(e) = state { + // The thread was detached. Prepare to free the memory. First read out + // all the fields that we'll need before freeing it. + #[cfg(feature = "log")] + let current_thread_id = current.0.as_ref().thread_id.load(SeqCst); + let current_map_size = current.0.as_ref().map_size; + let current_stack_addr = current.0.as_ref().stack_addr; + let current_guard_size = current.0.as_ref().guard_size; - // Free the thread's `mmap` region, if we allocated it. - let map_size = current_map_size; - if map_size != 0 { - // Null out the tid address so that the kernel doesn't write to - // memory that we've freed trying to clear our tid when we exit. - let _ = set_tid_address(null_mut()); + #[cfg(feature = "log")] + log::trace!("Thread[{:?}] exiting as detached", current_thread_id); + debug_assert_eq!(e, DETACHED); + + // Deallocate the `ThreadData`. + drop_in_place(current.0.as_ptr()); + + // Free the thread's `mmap` region, if we allocated it. + let map_size = current_map_size; + if map_size != 0 { + // Null out the tid address so that the kernel doesn't write to + // memory that we've freed trying to clear our tid when we exit. + let _ = set_tid_address(null_mut()); + + // In preparation for freeing the stack, block all signals, so that + // no signals for the process are delivered to this thread. + #[cfg(feature = "signal")] + { + let all = KernelSigSet::all(); + kernel_sigprocmask(How::BLOCK, Some(&all)).ok(); + } - // In preparation for freeing the stack, block all signals, so that - // no signals for the process are delivered to this thread. - #[cfg(feature = "signal")] - { - let all = KernelSigSet::all(); - kernel_sigprocmask(How::BLOCK, Some(&all)).ok(); + // `munmap` the memory, which also frees the stack we're currently + // on, and do an `exit` carefully without touching the stack. + let map = current_stack_addr.byte_sub(current_guard_size); + munmap_and_exit_thread(map, map_size); + } + } else { + // The thread was not detached, so its memory will be freed when it's + // joined. + #[cfg(feature = "log")] + if log::log_enabled!(log::Level::Trace) { + log::trace!( + "Thread[{:?}] exiting as joinable", + current.0.as_ref().thread_id.load(SeqCst) + ); } - // `munmap` the memory, which also frees the stack we're currently - // on, and do an `exit` carefully without touching the stack. - let map = current_stack_addr.byte_sub(current_guard_size); - munmap_and_exit_thread(map, map_size); - } - } else { - // The thread was not detached, so its memory will be freed when it's - // joined. - #[cfg(feature = "log")] - if log::log_enabled!(log::Level::Trace) { - log::trace!( - "Thread[{:?}] exiting as joinable", - current.0.as_ref().thread_id.load(SeqCst) - ); - } + // Convert `return_value` into a `*mut c_void` so that we can store it + // in an `AtomicPtr`. + let return_value = match return_value { + Some(return_value) => return_value.as_ptr(), + None => null_mut(), + }; - // Convert `return_value` into a `*mut c_void` so that we can store it - // in an `AtomicPtr`. - let return_value = match return_value { - Some(return_value) => return_value.as_ptr(), - None => null_mut(), - }; + // Store the return value in the thread for `join_thread` to read. + current.0.as_ref().return_value.store(return_value, SeqCst); + } - // Store the return value in the thread for `join_thread` to read. - current.0.as_ref().return_value.store(return_value, SeqCst); + // Terminate the thread. + rustix::runtime::exit_thread(0) } - - // Terminate the thread. - rustix::runtime::exit_thread(0) } /// Call the destructors registered with [`at_exit`]. @@ -801,25 +810,27 @@ pub(crate) fn call_dtors(current: Thread) { /// and will not be joined. #[inline] pub unsafe fn detach(thread: Thread) { - #[cfg(feature = "log")] - let thread_id = thread.0.as_ref().thread_id.load(SeqCst); + unsafe { + #[cfg(feature = "log")] + let thread_id = thread.0.as_ref().thread_id.load(SeqCst); - #[cfg(feature = "log")] - if log::log_enabled!(log::Level::Trace) { - log::trace!( - "Thread[{:?}] marked as detached by Thread[{:?}]", - thread_id, - current_id().as_raw_nonzero() - ); - } + #[cfg(feature = "log")] + if log::log_enabled!(log::Level::Trace) { + log::trace!( + "Thread[{:?}] marked as detached by Thread[{:?}]", + thread_id, + current_id().as_raw_nonzero() + ); + } - if thread.0.as_ref().detached.swap(DETACHED, SeqCst) == ABANDONED { - wait_for_exit(thread); + if thread.0.as_ref().detached.swap(DETACHED, SeqCst) == ABANDONED { + wait_for_exit(thread); - #[cfg(feature = "log")] - log_thread_to_be_freed(thread_id); + #[cfg(feature = "log")] + log_thread_to_be_freed(thread_id); - free_memory(thread); + free_memory(thread); + } } } @@ -833,36 +844,38 @@ pub unsafe fn detach(thread: Thread) { /// `thread` must point to a valid thread record that has not already been /// detached or joined. pub unsafe fn join(thread: Thread) -> Option> { - let thread_data = thread.0.as_ref(); + unsafe { + let thread_data = thread.0.as_ref(); - #[cfg(feature = "log")] - let thread_id = thread_data.thread_id.load(SeqCst); + #[cfg(feature = "log")] + let thread_id = thread_data.thread_id.load(SeqCst); - #[cfg(feature = "log")] - if log::log_enabled!(log::Level::Trace) { - log::trace!( - "Thread[{:?}] is being joined by Thread[{:?}]", - thread_id, - current_id().as_raw_nonzero() - ); - } + #[cfg(feature = "log")] + if log::log_enabled!(log::Level::Trace) { + log::trace!( + "Thread[{:?}] is being joined by Thread[{:?}]", + thread_id, + current_id().as_raw_nonzero() + ); + } - wait_for_exit(thread); - debug_assert_eq!(thread_data.detached.load(SeqCst), ABANDONED); + wait_for_exit(thread); + debug_assert_eq!(thread_data.detached.load(SeqCst), ABANDONED); - #[cfg(feature = "log")] - log_thread_to_be_freed(thread_id); + #[cfg(feature = "log")] + log_thread_to_be_freed(thread_id); - // Load the return value stored by `exit_thread`, before we free the - // thread's memory. - let return_value = thread_data.return_value.load(SeqCst); + // Load the return value stored by `exit_thread`, before we free the + // thread's memory. + let return_value = thread_data.return_value.load(SeqCst); - // `munmap` the stack and metadata for the thread. - free_memory(thread); + // `munmap` the stack and metadata for the thread. + free_memory(thread); - // Convert the `*mut c_void` we stored in the `AtomicPtr` back into an - // `Option>`. - NonNull::new(return_value) + // Convert the `*mut c_void` we stored in the `AtomicPtr` back into an + // `Option>`. + NonNull::new(return_value) + } } /// Wait until `thread` has exited. @@ -870,27 +883,29 @@ pub unsafe fn join(thread: Thread) -> Option> { /// `thread` must point to a valid thread record that has not already been /// detached or joined. unsafe fn wait_for_exit(thread: Thread) { - use rustix::thread::futex; - - // Check whether the thread has exited already; we set the - // `CloneFlags::CHILD_CLEARTID` flag on the clone syscall, so we can test - // for `NONE` here. - let thread_data = thread.0.as_ref(); - let thread_id = &thread_data.thread_id; - while let Some(id_value) = ThreadId::from_raw(thread_id.load(SeqCst)) { - // This doesn't use any shared memory, but we can't use - // `FutexFlags::PRIVATE` because the wake comes from Linux - // as arranged by the `CloneFlags::CHILD_CLEARTID` flag, - // and Linux doesn't use the private flag for the wake. - match futex::wait( - AtomicU32::from_ptr(thread_id.as_ptr().cast()), - futex::Flags::empty(), - id_value.as_raw_nonzero().get() as u32, - None, - ) { - Ok(_) => break, - Err(io::Errno::INTR) => continue, - Err(e) => debug_assert_eq!(e, io::Errno::AGAIN), + unsafe { + use rustix::thread::futex; + + // Check whether the thread has exited already; we set the + // `CloneFlags::CHILD_CLEARTID` flag on the clone syscall, so we can test + // for `NONE` here. + let thread_data = thread.0.as_ref(); + let thread_id = &thread_data.thread_id; + while let Some(id_value) = ThreadId::from_raw(thread_id.load(SeqCst)) { + // This doesn't use any shared memory, but we can't use + // `FutexFlags::PRIVATE` because the wake comes from Linux + // as arranged by the `CloneFlags::CHILD_CLEARTID` flag, + // and Linux doesn't use the private flag for the wake. + match futex::wait( + AtomicU32::from_ptr(thread_id.as_ptr().cast()), + futex::Flags::empty(), + id_value.as_raw_nonzero().get() as u32, + None, + ) { + Ok(_) => break, + Err(io::Errno::INTR) => continue, + Err(e) => debug_assert_eq!(e, io::Errno::AGAIN), + } } } } @@ -909,21 +924,23 @@ fn log_thread_to_be_freed(thread_id: i32) { /// `thread` must point to a valid thread record for a thread that has /// already exited. unsafe fn free_memory(thread: Thread) { - use rustix::mm::munmap; + unsafe { + use rustix::mm::munmap; - // The thread was detached. Prepare to free the memory. First read out - // all the fields that we'll need before freeing it. - let map_size = thread.0.as_ref().map_size; - let stack_addr = thread.0.as_ref().stack_addr; - let guard_size = thread.0.as_ref().guard_size; + // The thread was detached. Prepare to free the memory. First read out + // all the fields that we'll need before freeing it. + let map_size = thread.0.as_ref().map_size; + let stack_addr = thread.0.as_ref().stack_addr; + let guard_size = thread.0.as_ref().guard_size; - // Deallocate the `ThreadData`. - drop_in_place(thread.0.as_ptr()); + // Deallocate the `ThreadData`. + drop_in_place(thread.0.as_ptr()); - // Free the thread's `mmap` region, if we allocated it. - if map_size != 0 { - let map = stack_addr.byte_sub(guard_size); - munmap(map, map_size).unwrap(); + // Free the thread's `mmap` region, if we allocated it. + if map_size != 0 { + let map = stack_addr.byte_sub(guard_size); + munmap(map, map_size).unwrap(); + } } } @@ -987,18 +1004,20 @@ pub fn current_id() -> ThreadId { #[doc(hidden)] #[inline] pub unsafe fn set_current_id_after_a_fork(tid: ThreadId) { - let current = current(); - debug_assert_ne!( - tid.as_raw_nonzero().get(), - current.0.as_ref().thread_id.load(SeqCst), - "current thread ID already matches new thread ID" - ); - debug_assert_eq!(tid, gettid(), "new thread ID disagrees with `gettid`"); - current - .0 - .as_ref() - .thread_id - .store(tid.as_raw_nonzero().get(), SeqCst); + unsafe { + let current = current(); + debug_assert_ne!( + tid.as_raw_nonzero().get(), + current.0.as_ref().thread_id.load(SeqCst), + "current thread ID already matches new thread ID" + ); + debug_assert_eq!(tid, gettid(), "new thread ID disagrees with `gettid`"); + current + .0 + .as_ref() + .thread_id + .store(tid.as_raw_nonzero().get(), SeqCst); + } } /// Return the address of the thread-local `errno` state. @@ -1049,8 +1068,10 @@ pub fn current_tls_addr(module: usize, offset: usize) -> *mut c_void { #[inline] #[cfg_attr(docsrs, doc(cfg(feature = "take-charge")))] pub unsafe fn id(thread: Thread) -> Option { - let raw = thread.0.as_ref().thread_id.load(SeqCst); - ThreadId::from_raw(raw) + unsafe { + let raw = thread.0.as_ref().thread_id.load(SeqCst); + ThreadId::from_raw(raw) + } } /// Return the current thread's stack address (lowest address), size, and guard @@ -1062,8 +1083,10 @@ pub unsafe fn id(thread: Thread) -> Option { #[inline] #[must_use] pub unsafe fn stack(thread: Thread) -> (*mut c_void, usize, usize) { - let data = thread.0.as_ref(); - (data.stack_addr, data.stack_size, data.guard_size) + unsafe { + let data = thread.0.as_ref(); + (data.stack_addr, data.stack_size, data.guard_size) + } } /// Return the default stack size for new threads. @@ -1093,13 +1116,13 @@ pub fn yield_current() { /// The ARM ABI expects this to be defined. #[cfg(target_arch = "arm")] -#[no_mangle] +#[unsafe(no_mangle)] extern "C" fn __aeabi_read_tp() -> *mut c_void { thread_pointer() } /// Some targets use this global variable instead of the TLS `canary` field. -#[no_mangle] +#[unsafe(no_mangle)] static mut __stack_chk_guard: usize = 0; const fn round_up(addr: usize, boundary: usize) -> usize { diff --git a/src/unwind_unimplemented.rs b/src/unwind_unimplemented.rs index 19d4ac9..65c9cbb 100644 --- a/src/unwind_unimplemented.rs +++ b/src/unwind_unimplemented.rs @@ -3,102 +3,102 @@ //! //! Entirely `unimplemented!`. -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_Backtrace() { unimplemented!("_Unwind_Backtrace") } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_DeleteException() { unimplemented!("_Unwind_DeleteException") } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_GetDataRelBase() { unimplemented!("_Unwind_GetDataRelBase") } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_GetIP() { unimplemented!("_Unwind_GetIP") } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_GetIPInfo() { unimplemented!("_Unwind_GetIPInfo") } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_GetLanguageSpecificData() { unimplemented!("_Unwind_GetLanguageSpecificData") } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_GetRegionStart() { unimplemented!("_Unwind_GetRegionStart") } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_GetTextRelBase() { unimplemented!("_Unwind_GetTextRelBase") } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_RaiseException() { unimplemented!("_Unwind_RaiseException") } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_Resume() { unimplemented!("_Unwind_Resume") } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_SetGR() { unimplemented!("_Unwind_SetGR") } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_SetIP() { unimplemented!("_Unwind_SetIP") } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_FindEnclosingFunction() { unimplemented!("_Unwind_FindEnclosingFunction") } -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_GetCFA() { unimplemented!("_Unwind_GetCFA") } #[cfg(target_arch = "arm")] -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_VRS_Get() { unimplemented!("_Unwind_VRS_Get") } #[cfg(target_arch = "arm")] -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn _Unwind_VRS_Set() { unimplemented!("_Unwind_VRS_Set") } #[cfg(target_arch = "arm")] -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn __aeabi_unwind_cpp_pr0() { unimplemented!("__aeabi_unwind_cpp_pr0") } #[cfg(target_arch = "arm")] -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn __aeabi_unwind_cpp_pr1() { unimplemented!("__aeabi_unwind_cpp_pr1") } #[cfg(target_arch = "arm")] -#[no_mangle] +#[unsafe(no_mangle)] unsafe extern "C" fn __gnu_unwind_frame() { unimplemented!("__gnu_unwind_frame") } diff --git a/test-crates/origin-start/Cargo.toml b/test-crates/origin-start/Cargo.toml index 03ea686..ba1593a 100644 --- a/test-crates/origin-start/Cargo.toml +++ b/test-crates/origin-start/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "origin-start-tests" version = "0.0.0" -edition = "2021" +edition = "2024" publish = false [dependencies] diff --git a/test-crates/origin-start/src/bin/abort-via-raise.rs b/test-crates/origin-start/src/bin/abort-via-raise.rs index 4253840..db46cb3 100644 --- a/test-crates/origin-start/src/bin/abort-via-raise.rs +++ b/test-crates/origin-start/src/bin/abort-via-raise.rs @@ -11,7 +11,7 @@ use origin::{program, signal}; #[global_allocator] static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::GlobalDlmalloc; -#[no_mangle] +#[unsafe(no_mangle)] unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { // Terminate the current process using `Signal::ABORT`. rustix::process::kill_process(rustix::process::getpid(), signal::Signal::ABORT).ok(); diff --git a/test-crates/origin-start/src/bin/canary.rs b/test-crates/origin-start/src/bin/canary.rs index 7afb613..83d7b2e 100644 --- a/test-crates/origin-start/src/bin/canary.rs +++ b/test-crates/origin-start/src/bin/canary.rs @@ -13,7 +13,7 @@ use origin::{program, thread}; #[global_allocator] static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::GlobalDlmalloc; -extern "C" { +unsafe extern "C" { static __stack_chk_guard: UnsafeCell; } @@ -40,27 +40,29 @@ fn tls_guard() -> usize { ret } -#[no_mangle] +#[unsafe(no_mangle)] unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { - assert_ne!(*__stack_chk_guard.get(), 0); - assert_eq!(*__stack_chk_guard.get(), tls_guard()); + unsafe { + assert_ne!(*__stack_chk_guard.get(), 0); + assert_eq!(*__stack_chk_guard.get(), tls_guard()); - let thread = thread::create( - |_args| { - assert_ne!(*__stack_chk_guard.get(), 0); - assert_eq!(*__stack_chk_guard.get(), tls_guard()); - None - }, - &[], - thread::default_stack_size(), - thread::default_guard_size(), - ) - .unwrap(); + let thread = thread::create( + |_args| { + assert_ne!(*__stack_chk_guard.get(), 0); + assert_eq!(*__stack_chk_guard.get(), tls_guard()); + None + }, + &[], + thread::default_stack_size(), + thread::default_guard_size(), + ) + .unwrap(); - thread::join(thread); + thread::join(thread); - assert_ne!(*__stack_chk_guard.get(), 0); - assert_eq!(*__stack_chk_guard.get(), tls_guard()); + assert_ne!(*__stack_chk_guard.get(), 0); + assert_eq!(*__stack_chk_guard.get(), tls_guard()); - program::exit(203); + program::exit(203); + } } diff --git a/test-crates/origin-start/src/bin/detach.rs b/test-crates/origin-start/src/bin/detach.rs index c49135d..5899e06 100644 --- a/test-crates/origin-start/src/bin/detach.rs +++ b/test-crates/origin-start/src/bin/detach.rs @@ -10,37 +10,39 @@ use origin::{program, thread}; #[global_allocator] static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::GlobalDlmalloc; -#[no_mangle] +#[unsafe(no_mangle)] unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { - let long_thread = thread::create( - |_args| None, - &[], - thread::default_stack_size(), - thread::default_guard_size(), - ) - .unwrap(); - - let short_thread = thread::create( - |_args| None, - &[], - thread::default_stack_size(), - thread::default_guard_size(), - ) - .unwrap(); - thread::detach(short_thread); - - let _thread = thread::create( - |_args| { - thread::detach(thread::current()); - None - }, - &[], - thread::default_stack_size(), - thread::default_guard_size(), - ) - .unwrap(); - - thread::detach(long_thread); - - program::exit(202); + unsafe { + let long_thread = thread::create( + |_args| None, + &[], + thread::default_stack_size(), + thread::default_guard_size(), + ) + .unwrap(); + + let short_thread = thread::create( + |_args| None, + &[], + thread::default_stack_size(), + thread::default_guard_size(), + ) + .unwrap(); + thread::detach(short_thread); + + let _thread = thread::create( + |_args| { + thread::detach(thread::current()); + None + }, + &[], + thread::default_stack_size(), + thread::default_guard_size(), + ) + .unwrap(); + + thread::detach(long_thread); + + program::exit(202); + } } diff --git a/test-crates/origin-start/src/bin/main-thread-dtors-adding-dtors.rs b/test-crates/origin-start/src/bin/main-thread-dtors-adding-dtors.rs index ab18ceb..8dc4878 100644 --- a/test-crates/origin-start/src/bin/main-thread-dtors-adding-dtors.rs +++ b/test-crates/origin-start/src/bin/main-thread-dtors-adding-dtors.rs @@ -14,7 +14,7 @@ static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::Glob static FLAG: AtomicBool = AtomicBool::new(false); -#[no_mangle] +#[unsafe(no_mangle)] unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { thread::at_exit(Box::new(|| { assert!(FLAG.load(Ordering::Relaxed)); diff --git a/test-crates/origin-start/src/bin/program-dtors-adding-dtors.rs b/test-crates/origin-start/src/bin/program-dtors-adding-dtors.rs index 0560609..d00490b 100644 --- a/test-crates/origin-start/src/bin/program-dtors-adding-dtors.rs +++ b/test-crates/origin-start/src/bin/program-dtors-adding-dtors.rs @@ -14,7 +14,7 @@ static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::Glob static FLAG: AtomicBool = AtomicBool::new(false); -#[no_mangle] +#[unsafe(no_mangle)] unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { program::at_exit(Box::new(|| { assert!(FLAG.load(Ordering::Relaxed)); diff --git a/test-crates/origin-start/src/bin/thread-dtors-adding-dtors.rs b/test-crates/origin-start/src/bin/thread-dtors-adding-dtors.rs index 0f855ab..846e34a 100644 --- a/test-crates/origin-start/src/bin/thread-dtors-adding-dtors.rs +++ b/test-crates/origin-start/src/bin/thread-dtors-adding-dtors.rs @@ -14,35 +14,37 @@ static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::Glob static FLAG: AtomicBool = AtomicBool::new(false); -#[no_mangle] +#[unsafe(no_mangle)] unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { - let thread = thread::create( - |_args| { - thread::at_exit(Box::new(|| { - assert!(FLAG.load(Ordering::Relaxed)); - })); + unsafe { + let thread = thread::create( + |_args| { + thread::at_exit(Box::new(|| { + assert!(FLAG.load(Ordering::Relaxed)); + })); - thread::at_exit(Box::new(|| { thread::at_exit(Box::new(|| { thread::at_exit(Box::new(|| { - FLAG.store(true, Ordering::Relaxed); + thread::at_exit(Box::new(|| { + FLAG.store(true, Ordering::Relaxed); + })); })); })); - })); - thread::at_exit(Box::new(|| { - assert!(!FLAG.load(Ordering::Relaxed)); - })); + thread::at_exit(Box::new(|| { + assert!(!FLAG.load(Ordering::Relaxed)); + })); - None - }, - &[], - thread::default_stack_size(), - thread::default_guard_size(), - ) - .unwrap(); + None + }, + &[], + thread::default_stack_size(), + thread::default_guard_size(), + ) + .unwrap(); - thread::join(thread); + thread::join(thread); - 0 + 0 + } } diff --git a/test-crates/origin-start/src/bin/thread-id.rs b/test-crates/origin-start/src/bin/thread-id.rs index 3c06f4d..471d0bc 100644 --- a/test-crates/origin-start/src/bin/thread-id.rs +++ b/test-crates/origin-start/src/bin/thread-id.rs @@ -8,7 +8,7 @@ extern crate alloc; use alloc::boxed::Box; use alloc::sync::Arc; use core::ffi::c_void; -use core::ptr::{null_mut, NonNull}; +use core::ptr::{NonNull, null_mut}; use origin::{program, thread}; use rustix_futex_sync::{Condvar, Mutex}; @@ -21,92 +21,94 @@ enum Id { Child(thread::ThreadId), } -#[no_mangle] +#[unsafe(no_mangle)] unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { - assert_eq!(thread::current_id(), thread::id(thread::current()).unwrap()); - program::at_exit(Box::new(|| { - assert_eq!(thread::current_id(), thread::id(thread::current()).unwrap()) - })); - thread::at_exit(Box::new(|| { + unsafe { assert_eq!(thread::current_id(), thread::id(thread::current()).unwrap()); - })); + program::at_exit(Box::new(|| { + assert_eq!(thread::current_id(), thread::id(thread::current()).unwrap()) + })); + thread::at_exit(Box::new(|| { + assert_eq!(thread::current_id(), thread::id(thread::current()).unwrap()); + })); - let cond = Arc::new((Mutex::new(None), Condvar::new())); - let cond_clone = NonNull::new(Box::into_raw(Box::new(Arc::clone(&cond))).cast()); + let cond = Arc::new((Mutex::new(None), Condvar::new())); + let cond_clone = NonNull::new(Box::into_raw(Box::new(Arc::clone(&cond))).cast()); - let thread = thread::create( - move |args| { - assert_eq!(thread::current_id(), thread::id(thread::current()).unwrap()); - thread::at_exit(Box::new(|| { + let thread = thread::create( + move |args| { assert_eq!(thread::current_id(), thread::id(thread::current()).unwrap()); - })); - - let unpack = |x: Option>| match x { - Some(p) => p.as_ptr().cast::>, Condvar)>>(), - None => null_mut(), - }; - let cond = Box::from_raw(unpack(args[0])); - - // Wait for the parent to tell the child its id, and check it. - let (lock, cvar) = &**cond; - { - let mut id = lock.lock(); - while (*id).is_none() { - id = cvar.wait(id); + thread::at_exit(Box::new(|| { + assert_eq!(thread::current_id(), thread::id(thread::current()).unwrap()); + })); + + let unpack = |x: Option>| match x { + Some(p) => p.as_ptr().cast::>, Condvar)>>(), + None => null_mut(), + }; + let cond = Box::from_raw(unpack(args[0])); + + // Wait for the parent to tell the child its id, and check it. + let (lock, cvar) = &**cond; + { + let mut id = lock.lock(); + while (*id).is_none() { + id = cvar.wait(id); + } + assert_ne!(Id::Parent(thread::current_id()), *id.as_ref().unwrap()); } - assert_ne!(Id::Parent(thread::current_id()), *id.as_ref().unwrap()); - } - // Tell the parent the child's id. - { - let mut id = lock.lock(); - *id = Some(Id::Child(thread::current_id())); - cvar.notify_one(); - } + // Tell the parent the child's id. + { + let mut id = lock.lock(); + *id = Some(Id::Child(thread::current_id())); + cvar.notify_one(); + } - assert_eq!(thread::current_id(), thread::id(thread::current()).unwrap()); - None - }, - &[cond_clone], - thread::default_stack_size(), - thread::default_guard_size(), - ) - .unwrap(); - - // While the child is still running, examine its id. - let child_thread_id_from_main = thread::id(thread).unwrap(); - - // Tell the child the main thread id. - let (lock, cvar) = &*cond; - { - let mut id = lock.lock(); - *id = Some(Id::Parent(thread::current_id())); - cvar.notify_one(); - } + assert_eq!(thread::current_id(), thread::id(thread::current()).unwrap()); + None + }, + &[cond_clone], + thread::default_stack_size(), + thread::default_guard_size(), + ) + .unwrap(); + + // While the child is still running, examine its id. + let child_thread_id_from_main = thread::id(thread).unwrap(); + + // Tell the child the main thread id. + let (lock, cvar) = &*cond; + { + let mut id = lock.lock(); + *id = Some(Id::Parent(thread::current_id())); + cvar.notify_one(); + } - // Wait for the child to tell the main thread its id, and check it. - { - let mut id = lock.lock(); - loop { - if let Some(Id::Child(id)) = *id { - assert_eq!(child_thread_id_from_main, id); - break; + // Wait for the child to tell the main thread its id, and check it. + { + let mut id = lock.lock(); + loop { + if let Some(Id::Child(id)) = *id { + assert_eq!(child_thread_id_from_main, id); + break; + } + id = cvar.wait(id); } - id = cvar.wait(id); } - } - // At some point the child thread should exit, at which point `thread::id` - // will return `None`. - while thread::id(thread).is_some() { - let _ = rustix::thread::nanosleep(&rustix::thread::Timespec { - tv_sec: 0, - tv_nsec: 100_000_000, - }); - } + // At some point the child thread should exit, at which point `thread::id` + // will return `None`. + while thread::id(thread).is_some() { + let _ = rustix::thread::nanosleep(&rustix::thread::Timespec { + tv_sec: 0, + tv_nsec: 100_000_000, + }); + } - thread::join(thread); + thread::join(thread); - assert_eq!(thread::current_id(), thread::id(thread::current()).unwrap()); - program::exit(201); + assert_eq!(thread::current_id(), thread::id(thread::current()).unwrap()); + program::exit(201); + } } diff --git a/test-crates/origin-start/src/bin/tls.rs b/test-crates/origin-start/src/bin/tls.rs index 3f2b85b..c5b5a8c 100644 --- a/test-crates/origin-start/src/bin/tls.rs +++ b/test-crates/origin-start/src/bin/tls.rs @@ -16,69 +16,71 @@ use origin::{program, thread}; #[global_allocator] static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::GlobalDlmalloc; -#[no_mangle] +#[unsafe(no_mangle)] unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { - // Assert that the main thread initialized its TLS properly. - check_eq(TEST_DATA.0); - - // Mutate one of the TLS fields. - (*THREAD_LOCAL.get())[1] = without_provenance_mut(77); - - // Assert that the mutation happened properly. - check_eq([TEST_DATA.0[0], without_provenance_mut(77), TEST_DATA.0[2]]); - - program::at_exit(Box::new(|| { - // This is the last thing to run. Assert that we see the value stored - // by the `at_thread_exit` callback. - check_eq([TEST_DATA.0[0], without_provenance_mut(79), TEST_DATA.0[2]]); + unsafe { + // Assert that the main thread initialized its TLS properly. + check_eq(TEST_DATA.0); // Mutate one of the TLS fields. - (*THREAD_LOCAL.get())[1] = without_provenance_mut(80); - })); - thread::at_exit(Box::new(|| { - // Assert that we see the value stored at the end of `main`. - check_eq([TEST_DATA.0[0], without_provenance_mut(78), TEST_DATA.0[2]]); + (*THREAD_LOCAL.get())[1] = without_provenance_mut(77); - // Mutate one of the TLS fields. - (*THREAD_LOCAL.get())[1] = without_provenance_mut(79); - })); + // Assert that the mutation happened properly. + check_eq([TEST_DATA.0[0], without_provenance_mut(77), TEST_DATA.0[2]]); - let thread = thread::create( - |_args| { - // Assert that the new thread initialized its TLS properly. - check_eq(TEST_DATA.0); + program::at_exit(Box::new(|| { + // This is the last thing to run. Assert that we see the value stored + // by the `at_thread_exit` callback. + check_eq([TEST_DATA.0[0], without_provenance_mut(79), TEST_DATA.0[2]]); // Mutate one of the TLS fields. - (*THREAD_LOCAL.get())[1] = without_provenance_mut(175); + (*THREAD_LOCAL.get())[1] = without_provenance_mut(80); + })); + thread::at_exit(Box::new(|| { + // Assert that we see the value stored at the end of `main`. + check_eq([TEST_DATA.0[0], without_provenance_mut(78), TEST_DATA.0[2]]); - // Assert that the mutation happened properly. - check_eq([TEST_DATA.0[0], without_provenance_mut(175), TEST_DATA.0[2]]); + // Mutate one of the TLS fields. + (*THREAD_LOCAL.get())[1] = without_provenance_mut(79); + })); - thread::at_exit(Box::new(|| { - // Assert that we still see the value stored in the thread. + let thread = thread::create( + |_args| { + // Assert that the new thread initialized its TLS properly. + check_eq(TEST_DATA.0); + + // Mutate one of the TLS fields. + (*THREAD_LOCAL.get())[1] = without_provenance_mut(175); + + // Assert that the mutation happened properly. check_eq([TEST_DATA.0[0], without_provenance_mut(175), TEST_DATA.0[2]]); - })); - None - }, - &[], - thread::default_stack_size(), - thread::default_guard_size(), - ) - .unwrap(); + thread::at_exit(Box::new(|| { + // Assert that we still see the value stored in the thread. + check_eq([TEST_DATA.0[0], without_provenance_mut(175), TEST_DATA.0[2]]); + })); + + None + }, + &[], + thread::default_stack_size(), + thread::default_guard_size(), + ) + .unwrap(); - thread::join(thread); + thread::join(thread); - // Assert that the main thread's TLS is still in place. - check_eq([TEST_DATA.0[0], without_provenance_mut(77), TEST_DATA.0[2]]); + // Assert that the main thread's TLS is still in place. + check_eq([TEST_DATA.0[0], without_provenance_mut(77), TEST_DATA.0[2]]); - // Mutate one of the TLS fields. - (*THREAD_LOCAL.get())[1] = without_provenance_mut(78); + // Mutate one of the TLS fields. + (*THREAD_LOCAL.get())[1] = without_provenance_mut(78); - // Assert that the mutation happened properly. - check_eq([TEST_DATA.0[0], without_provenance_mut(78), TEST_DATA.0[2]]); + // Assert that the mutation happened properly. + check_eq([TEST_DATA.0[0], without_provenance_mut(78), TEST_DATA.0[2]]); - program::exit(200); + program::exit(200); + } } struct SyncTestData([*const u32; 3]); diff --git a/test-crates/origin-start/src/bin/trap.rs b/test-crates/origin-start/src/bin/trap.rs index aeadcca..2136f41 100644 --- a/test-crates/origin-start/src/bin/trap.rs +++ b/test-crates/origin-start/src/bin/trap.rs @@ -11,7 +11,7 @@ use origin::program; #[global_allocator] static GLOBAL_ALLOCATOR: rustix_dlmalloc::GlobalDlmalloc = rustix_dlmalloc::GlobalDlmalloc; -#[no_mangle] +#[unsafe(no_mangle)] unsafe fn origin_main(_argc: usize, _argv: *mut *mut u8, _envp: *mut *mut u8) -> i32 { program::trap() }