Skip to content

WIP 'opaque' pointers, no atomics yet #323

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
FractalFir wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

WIP 'opaque' pointers, no atomics yet #323

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
48bb9e6
WIP 'opaque' pointers, no atomics yet
FractalFir Nov 26, 2025
eed781a
Fully working atomics with opaque pointers
FractalFir Nov 29, 2025
ecb7537
clippy fixes + added an assertion to ensure typed pointers are *NOT* …
FractalFir Nov 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 4 additions & 16 deletions crates/rustc_codegen_nvvm/src/abi.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use libc::c_uint;
use rustc_abi::BackendRepr::Scalar;
use rustc_abi::CanonAbi;
use rustc_abi::Size;
use rustc_abi::{HasDataLayout, Primitive, Reg, RegKind};
use rustc_abi::{Primitive, Reg, RegKind};
use rustc_codegen_ssa::mir::operand::OperandRef;
use rustc_codegen_ssa::mir::operand::OperandValue;
use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue};
Expand Down Expand Up @@ -305,7 +305,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
PassMode::Cast { cast, .. } => cast.llvm_type(cx),
PassMode::Indirect { .. } => {
idx += 1;
llargument_tys.push(cx.type_ptr_to(self.ret.memory_ty(cx)));
llargument_tys.push(cx.type_i8p());
cx.type_void()
}
};
Expand Down Expand Up @@ -353,7 +353,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
attrs: _,
meta_attrs: None,
on_stack: _,
} => cx.type_ptr_to(arg.memory_ty(cx)),
} => cx.type_i8p(),
};
let (new, changed) = get_transformed_type(cx, llarg_ty);
if changed {
Expand All @@ -378,12 +378,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
}

fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type {
unsafe {
llvm::LLVMPointerType(
self.llvm_type(cx),
cx.data_layout().instruction_address_space.0 as c_uint,
)
}
cx.type_i8p()
}

fn apply_attrs_llfn(&self, cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value) {
Expand Down Expand Up @@ -569,7 +564,6 @@ impl<'tcx> AbiBuilderMethods for Builder<'_, '_, 'tcx> {
}

pub(crate) trait ArgAbiExt<'ll, 'tcx> {
fn memory_ty(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type;
fn store(
&self,
bx: &mut Builder<'_, 'll, 'tcx>,
Expand All @@ -585,12 +579,6 @@ pub(crate) trait ArgAbiExt<'ll, 'tcx> {
}

impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
/// Gets the LLVM type for a place of the original Rust type of
/// this argument/return, i.e., the result of `type_of::type_of`.
fn memory_ty(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type {
self.layout.llvm_type(cx)
}

/// Stores a direct/indirect value described by this ArgAbi into a
/// place for the original Rust type of this argument/return.
/// Can be used for both storing formal arguments into Rust variables
Expand Down
6 changes: 2 additions & 4 deletions crates/rustc_codegen_nvvm/src/allocator.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,14 +140,12 @@ pub(crate) unsafe fn codegen(
unsafe { llvm::LLVMBuildRetVoid(llbuilder) };
unsafe { llvm::LLVMDisposeBuilder(llbuilder) };

let ptr_ty = unsafe { llvm::LLVMPointerType(llvm::LLVMInt8TypeInContext(llcx), 0) };

for used in &mut used {
*used = unsafe { llvm::LLVMConstBitCast(used, ptr_ty) };
*used = unsafe { llvm::LLVMConstBitCast(used, i8p) };
}

let section = c"llvm.metadata";
let array = unsafe { llvm::LLVMConstArray(ptr_ty, used.as_ptr(), used.len() as u32) };
let array = unsafe { llvm::LLVMConstArray(i8p, used.as_ptr(), used.len() as u32) };
let g = unsafe {
llvm::LLVMAddGlobal(llmod, llvm::LLVMTypeOf(array), c"llvm.used".as_ptr().cast())
};
Expand Down
132 changes: 87 additions & 45 deletions crates/rustc_codegen_nvvm/src/builder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use std::ptr;

use libc::{c_char, c_uint};
use rustc_abi as abi;
use rustc_abi::{AddressSpace, Align, HasDataLayout, Size, TargetDataLayout, WrappingRange};
use rustc_abi::{Align, HasDataLayout, Size, TargetDataLayout, WrappingRange};
use rustc_codegen_ssa::MemFlags;
use rustc_codegen_ssa::common::{AtomicRmwBinOp, IntPredicate, RealPredicate, TypeKind};
use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
Expand Down Expand Up @@ -481,6 +481,10 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
// Cast to default addrspace if necessary
let alloca_ty = llvm::LLVMTypeOf(alloca);
let alloca_addrspace = llvm::LLVMGetPointerAddressSpace(alloca_ty);
let alloca = self.pointercast(
alloca,
self.type_i8p_ext(rustc_abi::AddressSpace(alloca_addrspace)),
);
let dest_ty = self.cx().type_ptr();
let dest_addrspace = llvm::LLVMGetPointerAddressSpace(dest_ty);
if alloca_addrspace != dest_addrspace {
Expand All @@ -493,7 +497,9 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {

fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
trace!("Load {ty:?} {:?}", ptr);
let ptr = self.pointercast(ptr, self.cx.type_ptr_to(ty));
let ptr = self.pointercast(ptr, unsafe {
llvm::LLVMPointerType(ty, llvm::LLVMGetPointerAddressSpace(self.val_ty(ptr)))
});
unsafe {
let load = llvm::LLVMBuildLoad(self.llbuilder, ptr, UNNAMED);
llvm::LLVMSetAlignment(load, align.bytes() as c_uint);
Expand All @@ -503,7 +509,9 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {

fn volatile_load(&mut self, ty: &'ll Type, ptr: &'ll Value) -> &'ll Value {
trace!("Volatile load `{:?}`", ptr);
let ptr = self.pointercast(ptr, self.cx.type_ptr_to(ty));
let ptr = self.pointercast(ptr, unsafe {
llvm::LLVMPointerType(ty, llvm::LLVMGetPointerAddressSpace(self.val_ty(ptr)))
});
unsafe {
let load = llvm::LLVMBuildLoad(self.llbuilder, ptr, UNNAMED);
llvm::LLVMSetVolatile(load, llvm::True);
Expand Down Expand Up @@ -711,14 +719,21 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
align: Align,
flags: MemFlags,
) -> &'ll Value {
assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);

let address_space = unsafe { llvm::LLVMGetPointerAddressSpace(self.val_ty(ptr)) };
let store_pointer_ty = unsafe { llvm::LLVMPointerType(self.val_ty(val), address_space) };

let ptr = unsafe {
llvm::LLVMBuildBitCast(self.llbuilder, ptr, store_pointer_ty, c"NAME".as_ptr())
};
trace!(
"store_with_flags: {:?} into {:?} with align {:?}",
val,
ptr,
align.bytes()
);
assert_eq!(self.cx.type_kind(self.cx.val_ty(ptr)), TypeKind::Pointer);
let ptr = self.check_store(val, ptr);

unsafe {
let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr);
let align = if flags.contains(MemFlags::UNALIGNED) {
Expand Down Expand Up @@ -757,15 +772,20 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {

fn gep(&mut self, ty: &'ll Type, ptr: &'ll Value, indices: &[&'ll Value]) -> &'ll Value {
trace!("gep: {ty:?} {:?} with indices {:?}", ptr, indices);
let ptr = self.pointercast(ptr, self.cx().type_ptr_to(ty));
let address_space = unsafe { llvm::LLVMGetPointerAddressSpace(self.val_ty(ptr)) };
let ptr = self.pointercast(ptr, unsafe { llvm::LLVMPointerType(ty, address_space) });
unsafe {
llvm::LLVMBuildGEP2(
let res = llvm::LLVMBuildGEP2(
self.llbuilder,
ty,
ptr,
indices.as_ptr(),
indices.len() as c_uint,
UNNAMED,
);
self.pointercast(
res,
self.type_i8p_ext(rustc_abi::AddressSpace(address_space)),
)
}
}
Expand All @@ -777,15 +797,20 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
indices: &[&'ll Value],
) -> &'ll Value {
trace!("gep inbounds: {ty:?} {:?} with indices {:?}", ptr, indices);
let ptr = self.pointercast(ptr, self.cx().type_ptr_to(ty));
let address_space = unsafe { llvm::LLVMGetPointerAddressSpace(self.val_ty(ptr)) };
let ptr = self.pointercast(ptr, unsafe { llvm::LLVMPointerType(ty, address_space) });
unsafe {
llvm::LLVMBuildInBoundsGEP2(
let res = llvm::LLVMBuildInBoundsGEP2(
self.llbuilder,
ty,
ptr,
indices.as_ptr(),
indices.len() as c_uint,
UNNAMED,
);
self.pointercast(
res,
self.type_i8p_ext(rustc_abi::AddressSpace(address_space)),
)
}
}
Expand Down Expand Up @@ -892,20 +917,20 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {

fn bitcast(&mut self, mut val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
trace!("Bitcast `{:?}` to ty `{:?}`", val, dest_ty);
unsafe {
let ty = self.val_ty(val);
let kind = llvm::LLVMRustGetTypeKind(ty);
if kind == llvm::TypeKind::Pointer {
let element = self.element_type(ty);
let addrspace = llvm::LLVMGetPointerAddressSpace(ty);
let new_ty = self.type_ptr_to_ext(element, AddressSpace::ZERO);
if addrspace != 0 {
trace!("injecting addrspace cast for `{:?}` to `{:?}`", ty, new_ty);
val = llvm::LLVMBuildAddrSpaceCast(self.llbuilder, val, new_ty, UNNAMED);
}

let ty = self.val_ty(val);
let kind = unsafe { llvm::LLVMRustGetTypeKind(ty) };

if kind == llvm::TypeKind::Pointer {
let element = self.element_type(ty);
let addrspace = unsafe { llvm::LLVMGetPointerAddressSpace(ty) };
let new_ty = unsafe { llvm::LLVMPointerType(element, 0) };
if addrspace != 0 {
trace!("injecting addrspace cast for `{:?}` to `{:?}`", ty, new_ty);
val = unsafe { llvm::LLVMBuildAddrSpaceCast(self.llbuilder, val, new_ty, UNNAMED) };
}
llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED)
}
unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) }
}

fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value {
Expand Down Expand Up @@ -1066,6 +1091,7 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {

fn insert_value(&mut self, agg_val: &'ll Value, mut elt: &'ll Value, idx: u64) -> &'ll Value {
trace!("insert value {:?}, {:?}, {:?}", agg_val, elt, idx);

assert_eq!(idx as c_uint as u64, idx);

let elt_ty = self.cx.val_ty(elt);
Expand Down Expand Up @@ -1168,9 +1194,15 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
);
}
};
let tuple = self.type_struct(&[self.val_ty(src), self.type_i1()], false);
let res = self.atomic_op(
dst,
|builder, dst| {
tuple,
|builder, dst, _| {
let address_space =
unsafe { llvm::LLVMGetPointerAddressSpace(builder.val_ty(dst)) };
let dst_ty = unsafe { llvm::LLVMPointerType(builder.val_ty(cmp), address_space) };
let dst = builder.pointercast(dst, dst_ty);
// We are in a supported address space - just use ordinary atomics
unsafe {
llvm::LLVMRustBuildAtomicCmpXchg(
Expand All @@ -1184,7 +1216,13 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
)
}
},
|builder, dst| {
|builder, dst, _| {
let dst = builder.pointercast(dst, unsafe {
llvm::LLVMPointerType(
builder.val_ty(cmp),
llvm::LLVMGetPointerAddressSpace(builder.val_ty(dst)),
)
});
// Local space is only accessible to the current thread.
// So, there are no synchronization issues, and we can emulate it using a simple load / compare / store.
let load: &'ll Value =
Expand Down Expand Up @@ -1221,8 +1259,13 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
}
self.atomic_op(
dst,
|builder, dst| {
self.val_ty(src),
|builder, dst, ty| {
// We are in a supported address space - just use ordinary atomics
let address_space =
unsafe { llvm::LLVMGetPointerAddressSpace(builder.val_ty(dst)) };
let dst_ty = unsafe { llvm::LLVMPointerType(ty, address_space) };
let dst = builder.pointercast(dst, dst_ty);
unsafe {
llvm::LLVMBuildAtomicRMW(
builder.llbuilder,
Expand All @@ -1234,9 +1277,13 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
)
}
},
|builder, dst| {
|builder, dst, ty| {
// Local space is only accessible to the current thread.
// So, there are no synchronization issues, and we can emulate it using a simple load / compare / store.
let dst = builder.pointercast(dst, unsafe {
llvm::LLVMPointerType(ty, llvm::LLVMGetPointerAddressSpace(builder.val_ty(dst)))
});

let load: &'ll Value =
unsafe { llvm::LLVMBuildLoad(builder.llbuilder, dst, UNNAMED) };
let next_val = match op {
Expand Down Expand Up @@ -1314,6 +1361,15 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
let args = self.check_call("call", llty, llfn, args);

let mut call = unsafe {
let llfn = if self.cx.type_kind(llty) == TypeKind::Pointer {
self.pointercast(llfn, llty)
} else if self.cx.type_kind(self.val_ty(llfn)) == TypeKind::Pointer {
let target_fnptr = llvm::LLVMPointerType(llty, 0);
self.pointercast(llfn, target_fnptr)
} else {
llfn
};

llvm::LLVMRustBuildCall(
self.llbuilder,
llfn,
Expand Down Expand Up @@ -1642,20 +1698,6 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {

fn noundef_metadata(&mut self, _load: &'ll Value) {}

fn check_store(&mut self, val: &'ll Value, ptr: &'ll Value) -> &'ll Value {
let dest_ptr_ty = self.cx.val_ty(ptr);
let stored_ty = self.cx.val_ty(val);
let stored_ptr_ty = self.cx.type_ptr_to(stored_ty);

assert_eq!(self.cx.type_kind(dest_ptr_ty), TypeKind::Pointer);

if dest_ptr_ty == stored_ptr_ty {
ptr
} else {
self.bitcast(ptr, stored_ptr_ty)
}
}

fn check_call<'b>(
&mut self,
typ: &str,
Expand Down Expand Up @@ -1719,7 +1761,6 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
if !self.cx().sess().emit_lifetime_markers() {
return;
}

self.call_intrinsic(intrinsic, &[self.cx.const_u64(size), ptr]);
}

Expand Down Expand Up @@ -1750,8 +1791,9 @@ impl<'ll, 'tcx, 'a> Builder<'a, 'll, 'tcx> {
fn atomic_op(
&mut self,
dst: &'ll Value,
atomic_supported: impl FnOnce(&mut Builder<'a, 'll, 'tcx>, &'ll Value) -> &'ll Value,
emulate_local: impl FnOnce(&mut Builder<'a, 'll, 'tcx>, &'ll Value) -> &'ll Value,
ty: &'ll Type,
atomic_supported: impl FnOnce(&mut Builder<'a, 'll, 'tcx>, &'ll Value, &'ll Type) -> &'ll Value,
emulate_local: impl FnOnce(&mut Builder<'a, 'll, 'tcx>, &'ll Value, &'ll Type) -> &'ll Value,
) -> &'ll Value {
// (FractalFir) Atomics in CUDA have some limitations, and we have to work around them.
// For example, they are restricted in what address space they operate on.
Expand Down Expand Up @@ -1803,7 +1845,7 @@ impl<'ll, 'tcx, 'a> Builder<'a, 'll, 'tcx> {
let merge_bb = self.append_sibling_block("atomic_op_done");
// Execute atomic op if supported, then jump to merge
self.switch_to_block(supported_bb);
let supported_res = atomic_supported(self, dst);
let supported_res = atomic_supported(self, dst, ty);
self.br(merge_bb);
// Check if the pointer is in the thread space. If so, we can emulate it.
self.switch_to_block(unsupported_bb);
Expand All @@ -1822,15 +1864,15 @@ impl<'ll, 'tcx, 'a> Builder<'a, 'll, 'tcx> {
self.cond_br(isspacep_local, local_bb, atomic_ub_bb);
// The pointer is in the thread(local) space.
self.switch_to_block(local_bb);
let local_res = emulate_local(self, dst);
let local_res = emulate_local(self, dst, ty);
self.br(merge_bb);
// The pointer is neither in the supported address space, nor the local space.
// This is very likely UB. So, we trap here.
// TODO: should we print some kind of a message here? NVVM supports printf.
self.switch_to_block(atomic_ub_bb);
self.abort();
self.unreachable();
// Atomic is impl has finished, and we can now switch to the merge_bb
// Atomic impl has finished, and we can now switch to the merge_bb
self.switch_to_block(merge_bb);
self.phi(
self.val_ty(local_res),
Expand Down
Loading
Loading