From 7611cf3fe811d1eacd4edd0a60b4033682f0245d Mon Sep 17 00:00:00 2001 From: Michael Selehov Date: Fri, 24 Apr 2026 08:50:55 +0000 Subject: [PATCH 1/7] perf(amdgpu): add subgroupDppSwapPairs for intra-wavefront pair exchange Expose the AMDGPU v_mov_b32_dpp quad_perm:[1,0,3,2] instruction as a new subgroup intrinsic. This enables sub-wave parallelism patterns where adjacent lanes exchange and reduce values without going through LDS, cutting the reduction to a single VALU-pipe cycle. - Register the op in internal_ops.inc.h and type_system.cpp - AMDGPU codegen: emit llvm.amdgcn.update.dpp with ctrl=0xB1 (32-bit native, 64-bit via lo/hi split) - CPU/base codegen: guard with QD_ERROR + actionable message - Python binding: qd.simt.subgroup.dpp_swap_pairs(value) Assisted-by: Claude Opus --- python/quadrants/lang/simt/subgroup.py | 5 +++ quadrants/codegen/amdgpu/codegen_amdgpu.cpp | 49 +++++++++++++++++++++ quadrants/codegen/llvm/codegen_llvm.cpp | 9 ++++ quadrants/inc/internal_ops.inc.h | 1 + quadrants/ir/type_system.cpp | 1 + 5 files changed, 65 insertions(+) diff --git a/python/quadrants/lang/simt/subgroup.py b/python/quadrants/lang/simt/subgroup.py index edec8978d8..046dfae7f1 100644 --- a/python/quadrants/lang/simt/subgroup.py +++ b/python/quadrants/lang/simt/subgroup.py @@ -147,6 +147,10 @@ def shuffle_xor(value, mask): pass +def dpp_swap_pairs(value): + return impl.call_internal("subgroupDppSwapPairs", value, with_runtime_context=False) + + def shuffle_up(value, offset): return impl.call_internal("subgroupShuffleUp", value, offset, with_runtime_context=False) @@ -188,4 +192,5 @@ def shuffle_down(value, offset): "shuffle_xor", "shuffle_up", "shuffle_down", + "dpp_swap_pairs", ] diff --git a/quadrants/codegen/amdgpu/codegen_amdgpu.cpp b/quadrants/codegen/amdgpu/codegen_amdgpu.cpp index b572c94c80..e5125fe02b 100644 --- a/quadrants/codegen/amdgpu/codegen_amdgpu.cpp +++ b/quadrants/codegen/amdgpu/codegen_amdgpu.cpp @@ -545,6 +545,16 @@ class TaskCodeGenAMDGPU : public TaskCodeGenLLVM { } } + + void visit(InternalFuncStmt *stmt) override { + if (stmt->func_name == "subgroupDppSwapPairs") { + llvm_val[stmt] = emit_amdgpu_dpp_swap_pairs( + llvm_val[stmt->args[0]], stmt->args[0]->ret_type); + } else { + TaskCodeGenLLVM::visit(stmt); + } + } + void visit(BinaryOpStmt *stmt) override { auto op = stmt->op_type; auto ret_quadrants_type = stmt->ret_type; @@ -586,6 +596,45 @@ class TaskCodeGenAMDGPU : public TaskCodeGenLLVM { } private: + llvm::Value *emit_amdgpu_dpp_swap_pairs(llvm::Value *value, DataType dt) { + auto *i32_ty = llvm::Type::getInt32Ty(*llvm_context); + auto *i1_ty = llvm::Type::getInt1Ty(*llvm_context); + auto *ctrl = llvm::ConstantInt::get(i32_ty, 0xB1); + auto *rmask = llvm::ConstantInt::get(i32_ty, 0xF); + auto *bmask = llvm::ConstantInt::get(i32_ty, 0xF); + auto *bctrl = llvm::ConstantInt::getFalse(i1_ty); + + auto emit_dpp_32 = [&](llvm::Value *v) -> llvm::Value * { + auto *ty = v->getType(); + return builder->CreateIntrinsic( + Intrinsic::amdgcn_update_dpp, {ty}, + {llvm::Constant::getNullValue(ty), v, ctrl, rmask, bmask, bctrl}); + }; + + if (dt->is_primitive(PrimitiveTypeID::i32) || + dt->is_primitive(PrimitiveTypeID::u32) || + dt->is_primitive(PrimitiveTypeID::f32)) { + return emit_dpp_32(value); + } + if (dt->is_primitive(PrimitiveTypeID::f64) || + dt->is_primitive(PrimitiveTypeID::i64) || + dt->is_primitive(PrimitiveTypeID::u64)) { + auto *i64_ty = llvm::Type::getInt64Ty(*llvm_context); + auto *i64_val = builder->CreateBitCast(value, i64_ty); + auto *lo = builder->CreateTrunc(i64_val, i32_ty); + auto *hi = builder->CreateTrunc(builder->CreateLShr(i64_val, 32), i32_ty); + lo = emit_dpp_32(lo); + hi = emit_dpp_32(hi); + auto *result = builder->CreateOr( + builder->CreateZExt(lo, i64_ty), + builder->CreateShl(builder->CreateZExt(hi, i64_ty), 32)); + return builder->CreateBitCast(result, value->getType()); + } + QD_ERROR("subgroupDppSwapPairs: unsupported type {} on AMDGPU", + data_type_name(dt)); + return nullptr; + } + std::tuple get_spmd_info() override { auto thread_idx = builder->CreateIntrinsic(Intrinsic::amdgcn_workitem_id_x, ArrayRef{}); diff --git a/quadrants/codegen/llvm/codegen_llvm.cpp b/quadrants/codegen/llvm/codegen_llvm.cpp index 2887a4c601..01aa641e97 100644 --- a/quadrants/codegen/llvm/codegen_llvm.cpp +++ b/quadrants/codegen/llvm/codegen_llvm.cpp @@ -2446,6 +2446,15 @@ void TaskCodeGenLLVM::visit(ClearListStmt *stmt) { } void TaskCodeGenLLVM::visit(InternalFuncStmt *stmt) { + if (stmt->func_name == "subgroupDppSwapPairs") { + QD_ERROR("Internal op \"{}\" requires a GPU backend (AMDGPU or CUDA). " + "Wrap the call site with a backend guard such as " + "qd.static(backend == gs.amdgpu) so the CPU path never reaches " + "it.", + stmt->func_name); + return; + } + std::vector args; if (stmt->with_runtime_context) diff --git a/quadrants/inc/internal_ops.inc.h b/quadrants/inc/internal_ops.inc.h index bef62c0d05..1b331a82d5 100644 --- a/quadrants/inc/internal_ops.inc.h +++ b/quadrants/inc/internal_ops.inc.h @@ -27,6 +27,7 @@ PER_INTERNAL_OP(subgroupBarrier) PER_INTERNAL_OP(subgroupMemoryBarrier) PER_INTERNAL_OP(subgroupElect) PER_INTERNAL_OP(subgroupBroadcast) +PER_INTERNAL_OP(subgroupDppSwapPairs) PER_INTERNAL_OP(subgroupSize) PER_INTERNAL_OP(subgroupInvocationId) PER_INTERNAL_OP(subgroupAdd) diff --git a/quadrants/ir/type_system.cpp b/quadrants/ir/type_system.cpp index 97a6a33704..baaaa8522d 100644 --- a/quadrants/ir/type_system.cpp +++ b/quadrants/ir/type_system.cpp @@ -379,6 +379,7 @@ void Operations::init_internals() { PLAIN_OP(subgroupMemoryBarrier, i32_void, false); PLAIN_OP(subgroupElect, i32, false); POLY_OP(subgroupBroadcast, false, Signature({}, {ValueT, !u32}, ValueT)); + POLY_OP(subgroupDppSwapPairs, false, Signature({}, {ValueT}, ValueT)); PLAIN_OP(subgroupSize, i32, false); PLAIN_OP(subgroupInvocationId, i32, false); POLY_OP(subgroupAdd, false, Signature({}, {ValueT}, ValueT)); From 7cd64adb464d64a37d4f1abd119eaa5695fac2a3 Mon Sep 17 00:00:00 2001 From: Michael Selehov Date: Fri, 24 Apr 2026 09:23:49 +0000 Subject: [PATCH 2/7] feat: support range(start, stop, step) in for-loops Desugar 3-argument range() into a while-loop at the AST level. The Quadrants IR does not natively support a step parameter in range-for, so range(start, stop, step) is lowered to: i = start while i < stop: i += step This eliminates the need for manual while-loop workarounds when writing strided iteration patterns (e.g. sub-wave parallelism). --- python/quadrants/lang/ast/ast_transformer.py | 51 +++++++++++++++++++- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/python/quadrants/lang/ast/ast_transformer.py b/python/quadrants/lang/ast/ast_transformer.py index 79c639761e..52034049d4 100644 --- a/python/quadrants/lang/ast/ast_transformer.py +++ b/python/quadrants/lang/ast/ast_transformer.py @@ -905,13 +905,15 @@ def build_static_for(ctx: ASTTransformerFuncContext, node: ast.For, is_grouped: @staticmethod def build_range_for(ctx: ASTTransformerFuncContext, node: ast.For) -> None: + if len(node.iter.args) not in [1, 2, 3]: + raise QuadrantsSyntaxError(f"Range should have 1, 2, or 3 arguments, found {len(node.iter.args)}") + if len(node.iter.args) == 3: + return ASTTransformer.build_strided_range_for(ctx, node) with ctx.variable_scope_guard(): loop_name = node.target.id ctx.check_loop_var(loop_name) loop_var = expr.Expr(ctx.ast_builder.make_id_expr("")) ctx.create_variable(loop_name, loop_var) - if len(node.iter.args) not in [1, 2]: - raise QuadrantsSyntaxError(f"Range should have 1 or 2 arguments, found {len(node.iter.args)}") if len(node.iter.args) == 2: begin_expr = expr.Expr(build_stmt(ctx, node.iter.args[0])) end_expr = expr.Expr(build_stmt(ctx, node.iter.args[1])) @@ -940,6 +942,51 @@ def build_range_for(ctx: ASTTransformerFuncContext, node: ast.For) -> None: ctx.ast_builder.end_frontend_range_for() return None + @staticmethod + def build_strided_range_for(ctx, node): + """Desugar `for i in range(start, stop, step)` into a while loop. + + The Quadrants IR does not natively support a step parameter in + range-for loops. We lower `range(start, stop, step)` into:: + + i = start + while i < stop: # (or i > stop when step < 0) + + i = i + step + """ + with ctx.variable_scope_guard(): + loop_name = node.target.id + + begin_expr = expr.Expr(build_stmt(ctx, node.iter.args[0])) + end_expr = expr.Expr(build_stmt(ctx, node.iter.args[1])) + step_expr = expr.Expr(build_stmt(ctx, node.iter.args[2])) + + begin = qd_ops.cast(begin_expr, primitive_types.i32) + end = qd_ops.cast(end_expr, primitive_types.i32) + step = qd_ops.cast(step_expr, primitive_types.i32) + + loop_var = impl.expr_init(begin) + ctx.create_variable(loop_name, loop_var) + + with ctx.loop_scope_guard(): + stmt_dbg_info = _qd_core.DebugInfo(ctx.get_pos_info(node)) + ctx.ast_builder.begin_frontend_while( + expr.Expr(1, dtype=primitive_types.i32).ptr, stmt_dbg_info) + + cond = loop_var < end + impl.begin_frontend_if(ctx.ast_builder, cond, stmt_dbg_info) + ctx.ast_builder.begin_frontend_if_true() + ctx.ast_builder.pop_scope() + ctx.ast_builder.begin_frontend_if_false() + ctx.ast_builder.insert_break_stmt(stmt_dbg_info) + ctx.ast_builder.pop_scope() + + build_stmts(ctx, node.body) + + loop_var._assign(loop_var + step) + ctx.ast_builder.pop_scope() + return None + @staticmethod def build_ndrange_for(ctx: ASTTransformerFuncContext, node: ast.For) -> None: with ctx.variable_scope_guard(): From 9cc89fffcccb48d82ddfe8dded1b556b9a391fd0 Mon Sep 17 00:00:00 2001 From: Michael Selehov Date: Fri, 24 Apr 2026 09:47:17 +0000 Subject: [PATCH 3/7] chore: fix linter warnings (black, clang-format, ruff, trailing-ws) --- Dockerfile.rocm | 4 ++-- python/quadrants/lang/_func_base.py | 1 - python/quadrants/lang/ast/ast_transformer.py | 3 +-- quadrants/codegen/amdgpu/codegen_amdgpu.cpp | 21 ++++++++----------- quadrants/codegen/llvm/codegen_llvm.cpp | 19 +++++++++-------- quadrants/rhi/amdgpu/amdgpu_context.cpp | 3 +-- .../rhi/amdgpu/amdgpu_driver_functions.inc.h | 6 +++++- quadrants/runtime/amdgpu/jit_amdgpu.cpp | 14 ++++++------- quadrants/runtime/amdgpu/kernel_launcher.cpp | 8 +++---- quadrants/runtime/llvm/llvm_context.cpp | 8 +++---- quadrants/runtime/llvm/llvm_context_pass.h | 16 +++++++------- 11 files changed, 52 insertions(+), 51 deletions(-) diff --git a/Dockerfile.rocm b/Dockerfile.rocm index c1e48190c1..76f6b22f9d 100644 --- a/Dockerfile.rocm +++ b/Dockerfile.rocm @@ -29,14 +29,14 @@ RUN uv venv -p ${PY_VERSION} /opt/venv ENV VIRTUAL_ENV=/opt/venv ENV PATH="/opt/venv/bin:$PATH" -# rocm and torch MUST be installed together or torch nightlies will +# rocm and torch MUST be installed together or torch nightlies will # mess up the rocm version RUN uv pip install pip cmake pybind11 build ninja scikit-build-core setuptools-scm numpy pytest && \ uv pip install --index-url https://repo.amd.com/rocm/whl/${GFX_FAMILY}/ "rocm[libraries,devel]"==${THE_ROCK_VERSION} torch==${TORCH_VERSION} && \ rocm-sdk init # Setup SRC -ENV QUADRANTS_SRC_DIR=/src/quadrants/ +ENV QUADRANTS_SRC_DIR=/src/quadrants/ COPY . ${QUADRANTS_SRC_DIR} RUN git config --global --add safe.directory ${QUADRANTS_SRC_DIR} diff --git a/python/quadrants/lang/_func_base.py b/python/quadrants/lang/_func_base.py index 15067182f3..c285ea072b 100644 --- a/python/quadrants/lang/_func_base.py +++ b/python/quadrants/lang/_func_base.py @@ -1,6 +1,5 @@ import ast import inspect -import math import sys import textwrap import types diff --git a/python/quadrants/lang/ast/ast_transformer.py b/python/quadrants/lang/ast/ast_transformer.py index 52034049d4..64acc8d79e 100644 --- a/python/quadrants/lang/ast/ast_transformer.py +++ b/python/quadrants/lang/ast/ast_transformer.py @@ -970,8 +970,7 @@ def build_strided_range_for(ctx, node): with ctx.loop_scope_guard(): stmt_dbg_info = _qd_core.DebugInfo(ctx.get_pos_info(node)) - ctx.ast_builder.begin_frontend_while( - expr.Expr(1, dtype=primitive_types.i32).ptr, stmt_dbg_info) + ctx.ast_builder.begin_frontend_while(expr.Expr(1, dtype=primitive_types.i32).ptr, stmt_dbg_info) cond = loop_var < end impl.begin_frontend_if(ctx.ast_builder, cond, stmt_dbg_info) diff --git a/quadrants/codegen/amdgpu/codegen_amdgpu.cpp b/quadrants/codegen/amdgpu/codegen_amdgpu.cpp index e5125fe02b..b4672cc742 100644 --- a/quadrants/codegen/amdgpu/codegen_amdgpu.cpp +++ b/quadrants/codegen/amdgpu/codegen_amdgpu.cpp @@ -72,8 +72,7 @@ class TaskCodeGenAMDGPU : public TaskCodeGenLLVM { auto base = new llvm::GlobalVariable( *module, type, false, llvm::GlobalValue::ExternalLinkage, nullptr, fmt::format("shared_array_t{}_s{}", task_codegen_id, stmt->id), - nullptr, llvm::GlobalVariable::NotThreadLocal, - 3 /*addrspace=LDS*/); + nullptr, llvm::GlobalVariable::NotThreadLocal, 3 /*addrspace=LDS*/); base->setAlignment(llvm::MaybeAlign(8)); auto ptr_type = llvm::PointerType::get(type, 0); llvm_val[stmt] = builder->CreatePointerCast(base, ptr_type); @@ -338,8 +337,7 @@ class TaskCodeGenAMDGPU : public TaskCodeGenLLVM { if (input && input->getType()->isPointerTy() && input->getType()->getPointerAddressSpace() == 1) { auto *ptr_as0 = llvm::PointerType::getUnqual(*llvm_context); - llvm_val[stmt->input_ptr] = - builder->CreateAddrSpaceCast(input, ptr_as0); + llvm_val[stmt->input_ptr] = builder->CreateAddrSpaceCast(input, ptr_as0); } TaskCodeGenLLVM::visit(stmt); llvm_val[stmt->input_ptr] = input; @@ -392,9 +390,9 @@ class TaskCodeGenAMDGPU : public TaskCodeGenLLVM { tlctx->get_data_type(stmt->origin->ret_type.ptr_removed()); auto *casted_ptr = builder->CreateBitCast( origin_ptr, llvm::PointerType::get(origin_pointee_ty, origin_as)); - llvm_val[stmt] = builder->CreateGEP( - origin_pointee_ty, casted_ptr, - {tlctx->get_constant(0), llvm_val[stmt->offset]}); + llvm_val[stmt] = + builder->CreateGEP(origin_pointee_ty, casted_ptr, + {tlctx->get_constant(0), llvm_val[stmt->offset]}); } else { // Byte-offset GEP preserves pointer provenance and address space, // avoiding the PtrToInt/IntToPtr round-trip that breaks addrspace @@ -471,8 +469,8 @@ class TaskCodeGenAMDGPU : public TaskCodeGenLLVM { // BLS / shared memory buffer allocation void create_bls_buffer(OffloadedStmt *stmt) { - auto type = llvm::ArrayType::get( - llvm::Type::getInt8Ty(*llvm_context), stmt->bls_size); + auto type = llvm::ArrayType::get(llvm::Type::getInt8Ty(*llvm_context), + stmt->bls_size); bls_buffer = new llvm::GlobalVariable( *module, type, false, llvm::GlobalValue::ExternalLinkage, nullptr, "bls_buffer", nullptr, llvm::GlobalVariable::NotThreadLocal, @@ -545,11 +543,10 @@ class TaskCodeGenAMDGPU : public TaskCodeGenLLVM { } } - void visit(InternalFuncStmt *stmt) override { if (stmt->func_name == "subgroupDppSwapPairs") { - llvm_val[stmt] = emit_amdgpu_dpp_swap_pairs( - llvm_val[stmt->args[0]], stmt->args[0]->ret_type); + llvm_val[stmt] = emit_amdgpu_dpp_swap_pairs(llvm_val[stmt->args[0]], + stmt->args[0]->ret_type); } else { TaskCodeGenLLVM::visit(stmt); } diff --git a/quadrants/codegen/llvm/codegen_llvm.cpp b/quadrants/codegen/llvm/codegen_llvm.cpp index 01aa641e97..62ac93db4a 100644 --- a/quadrants/codegen/llvm/codegen_llvm.cpp +++ b/quadrants/codegen/llvm/codegen_llvm.cpp @@ -1954,7 +1954,8 @@ void TaskCodeGenLLVM::visit(ExternalPtrStmt *stmt) { // Indexing array dimensions linear_index = builder->CreateMul(linear_index, sizes[size_var_index++]); } - auto index = builder->CreateSExtOrBitCast(llvm_val[stmt->indices[i]], i64_ty); + auto index = + builder->CreateSExtOrBitCast(llvm_val[stmt->indices[i]], i64_ty); linear_index = builder->CreateAdd(linear_index, index); } QD_ASSERT(size_var_index == num_indices - num_element_indices); @@ -2030,9 +2031,8 @@ std::string TaskCodeGenLLVM::init_offloaded_task_function(OffloadedStmt *stmt, } else { context_param_type = llvm::PointerType::get(context_ty, 0); } - task_function_type = - llvm::FunctionType::get(llvm::Type::getVoidTy(*llvm_context), - {context_param_type}, false); + task_function_type = llvm::FunctionType::get( + llvm::Type::getVoidTy(*llvm_context), {context_param_type}, false); auto task_kernel_name = fmt::format( "{}_{}_{}{}", kernel_name, task_codegen_id, stmt->task_name(), suffix); @@ -2447,11 +2447,12 @@ void TaskCodeGenLLVM::visit(ClearListStmt *stmt) { void TaskCodeGenLLVM::visit(InternalFuncStmt *stmt) { if (stmt->func_name == "subgroupDppSwapPairs") { - QD_ERROR("Internal op \"{}\" requires a GPU backend (AMDGPU or CUDA). " - "Wrap the call site with a backend guard such as " - "qd.static(backend == gs.amdgpu) so the CPU path never reaches " - "it.", - stmt->func_name); + QD_ERROR( + "Internal op \"{}\" requires a GPU backend (AMDGPU or CUDA). " + "Wrap the call site with a backend guard such as " + "qd.static(backend == gs.amdgpu) so the CPU path never reaches " + "it.", + stmt->func_name); return; } diff --git a/quadrants/rhi/amdgpu/amdgpu_context.cpp b/quadrants/rhi/amdgpu/amdgpu_context.cpp index 4880cc1fb0..a3f1630099 100644 --- a/quadrants/rhi/amdgpu/amdgpu_context.cpp +++ b/quadrants/rhi/amdgpu/amdgpu_context.cpp @@ -190,8 +190,7 @@ void AMDGPUContext::launch(void *func, bool valid = offline_cache::try_demangle_name(task_name, primal_task_name, key); profiler_amdgpu->trace(task_handle, valid ? primal_task_name : task_name, - func, grid_dim, block_dim, - dynamic_shared_mem_bytes); + func, grid_dim, block_dim, dynamic_shared_mem_bytes); } auto context_guard = AMDGPUContext::get_instance().get_guard(); diff --git a/quadrants/rhi/amdgpu/amdgpu_driver_functions.inc.h b/quadrants/rhi/amdgpu/amdgpu_driver_functions.inc.h index 57446c6aa1..58f3c626ef 100644 --- a/quadrants/rhi/amdgpu/amdgpu_driver_functions.inc.h +++ b/quadrants/rhi/amdgpu/amdgpu_driver_functions.inc.h @@ -69,7 +69,11 @@ PER_AMDGPU_FUNCTION(memcpy_device_to_host_async, std::size_t, void *); PER_AMDGPU_FUNCTION(malloc, hipMalloc, void **, std::size_t); -PER_AMDGPU_FUNCTION(malloc_async_impl, hipMallocAsync, void **, std::size_t, void *); +PER_AMDGPU_FUNCTION(malloc_async_impl, + hipMallocAsync, + void **, + std::size_t, + void *); PER_AMDGPU_FUNCTION(malloc_managed, hipMallocManaged, void **, diff --git a/quadrants/runtime/amdgpu/jit_amdgpu.cpp b/quadrants/runtime/amdgpu/jit_amdgpu.cpp index 8898191449..3bf59fe609 100644 --- a/quadrants/runtime/amdgpu/jit_amdgpu.cpp +++ b/quadrants/runtime/amdgpu/jit_amdgpu.cpp @@ -17,8 +17,8 @@ namespace quadrants { namespace lang { #if defined(QD_WITH_AMDGPU) -JITModule *JITSessionAMDGPU ::add_module(std::unique_ptr M, - int max_reg) { +JITModule *JITSessionAMDGPU::add_module(std::unique_ptr M, + int max_reg) { // HSACo caching auto cache_key = compute_module_cache_key(M.get()); auto cache_it = hsaco_cache_.find(cache_key); @@ -50,8 +50,7 @@ std::string JITSessionAMDGPU::compile_module_to_hsaco( std::unique_ptr &llvm_module) { static std::once_flag amdgpu_cl_flags; std::call_once(amdgpu_cl_flags, [] { - const char *args[] = {"quadrants", - "-force-vector-interleave=8"}; + const char *args[] = {"quadrants", "-force-vector-interleave=8"}; llvm::cl::ParseCommandLineOptions(2, args); }); @@ -128,10 +127,11 @@ std::string JITSessionAMDGPU::compile_module_to_hsaco( if (CB->getCalledOperand() != &F) continue; auto *Caller = CB->getFunction(); - if (Caller && Caller->getCallingConv() == llvm::CallingConv::AMDGPU_KERNEL && + if (Caller && + Caller->getCallingConv() == llvm::CallingConv::AMDGPU_KERNEL && Caller->hasFnAttribute("amdgpu-flat-work-group-size")) { - inherited = - Caller->getFnAttribute("amdgpu-flat-work-group-size").getValueAsString(); + inherited = Caller->getFnAttribute("amdgpu-flat-work-group-size") + .getValueAsString(); break; } } diff --git a/quadrants/runtime/amdgpu/kernel_launcher.cpp b/quadrants/runtime/amdgpu/kernel_launcher.cpp index 7255595639..cf20bbec00 100644 --- a/quadrants/runtime/amdgpu/kernel_launcher.cpp +++ b/quadrants/runtime/amdgpu/kernel_launcher.cpp @@ -212,14 +212,14 @@ void KernelLauncher::launch_llvm_kernel(Handle handle, LaunchContextBuilder::DevAllocType::kNone) { if (on_amdgpu_device(data_ptr)) { if (branch_counts) { - branch_counts->kNone_on_device.fetch_add( - 1, std::memory_order_relaxed); + branch_counts->kNone_on_device.fetch_add(1, + std::memory_order_relaxed); } resolved_dev_ptr = data_ptr; } else { if (branch_counts) { - branch_counts->kNone_host_copy.fetch_add( - 1, std::memory_order_relaxed); + branch_counts->kNone_host_copy.fetch_add(1, + std::memory_order_relaxed); } DeviceAllocation devalloc = executor->allocate_memory_on_device( arr_sz, (uint64 *)device_result_buffer); diff --git a/quadrants/runtime/llvm/llvm_context.cpp b/quadrants/runtime/llvm/llvm_context.cpp index 2e03bc104e..30fe3c8c91 100644 --- a/quadrants/runtime/llvm/llvm_context.cpp +++ b/quadrants/runtime/llvm/llvm_context.cpp @@ -1142,8 +1142,8 @@ void QuadrantsLLVMContext::mark_function_as_cuda_kernel(llvm::Function *func, } } -void QuadrantsLLVMContext::mark_function_as_amdgpu_kernel( - llvm::Function *func, int block_dim) { +void QuadrantsLLVMContext::mark_function_as_amdgpu_kernel(llvm::Function *func, + int block_dim) { func->setCallingConv(llvm::CallingConv::AMDGPU_KERNEL); if (block_dim > 0) { // Note: hardcoded wavefront size of 64 matches CDNA3. RDNA in wave32 @@ -1152,8 +1152,8 @@ void QuadrantsLLVMContext::mark_function_as_amdgpu_kernel( constexpr int kAmdgpuWavefrontSize = 64; int min_block_dim = std::max(block_dim, kAmdgpuWavefrontSize); int max_block_dim = std::max(block_dim, kAmdgpuWavefrontSize); - std::string size_str = std::to_string(min_block_dim) + "," + - std::to_string(max_block_dim); + std::string size_str = + std::to_string(min_block_dim) + "," + std::to_string(max_block_dim); func->addFnAttr("amdgpu-flat-work-group-size", size_str); } } diff --git a/quadrants/runtime/llvm/llvm_context_pass.h b/quadrants/runtime/llvm/llvm_context_pass.h index d686c8cc41..c707f48e78 100644 --- a/quadrants/runtime/llvm/llvm_context_pass.h +++ b/quadrants/runtime/llvm/llvm_context_pass.h @@ -122,10 +122,12 @@ struct AMDGPUConvertAllocaInstAddressSpacePass : public FunctionPass { // (scratch), the load/store is left as flat. struct AMDGPUFlatToGlobalLoadStorePass : public FunctionPass { static inline char ID{0}; - AMDGPUFlatToGlobalLoadStorePass() : FunctionPass(ID) {} + AMDGPUFlatToGlobalLoadStorePass() : FunctionPass(ID) { + } - static bool originatesFromScratch(llvm::Value *ptr, - llvm::SmallPtrSetImpl &Visited) { + static bool originatesFromScratch( + llvm::Value *ptr, + llvm::SmallPtrSetImpl &Visited) { auto *origin = ptr->stripPointerCasts(); if (!Visited.insert(origin).second) return false; // already on the walk path — break the cycle @@ -205,13 +207,13 @@ struct AMDGPUFlatToGlobalLoadStorePass : public FunctionPass { for (auto *I : to_convert) { llvm::IRBuilder<> B(I); if (auto *LI = llvm::dyn_cast(I)) { - auto *cast = B.CreateAddrSpaceCast(LI->getPointerOperand(), - ptr_global_ty); + auto *cast = + B.CreateAddrSpaceCast(LI->getPointerOperand(), ptr_global_ty); LI->setOperand(LI->getPointerOperandIndex(), cast); modified = true; } else if (auto *SI = llvm::dyn_cast(I)) { - auto *cast = B.CreateAddrSpaceCast(SI->getPointerOperand(), - ptr_global_ty); + auto *cast = + B.CreateAddrSpaceCast(SI->getPointerOperand(), ptr_global_ty); SI->setOperand(SI->getPointerOperandIndex(), cast); modified = true; } From 71f65546729a977e5e4ba309cf682fca30d0f4ea Mon Sep 17 00:00:00 2001 From: Michael Selehov Date: Fri, 24 Apr 2026 11:22:26 +0000 Subject: [PATCH 4/7] test: update tests for range(start, stop, step) support test_range_for_three_arguments now verifies correct strided iteration instead of expecting QuadrantsCompilationError. test_exception_in_node_with_body uses range() (0 args) as the invalid construct instead of range(1, 2, 3) which is now valid. --- tests/python/test_ast_refactor.py | 19 +++++++++++-------- tests/python/test_exception.py | 12 ++++++------ 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/tests/python/test_ast_refactor.py b/tests/python/test_ast_refactor.py index 0a49e55375..fa108e49da 100644 --- a/tests/python/test_ast_refactor.py +++ b/tests/python/test_ast_refactor.py @@ -399,15 +399,18 @@ def foo(x: qd.i32): def test_range_for_three_arguments(): a = qd.field(qd.i32, shape=(10,)) - with pytest.raises(qd.QuadrantsCompilationError, match="Range should have 1 or 2 arguments, found 3"): - - @qd.kernel - def foo(x: qd.i32): - for i in range(3, 7, 2): - a[i] = x + @qd.kernel + def foo(x: qd.i32): + for i in range(3, 7, 2): + a[i] = x - x = 5 - foo(x) + a.fill(0) + foo(5) + for i in range(10): + if i in (3, 5): + assert a[i] == 5 + else: + assert a[i] == 0 @test_utils.test(print_preprocessed_ir=True) diff --git a/tests/python/test_exception.py b/tests/python/test_exception.py index df145e155e..cf796c1b17 100644 --- a/tests/python/test_exception.py +++ b/tests/python/test_exception.py @@ -97,7 +97,7 @@ def foo(): msg = f""" File "{file}", line {lineno + 5}, in foo: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbaaaaaa - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ bbbbbbbbbbbbbbbbbbbbbaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa(111) @@ -108,9 +108,10 @@ def foo(): @test_utils.test(print_full_traceback=False) def test_exception_in_node_with_body(): frameinfo = getframeinfo(currentframe()) + @qd.kernel def foo(): - for i in range(1, 2, 3): + for i in range(): a = 1 b = 1 c = 1 @@ -122,8 +123,7 @@ def foo(): file = frameinfo.filename msg = f""" File "{file}", line {lineno + 3}, in foo: - for i in range(1, 2, 3): - ^^^^^^^^^^^^^^^^^^^^^^^^ -Range should have 1 or 2 arguments, found 3""" + for i in range(): + ^^^^^^^^^^^^^^^^^ +Range should have 1, 2, or 3 arguments, found 0""" assert msg in e.value.args[0] - From 63c9f68319c0041f639746ad1bb0305ec9aab66d Mon Sep 17 00:00:00 2001 From: Michael Selehov Date: Fri, 24 Apr 2026 07:44:48 -0500 Subject: [PATCH 5/7] test: fix caret count and line offset in test_exception.py --- tests/python/test_exception.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/python/test_exception.py b/tests/python/test_exception.py index cf796c1b17..32c7e73fb0 100644 --- a/tests/python/test_exception.py +++ b/tests/python/test_exception.py @@ -97,7 +97,7 @@ def foo(): msg = f""" File "{file}", line {lineno + 5}, in foo: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbaaaaaa - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ bbbbbbbbbbbbbbbbbbbbbaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa(111) @@ -122,7 +122,7 @@ def foo(): lineno = frameinfo.lineno file = frameinfo.filename msg = f""" -File "{file}", line {lineno + 3}, in foo: +File "{file}", line {lineno + 4}, in foo: for i in range(): ^^^^^^^^^^^^^^^^^ Range should have 1, 2, or 3 arguments, found 0""" From 1fbd8444d42ec7de860302abb56e48f594fdf539 Mon Sep 17 00:00:00 2001 From: carlobertolli Date: Sat, 25 Apr 2026 21:11:02 -0500 Subject: [PATCH 6/7] Reduce GPU oversubscription. This patch reduces the maximum amount of threablocks launched per CU to 8, instead of 32. The result is a smaller number of threadblocks that have no work to do, on average testing. I see a 7% improvement in my local system. --- quadrants/runtime/llvm/llvm_runtime_executor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quadrants/runtime/llvm/llvm_runtime_executor.cpp b/quadrants/runtime/llvm/llvm_runtime_executor.cpp index 6f7c40596c..3a9f72b465 100644 --- a/quadrants/runtime/llvm/llvm_runtime_executor.cpp +++ b/quadrants/runtime/llvm/llvm_runtime_executor.cpp @@ -143,7 +143,7 @@ LlvmRuntimeExecutor::LlvmRuntimeExecutor(CompileConfig &config, // magic number 32 // I didn't find the relevant parameter to limit the max block num per CU // So .... - int query_max_block_per_cu{32}; + int query_max_block_per_cu{8}; if (config.max_block_dim == 0) { config.max_block_dim = query_max_block_dim; } From 673d7bd00e1d9fc2fe64340df4ab036802b43463 Mon Sep 17 00:00:00 2001 From: Michael Selehov Date: Wed, 6 May 2026 04:53:36 -0500 Subject: [PATCH 7/7] chore: fix clang-format violations in our changed files --- quadrants/codegen/amdgpu/codegen_amdgpu.cpp | 2 +- quadrants/runtime/llvm/llvm_context.cpp | 22 +++++++++++---------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/quadrants/codegen/amdgpu/codegen_amdgpu.cpp b/quadrants/codegen/amdgpu/codegen_amdgpu.cpp index b4672cc742..d9480f4c4e 100644 --- a/quadrants/codegen/amdgpu/codegen_amdgpu.cpp +++ b/quadrants/codegen/amdgpu/codegen_amdgpu.cpp @@ -399,7 +399,7 @@ class TaskCodeGenAMDGPU : public TaskCodeGenLLVM { // tagging and confuses InferAddressSpaces. auto *byte_ptr = builder->CreateBitCast( origin_ptr, llvm::PointerType::get( - llvm::Type::getInt8Ty(*llvm_context), origin_as)); + llvm::Type::getInt8Ty(*llvm_context), origin_as)); auto *address_offset = builder->CreateSExt( llvm_val[stmt->offset], llvm::Type::getInt64Ty(*llvm_context)); auto *offset_ptr = builder->CreateGEP( diff --git a/quadrants/runtime/llvm/llvm_context.cpp b/quadrants/runtime/llvm/llvm_context.cpp index 30fe3c8c91..2efa096ffd 100644 --- a/quadrants/runtime/llvm/llvm_context.cpp +++ b/quadrants/runtime/llvm/llvm_context.cpp @@ -630,15 +630,16 @@ std::unique_ptr QuadrantsLLVMContext::module_from_file( auto *wave_size = llvm::ConstantInt::get(i32_ty, 64); auto *in_wave = builder.CreateICmpULT(src_lane, wave_size); - auto *addr = builder.CreateShl( - src_lane, llvm::ConstantInt::get(i32_ty, 2)); + auto *addr = + builder.CreateShl(src_lane, llvm::ConstantInt::get(i32_ty, 2)); auto *bpermute_i32 = builder.CreateIntrinsic( llvm::Intrinsic::amdgcn_ds_bpermute, {}, {addr, val_i32}); auto *picked = builder.CreateSelect(in_wave, bpermute_i32, val_i32); if (is_float) { - builder.CreateRet(builder.CreateBitCast(picked, builder.getFloatTy())); + builder.CreateRet( + builder.CreateBitCast(picked, builder.getFloatTy())); } else { builder.CreateRet(picked); } @@ -681,13 +682,14 @@ std::unique_ptr QuadrantsLLVMContext::module_from_file( auto *src_lane = builder.CreateSub(lane_id, delta_arg); // Wave-wide bound: src_lane must be >= 0. auto *in_wave = builder.CreateICmpSGE(src_lane, zero); - auto *addr = builder.CreateShl( - src_lane, llvm::ConstantInt::get(i32_ty, 2)); + auto *addr = + builder.CreateShl(src_lane, llvm::ConstantInt::get(i32_ty, 2)); auto *bpermute_i32 = builder.CreateIntrinsic( llvm::Intrinsic::amdgcn_ds_bpermute, {}, {addr, val_i32}); auto *picked = builder.CreateSelect(in_wave, bpermute_i32, val_i32); if (is_float) { - builder.CreateRet(builder.CreateBitCast(picked, builder.getFloatTy())); + builder.CreateRet( + builder.CreateBitCast(picked, builder.getFloatTy())); } else { builder.CreateRet(picked); } @@ -725,8 +727,8 @@ std::unique_ptr QuadrantsLLVMContext::module_from_file( if (is_float) { val_i32 = builder.CreateBitCast(val_arg, i32_ty); } - auto *addr = builder.CreateShl( - src_lane_arg, llvm::ConstantInt::get(i32_ty, 2)); + auto *addr = + builder.CreateShl(src_lane_arg, llvm::ConstantInt::get(i32_ty, 2)); auto *bpermute_i32 = builder.CreateIntrinsic( llvm::Intrinsic::amdgcn_ds_bpermute, {}, {addr, val_i32}); if (is_float) { @@ -772,8 +774,8 @@ std::unique_ptr QuadrantsLLVMContext::module_from_file( auto *lane_id = builder.CreateIntrinsic( llvm::Intrinsic::amdgcn_mbcnt_hi, {}, {neg_one, mbcnt_lo}); auto *src_lane = builder.CreateXor(lane_id, delta_arg); - auto *addr = builder.CreateShl( - src_lane, llvm::ConstantInt::get(i32_ty, 2)); + auto *addr = + builder.CreateShl(src_lane, llvm::ConstantInt::get(i32_ty, 2)); auto *bpermute_i32 = builder.CreateIntrinsic( llvm::Intrinsic::amdgcn_ds_bpermute, {}, {addr, val_i32}); if (is_float) {