From 5c504963af98056503fa3df3ea28198ab887e14f Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 28 Oct 2026 13:57:35 +0100 Subject: [PATCH 1/6] Return a typed vector from `classify_arguments` The arguments were collected in a `Vector{Any}`, so every use of their fields in the back-ends (Metal's argument metadata and parameter attributes, `irgen`'s by-value rewriting) dispatched dynamically, and so did the LLVM.jl and Base methods called with them. Many of those are only inferred in an upstream package image, without native code, and a back-end's own image can't add native code to them (JuliaLang/julia#58465), so they were compiled in every session. --- src/irgen.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/irgen.jl b/src/irgen.jl index 679e8b23..b516e120 100644 --- a/src/irgen.jl +++ b/src/irgen.jl @@ -563,6 +563,8 @@ end # - `name`: the name of the argument # - `idx`: the index of the argument in the LLVM function type, or `nothing` if the argument # is not passed at the LLVM level. +const ArgumentInfo = @NamedTuple{cc::ArgumentCC, typ::Any, name::Symbol, + idx::Union{Nothing,Int}} function classify_arguments(@nospecialize(job::CompilerJob), codegen_ft::LLVM.FunctionType; post_optimization::Bool=false) source_sig = job.source.specTypes::DataType @@ -576,12 +578,11 @@ function classify_arguments(@nospecialize(job::CompilerJob), codegen_ft::LLVM.Fu codegen_types = codegen_ft.parameters + args = ArgumentInfo[] if post_optimization && kernel_state_type(job) !== Nothing - args = [] push!(args, (cc=KERNEL_STATE, typ=kernel_state_type(job), name=:kernel_state, idx=1)) codegen_i = 2 else - args = [] codegen_i = 1 end for (source_typ, source_name) in zip(source_types, source_argnames) From 13109adb857118d2ba0564ec3620b8e6bdea79f9 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 28 Oct 2026 13:57:35 +0100 Subject: [PATCH 2/6] Avoid dynamic dispatch when building vectors of IR objects Splatting a collection of LLVM values or types into a vector literal dispatches on the run-time types of the elements, compiling a method for every combination of instruction and type kinds encountered. Build these vectors with `collect`, `pushfirst!` or by concatenating concretely typed collections instead. Also assert the types of the kernel state getter call and of the caller instruction `backtrace` moves to, which kept every property access in its loop dynamic. --- src/debug.jl | 2 +- src/irgen.jl | 13 +++++++------ src/relocation.jl | 2 +- src/rtlib.jl | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/debug.jl b/src/debug.jl index 382865a2..3150f174 100644 --- a/src/debug.jl +++ b/src/debug.jl @@ -49,7 +49,7 @@ function backtrace(inst::LLVM.Instruction, bt = StackTraces.StackFrame[]) frame = StackTraces.StackFrame("multiple call sites", "unknown", 0) push!(bt, frame) elseif length(call_sites) == 1 - inst = first(call_sites).user + inst = first(call_sites).user::LLVM.CallInst continue end end diff --git a/src/irgen.jl b/src/irgen.jl index b516e120..1456c008 100644 --- a/src/irgen.jl +++ b/src/irgen.jl @@ -888,7 +888,7 @@ function (self::AddKernelState)(mod::LLVM.Module) f.name = fn * ".stateless" # create a new function - new_param_types = [T_state, ft.parameters...] + new_param_types = pushfirst!(collect(LLVMType, ft.parameters), T_state) new_ft = LLVM.FunctionType(ft.return_type, new_param_types) new_f = LLVM.Function(mod, fn, new_ft) new_f.parameters[1].name = "state" @@ -921,7 +921,8 @@ function (self::AddKernelState)(mod::LLVM.Module) # is all this even sound? typ = val.value_type::LLVM.PointerType ft = typ.element_type::LLVM.FunctionType - new_ft = LLVM.FunctionType(ft.return_type, [T_state, ft.parameters...]) + new_params = pushfirst!(collect(LLVMType, ft.parameters), T_state) + new_ft = LLVM.FunctionType(ft.return_type, new_params) return const_bitcast(workmap[target], LLVM.PointerType(new_ft, typ.addrspace)) end elseif val.opcode == LLVM.Opcode.PtrToInt @@ -989,9 +990,9 @@ function rewrite_state_uses!(f, ft, state_intr, state_intr_ft) # forward the state argument position!(builder, LLVM.before(val)) - state_arg = call!(builder, state_intr_ft, state_intr, Value[], "state") + state_arg = call!(builder, state_intr_ft, state_intr, Value[], "state")::LLVM.CallInst new_val = if val isa LLVM.CallInst - call!(builder, ft, f, Value[state_arg, val.arguments...], + call!(builder, ft, f, Value[state_arg; val.arguments], val.operand_bundles) else # TODO: invoke and callbr @@ -1390,7 +1391,7 @@ function add_input_arguments!(@nospecialize(job::CompilerJob), mod::LLVM.Module, ft = f.function_type f.name = fn * ".orig" # create a new function - new_param_types = LLVMType[ft.parameters...] + new_param_types = collect(LLVMType, ft.parameters) for intr_fn in used_intrinsics llvm_typ = convert(LLVMType, kernel_intrinsics[intr_fn].typ) @@ -1477,7 +1478,7 @@ function rewrite_input_uses!(f, new_f, nargs) position!(builder, LLVM.before(val)) new_val = if val isa LLVM.CallInst call!(builder, new_f.function_type, new_f, - Value[val.arguments..., callee_f.parameters[end-nargs+1:end]...], + Value[val.arguments; callee_f.parameters[end-nargs+1:end]], val.operand_bundles) else # TODO: invoke and callbr diff --git a/src/relocation.jl b/src/relocation.jl index 23243660..bd80a979 100644 --- a/src/relocation.jl +++ b/src/relocation.jl @@ -939,7 +939,7 @@ function check_relocation_slot_uses!(mod::LLVM.Module, slots::Vector{LLVM.Global dl = mod.datalayout loads = LLVM.LoadInst[] seen = Set{LLVM.Value}(slots) - worklist = LLVM.Value[slots...] + worklist = Vector{LLVM.Value}(slots) while !isempty(worklist) for use in pop!(worklist).uses val = use.user diff --git a/src/rtlib.jl b/src/rtlib.jl index dd389b84..e3f02db2 100644 --- a/src/rtlib.jl +++ b/src/rtlib.jl @@ -50,7 +50,7 @@ function LLVM.call!(builder, rt::Runtime.RuntimeMethodInstance, args=LLVM.Value[ # runtime functions are written in Julia, while we're calling from LLVM, # this often results in argument type mismatches. try to fix some here. - args = LLVM.Value[args...] + args = collect(LLVM.Value, args) if length(args) != length(ft.parameters) error("Incorrect number of arguments for runtime function: ", "passing ", length(args), " argument(s) to '", string(ft), " ", rt.name, "'") From 0e80a460d112b6dffba2c26de308ab83bdb5c201 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 28 Oct 2026 13:57:35 +0100 Subject: [PATCH 3/6] PTX: avoid dynamic dispatch when lowering local atomics The comprehension collecting the atomics widened its element type at run time, so it was inferred as an untyped vector and every helper called on its elements was dispatched dynamically. Collect into a `Vector{LLVM.Instruction}`, don't specialize the helpers on the kind of instruction or pointer (which compiled each of them again for every kind), and give the other vector literals built from IR objects an element type. --- src/ptx.jl | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/ptx.jl b/src/ptx.jl index 4eb1c3da..80ed8cc8 100644 --- a/src/ptx.jl +++ b/src/ptx.jl @@ -387,7 +387,7 @@ end # single block on targets without clusters, where NVPTX uses the block scope instead. const PTX_SYNCSCOPES = ("singlethread", "block", "cluster", "device", "system") -ptx_syncscope_error(inst::LLVM.Instruction, what="atomic operation") = +ptx_syncscope_error(@nospecialize(inst::LLVM.Instruction), what="atomic operation") = inst.syncscope.name in PTX_SYNCSCOPES ? nothing : "$what with synchronization scope $(repr(inst.syncscope.name))" @@ -396,7 +396,7 @@ ptx_syncscope_error(inst::LLVM.Instruction, what="atomic operation") = # floating-point ones, or `fadd` on older devices) are fine: NVPTX expands them to # compare-exchange loops. function ptx_atomic_error(@nospecialize(job::CompilerJob{PTXCompilerTarget}), - dl::LLVM.DataLayout, inst::LLVM.Instruction) + dl::LLVM.DataLayout, @nospecialize(inst::LLVM.Instruction)) target = job.config.target reason = ptx_syncscope_error(inst) reason === nothing || return reason @@ -450,7 +450,7 @@ end # Why the NVPTX back-end cannot compile the atomic memory operation `inst`, although the # target supports it, or `nothing` if it can. These work around bugs in the back-end, and # should be revisited when updating it. -function nvptx_atomic_error(inst::LLVM.Instruction) +function nvptx_atomic_error(@nospecialize(inst::LLVM.Instruction)) # NVPTX's instruction printer aborts on the ordering of the `atom.cas.b128` and # `atom.exch.b128` that it legalizes these to (LLVM 23.1, and main as of October 2026) T = atomic_value_type(inst) @@ -486,12 +486,12 @@ end # (`ld.relaxed` and `st.relaxed`) from sm_70 on, which hasn't been seen to fail. # Is `ptr` known to point to the thread's own memory? -ptx_thread_private(ptr::LLVM.Value) = +ptx_thread_private(@nospecialize(ptr::LLVM.Value)) = ptr.value_type.addrspace == #=local=# 5 || is_thread_private(ptr) # Can `ptr` point to local memory? Generic pointers can, unless every object they can be # derived from is in another address space. -function ptx_may_be_local(ptr::LLVM.Value) +function ptx_may_be_local(@nospecialize(ptr::LLVM.Value)) seen = Set{LLVM.Value}() worklist = LLVM.Value[ptr] while !isempty(worklist) @@ -523,7 +523,7 @@ end # Does the `!noalias.addrspace` metadata of `inst` rule out that it accesses local memory? # `guard_local_atomic!` adds it to the atomic it keeps for other memory. -function excludes_local(inst::LLVM.Instruction) +function excludes_local(@nospecialize(inst::LLVM.Instruction)) haskey(inst.metadata, "noalias.addrspace") || return false md = inst.metadata["noalias.addrspace"] bounds = [convert(Int, Value(op)) % UInt32 for op in md.operands] @@ -537,11 +537,11 @@ end # Replace the atomic memory operation `inst` by a call to a function that checks whether # its pointer points to local memory, and then performs it with plain accesses, or atomically # otherwise. The function is meant to be inlined (it is `alwaysinline`). -function guard_local_atomic!(inst::LLVM.Instruction) +function guard_local_atomic!(@nospecialize(inst::LLVM.Instruction)) mod = LLVM.parent(LLVM.parent(LLVM.parent(inst))) args = collect(inst.operands) T = inst.value_type - ft = LLVM.FunctionType(T, [arg.value_type for arg in args]) + ft = LLVM.FunctionType(T, LLVMType[arg.value_type for arg in args]) f = LLVM.Function(mod, "gpucompiler.local_atomic", ft) f.linkage = LLVM.Linkage.Internal push!(f.function_attributes, EnumAttribute(:alwaysinline)) @@ -556,7 +556,7 @@ function guard_local_atomic!(inst::LLVM.Instruction) end end other_atomic.metadata["noalias.addrspace"] = - MDNode([Metadata(ConstantInt(Int32(5))), Metadata(ConstantInt(Int32(6)))]) + MDNode(Metadata[ConstantInt(Int32(5)), ConstantInt(Int32(6))]) isspacep = LLVM.Function(mod, LLVM.Intrinsic("llvm.nvvm.isspacep.local")) isspacep_ft = isspacep.function_type @@ -585,7 +585,8 @@ function guard_local_atomic!(inst::LLVM.Instruction) ret!(builder) else result = phi!(builder, T) - append!(result.incoming, [(local_atomic, local_block), (other_atomic, other_block)]) + append!(result.incoming, Tuple{LLVM.Value,BasicBlock}[(local_atomic, local_block), + (other_atomic, other_block)]) ret!(builder, result) end end @@ -606,8 +607,8 @@ function lower_local_atomics!(mod::LLVM.Module) changed = false @tracepoint "lower local atomics" begin - atomics = [inst for f in mod.functions for bb in f.blocks for inst in bb.instructions - if is_atomic_memop(inst)] + atomics = LLVM.Instruction[inst for f in mod.functions for bb in f.blocks + for inst in bb.instructions if is_atomic_memop(inst)] guarded = false for inst in atomics if ptx_thread_private(inst.pointer_operand) From 62f507f29224b43f24515f0ed5db30d32d6eee18 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 28 Oct 2026 13:57:35 +0100 Subject: [PATCH 4/6] Metal: avoid dynamic dispatch in atomics lowering and argument metadata As for PTX, collect the atomics to lower into a typed vector, don't specialize the helpers on the kind of instruction or pointer, and build the argument and parameter-type vectors of the AIR atomic intrinsics by appending to typed literals rather than by splatting into untyped ones. Give the table of kernel intrinsics that become arguments concrete key and value types, which made all of `add_input_arguments!` dynamic. When emitting argument metadata, assert the types of sizes and address spaces computed from untyped values, so that the `Int32` and `ConstantInt` calls on them are resolved statically. --- src/metal.jl | 81 ++++++++++++++++++++++++++++------------------------ 1 file changed, 43 insertions(+), 38 deletions(-) diff --git a/src/metal.jl b/src/metal.jl index 30e54013..c3b47ed0 100644 --- a/src/metal.jl +++ b/src/metal.jl @@ -867,14 +867,15 @@ end # The MSL thread_scope of an atomic operation or fence (see `metal_syncscope`). Threadgroup # memory is only shared within a threadgroup, and MSL never uses a wider scope for it. -function metal_thread_scope(inst::LLVM.Instruction, as::Union{Nothing,Int}=nothing) +function metal_thread_scope(@nospecialize(inst::LLVM.Instruction), + as::Union{Nothing,Int}=nothing) ss = metal_syncscope(inst.syncscope.name) ss === nothing && return nothing return as == 3 && ss.scope == 2 ? 1 : ss.scope end # The MSL mem_flags of an atomic operation or fence (see `metal_syncscope`) -function metal_memory_flags(inst::LLVM.Instruction) +function metal_memory_flags(@nospecialize(inst::LLVM.Instruction)) ss = metal_syncscope(inst.syncscope.name) # (`validate_ir` rejects unknown scopes; without validation, use the default flags) return ss === nothing ? METAL_MEM_FLAGS : ss.flags @@ -906,7 +907,7 @@ const EXPANDABLE_ATOMICRMW_OPS = # compare-exchange loop (`AtomicExpand`'s `insertRMWCmpXchgLoop`); # - `:select`: an operation AIR can express, which becomes an `air.atomic.*` call. function metal_atomic_action(@nospecialize(job::CompilerJob{MetalCompilerTarget}), - inst::LLVM.Instruction) + @nospecialize(inst::LLVM.Instruction)) target = job.config.target op = inst isa LLVM.AtomicRMWInst ? inst.binop : nothing if op !== nothing && !haskey(AIR_ATOMICRMW_OPS, op) && !(op in EXPANDABLE_ATOMICRMW_OPS) @@ -981,7 +982,7 @@ lowered_ordering(@nospecialize(job::CompilerJob{MetalCompilerTarget}), # Give the operation the ordering it is lowered with, before expanding it into other atomics. function lower_ordering!(@nospecialize(job::CompilerJob{MetalCompilerTarget}), - inst::LLVM.Instruction) + @nospecialize(inst::LLVM.Instruction)) if inst isa LLVM.AtomicCmpXchgInst inst.success_ordering = lowered_ordering(job, inst.success_ordering) inst.failure_ordering = lowered_ordering(job, inst.failure_ordering) @@ -993,7 +994,7 @@ end # Cast a floating-point load, store or exchange to an integer one, returning the new operation. cast_atomic_to_int!(@nospecialize(job::CompilerJob{MetalCompilerTarget}), - inst::LLVM.Instruction) = + @nospecialize(inst::LLVM.Instruction)) = cast_atomic_to_integer!(lower_ordering!(job, inst)) # Expand a 32-bit read-modify-write operation to a compare-exchange loop. @@ -1005,7 +1006,7 @@ expand_to_cmpxchg_loop!(@nospecialize(job::CompilerJob{MetalCompilerTarget}), # the whole word can be accessed: true for device buffers, which Metal allocates in pages, and # for threadgroup arrays, which Metal.jl aligns and pads to 4 bytes. expand_partword_atomic!(@nospecialize(job::CompilerJob{MetalCompilerTarget}), - inst::LLVM.Instruction) = + @nospecialize(inst::LLVM.Instruction)) = expand_partword_atomic!(lower_ordering!(job, inst), 4) # On targets without ordered atomics (MSL < 4.1), bracket the operation with fences @@ -1013,7 +1014,7 @@ expand_partword_atomic!(@nospecialize(job::CompilerJob{MetalCompilerTarget}), # `emitTrailingFence`), which `lower_fences!` turns into `air.atomic.fence` calls; the # operation itself is then lowered as a relaxed one (see `lowered_ordering`). The ordering is # not reset here. -function insert_atomic_fences!(inst::LLVM.Instruction) +function insert_atomic_fences!(@nospecialize(inst::LLVM.Instruction)) order = atomic_ordering(inst) is_ordered(order) || return false scope = inst.syncscope @@ -1040,7 +1041,7 @@ end # device memory as acquire ones (which on targets without ordered atomics adds a fence; before # MSL 3.2, there are no fences to lower that to). function strengthen_relaxed_load!(@nospecialize(job::CompilerJob{MetalCompilerTarget}), - inst::LLVM.Instruction) + @nospecialize(inst::LLVM.Instruction)) job.config.target.metal >= v"3.2" && inst isa LLVM.LoadInst && atomic_ordering(inst) == LLVM.AtomicOrdering.Monotonic && inst.pointer_operand.value_type.addrspace == 1 && @@ -1060,7 +1061,7 @@ end # buffering, ~0.5% of the time on an M1), which sequential consistency forbids. # Read-modify-writes and compare-exchanges wait for their result and don't need this, and # fences before loads instead would cost more (loads are more common than stores). -function insert_trailing_seq_cst_fence!(inst::LLVM.Instruction) +function insert_trailing_seq_cst_fence!(@nospecialize(inst::LLVM.Instruction)) inst isa LLVM.StoreInst && atomic_ordering(inst) == LLVM.AtomicOrdering.SequentiallyConsistent || return false @@ -1085,7 +1086,7 @@ air_atomic_function(mod::LLVM.Module, name::String, ft::LLVM.FunctionType) = # Replace an atomic operation by the equivalent `air.atomic.*` call, in the form the # target's AIR and MSL versions use. function select_atomic!(@nospecialize(job::CompilerJob{MetalCompilerTarget}), - mod::LLVM.Module, inst::LLVM.Instruction) + mod::LLVM.Module, @nospecialize(inst::LLVM.Instruction)) target = job.config.target ptr = inst.pointer_operand T_ptr = ptr.value_type @@ -1111,30 +1112,31 @@ function select_atomic!(@nospecialize(job::CompilerJob{MetalCompilerTarget}), # read-modify-writes; stores and compare-exchanges are compiled the same either way. volatile = ConstantInt(T_i1, target.metal < v"4.1" || inst.volatile || inst isa LLVM.LoadInst || inst isa LLVM.AtomicRMWInst) - trailing_types = target.air >= v"2.9" ? [T_i32, T_i32, T_i1] : [T_i32, T_i1] - trailing = target.air >= v"2.9" ? [scope, flags, volatile] : [scope, volatile] + trailing_types = target.air >= v"2.9" ? LLVMType[T_i32, T_i32, T_i1] : LLVMType[T_i32, T_i1] + trailing = target.air >= v"2.9" ? Value[scope, flags, volatile] : Value[scope, volatile] memory_order(order) = ConstantInt(T_i32, metal_memory_order(lowered_ordering(job, order))) @dispose builder=IRBuilder() begin position!(builder, LLVM.before(inst)) builder.debug_location = inst.debug_location if inst isa LLVM.LoadInst - ft = LLVM.FunctionType(T, [T_ptr, T_i32, trailing_types...]) + ft = LLVM.FunctionType(T, append!(LLVMType[T_ptr, T_i32], trailing_types)) f = air_atomic_function(mod, "air.atomic.$mem.load.$suffix", ft) - new = call!(builder, ft, f, [ptr, memory_order(order), trailing...]) + new = call!(builder, ft, f, append!(Value[ptr, memory_order(order)], trailing)) elseif inst isa LLVM.StoreInst - ft = LLVM.FunctionType(LLVM.VoidType(), [T_ptr, T, T_i32, trailing_types...]) + ft = LLVM.FunctionType(LLVM.VoidType(), + append!(LLVMType[T_ptr, T, T_i32], trailing_types)) f = air_atomic_function(mod, "air.atomic.$mem.store.$suffix", ft) - new = call!(builder, ft, f, [ptr, inst.value_operand, memory_order(order), - trailing...]) + new = call!(builder, ft, f, + append!(Value[ptr, inst.value_operand, memory_order(order)], trailing)) elseif inst isa LLVM.AtomicRMWInst op = AIR_ATOMICRMW_OPS[inst.binop] # AIR's 64-bit min/max don't return the old value T_ret = suffix == "i64" ? LLVM.VoidType() : T - ft = LLVM.FunctionType(T_ret, [T_ptr, T, T_i32, trailing_types...]) + ft = LLVM.FunctionType(T_ret, append!(LLVMType[T_ptr, T, T_i32], trailing_types)) f = air_atomic_function(mod, "air.atomic.$mem.$op.$suffix", ft) - new = call!(builder, ft, f, [ptr, inst.value_operand, memory_order(order), - trailing...]) + new = call!(builder, ft, f, + append!(Value[ptr, inst.value_operand, memory_order(order)], trailing)) else # AIR only has a weak compare-exchange, which takes the expected value by # reference. Like MSL, derive the success flag from the returned old value. @@ -1145,12 +1147,13 @@ function select_atomic!(@nospecialize(job::CompilerJob{MetalCompilerTarget}), alloca!(entry_builder, T) end store!(builder, cmp, expected) - ft = LLVM.FunctionType(T, [T_ptr, expected.value_type, T, T_i32, T_i32, - trailing_types...]) + ft = LLVM.FunctionType(T, append!(LLVMType[T_ptr, expected.value_type, T, + T_i32, T_i32], trailing_types)) f = air_atomic_function(mod, "air.atomic.$mem.cmpxchg.weak.$suffix", ft) old = call!(builder, ft, f, - [ptr, expected, desired, memory_order(inst.success_ordering), - memory_order(inst.failure_ordering), trailing...]) + append!(Value[ptr, expected, desired, + memory_order(inst.success_ordering), + memory_order(inst.failure_ordering)], trailing)) success = icmp!(builder, LLVM.IntPredicate.EQ, old, cmp) new = insert_value!(builder, UndefValue(inst.value_type), old, 0) new = insert_value!(builder, new, success, 1) @@ -1168,7 +1171,7 @@ function annotate_atomic_eltypes!(mod::LLVM.Module) m = match(r"^air\.atomic\.(global|local)\..+\.(i32|f32|i64)$", f.name) m === nothing && continue T = m.captures[2] == "f32" ? LLVM.FloatType() : LLVM.IntType(parse(Int, m.captures[2][2:end])) - mds = [] + mds = Metadata[] for (i, param) in enumerate(f.function_type.parameters) i <= 2 && param isa LLVM.PointerType || continue push!(mds, ConstantInt(Int32(i - 1))) @@ -1180,8 +1183,8 @@ end function lower_atomics!(@nospecialize(job::CompilerJob{MetalCompilerTarget}), mod::LLVM.Module) - atomics = [inst for f in mod.functions for bb in f.blocks for inst in bb.instructions - if is_atomic_memop(inst)] + atomics = LLVM.Instruction[inst for f in mod.functions for bb in f.blocks + for inst in bb.instructions if is_atomic_memop(inst)] isempty(atomics) && return false for inst in atomics @@ -1596,7 +1599,7 @@ end # whether `ptr` is only used as the address of plain loads, directly or through GEPs and # bitcasts, all of which address-space inference can move to another address space -function only_loaded_from(ptr::LLVM.Value) +function only_loaded_from(@nospecialize(ptr::LLVM.Value)) for use in ptr.uses u = LLVM.user(use) if u isa LLVM.GetElementPtrInst || u isa LLVM.BitCastInst @@ -1625,14 +1628,14 @@ function add_byref_parameter_attributes!(@nospecialize(job::CompilerJob), f::LLV # argument is bound as its address word T = arg.cc == MUT_REF ? UInt : arg.typ attrs = f.parameter_attributes[arg.idx] - push!(attrs, EnumAttribute("dereferenceable", sizeof(T))) + push!(attrs, EnumAttribute("dereferenceable", sizeof(T)::Int)) only_read_through(param) && push!(attrs, EnumAttribute("readonly")) end return end # whether memory is only read through `ptr` and the pointers derived from it -function only_read_through(ptr::LLVM.Value, seen=Set{LLVM.Value}()) +function only_read_through(@nospecialize(ptr::LLVM.Value), seen=Set{LLVM.Value}()) ptr in seen && return true push!(seen, ptr) for use in ptr.uses @@ -2174,7 +2177,7 @@ end # hardware index counters (thread id, group id, etc) aren't accessed via intrinsics, # but using special arguments to the kernel function. -const kernel_intrinsics = Dict() +const kernel_intrinsics = Dict{String,@NamedTuple{name::String, typ::DataType}}() for intr in [ "dispatch_quadgroups_per_threadgroup", "dispatch_simdgroups_per_threadgroup", "quadgroup_index_in_threadgroup", "quadgroups_per_threadgroup", @@ -2301,7 +2304,8 @@ function msl_struct_type_info(@nospecialize(T)) for i in 1:fieldcount(T) FT = fieldtype(T, i) # inline unions have no MSL equivalent; ghost fields take no space - FT isa DataType && sizeof(FT) > 0 || continue + FT isa DataType || continue + sizeof(FT) > 0 || continue # homogeneous tuples are laid out like MSL arrays elt, count = FT, 0 @@ -2316,7 +2320,7 @@ function msl_struct_type_info(@nospecialize(T)) push!(info, nested) end push!(info, Metadata(ConstantInt(Int32(fieldoffset(T, i))))) - push!(info, Metadata(ConstantInt(Int32(sizeof(elt))))) + push!(info, Metadata(ConstantInt(Int32(sizeof(elt)::Int)))) push!(info, Metadata(ConstantInt(Int32(count)))) push!(info, MDString(msl_type_name(elt))) push!(info, MDString(string(fieldname(T, i)))) @@ -2373,7 +2377,7 @@ function add_argument_metadata!(@nospecialize(job::CompilerJob), mod::LLVM.Modul if by_reference push!(md, MDString("air.buffer_size")) - push!(md, Metadata(ConstantInt(Int32(sizeof(arg_type))))) + push!(md, Metadata(ConstantInt(Int32(sizeof(arg_type)::Int)))) end push!(md, MDString("air.location_index")) @@ -2387,7 +2391,8 @@ function add_argument_metadata!(@nospecialize(job::CompilerJob), mod::LLVM.Modul push!(md, MDString(by_reference ? "air.read" : "air.read_write")) push!(md, MDString("air.address_space")) - push!(md, Metadata(ConstantInt(Int32(entry_ft.parameters[arg.idx].addrspace)))) + ptr_type = entry_ft.parameters[arg.idx]::LLVM.PointerType + push!(md, Metadata(ConstantInt(Int32(ptr_type.addrspace)))) struct_info = msl_struct_type_info(arg_type) if struct_info !== nothing @@ -2396,10 +2401,10 @@ function add_argument_metadata!(@nospecialize(job::CompilerJob), mod::LLVM.Modul end push!(md, MDString("air.arg_type_size")) - push!(md, Metadata(ConstantInt(Int32(sizeof(arg_type))))) + push!(md, Metadata(ConstantInt(Int32(sizeof(arg_type)::Int)))) push!(md, MDString("air.arg_type_align_size")) - push!(md, Metadata(ConstantInt(Int32(Base.datatype_alignment(arg_type))))) + push!(md, Metadata(ConstantInt(Int32(Base.datatype_alignment(arg_type)::Int)))) # like MSL, name the pointee type for pointer arguments type_name = if arg.cc == BITS_VALUE && (arg.typ <: Ptr || arg.typ <: Core.LLVMPtr) @@ -2638,7 +2643,7 @@ intrinsic_table(table) = Dict(intr => val for (intr, val) in ((tryparse(LLVM.Intrinsic, name), val) for (name, val) in table) if intr !== nothing) -function called_intrinsic(inst::LLVM.Instruction) +function called_intrinsic(@nospecialize(inst::LLVM.Instruction)) inst isa LLVM.CallBase || return nothing callee = inst.called_function return callee === nothing ? nothing : callee.intrinsic From 29cdfd70252516e071bdf4cee4a29292276f1eda Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 28 Oct 2026 13:57:35 +0100 Subject: [PATCH 5/6] SPIR-V: collect the atomics to lower into a typed vector Like for PTX and Metal, the untyped comprehension made every call on the collected instructions dispatch dynamically. --- src/spirv.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/spirv.jl b/src/spirv.jl index f23c1779..b5156f47 100644 --- a/src/spirv.jl +++ b/src/spirv.jl @@ -837,8 +837,8 @@ function lower_atomics!(@nospecialize(job::CompilerJob{SPIRVCompilerTarget}), @tracepoint "lower atomics" begin # (the expansions split blocks, so collect the atomics first) - atomics = [inst for f in mod.functions for bb in f.blocks for inst in bb.instructions - if is_atomic_memop(inst)] + atomics = LLVM.Instruction[inst for f in mod.functions for bb in f.blocks + for inst in bb.instructions if is_atomic_memop(inst)] # pad the local memory that 8- and 16-bit atomics may access variables = Set{LLVM.GlobalVariable}() From 9085f990a1d09b07ed4cab69b7c160d8aca74e20 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 28 Oct 2026 14:07:57 +0100 Subject: [PATCH 6/6] Don't specialize the shared atomics helpers on the instruction kind Their callers only know they have an `LLVM.Instruction` or `LLVM.Value`, so every call dispatched dynamically and compiled the helper again for each concrete kind of instruction or pointer it met. With `@nospecialize`, a single method instance serves all of them and the calls resolve statically. --- src/atomics.jl | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/atomics.jl b/src/atomics.jl index 9ed52420..808f7f22 100644 --- a/src/atomics.jl +++ b/src/atomics.jl @@ -8,7 +8,8 @@ is_ordered(order::LLVM.AtomicOrdering.T) = is_stronger(order, LLVM.AtomicOrdering.Monotonic) # (`atomicrmw` and `cmpxchg` are always atomic) -is_atomic_memop(inst::LLVM.Instruction) = inst isa LLVM.MemAccessInst && isatomic(inst) +is_atomic_memop(@nospecialize(inst::LLVM.Instruction)) = + inst isa LLVM.MemAccessInst && isatomic(inst) # the type of the value an atomic memory operation accesses atomic_value_type(inst::LLVM.LoadInst) = inst.value_type @@ -67,11 +68,11 @@ end # the ordering of an atomic memory operation, merging a compare-exchange's orderings atomic_ordering(inst::LLVM.AtomicCmpXchgInst) = merged_ordering(inst) -atomic_ordering(inst::LLVM.Instruction) = inst.ordering +atomic_ordering(@nospecialize(inst::LLVM.Instruction)) = inst.ordering # the size in bits of the value an atomic memory operation accesses, with pointers as large as # the module's data layout says, or `nothing` if it isn't a scalar -function atomic_bits(inst::LLVM.Instruction) +function atomic_bits(@nospecialize(inst::LLVM.Instruction)) T = atomic_value_type(inst) T isa Union{LLVM.IntegerType,LLVM.FloatingPointType,LLVM.PointerType} || return nothing mod = LLVM.parent(LLVM.parent(LLVM.parent(inst))) @@ -86,7 +87,7 @@ end # when it has a pointer to it (thread memory is private to every thread), so plain accesses # behave the same. Anything this cannot trace back to an `alloca`, e.g., a function argument # or a loaded pointer, is not known to be private. -function is_thread_private(ptr::LLVM.Value) +function is_thread_private(@nospecialize(ptr::LLVM.Value)) seen = Set{LLVM.Value}() worklist = LLVM.Value[ptr] while !isempty(worklist) @@ -113,7 +114,7 @@ end # Replace an atomic operation on the thread's own memory (see `is_thread_private`) by plain # accesses. Its ordering and scope don't matter either: no other thread can observe the memory # it accesses, so it cannot synchronize with any. -function demote_private_atomic!(inst::LLVM.Instruction) +function demote_private_atomic!(@nospecialize(inst::LLVM.Instruction)) if inst isa LLVM.LoadInst || inst isa LLVM.StoreInst # (a plain access has the default, system scope) inst.syncscope = SyncScope("system") @@ -154,7 +155,7 @@ end # on the word containing it, like `AtomicExpand` does for targets without atomics of that # size. Like `AtomicExpand`, this assumes that the whole word can be accessed, which the # target has to ensure. -function expand_partword_atomic!(inst::LLVM.Instruction, word_size::Integer) +function expand_partword_atomic!(@nospecialize(inst::LLVM.Instruction), word_size::Integer) if inst isa LLVM.LoadInst # a word-sized load, shifted @dispose builder=IRBuilder() begin