diff --git a/src/atomics.jl b/src/atomics.jl index 9ed52420..808f7f22 100644 --- a/src/atomics.jl +++ b/src/atomics.jl @@ -8,7 +8,8 @@ is_ordered(order::LLVM.AtomicOrdering.T) = is_stronger(order, LLVM.AtomicOrdering.Monotonic) # (`atomicrmw` and `cmpxchg` are always atomic) -is_atomic_memop(inst::LLVM.Instruction) = inst isa LLVM.MemAccessInst && isatomic(inst) +is_atomic_memop(@nospecialize(inst::LLVM.Instruction)) = + inst isa LLVM.MemAccessInst && isatomic(inst) # the type of the value an atomic memory operation accesses atomic_value_type(inst::LLVM.LoadInst) = inst.value_type @@ -67,11 +68,11 @@ end # the ordering of an atomic memory operation, merging a compare-exchange's orderings atomic_ordering(inst::LLVM.AtomicCmpXchgInst) = merged_ordering(inst) -atomic_ordering(inst::LLVM.Instruction) = inst.ordering +atomic_ordering(@nospecialize(inst::LLVM.Instruction)) = inst.ordering # the size in bits of the value an atomic memory operation accesses, with pointers as large as # the module's data layout says, or `nothing` if it isn't a scalar -function atomic_bits(inst::LLVM.Instruction) +function atomic_bits(@nospecialize(inst::LLVM.Instruction)) T = atomic_value_type(inst) T isa Union{LLVM.IntegerType,LLVM.FloatingPointType,LLVM.PointerType} || return nothing mod = LLVM.parent(LLVM.parent(LLVM.parent(inst))) @@ -86,7 +87,7 @@ end # when it has a pointer to it (thread memory is private to every thread), so plain accesses # behave the same. Anything this cannot trace back to an `alloca`, e.g., a function argument # or a loaded pointer, is not known to be private. -function is_thread_private(ptr::LLVM.Value) +function is_thread_private(@nospecialize(ptr::LLVM.Value)) seen = Set{LLVM.Value}() worklist = LLVM.Value[ptr] while !isempty(worklist) @@ -113,7 +114,7 @@ end # Replace an atomic operation on the thread's own memory (see `is_thread_private`) by plain # accesses. Its ordering and scope don't matter either: no other thread can observe the memory # it accesses, so it cannot synchronize with any. -function demote_private_atomic!(inst::LLVM.Instruction) +function demote_private_atomic!(@nospecialize(inst::LLVM.Instruction)) if inst isa LLVM.LoadInst || inst isa LLVM.StoreInst # (a plain access has the default, system scope) inst.syncscope = SyncScope("system") @@ -154,7 +155,7 @@ end # on the word containing it, like `AtomicExpand` does for targets without atomics of that # size. Like `AtomicExpand`, this assumes that the whole word can be accessed, which the # target has to ensure. -function expand_partword_atomic!(inst::LLVM.Instruction, word_size::Integer) +function expand_partword_atomic!(@nospecialize(inst::LLVM.Instruction), word_size::Integer) if inst isa LLVM.LoadInst # a word-sized load, shifted @dispose builder=IRBuilder() begin diff --git a/src/debug.jl b/src/debug.jl index 382865a2..3150f174 100644 --- a/src/debug.jl +++ b/src/debug.jl @@ -49,7 +49,7 @@ function backtrace(inst::LLVM.Instruction, bt = StackTraces.StackFrame[]) frame = StackTraces.StackFrame("multiple call sites", "unknown", 0) push!(bt, frame) elseif length(call_sites) == 1 - inst = first(call_sites).user + inst = first(call_sites).user::LLVM.CallInst continue end end diff --git a/src/irgen.jl b/src/irgen.jl index 679e8b23..1456c008 100644 --- a/src/irgen.jl +++ b/src/irgen.jl @@ -563,6 +563,8 @@ end # - `name`: the name of the argument # - `idx`: the index of the argument in the LLVM function type, or `nothing` if the argument # is not passed at the LLVM level. +const ArgumentInfo = @NamedTuple{cc::ArgumentCC, typ::Any, name::Symbol, + idx::Union{Nothing,Int}} function classify_arguments(@nospecialize(job::CompilerJob), codegen_ft::LLVM.FunctionType; post_optimization::Bool=false) source_sig = job.source.specTypes::DataType @@ -576,12 +578,11 @@ function classify_arguments(@nospecialize(job::CompilerJob), codegen_ft::LLVM.Fu codegen_types = codegen_ft.parameters + args = ArgumentInfo[] if post_optimization && kernel_state_type(job) !== Nothing - args = [] push!(args, (cc=KERNEL_STATE, typ=kernel_state_type(job), name=:kernel_state, idx=1)) codegen_i = 2 else - args = [] codegen_i = 1 end for (source_typ, source_name) in zip(source_types, source_argnames) @@ -887,7 +888,7 @@ function (self::AddKernelState)(mod::LLVM.Module) f.name = fn * ".stateless" # create a new function - new_param_types = [T_state, ft.parameters...] + new_param_types = pushfirst!(collect(LLVMType, ft.parameters), T_state) new_ft = LLVM.FunctionType(ft.return_type, new_param_types) new_f = LLVM.Function(mod, fn, new_ft) new_f.parameters[1].name = "state" @@ -920,7 +921,8 @@ function (self::AddKernelState)(mod::LLVM.Module) # is all this even sound? typ = val.value_type::LLVM.PointerType ft = typ.element_type::LLVM.FunctionType - new_ft = LLVM.FunctionType(ft.return_type, [T_state, ft.parameters...]) + new_params = pushfirst!(collect(LLVMType, ft.parameters), T_state) + new_ft = LLVM.FunctionType(ft.return_type, new_params) return const_bitcast(workmap[target], LLVM.PointerType(new_ft, typ.addrspace)) end elseif val.opcode == LLVM.Opcode.PtrToInt @@ -988,9 +990,9 @@ function rewrite_state_uses!(f, ft, state_intr, state_intr_ft) # forward the state argument position!(builder, LLVM.before(val)) - state_arg = call!(builder, state_intr_ft, state_intr, Value[], "state") + state_arg = call!(builder, state_intr_ft, state_intr, Value[], "state")::LLVM.CallInst new_val = if val isa LLVM.CallInst - call!(builder, ft, f, Value[state_arg, val.arguments...], + call!(builder, ft, f, Value[state_arg; val.arguments], val.operand_bundles) else # TODO: invoke and callbr @@ -1389,7 +1391,7 @@ function add_input_arguments!(@nospecialize(job::CompilerJob), mod::LLVM.Module, ft = f.function_type f.name = fn * ".orig" # create a new function - new_param_types = LLVMType[ft.parameters...] + new_param_types = collect(LLVMType, ft.parameters) for intr_fn in used_intrinsics llvm_typ = convert(LLVMType, kernel_intrinsics[intr_fn].typ) @@ -1476,7 +1478,7 @@ function rewrite_input_uses!(f, new_f, nargs) position!(builder, LLVM.before(val)) new_val = if val isa LLVM.CallInst call!(builder, new_f.function_type, new_f, - Value[val.arguments..., callee_f.parameters[end-nargs+1:end]...], + Value[val.arguments; callee_f.parameters[end-nargs+1:end]], val.operand_bundles) else # TODO: invoke and callbr diff --git a/src/metal.jl b/src/metal.jl index 30e54013..c3b47ed0 100644 --- a/src/metal.jl +++ b/src/metal.jl @@ -867,14 +867,15 @@ end # The MSL thread_scope of an atomic operation or fence (see `metal_syncscope`). Threadgroup # memory is only shared within a threadgroup, and MSL never uses a wider scope for it. -function metal_thread_scope(inst::LLVM.Instruction, as::Union{Nothing,Int}=nothing) +function metal_thread_scope(@nospecialize(inst::LLVM.Instruction), + as::Union{Nothing,Int}=nothing) ss = metal_syncscope(inst.syncscope.name) ss === nothing && return nothing return as == 3 && ss.scope == 2 ? 1 : ss.scope end # The MSL mem_flags of an atomic operation or fence (see `metal_syncscope`) -function metal_memory_flags(inst::LLVM.Instruction) +function metal_memory_flags(@nospecialize(inst::LLVM.Instruction)) ss = metal_syncscope(inst.syncscope.name) # (`validate_ir` rejects unknown scopes; without validation, use the default flags) return ss === nothing ? METAL_MEM_FLAGS : ss.flags @@ -906,7 +907,7 @@ const EXPANDABLE_ATOMICRMW_OPS = # compare-exchange loop (`AtomicExpand`'s `insertRMWCmpXchgLoop`); # - `:select`: an operation AIR can express, which becomes an `air.atomic.*` call. function metal_atomic_action(@nospecialize(job::CompilerJob{MetalCompilerTarget}), - inst::LLVM.Instruction) + @nospecialize(inst::LLVM.Instruction)) target = job.config.target op = inst isa LLVM.AtomicRMWInst ? inst.binop : nothing if op !== nothing && !haskey(AIR_ATOMICRMW_OPS, op) && !(op in EXPANDABLE_ATOMICRMW_OPS) @@ -981,7 +982,7 @@ lowered_ordering(@nospecialize(job::CompilerJob{MetalCompilerTarget}), # Give the operation the ordering it is lowered with, before expanding it into other atomics. function lower_ordering!(@nospecialize(job::CompilerJob{MetalCompilerTarget}), - inst::LLVM.Instruction) + @nospecialize(inst::LLVM.Instruction)) if inst isa LLVM.AtomicCmpXchgInst inst.success_ordering = lowered_ordering(job, inst.success_ordering) inst.failure_ordering = lowered_ordering(job, inst.failure_ordering) @@ -993,7 +994,7 @@ end # Cast a floating-point load, store or exchange to an integer one, returning the new operation. cast_atomic_to_int!(@nospecialize(job::CompilerJob{MetalCompilerTarget}), - inst::LLVM.Instruction) = + @nospecialize(inst::LLVM.Instruction)) = cast_atomic_to_integer!(lower_ordering!(job, inst)) # Expand a 32-bit read-modify-write operation to a compare-exchange loop. @@ -1005,7 +1006,7 @@ expand_to_cmpxchg_loop!(@nospecialize(job::CompilerJob{MetalCompilerTarget}), # the whole word can be accessed: true for device buffers, which Metal allocates in pages, and # for threadgroup arrays, which Metal.jl aligns and pads to 4 bytes. expand_partword_atomic!(@nospecialize(job::CompilerJob{MetalCompilerTarget}), - inst::LLVM.Instruction) = + @nospecialize(inst::LLVM.Instruction)) = expand_partword_atomic!(lower_ordering!(job, inst), 4) # On targets without ordered atomics (MSL < 4.1), bracket the operation with fences @@ -1013,7 +1014,7 @@ expand_partword_atomic!(@nospecialize(job::CompilerJob{MetalCompilerTarget}), # `emitTrailingFence`), which `lower_fences!` turns into `air.atomic.fence` calls; the # operation itself is then lowered as a relaxed one (see `lowered_ordering`). The ordering is # not reset here. -function insert_atomic_fences!(inst::LLVM.Instruction) +function insert_atomic_fences!(@nospecialize(inst::LLVM.Instruction)) order = atomic_ordering(inst) is_ordered(order) || return false scope = inst.syncscope @@ -1040,7 +1041,7 @@ end # device memory as acquire ones (which on targets without ordered atomics adds a fence; before # MSL 3.2, there are no fences to lower that to). function strengthen_relaxed_load!(@nospecialize(job::CompilerJob{MetalCompilerTarget}), - inst::LLVM.Instruction) + @nospecialize(inst::LLVM.Instruction)) job.config.target.metal >= v"3.2" && inst isa LLVM.LoadInst && atomic_ordering(inst) == LLVM.AtomicOrdering.Monotonic && inst.pointer_operand.value_type.addrspace == 1 && @@ -1060,7 +1061,7 @@ end # buffering, ~0.5% of the time on an M1), which sequential consistency forbids. # Read-modify-writes and compare-exchanges wait for their result and don't need this, and # fences before loads instead would cost more (loads are more common than stores). -function insert_trailing_seq_cst_fence!(inst::LLVM.Instruction) +function insert_trailing_seq_cst_fence!(@nospecialize(inst::LLVM.Instruction)) inst isa LLVM.StoreInst && atomic_ordering(inst) == LLVM.AtomicOrdering.SequentiallyConsistent || return false @@ -1085,7 +1086,7 @@ air_atomic_function(mod::LLVM.Module, name::String, ft::LLVM.FunctionType) = # Replace an atomic operation by the equivalent `air.atomic.*` call, in the form the # target's AIR and MSL versions use. function select_atomic!(@nospecialize(job::CompilerJob{MetalCompilerTarget}), - mod::LLVM.Module, inst::LLVM.Instruction) + mod::LLVM.Module, @nospecialize(inst::LLVM.Instruction)) target = job.config.target ptr = inst.pointer_operand T_ptr = ptr.value_type @@ -1111,30 +1112,31 @@ function select_atomic!(@nospecialize(job::CompilerJob{MetalCompilerTarget}), # read-modify-writes; stores and compare-exchanges are compiled the same either way. volatile = ConstantInt(T_i1, target.metal < v"4.1" || inst.volatile || inst isa LLVM.LoadInst || inst isa LLVM.AtomicRMWInst) - trailing_types = target.air >= v"2.9" ? [T_i32, T_i32, T_i1] : [T_i32, T_i1] - trailing = target.air >= v"2.9" ? [scope, flags, volatile] : [scope, volatile] + trailing_types = target.air >= v"2.9" ? LLVMType[T_i32, T_i32, T_i1] : LLVMType[T_i32, T_i1] + trailing = target.air >= v"2.9" ? Value[scope, flags, volatile] : Value[scope, volatile] memory_order(order) = ConstantInt(T_i32, metal_memory_order(lowered_ordering(job, order))) @dispose builder=IRBuilder() begin position!(builder, LLVM.before(inst)) builder.debug_location = inst.debug_location if inst isa LLVM.LoadInst - ft = LLVM.FunctionType(T, [T_ptr, T_i32, trailing_types...]) + ft = LLVM.FunctionType(T, append!(LLVMType[T_ptr, T_i32], trailing_types)) f = air_atomic_function(mod, "air.atomic.$mem.load.$suffix", ft) - new = call!(builder, ft, f, [ptr, memory_order(order), trailing...]) + new = call!(builder, ft, f, append!(Value[ptr, memory_order(order)], trailing)) elseif inst isa LLVM.StoreInst - ft = LLVM.FunctionType(LLVM.VoidType(), [T_ptr, T, T_i32, trailing_types...]) + ft = LLVM.FunctionType(LLVM.VoidType(), + append!(LLVMType[T_ptr, T, T_i32], trailing_types)) f = air_atomic_function(mod, "air.atomic.$mem.store.$suffix", ft) - new = call!(builder, ft, f, [ptr, inst.value_operand, memory_order(order), - trailing...]) + new = call!(builder, ft, f, + append!(Value[ptr, inst.value_operand, memory_order(order)], trailing)) elseif inst isa LLVM.AtomicRMWInst op = AIR_ATOMICRMW_OPS[inst.binop] # AIR's 64-bit min/max don't return the old value T_ret = suffix == "i64" ? LLVM.VoidType() : T - ft = LLVM.FunctionType(T_ret, [T_ptr, T, T_i32, trailing_types...]) + ft = LLVM.FunctionType(T_ret, append!(LLVMType[T_ptr, T, T_i32], trailing_types)) f = air_atomic_function(mod, "air.atomic.$mem.$op.$suffix", ft) - new = call!(builder, ft, f, [ptr, inst.value_operand, memory_order(order), - trailing...]) + new = call!(builder, ft, f, + append!(Value[ptr, inst.value_operand, memory_order(order)], trailing)) else # AIR only has a weak compare-exchange, which takes the expected value by # reference. Like MSL, derive the success flag from the returned old value. @@ -1145,12 +1147,13 @@ function select_atomic!(@nospecialize(job::CompilerJob{MetalCompilerTarget}), alloca!(entry_builder, T) end store!(builder, cmp, expected) - ft = LLVM.FunctionType(T, [T_ptr, expected.value_type, T, T_i32, T_i32, - trailing_types...]) + ft = LLVM.FunctionType(T, append!(LLVMType[T_ptr, expected.value_type, T, + T_i32, T_i32], trailing_types)) f = air_atomic_function(mod, "air.atomic.$mem.cmpxchg.weak.$suffix", ft) old = call!(builder, ft, f, - [ptr, expected, desired, memory_order(inst.success_ordering), - memory_order(inst.failure_ordering), trailing...]) + append!(Value[ptr, expected, desired, + memory_order(inst.success_ordering), + memory_order(inst.failure_ordering)], trailing)) success = icmp!(builder, LLVM.IntPredicate.EQ, old, cmp) new = insert_value!(builder, UndefValue(inst.value_type), old, 0) new = insert_value!(builder, new, success, 1) @@ -1168,7 +1171,7 @@ function annotate_atomic_eltypes!(mod::LLVM.Module) m = match(r"^air\.atomic\.(global|local)\..+\.(i32|f32|i64)$", f.name) m === nothing && continue T = m.captures[2] == "f32" ? LLVM.FloatType() : LLVM.IntType(parse(Int, m.captures[2][2:end])) - mds = [] + mds = Metadata[] for (i, param) in enumerate(f.function_type.parameters) i <= 2 && param isa LLVM.PointerType || continue push!(mds, ConstantInt(Int32(i - 1))) @@ -1180,8 +1183,8 @@ end function lower_atomics!(@nospecialize(job::CompilerJob{MetalCompilerTarget}), mod::LLVM.Module) - atomics = [inst for f in mod.functions for bb in f.blocks for inst in bb.instructions - if is_atomic_memop(inst)] + atomics = LLVM.Instruction[inst for f in mod.functions for bb in f.blocks + for inst in bb.instructions if is_atomic_memop(inst)] isempty(atomics) && return false for inst in atomics @@ -1596,7 +1599,7 @@ end # whether `ptr` is only used as the address of plain loads, directly or through GEPs and # bitcasts, all of which address-space inference can move to another address space -function only_loaded_from(ptr::LLVM.Value) +function only_loaded_from(@nospecialize(ptr::LLVM.Value)) for use in ptr.uses u = LLVM.user(use) if u isa LLVM.GetElementPtrInst || u isa LLVM.BitCastInst @@ -1625,14 +1628,14 @@ function add_byref_parameter_attributes!(@nospecialize(job::CompilerJob), f::LLV # argument is bound as its address word T = arg.cc == MUT_REF ? UInt : arg.typ attrs = f.parameter_attributes[arg.idx] - push!(attrs, EnumAttribute("dereferenceable", sizeof(T))) + push!(attrs, EnumAttribute("dereferenceable", sizeof(T)::Int)) only_read_through(param) && push!(attrs, EnumAttribute("readonly")) end return end # whether memory is only read through `ptr` and the pointers derived from it -function only_read_through(ptr::LLVM.Value, seen=Set{LLVM.Value}()) +function only_read_through(@nospecialize(ptr::LLVM.Value), seen=Set{LLVM.Value}()) ptr in seen && return true push!(seen, ptr) for use in ptr.uses @@ -2174,7 +2177,7 @@ end # hardware index counters (thread id, group id, etc) aren't accessed via intrinsics, # but using special arguments to the kernel function. -const kernel_intrinsics = Dict() +const kernel_intrinsics = Dict{String,@NamedTuple{name::String, typ::DataType}}() for intr in [ "dispatch_quadgroups_per_threadgroup", "dispatch_simdgroups_per_threadgroup", "quadgroup_index_in_threadgroup", "quadgroups_per_threadgroup", @@ -2301,7 +2304,8 @@ function msl_struct_type_info(@nospecialize(T)) for i in 1:fieldcount(T) FT = fieldtype(T, i) # inline unions have no MSL equivalent; ghost fields take no space - FT isa DataType && sizeof(FT) > 0 || continue + FT isa DataType || continue + sizeof(FT) > 0 || continue # homogeneous tuples are laid out like MSL arrays elt, count = FT, 0 @@ -2316,7 +2320,7 @@ function msl_struct_type_info(@nospecialize(T)) push!(info, nested) end push!(info, Metadata(ConstantInt(Int32(fieldoffset(T, i))))) - push!(info, Metadata(ConstantInt(Int32(sizeof(elt))))) + push!(info, Metadata(ConstantInt(Int32(sizeof(elt)::Int)))) push!(info, Metadata(ConstantInt(Int32(count)))) push!(info, MDString(msl_type_name(elt))) push!(info, MDString(string(fieldname(T, i)))) @@ -2373,7 +2377,7 @@ function add_argument_metadata!(@nospecialize(job::CompilerJob), mod::LLVM.Modul if by_reference push!(md, MDString("air.buffer_size")) - push!(md, Metadata(ConstantInt(Int32(sizeof(arg_type))))) + push!(md, Metadata(ConstantInt(Int32(sizeof(arg_type)::Int)))) end push!(md, MDString("air.location_index")) @@ -2387,7 +2391,8 @@ function add_argument_metadata!(@nospecialize(job::CompilerJob), mod::LLVM.Modul push!(md, MDString(by_reference ? "air.read" : "air.read_write")) push!(md, MDString("air.address_space")) - push!(md, Metadata(ConstantInt(Int32(entry_ft.parameters[arg.idx].addrspace)))) + ptr_type = entry_ft.parameters[arg.idx]::LLVM.PointerType + push!(md, Metadata(ConstantInt(Int32(ptr_type.addrspace)))) struct_info = msl_struct_type_info(arg_type) if struct_info !== nothing @@ -2396,10 +2401,10 @@ function add_argument_metadata!(@nospecialize(job::CompilerJob), mod::LLVM.Modul end push!(md, MDString("air.arg_type_size")) - push!(md, Metadata(ConstantInt(Int32(sizeof(arg_type))))) + push!(md, Metadata(ConstantInt(Int32(sizeof(arg_type)::Int)))) push!(md, MDString("air.arg_type_align_size")) - push!(md, Metadata(ConstantInt(Int32(Base.datatype_alignment(arg_type))))) + push!(md, Metadata(ConstantInt(Int32(Base.datatype_alignment(arg_type)::Int)))) # like MSL, name the pointee type for pointer arguments type_name = if arg.cc == BITS_VALUE && (arg.typ <: Ptr || arg.typ <: Core.LLVMPtr) @@ -2638,7 +2643,7 @@ intrinsic_table(table) = Dict(intr => val for (intr, val) in ((tryparse(LLVM.Intrinsic, name), val) for (name, val) in table) if intr !== nothing) -function called_intrinsic(inst::LLVM.Instruction) +function called_intrinsic(@nospecialize(inst::LLVM.Instruction)) inst isa LLVM.CallBase || return nothing callee = inst.called_function return callee === nothing ? nothing : callee.intrinsic diff --git a/src/ptx.jl b/src/ptx.jl index 4eb1c3da..80ed8cc8 100644 --- a/src/ptx.jl +++ b/src/ptx.jl @@ -387,7 +387,7 @@ end # single block on targets without clusters, where NVPTX uses the block scope instead. const PTX_SYNCSCOPES = ("singlethread", "block", "cluster", "device", "system") -ptx_syncscope_error(inst::LLVM.Instruction, what="atomic operation") = +ptx_syncscope_error(@nospecialize(inst::LLVM.Instruction), what="atomic operation") = inst.syncscope.name in PTX_SYNCSCOPES ? nothing : "$what with synchronization scope $(repr(inst.syncscope.name))" @@ -396,7 +396,7 @@ ptx_syncscope_error(inst::LLVM.Instruction, what="atomic operation") = # floating-point ones, or `fadd` on older devices) are fine: NVPTX expands them to # compare-exchange loops. function ptx_atomic_error(@nospecialize(job::CompilerJob{PTXCompilerTarget}), - dl::LLVM.DataLayout, inst::LLVM.Instruction) + dl::LLVM.DataLayout, @nospecialize(inst::LLVM.Instruction)) target = job.config.target reason = ptx_syncscope_error(inst) reason === nothing || return reason @@ -450,7 +450,7 @@ end # Why the NVPTX back-end cannot compile the atomic memory operation `inst`, although the # target supports it, or `nothing` if it can. These work around bugs in the back-end, and # should be revisited when updating it. -function nvptx_atomic_error(inst::LLVM.Instruction) +function nvptx_atomic_error(@nospecialize(inst::LLVM.Instruction)) # NVPTX's instruction printer aborts on the ordering of the `atom.cas.b128` and # `atom.exch.b128` that it legalizes these to (LLVM 23.1, and main as of October 2026) T = atomic_value_type(inst) @@ -486,12 +486,12 @@ end # (`ld.relaxed` and `st.relaxed`) from sm_70 on, which hasn't been seen to fail. # Is `ptr` known to point to the thread's own memory? -ptx_thread_private(ptr::LLVM.Value) = +ptx_thread_private(@nospecialize(ptr::LLVM.Value)) = ptr.value_type.addrspace == #=local=# 5 || is_thread_private(ptr) # Can `ptr` point to local memory? Generic pointers can, unless every object they can be # derived from is in another address space. -function ptx_may_be_local(ptr::LLVM.Value) +function ptx_may_be_local(@nospecialize(ptr::LLVM.Value)) seen = Set{LLVM.Value}() worklist = LLVM.Value[ptr] while !isempty(worklist) @@ -523,7 +523,7 @@ end # Does the `!noalias.addrspace` metadata of `inst` rule out that it accesses local memory? # `guard_local_atomic!` adds it to the atomic it keeps for other memory. -function excludes_local(inst::LLVM.Instruction) +function excludes_local(@nospecialize(inst::LLVM.Instruction)) haskey(inst.metadata, "noalias.addrspace") || return false md = inst.metadata["noalias.addrspace"] bounds = [convert(Int, Value(op)) % UInt32 for op in md.operands] @@ -537,11 +537,11 @@ end # Replace the atomic memory operation `inst` by a call to a function that checks whether # its pointer points to local memory, and then performs it with plain accesses, or atomically # otherwise. The function is meant to be inlined (it is `alwaysinline`). -function guard_local_atomic!(inst::LLVM.Instruction) +function guard_local_atomic!(@nospecialize(inst::LLVM.Instruction)) mod = LLVM.parent(LLVM.parent(LLVM.parent(inst))) args = collect(inst.operands) T = inst.value_type - ft = LLVM.FunctionType(T, [arg.value_type for arg in args]) + ft = LLVM.FunctionType(T, LLVMType[arg.value_type for arg in args]) f = LLVM.Function(mod, "gpucompiler.local_atomic", ft) f.linkage = LLVM.Linkage.Internal push!(f.function_attributes, EnumAttribute(:alwaysinline)) @@ -556,7 +556,7 @@ function guard_local_atomic!(inst::LLVM.Instruction) end end other_atomic.metadata["noalias.addrspace"] = - MDNode([Metadata(ConstantInt(Int32(5))), Metadata(ConstantInt(Int32(6)))]) + MDNode(Metadata[ConstantInt(Int32(5)), ConstantInt(Int32(6))]) isspacep = LLVM.Function(mod, LLVM.Intrinsic("llvm.nvvm.isspacep.local")) isspacep_ft = isspacep.function_type @@ -585,7 +585,8 @@ function guard_local_atomic!(inst::LLVM.Instruction) ret!(builder) else result = phi!(builder, T) - append!(result.incoming, [(local_atomic, local_block), (other_atomic, other_block)]) + append!(result.incoming, Tuple{LLVM.Value,BasicBlock}[(local_atomic, local_block), + (other_atomic, other_block)]) ret!(builder, result) end end @@ -606,8 +607,8 @@ function lower_local_atomics!(mod::LLVM.Module) changed = false @tracepoint "lower local atomics" begin - atomics = [inst for f in mod.functions for bb in f.blocks for inst in bb.instructions - if is_atomic_memop(inst)] + atomics = LLVM.Instruction[inst for f in mod.functions for bb in f.blocks + for inst in bb.instructions if is_atomic_memop(inst)] guarded = false for inst in atomics if ptx_thread_private(inst.pointer_operand) diff --git a/src/relocation.jl b/src/relocation.jl index 23243660..bd80a979 100644 --- a/src/relocation.jl +++ b/src/relocation.jl @@ -939,7 +939,7 @@ function check_relocation_slot_uses!(mod::LLVM.Module, slots::Vector{LLVM.Global dl = mod.datalayout loads = LLVM.LoadInst[] seen = Set{LLVM.Value}(slots) - worklist = LLVM.Value[slots...] + worklist = Vector{LLVM.Value}(slots) while !isempty(worklist) for use in pop!(worklist).uses val = use.user diff --git a/src/rtlib.jl b/src/rtlib.jl index dd389b84..e3f02db2 100644 --- a/src/rtlib.jl +++ b/src/rtlib.jl @@ -50,7 +50,7 @@ function LLVM.call!(builder, rt::Runtime.RuntimeMethodInstance, args=LLVM.Value[ # runtime functions are written in Julia, while we're calling from LLVM, # this often results in argument type mismatches. try to fix some here. - args = LLVM.Value[args...] + args = collect(LLVM.Value, args) if length(args) != length(ft.parameters) error("Incorrect number of arguments for runtime function: ", "passing ", length(args), " argument(s) to '", string(ft), " ", rt.name, "'") diff --git a/src/spirv.jl b/src/spirv.jl index f23c1779..b5156f47 100644 --- a/src/spirv.jl +++ b/src/spirv.jl @@ -837,8 +837,8 @@ function lower_atomics!(@nospecialize(job::CompilerJob{SPIRVCompilerTarget}), @tracepoint "lower atomics" begin # (the expansions split blocks, so collect the atomics first) - atomics = [inst for f in mod.functions for bb in f.blocks for inst in bb.instructions - if is_atomic_memop(inst)] + atomics = LLVM.Instruction[inst for f in mod.functions for bb in f.blocks + for inst in bb.instructions if is_atomic_memop(inst)] # pad the local memory that 8- and 16-bit atomics may access variables = Set{LLVM.GlobalVariable}()