Make all Level Zero calls GC-safe

lib/level-zero/pointer.jl

@@ -128,6 +128,10 @@ function Base.unsafe_convert(::Type{PtrOrZePtr{T}}, val) where {T}
     return Base.bitcast(PtrOrZePtr{T}, ptr)
 end
 
+# avoid ambiguities when passing PtrOrZePtr instances
+# NOTE: this happens now with `@gcsafe_ccall` due to the double `ccall`
+Base.unsafe_convert(::Type{PtrOrZePtr{T}}, x::PtrOrZePtr{T}) where {T} = x
+
 
 #
 # Device reference objects

lib/level-zero/synchronization.jl

@@ -27,16 +27,13 @@ Base.isdone(list::ZeImmediateCommandList) =
 Base.isdone(queue::ZeCommandQueue) =
     check_done(unchecked_zeCommandQueueSynchronize(queue, 0))
 
-# the blocking synchronization, marked GC-safe so that it doesn't keep the GC from running
+# the blocking synchronization (GC-safe, like all Level Zero calls)
 gcsafe_synchronize(list::ZeImmediateCommandList) =
-    @gcsafe_ccall libze_loader.zeCommandListHostSynchronize(
-        list::ze_command_list_handle_t, typemax(UInt64)::UInt64)::ze_result_t
+    unchecked_zeCommandListHostSynchronize(list, typemax(UInt64))
 gcsafe_synchronize(queue::ZeCommandQueue) =
-    @gcsafe_ccall libze_loader.zeCommandQueueSynchronize(
-        queue::ze_command_queue_handle_t, typemax(UInt64)::UInt64)::ze_result_t
+    unchecked_zeCommandQueueSynchronize(queue, typemax(UInt64))
 gcsafe_synchronize(event::ZeEvent) =
-    @gcsafe_ccall libze_loader.zeEventHostSynchronize(
-        event::ze_event_handle_t, typemax(UInt64)::UInt64)::ze_result_t
+    unchecked_zeEventHostSynchronize(event, typemax(UInt64))
 
 """
     nonblocking_synchronize(list_or_queue)

res/wrap.jl

@@ -56,6 +56,12 @@ function rewriter!(ctx, options)
             expr = node.exprs[1]
             call_expr = expr.args[2].args[1].args[3]    # assumes `@ccall`
 
+            # replace `@ccall` with `@gcsafe_ccall`, so that calls that block (e.g., module
+            # compilation or host synchronization) don't keep the GC from running
+            if haskey(options, "api") && get(options["api"], "gcsafe", false)
+                expr.args[2].args[1].args[1] = Symbol("@gcsafe_ccall")
+            end
+
             target_expr = call_expr.args[1].args[1]
             fn = String(target_expr.args[2].value)
 

res/ze.toml

@@ -9,6 +9,7 @@ use_ccall_macro = true
 
 [api]
 checked_rettypes = [ "ze_result_t" ]
+gcsafe = true
 
 
 [api.zeCommandListAppendMemoryCopy.argtypes]

test/execution.jl

@@ -1074,8 +1074,7 @@ end
             # running when this thread is blocked
             watchdog = @async oneL0.cooperative_wait(gate; spin=false) do gate
                 watching[] = true
-                res = oneL0.@gcsafe_ccall oneL0.libze_loader.zeEventHostSynchronize(
-                    gate::oneL0.ze_event_handle_t, timeout::UInt64)::oneL0.ze_result_t
+                res = oneL0.unchecked_zeEventHostSynchronize(gate, timeout)
                 res == oneL0.RESULT_NOT_READY || return false
                 oneL0.signal(gate)
                 return true

test/level-zero.jl

@@ -157,6 +157,32 @@ end
 end
 
 
+@testset "GC-safe blocking calls" begin
+# a thread blocked in the driver must not keep other threads from collecting garbage
+pool = ZeEventPool(ctx, 1)
+event = pool[1]
+waiting = Threads.Atomic{Bool}(false)
+# block on one of `cooperative_wait`'s worker threads. the timeout makes a collection that
+# has to wait for the driver call to return fail the test instead of deadlocking it.
+waiter = @async oneL0.cooperative_wait(event; spin=false) do event
+    waiting[] = true
+    try
+        wait(event, 10_000_000_000)
+        return true
+    catch
+        return false
+    end
+end
+while !waiting[]
+    yield()
+end
+sleep(0.5)  # let the worker enter the driver
+GC.gc()
+signal(event)
+@test something(fetch(waiter))
+end
+
+
 @testset "barrier" begin
 
 pool = ZeEventPool(ctx, 1)