diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 97714501b..e977441ed 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -14,47 +14,7 @@ steps: codecov: true commands: | unset LD_LIBRARY_PATH - julia -e 'using Pkg - - gpuarrays = pwd() - gpuarrayscore = joinpath(gpuarrays, "lib", "GPUArraysCore") - devdir = mktempdir() - package = joinpath(devdir, "CUDA") - - println("--- :julia: Installing TestEnv") - Pkg.activate(; temp=true) - Pkg.add("TestEnv") - using TestEnv - - println("--- :julia: Installing CUDA.jl") - withenv("JULIA_PKG_PRECOMPILE_AUTO" => 0, - "JULIA_PKG_DEVDIR" => devdir) do - try - Pkg.develop("CUDA") - Pkg.activate(package) - Pkg.develop([ - PackageSpec(path=gpuarrays), PackageSpec(path=gpuarrayscore), - PackageSpec(path=joinpath(package, "CUDACore")), - PackageSpec(path=joinpath(package, "CUDATools")), - PackageSpec(path=joinpath(package, "lib", "cupti")), - PackageSpec(path=joinpath(package, "lib", "nvml")), - PackageSpec(path=joinpath(package, "lib", "cublas")), - PackageSpec(path=joinpath(package, "lib", "cusparse")), - PackageSpec(path=joinpath(package, "lib", "cusolver")), - PackageSpec(path=joinpath(package, "lib", "cufft")), - PackageSpec(path=joinpath(package, "lib", "curand")), - ]) - TestEnv.activate() - catch err - @error "Could not install CUDA.jl" exception=(err,catch_backtrace()) - exit(3) - finally - Pkg.activate(package) - end - end - - println("+++ :julia: Running tests") - Pkg.test(; coverage="@" * gpuarrays)' + julia .buildkite/sparse-v13-stack.jl CUDA agents: queue: "cuda" if: | @@ -62,8 +22,6 @@ steps: (build.message =~ /\[only [^\]]*(tests|cuda)/ || build.message !~ /\[only / && !build.pull_request.draft) timeout_in_minutes: 120 - soft_fail: - - exit_status: 3 - label: "CUDA.jl -- Julia {{matrix.julia}}" plugins: @@ -73,36 +31,7 @@ steps: codecov: true commands: | unset LD_LIBRARY_PATH - julia -e 'using Pkg - - gpuarrays = pwd() - gpuarrayscore = joinpath(gpuarrays, "lib", "GPUArraysCore") - devdir = mktempdir() - package = joinpath(devdir, "CUDA") - - println("--- :julia: Installing TestEnv") - Pkg.activate(; temp=true) - Pkg.add("TestEnv") - using TestEnv - - println("--- :julia: Installing CUDA.jl") - withenv("JULIA_PKG_PRECOMPILE_AUTO" => 0, - "JULIA_PKG_DEVDIR" => devdir) do - try - Pkg.develop("CUDA") - Pkg.activate(package) - Pkg.develop([PackageSpec(path=gpuarrays), PackageSpec(path=gpuarrayscore)]) - TestEnv.activate() - catch err - @error "Could not install CUDA.jl" exception=(err,catch_backtrace()) - exit(3) - finally - Pkg.activate(package) - end - end - - println("+++ :julia: Running tests") - Pkg.test(; coverage="@" * gpuarrays)' + julia .buildkite/sparse-v13-stack.jl CUDA agents: queue: "cuda" if: | @@ -114,8 +43,6 @@ steps: setup: julia: - "1.13" - soft_fail: - - exit_status: 3 - label: "AMDGPU.jl -- Julia {{matrix.julia}}" plugins: @@ -124,36 +51,7 @@ steps: - JuliaCI/julia-coverage#v1: codecov: true command: | - julia -e 'using Pkg - - gpuarrays = pwd() - gpuarrayscore = joinpath(gpuarrays, "lib", "GPUArraysCore") - devdir = mktempdir() - package = joinpath(devdir, "AMDGPU") - - println("--- :julia: Installing TestEnv") - Pkg.activate(; temp=true) - Pkg.add("TestEnv") - using TestEnv - - println("--- :julia: Installing AMDGPU.jl") - withenv("JULIA_PKG_PRECOMPILE_AUTO" => 0, - "JULIA_PKG_DEVDIR" => devdir) do - try - Pkg.develop("AMDGPU") - Pkg.activate(package) - Pkg.develop([PackageSpec(path=gpuarrays), PackageSpec(path=gpuarrayscore)]) - TestEnv.activate() - catch err - @error "Could not install AMDGPU.jl" exception=(err,catch_backtrace()) - exit(3) - finally - Pkg.activate(package) - end - end - - println("+++ :julia: Running tests") - Pkg.test(; coverage="@" * gpuarrays)' + julia .buildkite/sparse-v13-stack.jl AMDGPU agents: queue: "rocm" rocmgpu: "*" @@ -167,8 +65,6 @@ steps: julia: - "1.10" - "1.13" - soft_fail: - - exit_status: 3 - label: "oneAPI.jl -- Julia {{matrix.julia}}" plugins: @@ -177,37 +73,7 @@ steps: - JuliaCI/julia-coverage#v1: codecov: true command: | - julia -e 'using Pkg - - gpuarrays = pwd() - gpuarrayscore = joinpath(gpuarrays, "lib", "GPUArraysCore") - devdir = mktempdir() - package = joinpath(devdir, "oneAPI") - - println("--- :julia: Installing TestEnv") - Pkg.activate(; temp=true) - Pkg.add("TestEnv") - using TestEnv - - println("--- :julia: Installing oneAPI.jl") - withenv("JULIA_PKG_PRECOMPILE_AUTO" => 0, - "JULIA_PKG_DEVDIR" => devdir) do - try - Pkg.develop("oneAPI") - include(joinpath(package, "deps", "build_ci.jl")) - Pkg.activate(package) - Pkg.develop([PackageSpec(path=gpuarrays), PackageSpec(path=gpuarrayscore)]) - TestEnv.activate() - catch err - @error "Could not install oneAPI.jl" exception=(err,catch_backtrace()) - exit(3) - finally - Pkg.activate(package) - end - end - - println("+++ :julia: Running tests") - Pkg.test(; coverage="@" * gpuarrays)' + julia .buildkite/sparse-v13-stack.jl oneAPI agents: queue: "oneapi" if: | @@ -220,8 +86,23 @@ steps: julia: - "1.10" - "1.13" - soft_fail: - - exit_status: 3 + + - label: "OpenCL.jl -- NVIDIA GPU" + plugins: + - JuliaCI/julia#v1: + version: "1.13" + - JuliaCI/julia-coverage#v1: + codecov: true + command: julia .buildkite/sparse-v13-stack.jl OpenCL --platform=cuda --verbose + agents: + queue: "cuda" + env: + OCL_ICD_FILENAMES: "libnvidia-opencl.so.1" + if: | + build.message !~ /\[skip [^\]]*(tests|opencl)/ && + (build.message =~ /\[only [^\]]*(tests|opencl)/ || + build.message !~ /\[only / && !build.pull_request.draft) + timeout_in_minutes: 120 - label: "Metal.jl -- Julia {{matrix.julia}}" plugins: @@ -230,36 +111,7 @@ steps: - JuliaCI/julia-coverage#v1: codecov: true command: | - julia -e 'using Pkg - - gpuarrays = pwd() - gpuarrayscore = joinpath(gpuarrays, "lib", "GPUArraysCore") - devdir = mktempdir() - package = joinpath(devdir, "Metal") - - println("--- :julia: Installing TestEnv") - Pkg.activate(; temp=true) - Pkg.add("TestEnv") - using TestEnv - - println("--- :julia: Installing Metal.jl") - withenv("JULIA_PKG_PRECOMPILE_AUTO" => 0, - "JULIA_PKG_DEVDIR" => devdir) do - try - Pkg.develop("Metal") - Pkg.activate(package) - Pkg.develop([PackageSpec(path=gpuarrays), PackageSpec(path=gpuarrayscore)]) - TestEnv.activate() - catch err - @error "Could not install Metal.jl" exception=(err,catch_backtrace()) - exit(3) - finally - Pkg.activate(package) - end - end - - println("+++ :julia: Running tests") - Pkg.test(; coverage="@" * gpuarrays, test_args=`--all`)' + julia .buildkite/sparse-v13-stack.jl Metal agents: queue: "metal" if: | @@ -272,8 +124,6 @@ steps: julia: - "1.10" - "1.13" - soft_fail: - - exit_status: 3 env: SECRET_CODECOV_TOKEN: "GrevHmzmr2Vt6UK4wbbTTB1+kcMcIlF6nCXVCk3Z0plHDimpD6BwdN9T2A+5J9k3I2em0xXUqpt+2qUSqM8Bn5mNdpjR0TvxVY3oYXc+qzvBXmcZJpuCgJeoTP1P+kVFwszUn4na3fohNq9Qffp6tXMn/j8yJQKOiiC8mkD0aPEI0zISHuDaa/7j7JYf0vTrMRRZ9BMUQHmFuVaIQN8FLGG2BiE3236rj4eHh0lj2IfekCG3wd/LUzAsMx0MC3kIR8WzOWW2rf6xUMPkjm5+NuHwhAOcZc0+LRM7GYIwoW/nHAgyIqjvLiInNFmaJk+7V/GAKtd+gSAIzmyBUHAy6A==;U2FsdGVkX1+4ZljneQoaNE295nRIx8D6+WoFIgT6Pg2BXHaTyhTL4sxEcG0jX0e7oq68uvi4bK7x7YMS4L0Kew==" diff --git a/.buildkite/sparse-v13-stack.jl b/.buildkite/sparse-v13-stack.jl new file mode 100644 index 000000000..844f37de6 --- /dev/null +++ b/.buildkite/sparse-v13-stack.jl @@ -0,0 +1,270 @@ +# TEMPORARY: validate the unreleased v13 stack, not registered release resolution. +# Explicit path development is intentional: Julia 1.10 ignores [sources]. +using Pkg, TOML + +# Preferences are keyed by package name but Base needs a UUID/name binding in +# each active project, including for transitive JLLs such as CUDA_Compiler_jll. +# Extras provide that binding without changing the resolved dependency graph. +function stack_bind_preferences() + project=Base.active_project() + preferences=joinpath(dirname(project),"LocalPreferences.toml") + isfile(preferences) || return + names=Dict(info.name=>string(uuid) for (uuid,info) in Pkg.dependencies()) + data=TOML.parsefile(project) + changed=false + for name in keys(TOML.parsefile(preferences)) + haskey(names,name) || continue + (get(data,"name",nothing)==name || haskey(get(data,"deps",Dict()),name)) && continue + extras=get!(data,"extras",Dict{String,Any}()) + if !haskey(extras,name) + extras[name]=names[name] + changed=true + println("STACK_PREFERENCE_BINDING ",name,' ',names[name]) + else + extras[name]==names[name] || error("conflicting preference UUID for $name") + end + end + if changed + open(project,"w") do io + TOML.print(io,data) + end + end +end + +gpuarrays = normpath(joinpath(@__DIR__, "..")) +pins = TOML.parsefile(joinpath(@__DIR__, "sparse-v13-stack.toml")) +backend = isempty(ARGS) ? "JLArrays" : first(ARGS) +setup_only = "--setup-only" in ARGS +scratch = mktempdir(get(ENV,"GPUARRAYS_V13_STACK_PARENT",tempdir()); + prefix="gpuarrays-v13-stack-",cleanup=!setup_only) + +function checkout(name) + pin = pins[name] + path = joinpath(scratch, name) + run(`git clone --quiet --filter=blob:none $(pin["url"]) $path`) + run(`git -C $path checkout --quiet --detach $(pin["rev"])`) + revision = readchomp(`git -C $path rev-parse HEAD`) + revision == pin["rev"] || error("Unexpected $name revision: $revision") + println("STACK_REVISION $name $revision") + path +end + +println("STACK_REVISION GPUArrays ", readchomp(`git -C $gpuarrays rev-parse HEAD`)) +println("STACK_DIRTY GPUArrays ", !isempty(readchomp(`git -C $gpuarrays status --porcelain`))) +println("STACK_JULIA ", VERSION) +specs = PackageSpec[ + PackageSpec(path=gpuarrays), + PackageSpec(path=joinpath(gpuarrays,"lib","GPUArraysCore")), + PackageSpec(path=checkout("AcceleratedKernels")), + PackageSpec(path=checkout("Atomix")), +] +backend in ("JLArrays","docs") && push!(specs,PackageSpec(path=joinpath(gpuarrays,"lib","JLArrays"))) +package = if backend in ("JLArrays","docs") + gpuarrays +elseif haskey(ENV,"GPUARRAYS_V13_STACK_BACKEND") + source=realpath(ENV["GPUARRAYS_V13_STACK_BACKEND"]) + TOML.parsefile(joinpath(source,"Project.toml"))["name"]==backend || error("wrong backend worktree") + isempty(readchomp(`git -C $source diff HEAD --`)) || error("commit backend changes before stack validation") + revision=readchomp(`git -C $source rev-parse HEAD`) + # Ignore stale workspace manifests and generated files in a reused CI checkout. + # Test the recorded commit, without modifying the supplied working tree. + path=joinpath(scratch,backend) + run(`git clone --quiet --shared --no-checkout $source $path`) + run(`git -C $path checkout --quiet --detach $revision`) + println("STACK_REVISION $backend ",readchomp(`git -C $path rev-parse HEAD`)) + path +else + checkout(backend) +end +if package != gpuarrays + push!(specs,PackageSpec(path=package)) +end +if backend == "CUDA" + # Include test-only workspace packages too. Otherwise 1.10 can silently resolve + # a registered library instead of the checked-out cuSPARSE/cuSOLVER stack. + for relative in ("CUDACore","CUDATools","lib/cupti","lib/nvml", + "lib/cublas","lib/cusparse","lib/cusolver","lib/cufft","lib/curand", + "lib/cudnn","lib/cutensor","lib/cutensornet","lib/custatevec") + push!(specs,PackageSpec(path=joinpath(package,relative))) + end +elseif backend == "OpenCL" + push!(specs,PackageSpec(path=joinpath(package,"lib","intrinsics"))) +end + +active = get(ENV,"GPUARRAYS_V13_STACK_SETUP_PROJECT", + backend == "docs" ? joinpath(gpuarrays,"docs") : joinpath(scratch,"env")) +filter!(spec -> normpath(spec.path) != active,specs) +Pkg.activate(active) +withenv("JULIA_PKG_PRECOMPILE_AUTO"=>"0") do + Pkg.develop(specs) +end +Pkg.status(; mode=Pkg.PKGMODE_MANIFEST) +if haskey(ENV,"GPUARRAYS_V13_STACK_PREFERENCES") + cp(ENV["GPUARRAYS_V13_STACK_PREFERENCES"],joinpath(active,"LocalPreferences.toml");force=true) + package==gpuarrays || cp(ENV["GPUARRAYS_V13_STACK_PREFERENCES"], + joinpath(package,"LocalPreferences.toml");force=true) +end +const stack_oneapi_debug=backend=="oneAPI" && + get(ENV,"GPUARRAYS_V13_STACK_ONEAPI_DEBUG","false")=="true" +stack_bind_preferences() + +if backend == "oneAPI" && (!setup_only || get(ENV,"GPUARRAYS_V13_STACK_BUILD_SUPPORT","false")=="true") + # This branch changes the support C++ wrappers. Build them for stack validation; + # the registered JLL has not incorporated those changes yet. + Pkg.activate(package) + Pkg.develop(filter(spec -> normpath(spec.path) != package,specs)) + Pkg.activate(joinpath(package,"deps")) + Pkg.instantiate() + @eval using Scratch + original_args=copy(ARGS) + empty!(ARGS) + push!(ARGS,joinpath(scratch,"support-build")) + try + Scratch.with_scratch_directory(joinpath(scratch,"support-scratch")) do + build_module=Module(gensym(:SparseV13SupportBuild)) + Core.eval(build_module,:(include(path)=Base.include($build_module,path))) + # This job requires the changed wrappers, regardless of JLL tag timestamps. + Base.include(build_module,joinpath(package,"deps","build_local.jl")) + end + finally + empty!(ARGS) + append!(ARGS,original_args) + end + Pkg.activate(active) + Pkg.add("oneAPI_Support_jll") + preferences=joinpath(package,"LocalPreferences.toml") + cp(preferences,joinpath(active,"LocalPreferences.toml");force=true) + support=TOML.parsefile(preferences)["oneAPI_Support_jll"]["liboneapi_support_path"] + ENV["GPUARRAYS_V13_STACK_SUPPORT"]=support + println("STACK_SUPPORT_LIBRARY ",support) +end +function stack_prepare_tests() + # CUDA's runner switches into each library test project. Julia 1.10 ignores + # [sources] and workspaces, so developing the libraries in `active` alone does + # not resolve test-only imports such as SparseMatricesCSR. Resolve every actual + # test project using the same pins, including the backend's root test project. + test_projects=String[] + package==gpuarrays || push!(test_projects,joinpath(package,"test")) + if backend=="CUDA" + append!(test_projects,[joinpath(package,"lib",name,"test") for name in + ("cublas","cusparse","cusolver","cufft","curand","cudnn", + "cutensor","cutensornet","custatevec")]) + end + for project in test_projects + isfile(joinpath(project,"Project.toml")) || continue + Pkg.activate(project) + withenv("JULIA_PKG_PRECOMPILE_AUTO"=>"0") do + Pkg.develop(specs) + end + preferences=joinpath(active,"LocalPreferences.toml") + isfile(preferences) && cp(preferences,joinpath(project,"LocalPreferences.toml");force=true) + stack_bind_preferences() + println("STACK_TEST_PROJECT ",project) + end + Pkg.activate(active) +end + +# Custom CI jobs (minimum dependencies, Enzyme) can prepare the pinned stack, +# change dependencies in its disposable checkout, then run the same checked runner. +function stack_load_backend(test_args) + stack_bind_preferences() + if backend=="CUDA" + @eval using CUDA,cuSPARSE,cuSOLVER + elseif backend=="AMDGPU" + @eval using AMDGPU + elseif backend=="Metal" + @eval using Metal + elseif backend=="oneAPI" + @eval using oneAPI + elseif backend=="OpenCL" + requested=findfirst(arg->startswith(arg,"--platform="),test_args) + request=requested===nothing ? "pocl" : split(test_args[requested],'=';limit=2)[2] + driver=request=="pocl_next" ? :pocl_next_jll : + request=="intel_cpu" ? :intel_opencl_rt_jll : + request=="pocl" ? :pocl_jll : nothing + driver === nothing || Pkg.add(string(driver)) + @eval using OpenCL + driver === nothing || @eval import $driver + end +end +function stack_run_tests(test_args) + stack_load_backend(test_args) + # Imports above can install new extension methods; enter their world explicitly. + Base.invokelatest(stack_run_loaded_tests,test_args) +end +function stack_run_loaded_tests(test_args) + # A backend runner may otherwise skip unavailable hardware/libraries and exit 0. + if backend=="CUDA" + CUDA.functional() && cuSPARSE.functional() && cuSOLVER.functional() || + error("CUDA/cuSPARSE/cuSOLVER must be functional for stack validation") + memory=get(Base.get_preferences(Base.PkgId(CUDA.CUDACore).uuid),"default_memory","device") + expected=Dict("device"=>CUDA.DeviceMemory,"unified"=>CUDA.UnifiedMemory,"host"=>CUDA.HostMemory)[memory] + CUDA.CUDACore.default_memory===expected || error("CUDA default memory preference was not propagated") + println("STACK_CUDA_MEMORY ",memory,' ',expected) + elseif backend=="AMDGPU" + all(AMDGPU.functional,(:hip,:rocsparse,:rocblas,:rocsolver)) || + error("HIP/rocSPARSE/rocBLAS/rocSOLVER must be functional for stack validation") + elseif backend=="Metal" + Metal.functional() || error("Metal must be functional for stack validation") + elseif backend=="oneAPI" + oneAPI.functional() || error("oneAPI must be functional for stack validation") + oneAPI.Support._check_sparse_abi() + if stack_oneapi_debug + for library in (oneAPI.oneL0.NEO_jll,oneAPI.oneL0.NEO_jll.libigc_jll) + get(Base.get_preferences(Base.PkgId(library).uuid),"debug",nothing)=="true" || + error("oneAPI debug preference was not propagated for $(nameof(library))") + println("STACK_DEBUG_PREFERENCE ",nameof(library)," true") + end + end + elseif backend=="OpenCL" + requested=findfirst(arg->startswith(arg,"--platform="),test_args) + request=requested===nothing ? "pocl" : split(test_args[requested],'=';limit=2)[2] + any(OpenCL.cl.platforms()) do candidate_platform + name=lowercase(candidate_platform.name*" "*candidate_platform.vendor) + matches=request in ("pocl","pocl_next") ? occursin("portable",name) : + request=="intel_cpu" ? candidate_platform.name=="Intel(R) OpenCL" : occursin(lowercase(request),name) + matches && !isempty(OpenCL.cl.devices(candidate_platform)) + end || error("requested OpenCL platform must expose a device for stack validation") + end + coverage=haskey(ENV,"GPUARRAYS_V13_STACK_BACKEND") ? "user" : "@"*gpuarrays + julia_args=Cmd(Base.shell_split(get(ENV,"GPUARRAYS_V13_STACK_JULIA_ARGS",""))) + allow_reresolve=parse(Bool,get(ENV,"GPUARRAYS_V13_STACK_ALLOW_RERESOLVE","true")) + try + if VERSION1 && (test_args=ARGS[2:end]) + stack_run_tests(test_args) +end diff --git a/.buildkite/sparse-v13-stack.toml b/.buildkite/sparse-v13-stack.toml new file mode 100644 index 000000000..8c00f8243 --- /dev/null +++ b/.buildkite/sparse-v13-stack.toml @@ -0,0 +1,22 @@ +# TEMPORARY coordinated-v13 stack validation. Remove after registered resolution works. +[AcceleratedKernels] +url = "https://github.com/JuliaGPU/AcceleratedKernels.jl" +rev = "a915b419728e3abd258bce78ea3d1d3255b88ed7" +[Atomix] +url = "https://github.com/JuliaConcurrent/Atomix.jl" +rev = "1ffd4ae4746fbc9463882c68563c6bdb81e0d9c0" +[CUDA] +url = "https://github.com/JuliaGPU/CUDA.jl" +rev = "a7336fd03673a9978f6cdd3bb2838da8801cac37" +[AMDGPU] +url = "https://github.com/JuliaGPU/AMDGPU.jl" +rev = "b518d77d41d35f916fc378cd02f0a22f698b8875" +[oneAPI] +url = "https://github.com/JuliaGPU/oneAPI.jl" +rev = "06f2f77021c105ab27c6c5e35af705426aaad730" +[Metal] +url = "https://github.com/JuliaGPU/Metal.jl" +rev = "4e837cb7db64ebd7315369a1521226f45eda80b3" +[OpenCL] +url = "https://github.com/JuliaGPU/OpenCL.jl" +rev = "94fae12aa6b88e3e3a39f7f5933ebb66fa9e6d29" diff --git a/.github/workflows/Test.yml b/.github/workflows/Test.yml index aa691a2a8..633c6d642 100644 --- a/.github/workflows/Test.yml +++ b/.github/workflows/Test.yml @@ -32,23 +32,8 @@ jobs: show-versioninfo: true - uses: julia-actions/cache@v3 timeout-minutes: 5 - - name: Develop subpackages - timeout-minutes: 5 - run: | - julia --project -e " - using Pkg - Pkg.develop([PackageSpec(; name=basename(path), path) for path in ARGS]) - " lib/GPUArraysCore lib/JLArrays - - name: "Set test arguments" - shell: bash - run: | - # Run tests in verbose mode - TEST_ARGS=(--verbose) - echo "runtest_test_args=${TEST_ARGS[@]}" >> "${GITHUB_ENV}" - - uses: julia-actions/julia-runtest@v1 - continue-on-error: ${{ matrix.version == 'nightly' }} - with: - test_args: ${{ env.runtest_test_args }} + - name: Test coordinated v13 stack + run: julia .buildkite/sparse-v13-stack.jl JLArrays - uses: julia-actions/julia-processcoverage@v1 with: directories: src,lib @@ -60,7 +45,8 @@ jobs: opencl: name: OpenCL.jl runs-on: ubuntu-latest - timeout-minutes: 60 + # Both IL and OpenCL-C suites compile many kernels on PoCL CPU runners. + timeout-minutes: 180 steps: - uses: actions/checkout@v6 - uses: julia-actions/setup-julia@v3 @@ -70,42 +56,7 @@ jobs: - uses: julia-actions/cache@v3 - name: Run tests run: | - julia -e 'using Pkg - - gpuarrays = pwd() - gpuarrayscore = joinpath(gpuarrays, "lib", "GPUArraysCore") - devdir = mktempdir() - package = joinpath(devdir, "OpenCL") - - println("--- :julia: Installing TestEnv") - Pkg.activate(; temp=true) - Pkg.add("TestEnv") - using TestEnv - - println("--- :julia: Installing OpenCL.jl") - withenv("JULIA_PKG_PRECOMPILE_AUTO" => 0, - "JULIA_PKG_DEVDIR" => devdir) do - try - Pkg.develop("OpenCL") - Pkg.activate(package) - Pkg.develop([PackageSpec(path=gpuarrays), PackageSpec(path=gpuarrayscore)]) - TestEnv.activate() - catch err - # OpenCL.jl may not support this GPUArrays version yet (e.g. after a - # breaking release): skip, like Buildkite soft-fails, but annotate - # the run so the skip is visible - @error "Could not install OpenCL.jl" exception=(err,catch_backtrace()) - println("::warning title=OpenCL.jl not tested::Could not install OpenCL.jl with this version of GPUArrays") - exit(0) - finally - Pkg.activate(package) - end - end - - println("+++ :julia: Running tests") - # `poclc/` repeats every test through the OpenCL C back-end, which exercises - # the compiler of OpenCL.jl rather than GPUArrays - Pkg.test(; coverage="@" * gpuarrays, test_args=`pocl/`)' + julia .buildkite/sparse-v13-stack.jl OpenCL - uses: julia-actions/julia-processcoverage@v1 with: directories: src,lib @@ -125,10 +76,7 @@ jobs: version: '1.10' - name: Develop packages run: | - julia -e " - using Pkg - Pkg.develop([PackageSpec(; name=basename(splitext(path)[1]), path) for path in ARGS]) - " ../GPUArrays.jl lib/GPUArraysCore + julia .buildkite/sparse-v13-stack.jl docs --setup-only - name: Install dependencies run: julia --project=docs/ -e 'using Pkg; Pkg.instantiate()' - name: Build and deploy diff --git a/HANDOFF.sparse-v13.md b/HANDOFF.sparse-v13.md new file mode 100644 index 000000000..b966575ea --- /dev/null +++ b/HANDOFF.sparse-v13.md @@ -0,0 +1,192 @@ +# GPUArrays v13 implementation handoff + +## Current state + +Implementation and backend migrations are committed. Final reviewed runtime is +GPUArrays34d125e0ce852c823c8aabcaf0c6bc88424bfe86, retained remotely as +backup/sparse-v13-all-dimensions-runtime-20261021. Stack-validation head +9a9126f36177d4230aaa77496c7d1bc1113537a4 differs only in dependency pins and +handoff/evidence; src/ext/lib/test/docs/src compare byte-identical. Later handoff-only +updates must not be mislabeled as literal tests of a different commit. +The tested head is also retained as tb/sparse-v13-validation so PR/handoff updates +and delayed webhooks cannot change that validation snapshot. + +Production ownership is settled: immutable canonical vector/CSR/CSC/row-major COO +with separate compressed-pointer/coordinate storage; checked widths and mutation; +borrowed raw buffers; ordered associative duplicates; centralized portable/vendor +selection and high-level vendor integration. CUDA/AMD native BSR, batch, solve, +factorization and preconditioner paths remain. oneAPI resources own context/queue/ +buffers and prepared analysis outside canonical containers. No scratch prototype +was merged. See PLAN.sparse-v13.md, docs/src/interface.md and the implementation. + +All ordinary broadcast dimensions now retain sparse validation: supported dense +vector/matrix expansion works; mixed sparse ranks, sparse expansion, GPU0 operands, +and higher-dimensional output explicitly reject before allocation/scalar capture. +Host-backed sparse inputs previously escaped into CPU Array output for 3D fusion; +this was reproduced and fixed. Negative-order norms include logical implicit zeros +on every Julia version, preserving NaN magnitudes. Nonfinite complex references use +scalar magnitudes rather than BLAS-dependent Inf/NaN handling. + +## Branches and CI pin graph + +All branches are topic branches, never main. Final intended pins: + +| Repository | Branch | Tip | +| --- | --- | --- | +| GPUArrays | tb/sparse and tb/sparse-v13 | stack-validation9a9126f; runtime34d125e | +| CUDA | tb/sparse-v13 | a7336fd03673a9978f6cdd3bb2838da8801cac37 | +| AMDGPU | tb/sparse-v13 | b518d77d41d35f916fc378cd02f0a22f698b8875 | +| oneAPI | tb/sparse-v13 | 06f2f77021c105ab27c6c5e35af705426aaad730 | +| Metal | tb/sparse and tb/sparse-v13 | 4e837cb7db64ebd7315369a1521226f45eda80b3 | +| OpenCL | tb/sparse-v13 | 94fae12aa6b88e3e3a39f7f5933ebb66fa9e6d29 | +| AcceleratedKernels | tb/sparse-v13 | a915b419728e3abd258bce78ea3d1d3255b88ed7 | +| Atomix | tb/sparse-v13 | 1ffd4ae4746fbc9463882c68563c6bdb81e0d9c0 | +| maleadt/Yggdrasil | tb/oneapi-sparse-v13-support | 31c79455bc9f0b415a75ceb9faac59a26148e8f4 | + +Final backend changes above only repin their TEMP bootstrap to frozen34d125e; +runtime/tests equal CUDAa3e0c60, AMDf5b6976, oneAPId1cb45d, Metal1aaa653 and +OpenCLe0a09f4 respectively. Central CI pins the resulting backend tips. This is a +finite graph: stop repinning for documentation/CI-only changes. All old tips are +preserved locally/remotely; rewrites check concurrent changes and use explicit +refspecs/leases. Useful contributor commits/authorship remain. Each repository has +ONE isolated TEMP CI layer, not an incremental debug-commit series. + +Exact setup includes all14 CUDA workspace packages and library test projects; +Julia1.10 uses explicit Pkg.develop because it ignores [sources]. Existing backend +CI variants/preferences remain. Installation and nightly failures are hard failures. +Custom min-dependency/Enzyme variants retain their selected dependency graphs. +The oneAPI CI override builds and verifies native support from source; this is not +published-artifact resolution. Remove temporary plumbing after coordinated releases. + +## Exact validation evidence + +Do not equate source equivalence, setup success, a focused test, and a full migration +suite. Older exact revisions remain useful evidence, not literal final-head passes. + +- Final34d125e local Julia1.13.2:2058 regressions,247 shared contracts,52 display, + 195 linalg,275 mixed broadcast,ambiguity1 pass. Logs + /Users/tim/Julia/tmp/ga13-broadcast-rank-green2.log. First added destination checks + expected ArgumentError instead of the existing pre-mutation DimensionMismatch; + corrected the test expectation and reran green. Initial 3D CPU fallback and GPU0 + wrong-route reproductions remain red evidence. +- Final34d125e Julia1.10.12:2058regressions,52display,195linalg,275broadcast and + ambiguity1 pass, local ga13-final34d110-focused.log. Julia1.15-DEV61 actual Pkg.test + passes2058regressions+ambiguity1, Pegasus ga13-nightly-final-env/final34d-regressions.log. + Nightly's separate closure-box check also passes1, final34d-closures.log. + CUDAa3e0c60 and AMDf5b6976 final shared contracts244+ambiguity1 pass each, + Pegasus ga13-{cuda,amdgpu-loadorder-final}-env/final34d-contracts.log. Final + oneAPI06f2f77 final contracts244+ambiguity1 pass with the generated JLL; OpenCL94fae12 + contracts247+ambiguity1 plus unsupported-Float16 capability1 pass. Metal4e837cb + contracts247+ambiguity1 pass with one evaluation thread; no tracked source changes, + existing untracked preferences/coverage left alone. Their logs are + ga13-{oneapi,opencl,metal}-final34d-env/contracts.log on the corresponding host. +- GA4e20de2/CUDAa3e0c60 and AMDf5b6976:181contracts+ambiguity1 each pass, + GPUArrays-first; GA64ea2d8/A3 focused151contracts+195linalg+275broadcast+ambiguity1. + Frozen GA0f2d80f/CUDAe37f85d full shared six-type automatic7712/portable7713 + (2004 portable selections); GA0f2d80f/AMDe3a3a604 shared four-type automatic5540/ + portable5541 (1557 selections), ambiguity1 each. Earlier actual native CUDA/AMD + sparse/solver/factorization/BSR/batch suites pass at revisions in HISTORY. +- GA4e20de2/oneAPId1cb45d, actual generated oneAPI_Support0.11.1+0 artifact on Arc A770: + Julia1.13.2 checked Pkg.test passes13167+24pre-existing broken (shared11671, + native1279,resources121,ABI96). Julia1.10.12 native/resources/ABI passes1496+24broken; + identity/marker/device checks3 each. Loaded artifact tree876c51345ae21089df99bd35dfc29cf292c2a4ce, + support SHA256d5e977efdfe577477c74827cd9b827b61d14f49fa2c8466fb3387a3e3c933fd7. + No source-build preference. Taurus ga13-oneapi-generated-jll{,110}-env/test.log. + See validation/REPORT.oneapi-support.md for recipe, tarball and audit limitations. +- Separately source-built oneAPId1cb45d/GA64ea2d8 checked Pkg.test passes13138+24broken, + including resources122 because an override-path check was enabled. Loaded support + SHA256c19915c2965bc4f7153e4c16d240deb107ecfe0a902f07936445d5f9969932c9. + Earlier oneAPI forced-portable shared suite5517/1560selections+ambiguity1 at + GAc8dd580/oneAPI26fb776. These are separate from generated-artifact evidence. +- Final actual pinned helper9a9126f/oneAPI06f2f77 passes13231+24pre-existing broken: + shared11734,native1279,resources122,ABI96. Source-build support SHA256 + b3f319f2972b03890f14b97c0616c67992ccaa1662075136db55fd684c89cd9a, loaded at + ~/Julia/tmp/gpuarrays-v13-stack-t12gvT/support-scratch/8f75cd03-7ff8-4ecb-9b8f-daf728133b1b/deps/lib/liboneapi_support.so. + Final actual helper9a9126f/OpenCL94fae12 passes11852 (shared11837+SVM15), strict + NVIDIA-only discovery. Both clean checkouts and all exact pins are logged. Logs: + Taurus ga13-oneapi-generated-jll-env/final9a-root.log and Pegasus + ga13-opencl-svm-env/final9a-root.log. These exercise the real checked Pkg.test + path, but are test-host stack validation, not hosted CI or registered resolution. +- GA4e20de2/OpenCLe0a09f4 actual root sparse/SVM11789 pass on Pegasus RTX5080 + (shared11774+SVM15). Prior GA64ea2d8/E0a root sparse/SVM11759 passes on EACH + Pegasus RTX5080 and Taurus RTX6000Ada, all ten supported types, normal GC/sync. + Strict NVIDIA-only discovery uses an empty OCL_ICD_VENDORS directory plus native + OCL_ICD_FILENAMES; actual enumeration is recorded. Logs both hosts + ga13-opencl-svm-env/actual-final-entry-full.log; latest Pegasus final4e-entry-full.log. + GA64edd6d/E0a actual final helper native array/memory/context/queue/event/sync155 + passes, final-native64.log. Not whole OpenCL suites. PoCL CPU IL/C portable tests + remain separate, not GPU correctness evidence. +- GA4e20de2/Metal1aaa653 actual root spot53 passes, one worker and explicit + --threads=1; /Users/tim/Julia/tmp/ga13-metal-final4e-spot.log. Daemon stopped. + No full local Metal. Final34d shared contract spot checks are recorded separately above. +- GA64ea2d8 Julia1.15-DEV61 full shared JL sparse14957+ambiguity1 passes. Corrected + nonfinite reference at4083b78 actual nightly Pkg.test1894+ambiguity1 passes. + Earlier full Julia1.10/1.13 dense+sparse runs and native test groups are in HISTORY. +- CUDA-only benchmark115 native cases: median migrated/baseline ratio1.029, + max1.171; measured SpGEMM578.21→583.60us, portable7369.04us; native device peak + 203605548 bytes both, portable553648304. Duplicate assembly1292.82→199.90us, + peak2.7→4.3MB. Exact revisions, methodology and limits: perf/REPORT.sparse-v13.md. + No performance claim for older cuSPARSE branch or other hardware. + +## Remaining gates + +1. Finish final exact-stack GitHub/Buildkite CI and inspect actual checkout and + STACK_REVISION lines, not PR head alone or a green pipeline-upload step. Current + final9a9126f Actions39646681744 tests the literal head; documentation passes and + its checkout/pins/doctests/document checks are verified in ga13-ci-docs-9a9126f.log. + Windows1.10 passes34480, Ubuntu nightly passes36814; both complete test logs verify + exact9a9126f and AK/Atomix pins,2058regressions and ambiguity1. Nightly closure boxes1 + also passes. Logs locally: ga13-ci-{windows110,nightly}-9a9126f.log. Other CPU/OpenCL + jobs are running or queued, not an aggregate pass. Buildkite2091 targets the same + 9a9126f; pipeline upload passes but all nine GPU jobs remain queued. Immutable + validation-branch Buildkite2092 targets9a9126f too and is the continuing GPU run; + no unrun step or pipeline-upload success is a backend-suite pass. + All matching GPU agents in the paginated131-agent query were busy on unrelated + jobs; do not cancel them. GHA macOS remains heavily queued. + Earlier ef7659a merge32c670f CI passed Ubuntu/Windows1.10–1.13 and docs, but both + nightlies failed21 (18 logical norms,3 CPU display references), fixed and tested + locally afterward. OpenCL hit60min while progressing; final unshortened IL/C + suite has180min. Never count canceled/broken jobs as a suite pass. +2. Resolve or explicitly review inherited BinaryBuilder audit-tool failures before + publishing support0.11.1: old binutils2.25 objdump aborts and standalone MKL probes + warn on baseline and patched builds. Normal generated-JLL native/resource checks + pass separately; interrupted audits did not pass. Test the published artifact, + then raise oneAPI's minimum to the actual registered version; retain marker2 gate. +3. Release compatibility-only AK0.5.9/Atomix1.6.1, GA13/JLArrays0.5 (Core0.2.1), then + coordinated CUDA7(all14), AMDGPU3, oneAPI3, OpenCL0.11, Metal1.12 as weak-dependency + ordering permits. Test every package/test project, especially Julia1.10. + Remove TEMP pins/bootstrap only once normal registered Pkg.test/docs resolve. + Retain hard installation, runtime/vendor-functionality and nightly failures; + removing temporary wiring must not restore soft_fail/continue-on-error behavior. +4. Complete downstream compatibility/API releases recorded in + validation/REPORT.sparse-v13-downstream.md (MLDataDevices, QuantumToolboxCore, + SparseMatrixColorings); no whole-consumer pass is claimed. +5. Keep the inherited NVIDIA dense-only SVM churn issue open. It fails on OpenCL + main9c27048/registered GA12.1.1 and v13; a native shim confirms NULL, including + strict NVIDIA-only discovery. Cause unresolved, not established v13 regression + or driver defect. Independent NULL/error guard passes15; do not mask with GC-off + or blocking defaults. See validation/REPORT.opencl-svm.md/reproducer. +6. Additional GPU generations, old-cuSPARSE algorithm performance and complete local + Metal are untested. Follow the user's Metal limit: single-threaded spot tests. + +## Continuation and review + +Local isolated repositories: /Users/tim/Julia/tmp/{gpuarrays,cuda,amdgpu,oneapi,metal,opencl,ak,atomix}-v13. +Remote scratch: ~/Julia/tmp only; existing ~/Julia/pkg checkouts untouched. +Pegasus direct SSH; Taurus via ssh -J pegasus.maleadt.lan taurus.maleadt.lan. +Use unique jld names, --no-revise, --idle-timeout=2h; clear ARGS before test runners. +Some user/untracked preferences and generated coverage files in Metal were left alone. + +Astra decisions through23 and Sol reviews through86 are in peer conversations +sparse-v13-architecture and sparse-v13-review; findings were independently checked. +Historical checkpoints and older exact native-suite evidence were moved intact to +validation/HISTORY.sparse-v13.md. PLAN/REPORT are the design and original investigation, +not implementations to merge. + +Existing PR745 and Metal982 remain draft, detached onto main. Both descriptions +were refreshed with concise, reviewed scope/coverage/release-gate text under the +user's explicit authorization. Previous bodies are backed up locally as +/Users/tim/Julia/tmp/sparse-v13-pr{745,982}-body-before-final-update.md. No GitHub +discussion/comment or new PR was posted. New backend/dependency +PR descriptions are local /Users/tim/Julia/tmp/sparse-v13-*-pr-description.md drafts; +do not post new PRs or discussions without authorization. diff --git a/PLAN.sparse-v13.md b/PLAN.sparse-v13.md new file mode 100644 index 000000000..c66d43985 --- /dev/null +++ b/PLAN.sparse-v13.md @@ -0,0 +1,326 @@ +# Productionize generic GPU sparse arrays for GPUArrays 13 + +This is the implementation handoff for [GPUArrays #745](https://github.com/JuliaGPU/GPUArrays.jl/pull/745). Read [REPORT.sparse-v13.md](REPORT.sparse-v13.md) for the design assessment, confirmed failures, source references and measured limits. Implementation is underway on `tb/sparse-v13`; see [HANDOFF.sparse-v13.md](HANDOFF.sparse-v13.md) for exact commits, reviews, tests and remaining work. Representation, mutation preflight, arithmetic corrections and the initial selector are implemented. The selector is not frozen: vendor migrations and release gates remain open. + +The objective is one coordinated backend migration, followed by portable sparse development and algorithm selection maintained in GPUArrays and its AK/KA dependencies. Keep backend-specific runtime and vendor bindings where they belong. New bindings or new hardware capabilities may require backend work; adding ordinary sparse operations or changing the preferred existing algorithm must not. + +Implementation evidence must distinguish independent runtime defects from redesign +regressions. OpenCL9e55fc3 fixes publication of failed SVM allocations (CLPtr NULL +was incorrectly compared with host C_NULL), using existing bounded reclamation and +preserving the native context until success. Fifteen focused checks pass; broader +NVIDIA allocation/transfer churn still fails on both baseline and v13. Astra20 and +Sol43 reviewed the focused fix. Do not mask the unresolved failure with GC-off or +blocking defaults. Actual root sparse/SVM tests now pass 11,759 checks on each of +Pegasus and Taurus (GA64ea2d8/OpenCLe0a09f4); keep this scoped GPU evidence distinct +from the unresolved dense-only churn reproducer and from whole-backend CI. + +Astra23/Sol65 corrected the stored-values norm assumption after actual Julia1.15 +CI: v13 norms cover logical entries on every Julia version. Negative orders include +implicit zeros, preserving a NaN magnitude via postprocessing the dense stored-buffer +norm. Use dense CPU references for this contract; older SparseArrays ignored those +zeros. This is a semantic correction, not a version-dependent dispatch policy. + +Astra22/Sol61 require native sparse wrapper contract version2, probed from the same +loaded handle as the entry points. Wide CSC symbols alone do not prove native +exception containment: published oneAPI_Support0.11.0 has those symbols but lacks +the catch boundary. Gate sparse wrappers without disabling dense oneAPI execution; +publish the patched0.11.1 artifact and then raise the dependency minimum. The Yggdrasil +recipe retains upstream source5093529 plus a focused native patch, not a topic-branch +source. Metadata generation and source-file equivalence are not an artifact build. + +## Decisions to implement + +### 1. Keep GPUArrays-owned immutable canonical containers + +Keep sparse vector, CSR, CSC and row-major COO, under `AbstractGPUSparseArray <: AbstractSparseArray`. Sorted, unique, one-based coordinates are invariants; stored zeros are valid. Unassembled inputs and unfinished vendor outputs must remain private buffers/descriptors until canonicalization, including internal intermediates and results of qualified vendor APIs. This replaces CUDA's former noncanonical raw-storage contract; raw borrowing constructors must not silently sort caller buffers. Raw constructors check metadata; `check_structure` validates contents explicitly. Do not launch a full structural validation on every operation. + +Canonicalization is a postcondition, not a requirement to run assembly after every vendor +operation. Astra turn13 and Sol turn21 (`sparse-v13-architecture`, `sparse-v13-review`) +agreed to publish completed cuSPARSE SpGEMM output directly: canonical one-based Int32 +inputs, independently allocated output buffers, contribution accumulation, and documented +sorted output establish the invariant. The fast path conservatively requires cuSPARSE +library version >=12.6; this is not a claim about when its guarantee first appeared. +Older versions retain assembly. Check the output count before allocation and initialize +empty output pointers explicitly. Test all available algorithms with repeated contributions, +cancellation, empty products, empty rows, structural validation, aliasing and GC. Do not +extend this argument to other vendor producers without verifying their guarantees. + +CSR/CSC become: + +```julia +struct GPUSparseMatrixCSR{Tv,Ti<:Integer, + Vp<:AbstractVector{<:Integer}, + Vi<:AbstractVector{Ti}, + Vv<:AbstractVector{Tv}} <: AbstractGPUSparseMatrix{Tv,Ti} + rowPtr::Vp + colVal::Vi + nzVal::Vv + dims::Dims{2} +end +``` + +CSC has the analogous fields. Vector and COO keep `{Tv,Ti,Vi,Vv}`. This is an interface sketch; retain checked inner constructors. No descriptor, generation, algorithm, padded capacity or mutable pattern object belongs in these structs. + +`indtype(A)` is the coordinate type; pointer arithmetic uses `eltype(compressed_ptr(A))`. Maintain these conversion rules: + +- Raw compressed constructors infer both integer types from their buffers. +- `similar(A)`, `similar(A,Tv)`, value-type-only constructors such as `GPUSparseMatrixCSR{Tv}(A)`, `copy`, and compressed-to-compressed conversion preserve both widths, where representable. Do not implement `{Tv}` by forwarding to the explicitly uniform `{Tv,Ti}` constructor. +- Explicit `{Tv,Ti}` construction and `similar(A,Tv,Ti)` request uniform `Ti` for coordinates and pointers, with checked narrowing. +- Binary operations promote coordinate types and pointer types independently. Expansion to COO preserves coordinates but necessarily loses pointer-width metadata. With no compressed pointer prototype, a new compressed output uses `promote_type(Int,Ti)` for pointers; this is type-stable and accommodates host-addressable stored counts. Explicit `{Tv,Ti}` still requests uniform widths. Internal algorithms must carry their chosen output pointer type separately through COO intermediates. COO has no compressed-pointer representability limit: its stored count may exceed `typemax(Ti)` when both coordinates fit. Vector constructors likewise must allow `nnz == typemax(Ti)` because they have no terminal pointer. Test such cases with narrow coordinates and wide pointers. +- Keep the conservative requirement that both dimensions fit the coordinate type in v13. This makes transpose and format conversion predictable. A later relaxation for huge implicit major dimensions is possible. +- Output counts, `nnz+1`, size products and allocation byte sizes are checked before narrowing or allocation. Using Int64 without checking overflow is insufficient. For v13, retain the selected output widths and throw a clear representability error rather than silently wrapping; callers can explicitly widen beforehand. +- Pointer-buffer length is a host dimension, not a value that must fit its element type. Compression's logical end-search key must not wrap into a narrow coordinate type. + +Update full-parameter aliases, constructor docs and dispatch examples together. Backends should constrain all buffer families while allowing their applicable memory/storage modes. Warn in migration notes that partially specified storage aliases are UnionAll types: downstream structs should parameterize on `typeof(A)` rather than assuming `CuSparseMatrixCSR{T,Ti}` is concrete. + +### 2. Define storage semantics without pretending to enforce ownership + +| Operation | Contract | +|---|---| +| Raw buffer constructor | Borrows supplied buffers. Caller supplies canonical compatible storage. | +| `copy`, `similar` with copied structure, allocating format constructors | Independent structure/value buffers as documented. | +| Identity `convert` | May return the input. | +| `adapt` | Preserves structure/format, adapts buffers; identity adaptation may share. GPU CSC/vector adapted to ordinary host vectors use SparseArrays types. For mixed-width CSC, SparseArrays requires a common integer type, so promote pointer and coordinate types, possibly allocating an index buffer. Adapting back preserves that unified type; the original width split is not recoverable. CSR/COO may remain generic format containers with host buffers. | +| `nonzeros(A)` updates | Values-only mutation; pattern remains valid. | +| Structural `copyto!`, filtering, sparse-output products/broadcast | Require supported resizable dense buffers and exclusive structural access; invalidate retained structural views/device representations/analysis. | + +Support raw views for reads and values-only operations. Before structural mutation writes anything, validate dimensions, representability and **all** destination storage restrictions. Reject unsupported views centrally; skip resizing buffers already at the required length. For GPU storage these are dense GPU array wrappers represented on the host; ordinary CPU vectors are also resizable. A dense-buffer capability distinction, if needed, belongs in the general dense interface, not a new sparse hook that every backend implements. + +Caller-guaranteed exclusivity remains necessary: buffers may be externally wrapped or shared even when their types look dense. Do not promise strong recovery from allocation failure. Do promise that known unsupported storage, invalid shapes and overflow do not leave a partially modified matrix. Tests must compare all buffers after rejected operations. + +Define component-based `Base.dataids` and appropriate unalias copying. Audit in-place broadcast and overlapping `copyto!` against main's alias handling. Do not indiscriminately implement sparse `unsafe_free!`; preserve existing backend deallocation APIs only with an explicit ownership/alias contract. Internal temporaries can be released when exclusively owned and safe with asynchronous execution. + +Remove the unconditional sparse host `assertscalar` from lookup. Host dense buffers already enforce their scalar-access policy; device buffers can then implement the lookup in kernels. Shared/host-accessible buffers may intentionally permit scalar access. Test implicit and stored zeros, bounds, direct kernel indexing, CSC `nzrange`, and host scalar rejection. Host-backed CSR/COO should remain usable through ordinary CPU array fallbacks; do not silently change format during `adapt(Array, A)` to avoid implementing that behavior. + +Check compatible storage/device placement at host construction boundaries where it can be determined safely. Do not call host runtime queries when reconstructing device-buffer structs during adaptation or inside kernels. Same backend type alone is not proof of the same physical device/context. Add mismatched-buffer tests and use the stack's existing dense/runtime checks rather than introducing a sparse-specific device registry. + +### 3. Make GPUArrays own the operation boundary + +Use this call path: + +```text +Julia public operation: mul!, *, constructor, sparse + -> GPUArrays validation and wrapper normalization + -> GPUArrays selection using operation/type/layout/device/library capabilities + -> portable executor OR an already-selected vendor executor +``` + +Implement a small typed selection interface first for SpMV and SpMM, and apply the same boundary to regroup/conversion, assembly and SpGEMM as their vendor migrations land. Names can follow GPUArrays conventions, but the semantics must be concrete: + +- Automatic selection may choose a supported portable or vendor algorithm. +- Forced portable execution bypasses vendor execution entirely. +- Forced vendor execution reports unsupported combinations before writing the output. +- Capability selection covers element and both index types, operand characters, layout/strides, device and library versions. +- Fall back because a capability is absent, not because an arbitrary vendor exception was caught after partial execution. +- Keep callable generic implementations for integrations and tests, but prevent recursive selection into the same public operation. Document whether subordinate operations may select implementations independently. + +A scoped operation-specific entry point can expose algorithm/workspace choices without changing LinearAlgebra's standard `mul!` API. Do not make backend packages extend version-specific LinearAlgebra internals. Keep the Julia <1.13 forwarding adapters in GPUArrays. + +Place high-level vendor integration in GPUArrays extensions. CUDA's extension should use cuSPARSE/CUDACore package boundaries; AMDGPU and oneAPI extensions can depend on their backend packages and invoke low-level vendor facilities. Remove or redirect old backend public methods that would bypass selection. Aliases and opinionated adaptors may remain backend conveniences, with their precision, format and index defaults preserved explicitly. + +For dense `sparse(A;fmt)` specifically, GPUArrays owns the entrypoint and two scoped +policies: `default_sparse_indtype(A)` and `sparse_format(fmt,A)`. GPUArrays extensions +preserve CUDA/AMD's uniform Int32 defaults and resolve BSR to the existing backend +constructor (including its block-size default). Canonical constructors and coordinate +assembly do not consult this policy. Astra turn14 and Sol turn24 approved this ownership +correction after final shared tests exposed remaining backend public overrides. Keep +the existing AbstractGPUMatrix domain; do not incidentally broaden wrapped-array support. +Shared tests verify the declared policy, while backend tests pin its Int32 default and +BSR functionality independently. Unknown-format rejection must not assume all backends +reject a format they already support. + +Do not invent a universal capability registry or a universal public planning API. The minimum proof is one real migrated vendor executor, forced-portable tests on the same buffers, and adding/changing a generic algorithm without editing that backend. + +### 4. Separate transient scratch from persistent analysis + +Reuse AK workspace facilities for temporary storage when their contracts fit. An AK workspace does not validate sparse pattern reuse. + +Ordinary oneAPI operations need a concrete handle owner during migration, including originating context/queue, retained arrays, exception cleanup and deferred destruction. Disposal must not block under allocator reclaim locks. Pre-release failures retain an open owner; uncertain native-release outcomes quarantine the full resource record without retry. A failed close does not lift borrowed-buffer restrictions. Existing optimized repeated operations need an explicit prepared object, not a fresh handle on every invocation that silently discards optimization. Document the transition from `sparse_optimize_*` entry points. + +For prepared operations, choose and document a pattern lifetime: copy/snapshot the pattern, or borrow it while forbidding structural edits and resize. Holding an array reference does not pin its allocation. Buffer identity, pointer and length checks are useful diagnostics but cannot detect in-place index writes. Specify whether values can be updated in place or rebound. Bind independent concurrent executions to independent workspaces/descriptors unless the vendor explicitly supports sharing. + +The oneAPI implementation borrows a fixed pattern and permits in-place source-value +updates between completed executions, but binds an independent numerical snapshot. +Intel's matrix-handle contract prohibits modification of bound data; hardware also +confirmed stale optimized CSC results after direct value edits. Before execution, +compare source and snapshot bitwise (including signed zero and NaN payloads). +Unchanged values reuse native analysis; changed values create and optimize a new +snapshot before disposing the old owner and installing the replacement. This costs +O(nnz) comparison work, a host-visible decision and snapshot storage; replacement +briefly retains two snapshots/analyses. Pattern sharing is read-only and serialized. +Preparation failure propagates without executing stale data; disposal failure +invalidates the prepared operation and preserves/quarantines complete resources. +This refinement was agreed with Astra in `sparse-v13-architecture` turns 8–10. +See Intel's [matrix-handle contract](https://www.intel.com/content/www/us/en/docs/onemkl/developer-reference-dpcpp/2026-0/sparse-blas-matrix-handle-contract-between-user.html). + +A future portable prepared conversion can retain an index permutation and gather **current** values. A future SpGEMM plan can retain symbolic structure. These can be added without changing sparse array fields. Do not require public APIs for every future plan before v13. + +### 5. Define the supported Julia surface + +Keep the PR's supported operations, subject to the corrections and assembly contract below. Do not claim generic SparseArrays inheritance makes everything work. + +Before removing backend methods, preserve existing user-visible operations: constructors, transfer policies, `findnz`, linear slicing, supported wrappers, vendor products, triangular solves, factorizations/preconditioners and vendor-only formats. Use a per-package checklist; backend migration cannot mean losing previously accelerated solves until GPUArrays implements a portable replacement. + +Add central value-based equality and alias-safe in-place broadcast for the advertised formats. A first implementation can materialize a canonical temporary and use checked `copyto!`; optimize values-only updates afterward. Handle zero-preserving and structure-changing cases explicitly. Never write a dense array through a scalar sparse loop. If nonzero fill or a dense broadcast into a sparse destination is intentionally unsupported, throw a precise error before mutation; don't quietly alter SparseArrays semantics. + +Support common transpose/adjoint metadata, host conversion, reductions, broadcast and range slicing centrally, initially by explicit materialization where necessary. Complete `A[:]` via `vec`/copy with the documented ownership semantics. Equality must compare values rather than demand identical stored-zero patterns. Do not implement `hash` by hashing storage order; either implement array-compatible semantics or document it outside the initial surface. + +Audit inherited paths including `hcat`, `vcat`, `blockdiag`, `UniformScaling`, `Diagonal`, views and mixed sparse/dense operations. Prefer straightforward central implementations where practical. Known unsupported paths must not silently download; give explicit errors and an accurately scoped operation table. Efficient sparse views, arbitrary index vectors, full concatenation and every structured wrapper need not all block v13. + +Retain existing GPU rectangular `A ± λI` / `λI ± A` diagonal behavior across Julia +versions. Julia1.15 SparseArrays newly requires square inputs; this does not remove +the GPU contract. Astra turn15 approved retaining and documenting this difference. +Use an explicit rectangular diagonal reference (including tall, wide and empty cases, +nonunit λ and reversed subtraction); keep square tests against SparseArrays directly. + +Keep these deliberate differences explicit: non-zero-preserving broadcast and dimensional reductions may return dense GPU arrays; stored zeros may be retained; individual sparse insertion is unsupported; reshape currently copies. Avoid a broader claim of complete SparseArrays compatibility. + +Implementation refinement (Astra architecture review, turn12; Sol review, turn16): +`*(α,A,B)` with a canonical sparse/transpose/adjoint left operand and sparse or +dense matrix RHS scales and promotes A before multiplication, equivalent to +`(α*A)*B`. The inherited fused method can instead accumulate Float32 products +before applying a Float64 scalar. This is an explicit numerical compatibility +change, not a consequence of the duplicate-combination contract. It can change +rounding/overflow/underflow and vendor eligibility through promotion; it is not +universally more accurate. Vector RHS, structured wrappers and five-argument +`mul!` are outside this refinement. Tests assert cancellation behavior and +promoted result type; backend comparisons use the same explicit association. + +## Implementation work packages + +Make each completed logical change reviewable and independently testable. Preserve the original author's commits where they still express the resulting design; avoid combining dependency reconciliation, interface changes and algorithm improvements into one commit. + +| Step | Package(s) and change | Completion evidence | +|---|---|---| +| 1 | Reconcile #795/#796/#798 and #745 with current GPUArrays main. Preserve AK dependency/current compat, dense alias and wrapper fixes, and current tests. Target GPUArrays 13.0 and the appropriate new JLArrays version. | Correct merge base; range-diff; dense regression tests; Julia 1.10 resolution plus separate KA 0.9/0.10 environments as supported. | +| 2 | GPUArrays representation and contracts: separate `Vp`, checked integer arithmetic, raw borrowing docs, mutation preflight, `dataids`, device lookup. | Mixed Int64-pointer/Int32-coordinate tests; all four formats; unchanged buffers on rejected mutation; real GPU kernel lookup; host guards. | +| 3 | GPUArrays semantic corrections: symmetric/Hermitian filtering, compression and broadcast sentinels/counts, nightly copy ambiguity, robust norms. | Red/green repros below, nonfinite/complex tests, overflow before allocation/write, current nightly ambiguity test. Keep inherited numerical fixes identified as such. | +| 4 | GPUArrays Julia surface and fallback control: equality, in-place broadcast, common wrappers, linear slicing, explicit unsupported paths. | Scalar indexing disabled; output device/type assertions; aliasing tests; no implicit host result from concatenation/structured paths. | +| 5 | GPUArrays selection seam plus one actual CUDA/cuSPARSE product integration; move high-level selection to extensions. | Same input exercises automatic, forced portable and forced vendor paths; unsupported vendor case cleanly falls back only in automatic mode; no dispatch ambiguities or bypasses. | +| 6 | Remove obsolete sparse sorting workaround/JLArrays shims. Use current stable AK sorting, including a checked lexicographic alternative to packed-key overflow. | Duplicate order and empty cases on Metal, CUDA, OpenCL; no second sort; distinguish assembly combine semantics from reassociation permitted for products. | +| 7 | Backend migrations, each separately reviewable: CUDA family, AMDGPU, oneAPI, Metal; OpenCL integration/testing. | Existing backend sparse suites plus GPUArrays shared suite; preserved vendor-only operations; extension load order; oneAPI handle lifetime/reuse evidence. | +| 8 | Memory/performance and CI release gates; documentation, migration notes, PR text. | Hard-failing reverse CI; reproducible benchmark report; bounded SpGEMM behavior; registered dependency resolution; no outdated v12/OpenCL/Metal claims. | + +Steps 2–4 can be developed alongside the selection prototype once step 1 establishes a trustworthy baseline. The interface is not frozen until the CUDA and oneAPI migration prototypes demonstrate it. AK enhancements below can proceed separately; do not make every optimization a release prerequisite. + +### Backend-specific work in step 7 + +**CUDA / CUDACore / cuSPARSE / cuSOLVER** + +Replace ordinary sparse formats with aliases plus compatible constructors. Update field users (`iPtr` to `nzInd`, `nnz` field to accessor), descriptors, `copyto!`, SpGEMM output mutation, type conversions, adaptation, precompilation and tests. Audit constructors taking explicit `nnz`/spare capacity: never silently treat extra buffer elements as stored entries. Preserve or explicitly deprecate those signatures with valid trimming/copy behavior. + +Retain BSR and batched CSR as separate formats and supply conversions. Port their device representations to GPUArrays-owned facilities or keep narrowly owned backend representations; the PR deletes their old shared device structs too. Preserve cuSOLVER and preconditioner consumers. Test both Int32/Int64 uniform storage and mixed widths with explicit capability rejection where vendor support is narrower. Preserve `cu` policy unless deliberately changed in release notes. + +**AMDGPU / rocSPARSE** + +Migrate formats, accessors, device adaptation, precompile signatures and vendor wrappers. Preserve BSR, solves and factorization functionality. Test on pegasus's AMD GPU, not merely by loading the package. Treat integrated-GPU timings as noisy; correctness and dispatch coverage are still useful. The new generic paths must work for types outside rocSPARSE's supported set. + +**oneAPI / oneMKL** + +Migrate the handle/finalizer/optimization lifecycle before removing the old containers. Update descriptor binding and release behavior, `copyto!`, transpose/conversion overrides and test registrations. Preserve device/context affinity, including when the active context differs at destruction time. Exercise the portable path for element/index types not supported by oneMKL. Test on taurus, including empty matrices, same-length structural changes, changed `nnz`, GC, repeated optimized calls and nondefault tasks/contexts. + +**Metal** + +Rework #982 onto current main. Its old zero-length resize fix is already upstream; do not replay it as a new fix. Keep aliases/constructors and the explicit `mtl` precision/CSR/index policy, updating them for `Vp`. Remove temporary `[sources]` pins when releasing. Current KA 0.10 supports empty launches; retain guards only where still needed for the supported KA 0.9 path. Validate on this M3 Pro and another supported Apple generation in CI when available. + +**OpenCL** + +No vendor sparse layer is required. Test the generic implementation with current AK on pegasus and taurus devices. The investigation passed on PoCL CPU but failed on NVIDIA OpenCL with transfer and kernel-build errors; reproduce/minimize those failures from fresh contexts against main before attributing them to the sparse rewrite. Test both relevant OpenCL compilation routes/devices when available. PoCL CPU execution is valuable portability evidence but must not be reported as a GPU test. Registry/backend compatibility still needs coordination for v13 even when no sparse source changes are needed. + +**Downstream users** + +Implementation audit (Astra turn16): retain deprecated CSR/CSC/COO abstract names as +aliases of the corresponding canonical containers, preserving ordinary dispatch but not +subtyping, introspection or independent-format coverage. Do not fake BSR or device-type +aliases. Keep the removed dense_array_type protocol removed: migrate value-based callers +to allocation from nonzeros(A) followed by explicit copyto!, and require an exemplar or +explicit destination for type-only callers. Document these limits and test the retained +MLDataDevices-style CPU-transfer method body, not merely its import. + +Audit users that subtype removed abstract types versus users that only dispatch on them. Deprecated abstract aliases preserve some dispatch but cannot preserve subtyping. Device-type aliases also have changed type parameters and storage predicates; do not add misleading host/device aliases simply to make imports succeed. Document the new kernel signatures and test representative consumers such as MLDataDevices, BSR/batched users and packages defining their own sparse formats. Version bumps for backend packages should reflect their public compatibility changes, not just GPUArrays' version number. + +## AK and KA roadmap + +Use the existing stable sort, scan, compaction, reduction and workspace facilities first. Keep matrix-specific formats, constructors, output policy and vendor selection in GPUArrays. + +Prioritize these reusable AK additions based on measurements: + +1. Stable radix permutation/key-value sorting, with workspace support, key-width/range handling and a portable merge-sort fallback. Current AK radix sort does not offer these paths. +2. Segmented reduction/scan and reduce-by-key, accepting compressed offsets or sorted keys. Define empty segments, output/count types, associativity, ordering and initialization. Use the ordered associative assembly contract below; do not accidentally promise the current strict sequential fold or permit arbitrary operand permutation. +3. Merge/union primitives useful for sparse-vector broadcast and work partitioning. Demonstrate at least the sparse consumer before generalizing an API. +4. Caller-provided output/count variants for compaction where fixed capacity or graph capture actually needs them. Dynamic exact-size constructors can legitimately synchronize for the resulting length. + +Use KA/KernelInterface's existing subgroup, workgroup and atomic capability interfaces. Verify current released APIs rather than designing from the stale local KA checkout. Add a KA primitive only for a demonstrated missing portable kernel operation. Portable fallbacks must not assume 32-lane warps, independent subgroup progress, Float64 support or universal 64-bit floating atomics. + +Subsequent GPUArrays work can then implement row-adaptive/workgroup SpMV, coalesced tiled SpMM, direct or prepared transposed products, faster regroup, merged sparse-vector broadcast, bounded symbolic/numeric SpGEMM, triangular solves, SDDMM and extra formats. These should not require backend sparse types or fresh allocation hooks. + +Do not freeze bitwise reproducibility across versions/backends. Settle assembly before v13 with this contract: + +> Duplicate values retain their original relative order and are combined using `combine`. Parenthesization is unspecified, so custom combiners must be associative; commutativity is not required. Floating-point addition is supported, but rounding may differ between algorithms, backends and versions. + +This deliberately replaces the PR's strict left-fold promise. SparseArrays documents duplicate combination, while its current sequential implementation provides stronger observable ordering than the public contract. A nonassociative custom combiner is outside the GPU contract even if it happens to work on CPU. Stable sorting plus an order-preserving segmented tree is legal; arbitrary operand permutation is not. Test an exactly associative, noncommutative combiner and cancellation-sensitive floats using justified absolute/error-scale bounds, not only relative tolerance against the CPU fold. Deterministic product algorithms can later be selected explicitly. + +## Validation and release gates + +Use scratch environments and the julia-daemon workflow, with distinct daemon names and `--idle-timeout=2h`. For tests involving module or kernel redefinition, use fresh/no-Revise daemons or restart deliberately. Never resolve into the default environment or change the user's backend checkouts. + +The shared suite must cover each generic format, Int32/Int64 coordinates, mixed pointer widths, supported value types including Bool/complex/Float16, zero dimensions, empty major slices, stored zeros, duplicate assembly, range boundaries, copy/adapt aliasing, values-only mutation, structural growth/shrinkage, rejected storage, wrappers and direct user kernels. Include small narrow-index tests to exercise limits without allocating billions of entries. + +Run three distinct layers and report them separately: + +1. **Generic semantics:** JLArrays on supported Julia releases and current nightly; preserve whole-suite dense tests affected by dispatch. Add sparse ambiguity checks with SparseArrays/LinearAlgebra and after loading each backend extension. +2. **Physical execution:** Metal locally; CUDA and AMDGPU on pegasus; oneAPI on taurus; OpenCL on both. The shared portable suite must also run with vendor execution explicitly disabled. Test actual migrated packages, not only CUDACore. +3. **Vendor/resource parity:** existing backend sparse/solver suites, descriptor lifetimes, context affinity, optimized repeated operations, unsupported capability fallback, extension load order and package resolution. A soft install failure is a release failure. + +Benchmark warm and first-use execution separately, synchronizing correctly. Use diagonal/banded, uniform-random, power-law/skewed, empty-row and duplicate-heavy structures, plus representative SuiteSparse workloads. Include both CSR/CSC, transposes, complex adjoints, several SpMM widths and SpGEMM with heavy candidate expansion. Report allocation and peak temporary memory alongside time. Compare the old backend vendor path, the migrated vendor path and portable execution on the same hardware/input. + +Before v13: + +- Existing accelerated backend functionality remains available and the selector cannot silently choose a known large regression. +- SpGEMM checks candidate-count arithmetic before allocating. Provide a bounded-memory path or an explicit documented resource limit/error; do not let a tiny final result require uncontrolled candidate materialization. +- No known silent wrong result, structural corruption or implicit download remains in the advertised surface. +- Reverse CI actually installs and tests the coordinated revisions. Latest-nightly failures are triaged explicitly, not hidden behind a green allowed-failure job. +- Documentation and PR descriptions target v13, accurately describe synchronization, and distinguish supported operations from deferred work. +- Tag/release ordering is coordinated across GPUArrays/JLArrays and the migrated backend packages. Test unreleased revisions with temporary sources first, then verify registered resolution. Publishing, posting and merging are separate maintainer actions, not part of this investigation. + +## Reproduction snippets for the next session + +On exact PR head with JLArrays and scalar indexing disabled: + +```julia +using GPUArrays, JLArrays, SparseArrays, LinearAlgebra, Adapt +using GPUArrays: GPUSparseMatrixCSC, GPUSparseVector +GPUArrays.allowscalar(false) + +# Excluded symmetric terms must not manufacture NaNs. +A = adapt(JLArray, sparse([1. 2.; 4. 3.])) +Array(Symmetric(A, :U) * JLArray([Inf, 1.])) # [NaN, NaN]; CPU: [Inf, Inf] + +# Legal coordinate collides with broadcast's absent-entry sentinel. +x = GPUSparseVector(JLArray(Int8[127]), JLArray(Float32[2]), 127) +y = x .+ x +(nnz(y), Array(y)[end]) # (0, 0), expected (1, 4) + +# Compression's final search key wraps. +p = GPUArrays.compress(JLArray(Int8[127]), 127) +Array(p)[end] # 1, expected 2 + +# Accepted view storage is corrupted before resize! throws. +p = JLArray([1, 2, 3]); i = JLArray([1, 2]) +v = JLArray([1., 7., 0., 9.]) +A = GPUSparseMatrixCSC(p, i, view(v, 1:2:4), (2, 2)) +try dropzeros!(A) catch err; @show typeof(err); end +(Array(A.colPtr), length(A.rowVal), length(A.nzVal)) # ([1,2,2], 1, 2) + +# Inherited norm instability; compare with norm(nonzeros(A)). +S = sparse([1], [1], [1f30], 1, 1) +(norm(S), norm(adapt(JLArray, S))) # (1f30, Inf) + +# Host fallback must be handled or explicitly rejected. +A = adapt(JLArray, sparse(Float32[1 0; 2 3])) +typeof(hcat(A, A)) # host SparseMatrixCSC +``` + +For device indexing, pass a sparse matrix to a KA kernel that assigns `out[i] = A[i,i]`. The unmodified PR fails compilation on Metal through `assertscalar`. A scratch prototype removing only the outer sparse checks preserved host scalar guards and enabled device lookup. Do not remove dense-buffer guards. + +The exact-PR baseline, source overlay, scripts and detailed logs remain in `/Users/tim/Julia/tmp/gpuarrays745-review`; remote portability scratch is `/home/tim/Julia/tmp/ga745-review` on pegasus. These are evidence, not branches to merge. Start implementation from fresh current revisions and this plan, and rerun every release-relevant claim. diff --git a/Project.toml b/Project.toml index 1bd6c0337..14737a093 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "GPUArrays" uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" -version = "12.1.1" +version = "13.0.0" [workspace] projects = ["lib/GPUArraysCore", "lib/JLArrays", "test", "docs"] @@ -21,14 +21,25 @@ SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" [weakdeps] +AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" +oneAPI = "8f75cd03-7ff8-4ecb-9b8f-daf728133b1b" +CUDACore = "bd0ed864-bdfe-4181-a5ed-ce625a5fdea2" +cuSPARSE = "b26da814-b3bc-49ef-b0ee-c816305aa060" JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" [extensions] +ROCSPARSEExt = "AMDGPU" +ONEMKLExt = "oneAPI" +CUSPARSEExt = ["CUDACore", "cuSPARSE"] JLD2Ext = "JLD2" [compat] +AMDGPU = "3" +oneAPI = "3" AcceleratedKernels = "0.5" Adapt = "4.7.3" +CUDACore = "7" +cuSPARSE = "7" GPUArraysCore = "= 0.2.1" JLD2 = "0.4, 0.5, 0.6" KernelAbstractions = "0.9.43, 0.10" diff --git a/README.md b/README.md index ebd08fbe3..e43cf6b29 100644 --- a/README.md +++ b/README.md @@ -47,52 +47,13 @@ without further methods. See the [interface documentation](https://juliagpu.github.io/GPUArrays.jl/dev/interface/) and `JLArrays` for a complete example. -### Sparse array support (optional) - -`GPUArrays.jl` provides **device-side** array types for `CSC`, `CSR`, `COO`, and `BSR` matrices, as well as sparse vectors. -It also provides abstract types for these layouts that you can create concrete child types of in order to benefit from the -backend-agnostic wrappers. In particular, `GPUArrays.jl` provides out-of-the-box support for broadcasting and `mapreduce` over -GPU sparse arrays. - -For **host-side** types, your custom sparse types should implement: - -- `dense_array_type` - the corresponding dense array type. For example, for a `CuSparseVector` or `CuSparseMatrixCXX`, the `dense_array_type` is `CuArray` -- `sparse_array_type` - the **untyped** sparse array type corresponding to a given parametrized type. A `CuSparseVector{Tv, Ti}` would have a `sparse_array_type` of `CuSparseVector` -- note the lack of type parameters! -- `csc_type(::Type{T})` - the compressed sparse column type for your backend. A `CuSparseMatrixCSR` would have a `csc_type` of `CuSparseMatrixCSC`. -- `csr_type(::Type{T})` - the compressed sparse row type for your backend. A `CuSparseMatrixCSC` would have a `csr_type` of `CuSparseMatrixCSR`. -- `coo_type(::Type{T})` - the coordinate sparse matrix type for your backend. A `CuSparseMatrixCSC` would have a `coo_type` of `CuSparseMatrixCOO`. - -Additionally, you need to teach `GPUArrays.jl` how to translate your backend's specific types onto the device. `GPUArrays.jl` provides the device-side types: - -- `GPUSparseDeviceVector` -- `GPUSparseDeviceMatrixCSC` -- `GPUSparseDeviceMatrixCSR` -- `GPUSparseDeviceMatrixBSR` -- `GPUSparseDeviceMatrixCOO` - -You will need to create a method of `Adapt.adapt_structure` for each format your backend supports. **Note** that if your backend supports separate address spaces, -as CUDA and ROCm do, you need to provide a parameter to these device-side arrays to indicate in which address space the underlying pointers live. An example of adapting -an array to the device-side struct: - -```julia -function GPUArrays.GPUSparseDeviceVector(iPtr::MyDeviceVector{Ti, A}, - nzVal::MyDeviceVector{Tv, A}, - len::Int, - nnz::Ti) where {Ti, Tv, A} - GPUArrays.GPUSparseDeviceVector{Tv, Ti, MyDeviceVector{Ti, A}, MyDeviceVector{Tv, A}, A}(iPtr, nzVal, len, nnz) -end - -function Adapt.adapt_structure(to::MyAdaptor, x::MySparseVector) - return GPUArrays.GPUSparseDeviceVector( - adapt(to, x.iPtr), - adapt(to, x.nzVal), - length(x), x.nnz - ) -end -``` - -You'll also need to inform `GPUArrays.jl` and `GPUCompiler.jl` how to adapt your sparse arrays by extending `KernelAbstractions.jl`'s `get_backend()`: - -```julia -KA.get_backend(::MySparseVector) = MyBackend() -``` +### Sparse array support + +`GPUArrays.jl` implements sparse vectors and CSR, CSC and COO matrices for every back-end +(`GPUSparseVector`, `GPUSparseMatrixCSR`, `GPUSparseMatrixCSC`, `GPUSparseMatrixCOO`). The +types are parametrized on the dense vectors that store their indices and values, so they +work with any back-end's arrays, and the same structs are used inside kernels. A back-end +needs no sparse-specific code; it can add aliases for its storage (e.g. +`const MtlSparseMatrixCSR{Tv,Ti} = GPUSparseMatrixCSR{Tv,Ti,<:MtlVector{<:Integer},<:MtlVector{Ti},<:MtlVector{Tv}}`), +a transfer policy for its adaptor, and vendor bindings used by GPUArrays-owned extensions. See the +[interface documentation](https://juliagpu.github.io/GPUArrays.jl/dev/interface/) for details. diff --git a/REPORT.sparse-v13.md b/REPORT.sparse-v13.md new file mode 100644 index 000000000..3126842ff --- /dev/null +++ b/REPORT.sparse-v13.md @@ -0,0 +1,142 @@ +# GPUArrays sparse arrays: v13 design review + +**Recommendation: keep PR #745's GPUArrays-owned formats, but do not release its current interface unchanged.** The storage-parametric design removes the right backend dependencies. Before switching backends, separate compressed-pointer storage from coordinate storage, make GPUArrays own vendor selection, and finish the ownership, mutation, device-access and Julia-array contracts. These are foundations; implementing every sparse operation or a universal planning framework is not a prerequisite. + +The implementation sequence and release gates are in [PLAN.sparse-v13.md](PLAN.sparse-v13.md). This report records the evidence behind that plan. No production code or remote branches were changed during this investigation. + +## Scope and revisions + +Investigated on 2026-10-21: + +- [GPUArrays #745](https://github.com/JuliaGPU/GPUArrays.jl/pull/745), head `2a50d28cbce47cf1a7fb471ca3d9a33f20236917`, merge base `47056426de4a165e95c400ef7f4a96ed6da5b567` on `tb/permutedims-empty`. +- GPUArrays main `6fa6490`, version 12.1.1. Main and the PR have 53 and 13 commits exclusive to their respective histories. The PR's intended 12.0 release has already been overtaken by the AK release; this work must target 13.0. +- The current PR supersedes its original backend-hook design. The seven inline review comments concern that older version; the rewrite addresses their substantive points. The requested CUDA #3189 `findnz` regression test is present. +- The existing `GPUArrays-sparse/{PLAN,HANDOFF}.md` and Metal sparse notes were read as historical context. Their September test counts, old hardware assumptions and earlier permissions are not evidence or authorization for this session. + +This is a design review and implementation handoff, not a completed migration. CUDA/cuSPARSE, AMDGPU/rocSPARSE and oneAPI vendor integrations have not been migrated or validated here. + +## What is sound + +Owning vector, CSR, CSC and COO containers in GPUArrays is the right direction. Backend-specific structs and format/allocation hooks are unnecessary when component vectors already determine allocation, device adaptation and element storage. The PR's `similar`-based allocation lets a new portable operation work on every participating dense backend. Direct adaptation of immutable containers into kernel arguments works; the Metal representation is isbits. + +Keep the canonical representation: one-based, sorted, unique coordinates, with explicit stored zeros permitted. COO is row-major. Unsorted or duplicate coordinates belong at the assembly boundary. This keeps ordinary operations simple and provides suitable input to vendor routines. A future uncoalesced assembly buffer can be a separate type; it need not weaken existing invariants. + +Keep one open `AbstractGPUSparseArray` root and closed unions for algorithms that specifically understand these four layouts. An arbitrary BSR subtype cannot inherit a CSR traversal algorithm merely by subtyping the root. Document conversions and basic accessors as the extension route. Portable BSR, batched arrays, alternative formats and additional operations can subsequently be added centrally. + +Keep vendor descriptors and analysis state out of the array fields. Keep `nnz(A) == length(nonzeros(A))`: backing allocation capacity is distinct from logical stored length. Preserve structural `adapt` rather than having it silently densify or change CSR into CSC. + +## What must change in the foundation + +### Separate pointer and coordinate storage before aliases spread + +The proposed CSR/CSC representation uses the same `Vi` for both compressed pointers and coordinates. That couples the representation of `nnz+1` to the representation of row/column indices and also prevents different vector wrappers for the two buffers. + +Use `GPUSparseMatrixCSR/CSC{Tv,Ti,Vp,Vi,Vv}`, with `Ti` describing coordinates and `eltype(ptr)` describing offsets. Keep vector and COO at four parameters. Existing compressed inputs preserve their widths; new compressed outputs without a pointer prototype default to `promote_type(Int,Ti)`. This permits Int64 pointers with Int32 coordinates, without adding a pattern object, capacity field or algorithm state. cuSPARSE already distinguishes the two index types in its CSR descriptor, although support must still be checked per operation. [CSR descriptor API](https://docs.nvidia.com/cuda/cusparse/index.html#cusparsecreatecsr). + +This is a deliberate v13 representation change. The PR advertises full type parameters and storage aliases as extension interfaces; postponing it would make a useful, existing representation requirement expensive to add. Exact width preservation and explicit conversion semantics are settled in the plan. + +### Keep selection and vendor integration under GPUArrays maintenance + +The PR recommends more-specific backend methods on public `mul!`, format constructors and assembly. Those methods would intercept future GPUArrays selection and semantic fixes. An `alg` keyword alone cannot fix this: Julia chooses the positional method before processing keywords. + +GPUArrays must own validation, wrapper normalization, selection and dispatch into portable or vendor implementations. Put vendor integration in GPUArrays package extensions, analogous to AK's backend extensions. Backends retain dense storage/runtime facilities, vendor bindings and resource-management primitives. Their sparse aliases must not create a competing selection layer. + +This requires coordinated migration, not simply moving files. CUDA has CUDACore/cuSPARSE/cuSOLVER packages; cuSPARSE's current Julia wrappers still assume its old sparse types. AMDGPU and oneAPI package their vendor APIs differently. Each needs a narrow executable integration seam. New vendor API bindings may still require backend changes; new portable algorithms and selection among existing bindings should not. + +### Specify borrowing and mutation honestly + +The documentation says constructors always return independent storage. Raw constructors actually borrow their buffers, as SparseArrays does. `convert` may return its argument, and `adapt` may preserve buffers. These are useful semantics, but require an operation-by-operation ownership contract. + +Retain immutable structs. Structural mutation is conditional on supported resizable dense buffers (GPU array wrappers represented on the host, or ordinary CPU vectors) and caller-guaranteed exclusive structural access. Raw views remain useful for reads and value updates; unsupported structural updates must be rejected before any destination write. Check every affected buffer first and avoid `resize!` when its length is unchanged. Do not infer ownership from `AbstractGPUVector` membership or from the existence of a `resize!` method. + +Add `dataids`/alias handling based on component buffers. Do not add a generic sparse `unsafe_free!` that blindly frees borrowed buffers. No public structural-generation counter can reliably detect user kernels editing exposed index arrays. Future reusable analysis must snapshot its pattern or use an explicit borrowing/invalidation contract. + +### oneAPI makes resource lifetime an immediate requirement + +oneAPI's sparse structs currently contain a lazy matrix handle, register finalizers, and invalidate it on storage replacement. `sparse_optimize_gemv!` also prepares reusable state. Aliasing those types to GPUArrays containers removes this machinery. + +Migrate ordinary operations to a resource owner with an explicit operation lifetime. Preserve optimized repeated execution through an explicit prepared vendor operation. Keep originating device/context/queue and buffers alive through asynchronous use and safe destruction. Existing oneAPI release code defers queue work out of finalizers; replacing its type must preserve that safety property. A global cache keyed only by pointers, object identity or `nnz` is not correct for mutable exposed indices. + +This requires a working oneAPI lifetime design now, not a universal public sparse-plan API now. + +## Confirmed findings + +Paths below refer to PR head `2a50d28`; line numbers are useful until rebasing. Full repro scripts and logs are under `/Users/tim/Julia/tmp/gpuarrays745-review/`. + +| Priority | Finding and trigger | Evidence / required correction | +|---|---|---| +| P1 | Structural mutation corrupts accepted view-backed input before throwing. | `types.jl:87`, `structure.jl:25–56`: CSC with dense pointer/index buffers and `view(values,1:2:4)` values; `dropzeros!` throws `MethodError(resize!, SubArray)` after pointers become `[1,2,2]` and index/value lengths become 1/2. Reproduced on JLArrays and Metal. Preflight the complete destination before writing. | +| P1 | Symmetric/Hermitian filtering multiplies excluded terms by zero. | `linalg.jl:60–67,76–124`: `Symmetric(sparse([1. 2.;4. 3.]),:U)*[Inf,1.]` gives `[NaN,NaN]`, versus `[Inf,Inf]` on SparseArrays. Reproduced on JLArrays and Metal. Skip excluded terms instead of forming `0*Inf`. | +| P1 | Compression narrows the end-search sentinel into the coordinate type. | `primitives.jl:24–26`: `compress(JLArray(Int8[127]),127)` ends in 1 rather than 2. The constructor accepts dimension 127. Use wide search arithmetic; narrow only representable stored pointers. | +| P1 | Sparse-vector broadcast loses a real maximum-valued index. | `broadcast.jl:311–330,587–590`: vector length 127 with `Int8[127]`, value 2; `x .+ x` has zero stored entries instead of value 4. `typemax(Ti)` is both a legal coordinate and the absent-entry sentinel. Use a separate flag or an impossible position. | +| P2 | The new public sparse indexing method cannot run inside kernels. | `indexing.jl:14,22,28,37`: unconditional host `assertscalar` produces Metal InvalidIR. Existing kernel tests access fields only. Removing the outer check and relying on component-buffer host guards was prototyped successfully. This is a missing device capability, not an established regression from the old bare-bones device types. | +| P2 | Alias detection misses shared structure. | `types.jl`: two CSC containers sharing pointers/indices but different values give `Base.mightalias(A,B) == false`. Expose component `dataids` and cover overlapping inputs and outputs. | +| P2 | Claimed no-host-fallback contract is not enforced. | `hcat(A,A)` on JLArrays-backed CSC returns a host `SparseMatrixCSC`. Explicit download constructors are reached through SparseArrays fallbacks. Implement promised operations centrally; explicitly reject other known host fallbacks until supported. | +| P2 | A recorded nightly ambiguity remains. | `conversions.jl:284` `copyto!(::Array, ::GPUSparseArray)` conflicts with the newer SparseArrays vector method. CI log on Julia 1.14.0-DEV.3366 shows `sparse/ambiguities` failing. Add the appropriate vector specialization and rerun on current nightly. | + +The narrow-index examples are small tests of general arithmetic errors. Int32 has the same sentinel problems at its boundary. Separately, broadcast counts and prefix sums use `Ti` before checking the resulting size (`broadcast.jl:343–352,583–594`); the implementation must check growth before narrowing, including allocation sizes and products. This latter finding is source-traced, not a large-allocation experiment. + +Several important gaps are inherited, rather than introduced by #745: + +- Sparse `norm` uses naive powers: `norm` of one stored Float32 `1f30` returns `Inf`, and `1f-30` returns zero. SparseArrays returns the finite magnitude. Delegate to the robust dense-buffer norm where its semantics agree. Main already has scaled dense norm code. +- `==`, in-place sparse broadcast, and common wrapper operations reach scalar fallbacks. Direct probes of `A == copy(A)`, `A .*= 2`, `sum(transpose(A))` fail with scalar indexing disabled; `nnz(transpose(A))` has no method. These are production surface gaps, not all PR regressions. +- `reshape` copies, unlike Base's shared-storage reshape. The old GPU implementation also copied. Document this existing limitation; do not introduce fragile shared structural storage to mimic CPU aliasing. +- Matrix linear slicing `A[:]` exists in CUDA and must not disappear during migration. Vendor solves, preconditioners, BSR and batched formats need a similar parity audit. + +## Julia and external alignment + +SparseArrays is the behavioral reference for stored zeros, duplicate combination, accessors, copying, `similar`, wrapper mathematics and index validation. It is not a safe source of inherited GPU implementations: many methods on `AbstractSparseVector` or `AbstractSparseMatrix` scalar-loop or download via constructors. GPU CSC cannot inherit both the current GPU root and `AbstractSparseMatrixCSC`; retain the GPU root and implement the required accessor/behavior contracts explicitly. [SparseArrays documentation](https://docs.julialang.org/en/v1/stdlib/SparseArrays/). + +| Reference | Relevant lesson for this design | +|---|---| +| [SciPy CSR](https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.csr_array.html), [CuPy sparse](https://docs.cupy.dev/en/stable/reference/scipy_sparse.html) | Familiar compressed storage and normalization are useful references. Their mutable canonicality flags are unnecessary if GPUArrays requires canonical arrays and confines duplicate handling to assembly. | +| [PyTorch sparse](https://docs.pytorch.org/docs/2.14/sparse.html) | Distinguish uncoalesced COO from canonical compressed storage. GPUArrays can deliberately offer a stricter COO contract without importing tensor batching or autograd conventions into its core matrix types. | +| [JAX sparse / BCOO](https://docs.jax.dev/en/latest/jax.experimental.sparse.html) | Static stored capacity supports tracing and transformations. It is not a reason to burden Julia's ordinary dynamic sparse containers with padding/sentinel semantics. A future capacity-aware type can be separate. | +| [cuSPARSE SpMV](https://docs.nvidia.com/cuda/cusparse/index.html#cusparsespmv) | Preprocessing state and workspace have lifetimes distinct from matrix storage. Pattern reuse permits value changes under documented conditions. Independent prepared operations should own their descriptors. | +| [rocSPARSE generic operations](https://rocm.docs.amd.com/projects/rocSPARSE/en/latest/reference/generic.html) | Buffer sizing, preprocessing and execution are distinct stages; capabilities and determinism vary by algorithm. Do not equate one sparse format with one universally valid vendor path. | +| [Kokkos symbolic SpGEMM](https://kokkos.org/kokkos-kernels/docs/API/sparse/spgemm_symbolic.html) and [numeric SpGEMM](https://kokkos.org/kokkos-kernels/docs/API/sparse/spgemm_numeric.html) | Separate reusable structural analysis from numeric work. Keep this possible without putting vendor handles into the array representation. | +| [oneMKL matmat](https://www.intel.com/content/www/us/en/docs/onemkl/developer-reference-dpcpp/2025-2/oneapi-mkl-sparse-matmat.html) | Its staged API reinforces the same separation. Source inspection of oneAPI.jl, rather than this page alone, establishes the concrete migration requirement. | + +The PR's strict left-fold duplicate-combination promise is stronger than SparseArrays' documented contract and would unnecessarily constrain future parallel assembly. Before v13, specify preserved relative operand order with unspecified parenthesization: custom combiners must be associative, not necessarily commutative; floating-point rounding may vary. This allows ordered segmented reductions while retaining meaningful noncommutative combiners. It explicitly excludes nonassociative custom combiners, and requires cancellation-aware numerical tests. + +The independent research tracks disagreed about adding public plans and algorithm knobs immediately. I agree with Astra's final assessment: establish central selection and implement the concrete oneAPI resource owner now; defer a universal plan API until a portable repeated-use implementation and a vendor implementation establish its contract. A scratch workspace is not a symbolic-analysis cache. + +## Performance foundations + +Rebasing onto main materially changes the prerequisite story. GPUArrays 12 already implements dense sort, scan, compaction and reductions through AK. The PR's JLArrays host shims and claim that OpenCL needs its own implementations are obsolete. AK's stable `sortperm` can remove the second sort used to restore input order. Its current radix implementation does not implement permutation/key-value sorting, so merely requesting radix is not a solution. + +The PR's algorithms are valid starting points, but several costs must remain replaceable: + +- CSR↔CSC uses expansion, packed-key sorting and gathers. CSC SpMV and transposed CSR SpMV repeat this conversion. A reusable permutation must gather current values, not cache stale values. +- SpMV assigns one thread per row; SpMM assigns one thread per output entry. Skewed rows and dense-access layout limit performance. Workgroup/subgroup algorithms can replace these centrally through KA. +- SpGEMM materializes every candidate scalar product before sorting/deduplication. Memory follows candidate count, not final `nnz`. Checked counts and bounded-memory handling are release concerns; an efficient symbolic/numeric algorithm is the subsequent optimization. +- Sparse-vector broadcast currently allocates/work-scans by logical length rather than stored length. A merged structural union belongs in GPUArrays; reusable merge/segmented primitives belong in AK. +- Packed keys impose `m*n < typemax(Int64)` even for extremely sparse matrices. Retain a fast packed path with a checked lexicographic fallback; stable sorting removes the second-sort size limit, not this limit. + +Measured on Apple M3 Pro, Julia 1.13.2, current-main dense code plus unchanged PR sparse sources, AK 0.5.8 / KA 0.10.1: a random 4096×4096 Float32 matrix with 65,777 stored entries took 0.350 ms for CSR SpMV, 1.721 ms for CSC SpMV, 1.563 ms for transposed CSR SpMV, and 1.512 ms for CSR→CSC. These are minima of seven synchronized warm calls after three warmups. They demonstrate conversion cost, not general performance or a controlled comparison with vendor libraries. + +## Validation and limits + +| Configuration | Result | +|---|---| +| Exact PR, JLArrays, Julia 1.13.2 / KA 0.9.45 | 1,724/1,724 existing focused Float32 assertions passed. Additional adversarial probes found the issues above. | +| Exact PR + Metal #982 snapshot `ce1ba502f`, M3 Pro | Basic generic SpMV and isbits adaptation work; nonfinite masking, device indexing and view-mutation failures reproduced. | +| Scratch current-main dense code + PR sparse sources, current local Metal snapshot `bf1f54660`, KA 0.10.1 | 31/31 portable checks passed across CSR/CSC/COO: construction, SpMV/SpMM/SpGEMM, transpose product, broadcast, reduction, filtering, duplicate assembly and structure checks. | +| Same scratch source on pegasus, CUDACore 6.4.2 / KA 0.10.1 | 31/31 portable checks passed using CuArray, without cuSPARSE. This proves generic portability, not CUDA.jl's vendor migration. | +| Same scratch source on pegasus, OpenCL 0.10.12 / PoCL 7.2.2 | 31/31 portable checks passed on the Ryzen 9950X CPU device. This is OpenCL CPU portability evidence, not a GPU result. | +| Same scratch source, OpenCL NVIDIA driver / RTX 5080 | Failed: 2 checks passed, 6 errored before completion. SpMM download reported `CL_INVALID_VALUE`; SpGEMM and broadcast kernels failed to compile. This path is not validated. | +| Scratch device-indexing fix on modern Metal | 15/15 checks passed: device lookup, host-backed arithmetic, permitted shared-memory reads and rejected PrivateStorage host scalar reads. Only the outer sparse `assertscalar` checks were removed. | + +The original JLArrays environment pins KA 0.9 while current Metal requires KA 0.10, so those runs used separate environments. The scratch overlay is **not** a completed rebase or a v13 implementation. Remote backend source checkouts were only read; environments and modified GPUArrays sources are under `~/Julia/tmp/ga745-review`. + +Buildkite build [1958](https://buildkite.com/julialang/gpuarrays-dot-jl/builds/1958) is green because all 16 downstream backend jobs soft-failed at installation with exit 3. No backend test suite passed there. Nightly Ubuntu's ambiguity failure is also hidden by allowed failure; macOS nightly additionally timed out. These checks are not release evidence. + +OpenCL NVIDIA failures are unresolved portability evidence, not attributed PR bugs: the failing session also stopped handling simple dense operations, while a fresh daemon passed dense round-trip, sortperm and accumulate probes. Reproduce the sparse failures from a fresh context and minimize each failure against main before choosing GPUArrays, AK, OpenCL or driver fixes. Detailed logs are `~/Julia/tmp/ga745-review/opencl-nvidia-tests.log` and `opencl-baseline*.log` on pegasus. PoCL required loading `pocl_jll` before the first platform enumeration in a fresh daemon with system ICDs excluded. + +No complete CUDA/cuSPARSE, AMDGPU, oneAPI, cross-context lifetime or full-version test matrix was run. The user's available hosts are pegasus for CUDA/OpenCL/AMDGPU and taurus for oneAPI/CUDA/OpenCL; both SSH connections were verified. The full matrix belongs to the implementation/release gates. + +## Design discussion and evidence locations + +Peer discussions are available with `peer log ga745-astra`, `ga745-history`, `ga745-julia`, `ga745-primitives` and `ga745-frameworks`. The initial Opus launches exited before producing research; retries completed. Findings used here were checked against sources, logs or direct probes. Astra changed its initial preference for public vendor overrides after examining the central-maintenance requirement, and recommended separate pointer storage before v13. Both conclusions are adopted. + +Local scratch root: `/Users/tim/Julia/tmp/gpuarrays745-review`. Useful files: `probes.jl`, `more-probes.jl`, `boundary-probes.jl`, `semantic-smoke.jl`, `focused-tests.jl`, `metal-probes.jl`, `portable-probes.jl`, `kernel-fixed.jl`, `bench.jl`, their logs, `evidence/bk1958.json`, and the nightly logs. The large `metal-probes.log` contains the full InvalidIR diagnostic; search it rather than printing it all. The plan contains self-contained reproduction snippets so the next session is not dependent on these temporary files. Sol's handoff review (`peer log ga745-plan-review`) identified missing mixed-width round-trip and value-only-constructor rules; these are now explicit. diff --git a/docs/src/interface.md b/docs/src/interface.md index 37663494f..5783ace54 100644 --- a/docs/src/interface.md +++ b/docs/src/interface.md @@ -42,6 +42,221 @@ precise: GPUArrays.memory_location ``` +## Sparse arrays + +GPUArrays implements the sparse array formats itself, for every back-end. There are four, +all of them subtypes of `AbstractGPUSparseArray{Tv,Ti,N}` (which subtypes SparseArrays' +`AbstractSparseArray`): + +| type | layout | fields | +|:--|:--|:--| +| `GPUSparseVector{Tv,Ti,Vi,Vv}` | like `SparseVector` | `nzInd`, `nzVal`, `len` | +| `GPUSparseMatrixCSC{Tv,Ti,Vp,Vi,Vv}` | compressed sparse columns, like `SparseMatrixCSC` | `colPtr`, `rowVal`, `nzVal`, `dims` | +| `GPUSparseMatrixCSR{Tv,Ti,Vp,Vi,Vv}` | compressed sparse rows | `rowPtr`, `colVal`, `nzVal`, `dims` | +| `GPUSparseMatrixCOO{Tv,Ti,Vi,Vv}` | coordinates, sorted in row-major order | `rowInd`, `colInd`, `nzVal`, `dims` | + +`Tv` is the value type and `Ti` the coordinate type (`indtype(A)`). Compressed pointers +have their own storage `Vp` and integer type `eltype(compressed_ptr(A))`. `Vi` and `Vv` are the types of the +dense vectors that store the indices and values, typically the back-end's GPU vectors +(`MtlVector{Int32,Metal.PrivateStorage}`, `CuVector{Float32}`, ...). They are part of the +type so that generic code can allocate results with `similar` on the buffers it already +has, and so that methods can dispatch on the storage. A matrix is constructed from its +buffers, with the dimensions last: + +```julia +A = GPUSparseMatrixCSR(rowPtr, colVal, nzVal, (m, n)) +``` + +The constructors only check host metadata: buffer lengths, dimensions fitting the +coordinate type, and compressed `nnz+1` fitting the pointer type. COO/vector entry +counts are not limited by their coordinate type. Like SparseArrays, the +formats are *canonical*: the indices within a row (CSR), column (CSC) or vector are sorted +and unique, and COO entries are sorted by row and then column without duplicates. +Explicitly stored zeros are allowed. Every operation relies on these invariants and +preserves them; `GPUArrays.check_structure(A)` verifies them on the device. + +The structs are immutable, and the same struct is used inside kernels: `adapt` converts +each buffer, so a kernel that receives a `GPUSparseMatrixCSR` can read `A.rowPtr`, +`A.colVal` and `A.nzVal` directly. Operations that change the structure of an existing +array (`copyto!` between sparse arrays, for example) resize its buffers, which is only +safe with exclusive structural access and storage supporting `GPUArrays.supports_resize`. +Raw buffer constructors borrow their inputs. Allocating format constructors and `copy` +return independent buffers; identity `convert` and `adapt` may share. Updating +`nonzeros(A)` preserves the pattern. Structural updates invalidate retained structural +views, device representations and prepared analyses. Known unsupported storage, shapes +and representability errors are rejected before any destination write. Allocation +failure does not promise transactional recovery. + +### What a back-end provides + +Nothing sparse-specific is required. The generic implementation relies on the back-end's +dense support: its vector type, with `similar`, `copyto!` from and to the host, `resize!`, +a KernelAbstractions back-end, and implementations of `sortperm` (at least for `Int64` +keys), `accumulate!` and `findall` that run on the device. + +A back-end will usually add the following, all optional: + +1. **Aliases** for its storage, for naming and dispatch, with constructors on them: + + ```julia + const MtlSparseMatrixCSR{Tv,Ti} = GPUSparseMatrixCSR{Tv,Ti,<:MtlVector{<:Integer},<:MtlVector{Ti},<:MtlVector{Tv}} + MtlSparseMatrixCSR(A::AbstractArray) = GPUSparseMatrixCSR(adapt(MtlArray, A)) + ``` + + Constructors must be defined on the alias itself (including the `{Tv}` and `{Tv,Ti}` + forms): methods of the generic type do not apply to a constrained alias. +2. **A transfer policy** for its opinionated adaptor (`mtl`, `cu`), one method per host + format on `adapt_structure`, e.g. producing CSR matrices with `Int32` indices and + narrowing the values as the dense adaptor does. Without it, those adaptors treat a host + sparse array like any other array. +3. **A dense constructor** that densifies on the device, e.g. + `MtlArray{T,2}(A::MtlSparseMatrixCSR) = copyto!(similar(nonzeros(A), T, size(A)), A)`. + Otherwise the generic dense constructor copies through the host. +4. **Vendor bindings and resource primitives** remain backend responsibilities. + GPUArrays extensions own public-operation validation and selection. They implement + `vendor_spmv`/`vendor_spmm` capability selection and the corresponding selected + executor, rather than overriding public `mul!` methods. `GPUArrays.spmv!` and + `GPUArrays.spmm!` accept `Automatic()`, `Portable()`, and `Vendor()` policies. + Forced portable execution includes subordinate sparse conversions. Forced vendor + execution rejects unsupported combinations before output writes; execution failures + propagate and never trigger a fallback retry. + + The callable `generic_spmv!`, `generic_spmm!`, `generic_regroup` and + `generic_assemble` implementations remain available. GPUArrays retains the + Julia <1.13 forwarding methods for LinearAlgebra internals. +5. **Formats that GPUArrays does not implement** (block-sparse formats, batched matrices) + as back-end structs subtyping `AbstractGPUSparseArray`, with constructors to and from + the generic formats. + +### Allocation and conversion + +Generic code never names a back-end type; it allocates through the buffers it has: + +| need | idiom | +|:--|:--| +| a dense array on the same device and storage | `similar(nonzeros(A), T, dims)` | +| a new index or value buffer | `similar(A.colVal, Ti, n)`, `similar(A.nzVal, Tv, n)` | +| an empty matrix of the same format | `similar(A, Tv, (m, n))` | + +`similar` follows SparseArrays: `similar(A)`, `similar(A, Tv)` and `similar(A, Tv, Ti)` +copy the structure and leave the values uninitialized, `similar(A, Tv, dims)` returns an +array without stored entries (a matrix of the same format for two dimensions, a vector for +one, a dense array for more), and `zero(A)` is an empty array of the same format. Copying, value-only type changes +and compressed-to-compressed conversion preserve both integer widths. Explicit +`{Tv,Ti}` constructors and `similar(A,Tv,Ti)` request uniform `Ti`, with checked +narrowing. Binary operations promote coordinate and pointer widths independently. +Without a compressed-pointer prototype, new compressed pointers use +`promote_type(Int,Ti)`. COO and vectors have no terminal-pointer count restriction. +Adapting split-width CSC to host SparseArrays promotes both index buffers to a common +type; adapting back cannot recover the original split. Storage aliases such as +`CuSparseMatrixCSR{T,Ti}` are UnionAll types: downstream structs should parameterize +on `typeof(A)` rather than assume that alias is concrete. + +Conversions: + +- between formats (on the device), with the target format's constructor: `GPUSparseMatrixCSC(A)`, and + `GPUSparseMatrixCSC{Tv,Ti}(A)` to also change the element and index types; + `convert(GPUSparseMatrixCSC, A)` returns `A` itself if it already has that format; +- to the host, with `SparseMatrixCSC(A)`, `SparseVector(x)` or `Array(A)`; +- from a dense GPU array, dropping its zeros, with `sparse(A; fmt=:csc)` or the format's + constructor (`GPUSparseMatrixCSR(A)`); canonical constructors default to `Int`. + Dense `sparse(A)` preserves CUDA/AMD's existing uniform `Int32` default and BSR option; + this convenience policy is maintained by GPUArrays' extensions, not backend overrides; +- to a dense array on the device, with `copyto!(similar(nonzeros(A), T, size(A)), A)`; +- transposes: `transpose(A)` and `adjoint(A)` stay lazy, `copy(transpose(A))` and + `permutedims(A)` materialize them in the same format, and a format constructor + (`GPUSparseMatrixCSC(transpose(A))`) in another one; +- between host and device, with `adapt`. `adapt(MtlArray, S)` and `adapt(Array, A)` move + the storage and keep the format, element and index type (a CSC matrix stays CSC, and + comes back as a `SparseMatrixCSC`); adapting never densifies. `adapt(T, S)` with a + sparse type `T` calls the constructor `T(S)`. + +The generic types cannot be constructed from host arrays, because they do not determine +where to store the result: use a back-end alias, `adapt`, or the back-end's adaptor. + +### Supported surface and limits + +The canonical formats support construction, transfer, conversions, products, reductions, +range slicing, stored-value updates, value equality and sparse in-place broadcast. +Stored zeros are permitted. Non-zero-preserving allocating broadcast and dimensional +reductions may return dense device arrays. Dense/non-zero-preserving broadcast into a +sparse destination and nonzero fill throw before mutation. Individual insertion, +concatenation and blockdiag are not implemented; reshape and linearization copy. +Broadcasting may expand dense vectors/matrices, but expansion of sparse operands, +mixing sparse vectors with sparse matrices, and zero-dimensional GPU operands are +explicitly rejected before scalar evaluation or mutation. Higher-dimensional +broadcast results also reject instead of falling back to CPU execution. +Arbitrary sparse views and complete SparseArrays compatibility are not promised. +Norms cover logical entries: negative orders include implicit zeros while preserving +NaN magnitudes, independently of the installed SparseArrays version. +Diagonal extraction follows the dense-array convention of returning an empty device +vector for offsets outside the matrix, including extreme signed offsets. +`A ± λI` and `λI ± A` remain supported for rectangular GPU sparse matrices: the +identity contributes through `min(size(A)...)`. This preserves existing CUDA/AMD +functionality independently of Julia version; newer SparseArrays restricts these +operations to square matrices. Square results follow SparseArrays. + +For `*(α, A, B)` with a canonical sparse matrix (or its transpose/adjoint) on +the left, GPUArrays scales `A` with normal element-type promotion before the +matrix product, including dense GPU right-hand sides. This is equivalent to +`(α*A)*B`; Julia's fused CPU method may instead accumulate in the input type +before scaling. The evaluation order can change rounding, overflow and underflow. + +Assembly preserves duplicate operand order with unspecified parenthesization. Custom +combiners must be associative; commutativity is not required. Floating-point rounding +can vary across algorithms, backends and releases. Stable sorting uses checked packed +keys with a lexicographic fallback. Generic ESC SpGEMM checks candidate counts and has +a default 1 GiB temporary-memory estimate limit, overridable through +`generic_spgemm(A,B; max_temp_bytes=...)`; its memory follows candidate products. + +### Migrating from backend-owned formats + +Backends now alias the canonical formats above. Compressed types gained a pointer-buffer +parameter independent of coordinate storage; parameterize downstream fields by `typeof(A)` +instead of assuming `CuSparseMatrixCSR{Tv,Ti}` is concrete. Use accessors such as `nnz(A)` +and `nonzeros(A)`, not removed count fields. The vector index field is `nzInd`. + +`AbstractGPUSparseMatrixCSR`, `AbstractGPUSparseMatrixCSC` and +`AbstractGPUSparseMatrixCOO` are deprecated names for the corresponding canonical +containers. Ordinary dispatch, including `{Tv,Ti}`, continues to work, but these are no +longer abstract extension points. They do not preserve subtyping, abstract-type introspection +or coverage of independently implemented formats. Such formats should subtype the open +dimensional `AbstractGPUSparseArray` root and implement their supported operations. There +is no replacement alias for the removed BSR abstract or separate device sparse structs. +Kernel signatures should use the canonical format with device-buffer parameters left free. + +The type-level `dense_array_type` protocol is removed. Allocate from an existing value +buffer and populate the destination explicitly, preserving its device and storage policy: + +```julia +dense = similar(nonzeros(A), T, size(A)) +copyto!(dense, A) +``` + +Allocation alone does not convert or initialize values. For a dense CPU result, use +`Array(A)`; for a host sparse result, use `SparseMatrixCSC(A)` or `SparseVector(A)`. +Type-only callers need a storage exemplar or an explicit destination type. A downstream +compatibility release may still be required for removed helpers, fields and type layouts, +even when its imports and dispatch work through the deprecated aliases. + +### Reference + +```@docs +GPUArrays.AbstractGPUSparseArray +GPUArrays.GPUSparseMatrixCSR +GPUArrays.GPUSparseMatrixCSC +GPUArrays.GPUSparseMatrixCOO +GPUArrays.GPUSparseVector +GPUArrays.check_structure +GPUArrays.generic_regroup +GPUArrays.generic_expand +GPUArrays.generic_assemble +GPUArrays.generic_spmv! +GPUArrays.generic_spmm! +GPUArrays.generic_spgemm +GPUArrays.generic_spgemm! +``` + ## Caching Allocator ```@docs diff --git a/ext/CUSPARSEExt.jl b/ext/CUSPARSEExt.jl new file mode 100644 index 000000000..de5f3ad14 --- /dev/null +++ b/ext/CUSPARSEExt.jl @@ -0,0 +1,184 @@ +module CUSPARSEExt + +using GPUArrays, CUDACore, cuSPARSE, LinearAlgebra, SparseArrays +using GPUArrays: UnsupportedSparseVendor +using cuSPARSE: CuCanonicalMatrix, CuCanonicalSparse +GPUArrays.product_buffers(A::cuSPARSE.CuSparseMatrixBSR) = (A.rowPtr,A.colVal,A.nzVal) +GPUArrays.default_sparse_indtype(::CuMatrix) = Cint +GPUArrays.sparse_format(fmt::Symbol,A::CuMatrix) = fmt===:bsr ? cuSPARSE.CuSparseMatrixBSR : + invoke(GPUArrays.sparse_format,Tuple{Symbol,GPUArrays.AbstractGPUMatrix},fmt,A) + +struct CuSparseSpMV end +struct CuSparseSpMM end +struct CuSparseSpGEMM end + +function GPUArrays.normalize_sparse_product_operand(A::cuSPARSE.CuSparseMatrixBSR,t,algorithm) + t in ('S','s','H','h') || return A,t + algorithm isa GPUArrays.Portable && + throw(ArgumentError("portable conversion of BSR storage is not implemented")) + # Vendor-only BSR conversion establishes canonical storage before GPUArrays + # materializes the wrapper and selects the numerical product algorithm. + canonical=cuSPARSE.CuSparseMatrixCSR(A) + return GPUArrays.materialize(LinearAlgebra.wrap(canonical,t)),'N' +end + +function GPUArrays.check_storage_compatibility(arrays::CuArray...) + ctx = CUDACore.context(first(arrays)) + all(A -> CUDACore.context(A) == ctx, arrays) || + throw(ArgumentError("CUDA sparse operands must belong to the same context")) + return nothing +end + +const VendorFloat = Union{Float16,ComplexF16,Float32,Float64,ComplexF32,ComplexF64} +const VendorNumeric = Union{Integer,AbstractFloat,Complex{<:AbstractFloat}} + +function uniform_vendor_indices(A) + Ti = SparseArrays.indtype(A) + Ti in (Int32,Int64) || return false + A isa Union{cuSPARSE.CuSparseMatrixCSR,cuSPARSE.CuSparseMatrixCSC} || return true + return eltype(GPUArrays.compressed_ptr(A)) === Ti +end + +function product_capability(C,A,B) + T = eltype(C) + T <: VendorFloat || return UnsupportedSparseVendor("cuSPARSE products require a supported floating output type") + eltype(A) <: VendorNumeric && eltype(B) <: VendorNumeric || + return UnsupportedSparseVendor("cuSPARSE cannot normalize these operand value types") + T <: Real && (eltype(A) <: Complex || eltype(B) <: Complex) && + return UnsupportedSparseVendor("cuSPARSE does not convert complex operands to real output") + GPUArrays.product_accumulator(eltype(A),eltype(B)) === GPUArrays.product_accumulator(T,T) || + return UnsupportedSparseVendor("normalizing to this output type would change sparse accumulation precision") + if T in (Float16,ComplexF16) + all(S -> S===T || S===Bool || (T===ComplexF16 && S===Float16),(eltype(A),eltype(B))) || + return UnsupportedSparseVendor("normalizing operands to half precision would lose input values") + end + cuSPARSE.functional() || return UnsupportedSparseVendor("cuSPARSE is unavailable") + T === Float16 && cuSPARSE.version() < v"11.4" && + return UnsupportedSparseVendor("Float16 products require cuSPARSE 11.4") + T === ComplexF16 && cuSPARSE.version() < v"11.7.2" && + return UnsupportedSparseVendor("ComplexF16 products require cuSPARSE 11.7.2") + return nothing +end + +function GPUArrays.vendor_spmv(y::CuVector,tA,A::CuCanonicalMatrix,x::CuVector,α,β) + reason = product_capability(y,A,x) + reason === nothing || return reason + uniform_vendor_indices(A) || return UnsupportedSparseVendor("this cuSPARSE executor requires uniform Int32 or Int64 indices") + tA in ('N','T','C') || return UnsupportedSparseVendor("this cuSPARSE executor does not materialize symmetric wrappers") + stride(x,1) == stride(y,1) == 1 || return UnsupportedSparseVendor("cuSPARSE vector descriptors require unit strides") + A isa cuSPARSE.CuSparseMatrixCSC && tA == 'C' && eltype(A) <: Complex && cuSPARSE.version() < v"12" && + return UnsupportedSparseVendor("complex CSC adjoint requires cuSPARSE 12") + # Floating mixed precision is not universally supported by cuSPARSE SpMV. Integer + # and Bool operands retain upstream's explicit normalization to the output type. + return CuSparseSpMV() +end + +function GPUArrays.execute_spmv!(::CuSparseSpMV,y,tA,A,x,α,β) + T = eltype(y) + A = vendor_values(A,T) + x = eltype(x) === T ? x : convert(CuVector{T},x) + cuSPARSE.mv!(Char(tA),α,A,x,β,y,'O',cuSPARSE.CUSPARSE_SPMV_ALG_DEFAULT) +end + +function spmm_capability(C,tA,tB,A,B) + reason = product_capability(C,A,B) + reason === nothing || return reason + S = A isa CuCanonicalMatrix ? A : B + uniform_vendor_indices(S) || return UnsupportedSparseVendor("this cuSPARSE executor requires uniform Int32 or Int64 indices") + tA in ('N','T','C') && tB in ('N','T','C') || + return UnsupportedSparseVendor("this cuSPARSE executor does not materialize symmetric wrappers") + D = A isa CuCanonicalMatrix ? B : A + sparse_trans = A isa CuCanonicalMatrix ? tA : tB + dense_trans = A isa CuCanonicalMatrix ? tB : tA + if eltype(C)<:Union{Float16,ComplexF16} + # Dense-times-sparse is executed as a transposed sparse-times-dense product. + # Half storage uses a wider accumulator with stricter vendor layout support. + supported = A isa CuCanonicalMatrix ? + !(S isa cuSPARSE.CuSparseMatrixCSC) && sparse_trans=='N' : + S isa cuSPARSE.CuSparseMatrixCSC ? sparse_trans=='N' : + S isa cuSPARSE.CuSparseMatrixCOO && sparse_trans!='N' + supported || return UnsupportedSparseVendor("cuSPARSE half-precision SpMM does not support this sparse layout or operation") + eltype(C)<:Complex && dense_trans=='C' && + return UnsupportedSparseVendor("cuSPARSE half-precision SpMM does not support dense conjugate transpose") + end + S isa cuSPARSE.CuSparseMatrixCSC && sparse_trans=='C' && eltype(S)<:Complex && cuSPARSE.version()= max(1,size(D,1)) && + stride(C,2) >= max(1,size(C,1)) || + return UnsupportedSparseVendor("cuSPARSE matrix descriptors require column-major dense storage") + return CuSparseSpMM() +end +GPUArrays.vendor_spmm(C::CuMatrix,tA,tB,A::CuCanonicalMatrix,B::CuMatrix,α,β) = + spmm_capability(C,tA,tB,A,B) +GPUArrays.vendor_spmm(C::CuMatrix,tA,tB,A::CuMatrix,B::CuCanonicalMatrix,α,β) = + spmm_capability(C,tA,tB,A,B) + +function GPUArrays.execute_spmm!(::CuSparseSpMM,C,tA,tB,A,B,α,β) + T = eltype(C) + normalize(X::Union{CuCanonicalMatrix,cuSPARSE.CuSparseMatrixBSR}) = vendor_values(X,T) + normalize(X::CuMatrix) = eltype(X) === T ? X : convert(CuMatrix{T},X) + cuSPARSE.mm!(Char(tA),Char(tB),α,normalize(A),normalize(B),β,C,'O',cuSPARSE.CUSPARSE_SPMM_ALG_DEFAULT) +end + +function GPUArrays.vendor_spgemm(A::CuCanonicalMatrix,B::CuCanonicalMatrix) + eltype(A) === eltype(B) && eltype(A) <: LinearAlgebra.BlasFloat || + return UnsupportedSparseVendor("cuSPARSE SpGEMM requires matching BLAS floating value types") + uniform_vendor_indices(A) && uniform_vendor_indices(B) && + SparseArrays.indtype(A) === Int32 && SparseArrays.indtype(B) === Int32 || + return UnsupportedSparseVendor("this cuSPARSE SpGEMM executor requires uniform Int32 storage") + max(nnz(A),nnz(B)) < typemax(Int32) || + return UnsupportedSparseVendor("this cuSPARSE SpGEMM executor requires Int32-representable stored counts") + cuSPARSE.functional() && cuSPARSE.version() >= v"11" || + return UnsupportedSparseVendor("cuSPARSE SpGEMM requires cuSPARSE 11") + return CuSparseSpGEMM() +end +function GPUArrays.execute_spgemm(::CuSparseSpGEMM,A,B;max_temp_bytes) + # Regroup explicitly here; the selected execution never calls public multiplication. + Acsr = A isa cuSPARSE.CuSparseMatrixCSR ? A : + A isa cuSPARSE.CuSparseMatrixCSC ? GPUArrays.generic_regroup(A) : GPUArrays.generic_compress(A) + Bcsr = B isa cuSPARSE.CuSparseMatrixCSR ? B : + B isa cuSPARSE.CuSparseMatrixCSC ? GPUArrays.generic_regroup(B) : GPUArrays.generic_compress(B) + # COO has no pointer prototype: portable compression chooses Int pointers. The + # selected vendor ABI requires Int32, already proved representable above. + A isa cuSPARSE.CuSparseMatrixCOO && (Acsr = GPUArrays.with_eltypes(Acsr,eltype(A),Int32)) + B isa cuSPARSE.CuSparseMatrixCOO && (Bcsr = GPUArrays.with_eltypes(Bcsr,eltype(B),Int32)) + C = cuSPARSE.gemm('N','N',one(eltype(A)),Acsr,Bcsr,'O',cuSPARSE.CUSPARSE_SPGEMM_DEFAULT) + convert(GPUArrays.matrix_format(A),C) +end + +function GPUArrays.vendor_spmv(y::CuVector,tA,A::cuSPARSE.CuSparseMatrixBSR,x::CuVector,α,β) + reason = product_capability(y,A,x) + reason === nothing || return reason + tA in ('N','T','C') && SparseArrays.indtype(A) === Int32 && + eltype(A.rowPtr) === Int32 && cuSPARSE.version() >= v"12.6.3" || + return UnsupportedSparseVendor("BSR products require Int32 storage, ordinary operands and cuSPARSE 12.6.3") + return CuSparseSpMV() +end +function GPUArrays.vendor_spmm(C::CuMatrix,tA,tB,A::cuSPARSE.CuSparseMatrixBSR,B::CuMatrix,α,β) + reason = product_capability(C,A,B) + reason === nothing || return reason + tA in ('N','T','C') && tB in ('N','T','C') && SparseArrays.indtype(A) === Int32 && + eltype(A.rowPtr) === Int32 && cuSPARSE.version() >= v"12.6.3" || + return UnsupportedSparseVendor("BSR products require Int32 storage, ordinary operands and cuSPARSE 12.6.3") + return CuSparseSpMM() +end + +vendor_values(A::CuCanonicalMatrix,T) = eltype(A) === T ? A : GPUArrays.with_values(A,T) +vendor_values(A::cuSPARSE.CuSparseMatrixBSR,T) = eltype(A) === T ? A : cuSPARSE.CuSparseMatrixBSR{T}(A) + +# Backend-only formats normalize once, then use centrally owned arithmetic. +for op in (:+,:-) + @eval begin + Base.$op(A::cuSPARSE.CuSparseMatrixBSR,B::GPUArrays.GPUSparseMatrix)=Base.$op(cuSPARSE.CuSparseMatrixCSR(A),B) + Base.$op(A::GPUArrays.GPUSparseMatrix,B::cuSPARSE.CuSparseMatrixBSR)=Base.$op(A,cuSPARSE.CuSparseMatrixCSR(B)) + Base.$op(A::cuSPARSE.CuSparseMatrixBSR,B::cuSPARSE.CuSparseMatrixBSR)=Base.$op(cuSPARSE.CuSparseMatrixCSR(A),cuSPARSE.CuSparseMatrixCSR(B)) + end +end + +include("CUSPARSEExt/solves.jl") + +end diff --git a/ext/CUSPARSEExt/solves.jl b/ext/CUSPARSEExt/solves.jl new file mode 100644 index 000000000..7a0fe41a5 --- /dev/null +++ b/ext/CUSPARSEExt/solves.jl @@ -0,0 +1,73 @@ +# Vendor triangular solves remain accelerated after replacing the ordinary containers. +# BSR identities preserve block layout, including padded final blocks. +function bsr_identity(unit,A::CuSparseMatrixBSR{<:Any,Ti}) where {Ti} + n=LinearAlgebra.checksquare(A) + b=Int(A.blockDim); mb=cld(n,b) + GPUArrays.check_sparse_nnz(Ti,mb) + p=GPUArrays.sparse_similar(A.rowPtr,Ti,Base.checked_add(mb,1)); p .= 1:mb+1 + i=GPUArrays.sparse_similar(A.colVal,Ti,mb); i .= 1:mb + values=fill!(GPUArrays.sparse_similar(A.nzVal,typeof(unit),Base.checked_mul(mb,Base.checked_mul(b,b))),zero(unit)) + diagonal=view(reshape(values,b*b,mb),1:b+1:b*b,:) + diagonal .= unit + pad=mb*b-n + pad>0 && (diagonal[end-pad+1:end,end] .= zero(unit)) + CuSparseMatrixBSR{typeof(unit),Ti}(p,i,values,size(A),b,A.dir,mb) +end +Base.one(A::CuSparseMatrixBSR)=bsr_identity(one(eltype(A)),A) +Base.oneunit(A::CuSparseMatrixBSR)=bsr_identity(oneunit(eltype(A)),A) +using LinearAlgebra: BlasFloat, AdjOrTrans +using cuSPARSE: CuSparseMatrixCSR,CuSparseMatrixCSC,CuSparseMatrixCOO,CuSparseMatrixBSR, + sv!,sm!,sv2!,sm2! + +# LinearAlgebra inspects triangular wrappers before reaching the solve executor. +# Vendor-only storage must not fall through its host scalar traversal. +LinearAlgebra.istriu(A::CuSparseMatrixBSR,k::Integer=0) = istriu(CuSparseMatrixCSR(A),k) +LinearAlgebra.istril(A::CuSparseMatrixBSR,k::Integer=0) = istril(CuSparseMatrixCSR(A),k) + +# triangular +for SparseMatrixType in (:CuSparseMatrixBSR,) + @eval begin + LinearAlgebra.generic_trimatdiv!(C::DenseCuVector{T}, uploc, isunitc, tfun::Function, A::$SparseMatrixType{T}, B::DenseCuVector{T}) where {T<:BlasFloat} = + sv2!(tfun === identity ? 'N' : tfun === transpose ? 'T' : 'C', uploc, isunitc, one(T), A, C === B ? C : copyto!(C, B), 'O') + LinearAlgebra.generic_trimatdiv!(C::DenseCuMatrix{T}, uploc, isunitc, tfun::Function, A::$SparseMatrixType{T}, B::DenseCuMatrix{T}) where {T<:BlasFloat} = + sm2!(tfun === identity ? 'N' : tfun === transpose ? 'T' : 'C', 'N', uploc, isunitc, one(T), A, C === B ? C : copyto!(C, B), 'O') + function LinearAlgebra.generic_trimatdiv!(C::DenseCuMatrix{T}, uploc, isunitc, tfun::Function, A::$SparseMatrixType{T}, B::AdjOrTrans{T,<:DenseCuMatrix{T}}) where {T<:BlasFloat} + transb = LinearAlgebra.wrapper_char(B) + (transb == 'C') && (T <: Complex) && throw(ErrorException("This operation is not supported by the current CUDA version.")) + C !== parent(B) && copyto!(C, B) + sm2!(tfun === identity ? 'N' : tfun === transpose ? 'T' : 'C', 'N', uploc, isunitc, one(T), A, C, 'O') + end + function LinearAlgebra.generic_trimatdiv!(C::Transpose{T,<:DenseCuMatrix{T}}, uploc, isunitc, tfun::Function, A::$SparseMatrixType{T}, B::Transpose{T,<:DenseCuMatrix{T}}) where {T<:BlasFloat} + B === C || throw(ErrorException("This operation is only supported if B and C are identical.")) + sm2!(tfun === identity ? 'N' : tfun === transpose ? 'T' : 'C', 'T', uploc, isunitc, one(T), A, parent(B), 'O') + end + function LinearAlgebra.generic_trimatdiv!(C::Adjoint{T,<:DenseCuMatrix{T}}, uploc, isunitc, tfun::Function, A::$SparseMatrixType{T}, B::Adjoint{T,<:DenseCuMatrix{T}}) where {T<:BlasFloat} + B === C || throw(ErrorException("This operation is only supported if B and C are identical.")) + sm2!(tfun === identity ? 'N' : tfun === transpose ? 'T' : 'C', 'C', uploc, isunitc, one(T), A, parent(B), 'O') + end + end +end # SparseMatrixType loop + +for SparseMatrixType in (:CuSparseMatrixCOO, :CuSparseMatrixCSR, :CuSparseMatrixCSC) + @eval begin + function LinearAlgebra.generic_trimatdiv!(C::DenseCuVector{T}, uploc, isunitc, tfun::Function, A::$SparseMatrixType{T}, B::DenseCuVector{T}) where {T<:BlasFloat} + sv!(tfun === identity ? 'N' : tfun === transpose ? 'T' : 'C', uploc, isunitc, one(T), A, B, C, 'O') + end + function LinearAlgebra.generic_trimatdiv!(C::DenseCuMatrix{T}, uploc, isunitc, tfun::Function, A::$SparseMatrixType{T}, B::DenseCuMatrix{T}) where {T<:BlasFloat} + sm!(tfun === identity ? 'N' : tfun === transpose ? 'T' : 'C', 'N', uploc, isunitc, one(T), A, B, C, 'O') + end + function LinearAlgebra.generic_trimatdiv!(C::DenseCuMatrix{T}, uploc, isunitc, tfun::Function, A::$SparseMatrixType{T}, B::AdjOrTrans{T,<:DenseCuMatrix{T}}) where {T<:BlasFloat} + transb = LinearAlgebra.wrapper_char(B) + (transb == 'C') && (T <: Complex) && throw(ErrorException("This operation is not supported by the current CUDA version.")) + sm!(tfun === identity ? 'N' : tfun === transpose ? 'T' : 'C', transb, uploc, isunitc, one(T), A, parent(B), C, 'O') + end + function LinearAlgebra.generic_trimatdiv!(C::Transpose{T,<:DenseCuMatrix{T}}, uploc, isunitc, tfun::Function, A::$SparseMatrixType{T}, B::Transpose{T,<:DenseCuMatrix{T}}) where {T<:BlasFloat} + (B !== C) && throw(ErrorException("This operation is only supported if B and C are identical.")) + sm!(tfun === identity ? 'N' : tfun === transpose ? 'T' : 'C', 'T', uploc, isunitc, one(T), A, parent(B), parent(C), 'O') + end + function LinearAlgebra.generic_trimatdiv!(C::Adjoint{T,<:DenseCuMatrix{T}}, uploc, isunitc, tfun::Function, A::$SparseMatrixType{T}, B::Adjoint{T,<:DenseCuMatrix{T}}) where {T<:BlasFloat} + (B !== C) && throw(ErrorException("This operation is only supported if B and C are identical.")) + sm!(tfun === identity ? 'N' : tfun === transpose ? 'T' : 'C', 'C', uploc, isunitc, one(T), A, parent(B), parent(C), 'O') + end + end +end # SparseMatrixType loop diff --git a/ext/ONEMKLExt.jl b/ext/ONEMKLExt.jl new file mode 100644 index 000000000..fd120ce20 --- /dev/null +++ b/ext/ONEMKLExt.jl @@ -0,0 +1,61 @@ +module ONEMKLExt +using GPUArrays,oneAPI,LinearAlgebra,SparseArrays +using GPUArrays: UnsupportedSparseVendor + +@static if Sys.islinux() +using oneAPI.oneMKL: oneAbstractSparseMatrix,oneSparseMatrixCSR +const MKL=oneAPI.oneMKL +struct OneSparseSpMV end +struct OneSparseSpMM end + +function GPUArrays.check_storage_compatibility(arrays::oneArray...) + first_array=first(arrays) + all(A -> oneAPI.context(A)==oneAPI.context(first_array) && + oneAPI.device(A)==oneAPI.device(first_array),arrays) || + throw(ArgumentError("oneAPI sparse buffers must share a context and device")) + nothing +end + +function capability(C,A,B) + eltype(A)===eltype(B)===eltype(C) && eltype(A)<:MKL.onemklFloat || + return UnsupportedSparseVendor("oneMKL sparse requires matching BLAS floating values") + Ti=SparseArrays.indtype(A) + Ti in (Int32,Int64) && eltype(first(GPUArrays.sparse_buffers(A)))===Ti || + return UnsupportedSparseVendor("oneMKL sparse requires uniform Int32 or Int64 indices") + oneAPI.functional() && oneAPI.Support._sparse_abi_ok[] || + return UnsupportedSparseVendor("oneMKL sparse support ABI is unavailable") + oneAPI.context()==oneAPI.context(A.nzVal) && oneAPI.device()==oneAPI.device(A.nzVal) || + return UnsupportedSparseVendor("oneMKL sparse execution requires the matrix context and device to be current") + for X in (B,C) + stride(X,1)==1 && (ndims(X)==1 || stride(X,2)>=max(1,size(X,1))) || + return UnsupportedSparseVendor("oneMKL sparse requires contiguous vectors and column-major matrices") + end + nothing +end +function GPUArrays.vendor_spmv(y::oneVector,trans,A::oneAbstractSparseMatrix,x::oneVector,alpha,beta) + reason=capability(y,A,x) + reason===nothing || return reason + trans in ('N','T','C') || return UnsupportedSparseVendor("oneMKL executor requires ordinary operand characters") + OneSparseSpMV() +end +GPUArrays.execute_spmv!(::OneSparseSpMV,y,trans,A,x,alpha,beta) = + MKL.sparse_gemv!(Char(trans),alpha,A,x,beta,y) +function GPUArrays.vendor_spmm(C::oneMatrix,trans,transB,A::oneAbstractSparseMatrix,B::oneMatrix,alpha,beta) + reason=capability(C,A,B) + reason===nothing || return reason + A isa MKL.oneSparseMatrixCOO && return UnsupportedSparseVendor("oneMKL SpMM requires CSR or CSC") + trans in ('N','T','C') && transB in ('N','T','C') || + return UnsupportedSparseVendor("oneMKL executor requires ordinary operand characters") + transB=='N' || return UnsupportedSparseVendor("oneMKL SpMM requires a nontransposed dense operand") + OneSparseSpMM() +end +GPUArrays.execute_spmm!(::OneSparseSpMM,C,trans,transB,A,B,alpha,beta) = + MKL.sparse_gemm!(Char(trans),Char(transB),alpha,A,B,beta,C) + +using LinearAlgebra: BlasFloat +LinearAlgebra.generic_trimatdiv!(C::oneVector{T},uplo,diag,f::Function,A::oneSparseMatrixCSR{T},B::oneVector{T}) where {T<:BlasFloat} = + MKL.sparse_trsv!(uplo,f===identity ? 'N' : f===transpose ? 'T' : 'C',diag,one(T),A,B,C) +LinearAlgebra.generic_trimatdiv!(C::oneMatrix{T},uplo,diag,f::Function,A::oneSparseMatrixCSR{T},B::oneMatrix{T}) where {T<:BlasFloat} = + MKL.sparse_trsm!(uplo,f===identity ? 'N' : f===transpose ? 'T' : 'C','N',diag,one(T),A,B,C) +end +end diff --git a/ext/ROCSPARSEExt.jl b/ext/ROCSPARSEExt.jl new file mode 100644 index 000000000..6ccfb7d66 --- /dev/null +++ b/ext/ROCSPARSEExt.jl @@ -0,0 +1,105 @@ +module ROCSPARSEExt +using GPUArrays,AMDGPU,SparseArrays,LinearAlgebra +using GPUArrays: UnsupportedSparseVendor +using AMDGPU.rocSPARSE: ROCCanonicalMatrix,ROCSparseMatrix,ROCSparseMatrixCSR, + ROCSparseMatrixCSC,ROCSparseMatrixCOO,ROCSparseMatrixBSR +const SP=AMDGPU.rocSPARSE +GPUArrays.default_sparse_indtype(::ROCMatrix) = Cint +GPUArrays.sparse_format(fmt::Symbol,A::ROCMatrix) = fmt===:bsr ? ROCSparseMatrixBSR : + invoke(GPUArrays.sparse_format,Tuple{Symbol,GPUArrays.AbstractGPUMatrix},fmt,A) +struct ROCSpMV end +struct ROCSpMM end +struct ROCSpGEMM end + +function GPUArrays.check_storage_compatibility(arrays::ROCArray...) + all(A -> AMDGPU.device(A)==AMDGPU.device(first(arrays)),arrays) || + throw(ArgumentError("ROCm sparse operands must share a device")) + nothing +end +GPUArrays.product_buffers(A::ROCSparseMatrixBSR)=(A.rowPtr,A.colVal,A.nzVal) +function capability(C,A,B) + eltype(A)===eltype(B)===eltype(C) && eltype(C)<:LinearAlgebra.BlasFloat || + return UnsupportedSparseVendor("rocSPARSE requires matching BLAS floating types") + AMDGPU.functional(:rocsparse) || return UnsupportedSparseVendor("rocSPARSE is unavailable") + sparse=A isa ROCSparseMatrix ? A : B + AMDGPU.device()==AMDGPU.device(nonzeros(sparse)) || + return UnsupportedSparseVendor("rocSPARSE requires the operands' device to be current") + Ti=SparseArrays.indtype(sparse) + Ti in (Int32,Int64) || return UnsupportedSparseVendor("rocSPARSE requires Int32 or Int64 indices") + p=sparse isa ROCSparseMatrixBSR ? sparse.rowPtr : first(GPUArrays.sparse_buffers(sparse)) + eltype(p)===Ti || return UnsupportedSparseVendor("this rocSPARSE executor requires uniform index widths") + if sparse isa ROCSparseMatrixBSR + Ti===Int32 && all(iszero(d%sparse.blockDim) for d in size(sparse)) || + return UnsupportedSparseVendor("BSR execution requires Int32 indices and block-aligned dimensions") + end + for X in (A,B,C) + X isa ROCSparseMatrix && continue + stride(X,1)==1 && (ndims(X)==1 || stride(X,2)>=max(1,size(X,1))) || + return UnsupportedSparseVendor("rocSPARSE requires contiguous vectors and column-major matrices") + end + nothing +end +function GPUArrays.normalize_sparse_product_operand(A::ROCSparseMatrixBSR,t,algorithm) + t in ('S','s','H','h') || return (A,t) + algorithm isa GPUArrays.Portable && throw(ArgumentError("portable BSR products are not implemented")) + (GPUArrays.materialize(LinearAlgebra.wrap(ROCSparseMatrixCSR(A),t)),'N') +end +function GPUArrays.vendor_spmv(y::ROCVector,t,A::ROCSparseMatrix,x::ROCVector,alpha,beta) + reason=capability(y,A,x) + reason===nothing || return reason + t in ('N','T','C') || return UnsupportedSparseVendor("rocSPARSE requires an ordinary operand") + A isa ROCSparseMatrixBSR && t!='N' && + return UnsupportedSparseVendor("rocSPARSE BSR SpMV requires a nontransposed matrix") + ROCSpMV() +end +GPUArrays.execute_spmv!(::ROCSpMV,y,t,A,x,alpha,beta)=SP.mv!(Char(t),alpha,A,x,beta,y,'O') +function spmm_capability(C,ta,tb,A,B) + reason=capability(C,A,B) + reason===nothing || return reason + ta in ('N','T','C') && tb in ('N','T','C') || + return UnsupportedSparseVendor("rocSPARSE requires ordinary operands") + eltype(C)<:Complex && ((A isa ROCSparseMatrixCSC && ta=='C') || + (B isa ROCSparseMatrixCSR && tb=='C')) && + return UnsupportedSparseVendor("rocSPARSE does not support this complex compressed adjoint SpMM") + (A isa ROCSparseMatrixBSR && ta!='N') && + return UnsupportedSparseVendor("rocSPARSE BSR SpMM requires a nontransposed sparse matrix") + ROCSpMM() +end +GPUArrays.vendor_spmm(C::ROCMatrix,ta,tb,A::ROCSparseMatrix,B::ROCMatrix,alpha,beta)= + spmm_capability(C,ta,tb,A,B) +GPUArrays.vendor_spmm(C::ROCMatrix,ta,tb,A::ROCMatrix,B::ROCCanonicalMatrix,alpha,beta)= + spmm_capability(C,ta,tb,A,B) +GPUArrays.execute_spmm!(::ROCSpMM,C,ta,tb,A,B,alpha,beta)= + SP.mm!(Char(ta),Char(tb),alpha,A,B,beta,C,'O') + +function GPUArrays.vendor_spgemm(A::ROCCanonicalMatrix,B::ROCCanonicalMatrix) + eltype(A)===eltype(B) && eltype(A)<:LinearAlgebra.BlasFloat && + SparseArrays.indtype(A)===SparseArrays.indtype(B)===Int32 || + return UnsupportedSparseVendor("this rocSPARSE SpGEMM executor requires matching BLAS values and Int32 indices") + for X in (A,B) + eltype(first(GPUArrays.sparse_buffers(X)))===Int32 || + return UnsupportedSparseVendor("rocSPARSE SpGEMM requires uniform indices") + end + AMDGPU.functional(:rocsparse) || return UnsupportedSparseVendor("rocSPARSE is unavailable") + AMDGPU.device()==AMDGPU.device(nonzeros(A)) || + return UnsupportedSparseVendor("rocSPARSE requires the operands' device to be current") + ROCSpGEMM() +end +function GPUArrays.execute_spgemm(::ROCSpGEMM,A,B;max_temp_bytes) + Acsr=ROCSparseMatrixCSR{eltype(A),Int32}(A) + Bcsr=ROCSparseMatrixCSR{eltype(B),Int32}(B) + C=SP.gemm('N','N',one(eltype(A)),Acsr,Bcsr,'O') + convert(GPUArrays.matrix_format(A),C) +end + +# Backend-only formats normalize once, then use centrally owned arithmetic. +for op in (:+,:-) + @eval begin + Base.$op(A::ROCSparseMatrixBSR,B::GPUArrays.GPUSparseMatrix)=Base.$op(ROCSparseMatrixCSR(A),B) + Base.$op(A::GPUArrays.GPUSparseMatrix,B::ROCSparseMatrixBSR)=Base.$op(A,ROCSparseMatrixCSR(B)) + Base.$op(A::ROCSparseMatrixBSR,B::ROCSparseMatrixBSR)=Base.$op(ROCSparseMatrixCSR(A),ROCSparseMatrixCSR(B)) + end +end + +include("ROCSPARSEExt/solves.jl") +end diff --git a/ext/ROCSPARSEExt/solves.jl b/ext/ROCSPARSEExt/solves.jl new file mode 100644 index 000000000..2555790c8 --- /dev/null +++ b/ext/ROCSPARSEExt/solves.jl @@ -0,0 +1,73 @@ +# Vendor triangular solves remain accelerated after replacing the ordinary containers. +# BSR identities preserve block layout, including padded final blocks. +function bsr_identity(unit,A::ROCSparseMatrixBSR{<:Any,Ti}) where {Ti} + n=LinearAlgebra.checksquare(A) + b=Int(A.blockDim); mb=cld(n,b) + GPUArrays.check_sparse_nnz(Ti,mb) + p=GPUArrays.sparse_similar(A.rowPtr,Ti,Base.checked_add(mb,1)); p .= 1:mb+1 + i=GPUArrays.sparse_similar(A.colVal,Ti,mb); i .= 1:mb + values=fill!(GPUArrays.sparse_similar(A.nzVal,typeof(unit),Base.checked_mul(mb,Base.checked_mul(b,b))),zero(unit)) + diagonal=view(reshape(values,b*b,mb),1:b+1:b*b,:) + diagonal .= unit + pad=mb*b-n + pad>0 && (diagonal[end-pad+1:end,end] .= zero(unit)) + ROCSparseMatrixBSR{typeof(unit),Ti}(p,i,values,size(A),b,A.dir,mb) +end +Base.one(A::ROCSparseMatrixBSR)=bsr_identity(one(eltype(A)),A) +Base.oneunit(A::ROCSparseMatrixBSR)=bsr_identity(oneunit(eltype(A)),A) +using LinearAlgebra: BlasFloat, AdjOrTrans +using AMDGPU.rocSPARSE: ROCSparseMatrixCSR,ROCSparseMatrixCSC,ROCSparseMatrixCOO,ROCSparseMatrixBSR, + sv!,sm!,sv2!,sm2! + +# LinearAlgebra inspects triangular wrappers before reaching the solve executor. +# Vendor-only storage must not fall through its host scalar traversal. +LinearAlgebra.istriu(A::ROCSparseMatrixBSR,k::Integer=0) = istriu(ROCSparseMatrixCSR(A),k) +LinearAlgebra.istril(A::ROCSparseMatrixBSR,k::Integer=0) = istril(ROCSparseMatrixCSR(A),k) + +# triangular +for SparseMatrixType in (:ROCSparseMatrixBSR,) + @eval begin + LinearAlgebra.generic_trimatdiv!(C::DenseROCVector{T}, uploc, isunitc, tfun::Function, A::$SparseMatrixType{T}, B::DenseROCVector{T}) where {T<:BlasFloat} = + sv2!(tfun === identity ? 'N' : tfun === transpose ? 'T' : 'C', uploc, isunitc, one(T), A, C === B ? C : copyto!(C, B), 'O') + LinearAlgebra.generic_trimatdiv!(C::DenseROCMatrix{T}, uploc, isunitc, tfun::Function, A::$SparseMatrixType{T}, B::DenseROCMatrix{T}) where {T<:BlasFloat} = + sm2!(tfun === identity ? 'N' : tfun === transpose ? 'T' : 'C', 'N', uploc, isunitc, one(T), A, C === B ? C : copyto!(C, B), 'O') + function LinearAlgebra.generic_trimatdiv!(C::DenseROCMatrix{T}, uploc, isunitc, tfun::Function, A::$SparseMatrixType{T}, B::AdjOrTrans{T,<:DenseROCMatrix{T}}) where {T<:BlasFloat} + transb = LinearAlgebra.wrapper_char(B) + (transb == 'C') && (T <: Complex) && throw(ErrorException("This operation is not supported by the current ROCm version.")) + C !== parent(B) && copyto!(C, B) + sm2!(tfun === identity ? 'N' : tfun === transpose ? 'T' : 'C', 'N', uploc, isunitc, one(T), A, C, 'O') + end + function LinearAlgebra.generic_trimatdiv!(C::Transpose{T,<:DenseROCMatrix{T}}, uploc, isunitc, tfun::Function, A::$SparseMatrixType{T}, B::Transpose{T,<:DenseROCMatrix{T}}) where {T<:BlasFloat} + B === C || throw(ErrorException("This operation is only supported if B and C are identical.")) + sm2!(tfun === identity ? 'N' : tfun === transpose ? 'T' : 'C', 'T', uploc, isunitc, one(T), A, parent(B), 'O') + end + function LinearAlgebra.generic_trimatdiv!(C::Adjoint{T,<:DenseROCMatrix{T}}, uploc, isunitc, tfun::Function, A::$SparseMatrixType{T}, B::Adjoint{T,<:DenseROCMatrix{T}}) where {T<:BlasFloat} + B === C || throw(ErrorException("This operation is only supported if B and C are identical.")) + sm2!(tfun === identity ? 'N' : tfun === transpose ? 'T' : 'C', 'C', uploc, isunitc, one(T), A, parent(B), 'O') + end + end +end # SparseMatrixType loop + +for SparseMatrixType in (:ROCSparseMatrixCOO, :ROCSparseMatrixCSR, :ROCSparseMatrixCSC) + @eval begin + function LinearAlgebra.generic_trimatdiv!(C::DenseROCVector{T}, uploc, isunitc, tfun::Function, A::$SparseMatrixType{T}, B::DenseROCVector{T}) where {T<:BlasFloat} + sv!(tfun === identity ? 'N' : tfun === transpose ? 'T' : 'C', uploc, isunitc, one(T), A, B, C, 'O') + end + function LinearAlgebra.generic_trimatdiv!(C::DenseROCMatrix{T}, uploc, isunitc, tfun::Function, A::$SparseMatrixType{T}, B::DenseROCMatrix{T}) where {T<:BlasFloat} + sm!(tfun === identity ? 'N' : tfun === transpose ? 'T' : 'C', 'N', uploc, isunitc, one(T), A, B, C, 'O') + end + function LinearAlgebra.generic_trimatdiv!(C::DenseROCMatrix{T}, uploc, isunitc, tfun::Function, A::$SparseMatrixType{T}, B::AdjOrTrans{T,<:DenseROCMatrix{T}}) where {T<:BlasFloat} + transb = LinearAlgebra.wrapper_char(B) + (transb == 'C') && (T <: Complex) && throw(ErrorException("This operation is not supported by the current ROCm version.")) + sm!(tfun === identity ? 'N' : tfun === transpose ? 'T' : 'C', transb, uploc, isunitc, one(T), A, parent(B), C, 'O') + end + function LinearAlgebra.generic_trimatdiv!(C::Transpose{T,<:DenseROCMatrix{T}}, uploc, isunitc, tfun::Function, A::$SparseMatrixType{T}, B::Transpose{T,<:DenseROCMatrix{T}}) where {T<:BlasFloat} + (B !== C) && throw(ErrorException("This operation is only supported if B and C are identical.")) + sm!(tfun === identity ? 'N' : tfun === transpose ? 'T' : 'C', 'T', uploc, isunitc, one(T), A, parent(B), parent(C), 'O') + end + function LinearAlgebra.generic_trimatdiv!(C::Adjoint{T,<:DenseROCMatrix{T}}, uploc, isunitc, tfun::Function, A::$SparseMatrixType{T}, B::Adjoint{T,<:DenseROCMatrix{T}}) where {T<:BlasFloat} + (B !== C) && throw(ErrorException("This operation is only supported if B and C are identical.")) + sm!(tfun === identity ? 'N' : tfun === transpose ? 'T' : 'C', 'C', uploc, isunitc, one(T), A, parent(B), parent(C), 'O') + end + end +end # SparseMatrixType loop diff --git a/lib/JLArrays/Project.toml b/lib/JLArrays/Project.toml index 698fb8ebd..94274447e 100644 --- a/lib/JLArrays/Project.toml +++ b/lib/JLArrays/Project.toml @@ -1,6 +1,6 @@ name = "JLArrays" uuid = "27aeb0d3-9eb9-45fb-866b-73c2ecf80fcb" -version = "0.4.2" +version = "0.5.0" authors = ["Tim Besard "] [deps] @@ -13,7 +13,7 @@ SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [compat] Adapt = "2.0, 3.0, 4.0" -GPUArrays = "12.1" +GPUArrays = "13" KernelAbstractions = "0.9" LinearAlgebra = "1" Random = "1" diff --git a/lib/JLArrays/src/JLArrays.jl b/lib/JLArrays/src/JLArrays.jl index c6caf8524..c5d9ebc17 100644 --- a/lib/JLArrays/src/JLArrays.jl +++ b/lib/JLArrays/src/JLArrays.jl @@ -6,7 +6,8 @@ module JLArrays -export JLArray, JLVector, JLMatrix, jl, JLBackend, JLSparseVector, JLSparseMatrixCSC, JLSparseMatrixCSR +export JLArray, JLVector, JLMatrix, jl, JLBackend, JLSparseVector, JLSparseMatrixCSC, JLSparseMatrixCSR, + JLSparseMatrixCOO using GPUArrays @@ -14,7 +15,7 @@ using Adapt using SparseArrays, LinearAlgebra using Random -import GPUArrays: dense_array_type +using GPUArrays: GPUSparseVector, GPUSparseMatrixCSR, GPUSparseMatrixCSC, GPUSparseMatrixCOO import KernelAbstractions diff --git a/lib/JLArrays/src/JLKernels.jl b/lib/JLArrays/src/JLKernels.jl index ca4d84d0f..a29241a88 100644 --- a/lib/JLArrays/src/JLKernels.jl +++ b/lib/JLArrays/src/JLKernels.jl @@ -21,7 +21,6 @@ struct JLBackend <: KernelAbstractions.GPU end KernelAbstractions.get_backend(a::JLA) where JLA <: JLArray = JLBackend() -KernelAbstractions.get_backend(a::JLA) where JLA <: Union{JLSparseMatrixCSC, JLSparseMatrixCSR, JLSparseVector} = JLBackend() function KernelAbstractions.mkcontext(kernel::Kernel{JLBackend}, I, _ndrange, iterspace, ::Dynamic) where Dynamic return KernelAbstractions.CompilerMetadata{KernelAbstractions.ndrange(kernel), Dynamic}(I, _ndrange, iterspace) diff --git a/lib/JLArrays/src/array.jl b/lib/JLArrays/src/array.jl index d47184a8b..f6e8f9b30 100644 --- a/lib/JLArrays/src/array.jl +++ b/lib/JLArrays/src/array.jl @@ -44,14 +44,8 @@ mutable struct JLArray{T, N} <: AbstractGPUArray{T, N} end GPUArrays.storage(a::JLArray) = a.data - GPUArrays.memory_location(a::JLArray) = (UInt(pointer(a.data[])), a.offset) -GPUArrays.dense_array_type(a::JLArray{T, N}) where {T, N} = JLArray{T, N} -GPUArrays.dense_array_type(::Type{JLArray{T, N}}) where {T, N} = JLArray{T, N} -GPUArrays.dense_vector_type(a::JLArray{T, N}) where {T, N} = JLArray{T, 1} -GPUArrays.dense_vector_type(::Type{JLArray{T, N}}) where {T, N} = JLArray{T, 1} - # conversion of untyped data to a typed Array function typed_data(x::JLArray{T}) where {T} unsafe_wrap(Array, pointer(x), x.dims) @@ -237,6 +231,8 @@ end Base.copyto!(dest::DenseJLArray{T}, source::DenseJLArray{T}) where {T} = copyto!(dest, 1, source, 1, length(source)) +GPUArrays.supports_resize(::JLVector) = true + function Base.resize!(a::DenseJLVector{T}, nl::Integer) where {T} # JLArrays aren't performance critical, so simply allocate a new one # instead of duplicating the underlying data allocation from the ctor. diff --git a/lib/JLArrays/src/sparse.jl b/lib/JLArrays/src/sparse.jl index 032bd422f..c48c9a4fa 100644 --- a/lib/JLArrays/src/sparse.jl +++ b/lib/JLArrays/src/sparse.jl @@ -1,217 +1,22 @@ -# sparse array types - -mutable struct JLSparseVector{Tv, Ti} <: GPUArrays.AbstractGPUSparseVector{Tv, Ti} - iPtr::JLArray{Ti, 1} - nzVal::JLArray{Tv, 1} - len::Int - nnz::Ti - - function JLSparseVector{Tv, Ti}(iPtr::JLArray{<:Integer, 1}, nzVal::JLArray{Tv, 1}, - len::Integer) where {Tv, Ti <: Integer} - new{Tv, Ti}(iPtr, nzVal, len, length(nzVal)) - end -end -SparseArrays.nnz(x::JLSparseVector) = x.nnz -SparseArrays.nonzeroinds(x::JLSparseVector) = x.iPtr -SparseArrays.nonzeros(x::JLSparseVector) = x.nzVal - -mutable struct JLSparseMatrixCSC{Tv, Ti} <: GPUArrays.AbstractGPUSparseMatrixCSC{Tv, Ti} - colPtr::JLArray{Ti, 1} - rowVal::JLArray{Ti, 1} - nzVal::JLArray{Tv, 1} - dims::NTuple{2,Int} - nnz::Ti - - function JLSparseMatrixCSC{Tv, Ti}(colPtr::JLArray{<:Integer, 1}, rowVal::JLArray{<:Integer, 1}, - nzVal::JLArray{Tv, 1}, dims::NTuple{2,<:Integer}) where {Tv, Ti <: Integer} - new{Tv, Ti}(colPtr, rowVal, nzVal, dims, length(nzVal)) - end -end -function JLSparseMatrixCSC(colPtr::JLArray{Ti, 1}, rowVal::JLArray{Ti, 1}, nzVal::JLArray{Tv, 1}, dims::NTuple{2,<:Integer}) where {Tv, Ti <: Integer} - return JLSparseMatrixCSC{Tv, Ti}(colPtr, rowVal, nzVal, dims) -end -SparseArrays.SparseMatrixCSC(x::JLSparseMatrixCSC) = SparseMatrixCSC(size(x)..., Array(x.colPtr), Array(x.rowVal), Array(x.nzVal)) - -JLSparseMatrixCSC(A::JLSparseMatrixCSC) = A - -function Base.getindex(A::JLSparseMatrixCSC{Tv, Ti}, i::Integer, j::Integer) where {Tv, Ti} - @boundscheck checkbounds(A, i, j) - r1 = Int(@inbounds A.colPtr[j]) - r2 = Int(@inbounds A.colPtr[j+1]-1) - (r1 > r2) && return zero(Tv) - r1 = searchsortedfirst(view(A.rowVal, r1:r2), i) + r1 - 1 - ((r1 > r2) || (A.rowVal[r1] != i)) ? zero(Tv) : A.nzVal[r1] -end - -mutable struct JLSparseMatrixCSR{Tv, Ti} <: GPUArrays.AbstractGPUSparseMatrixCSR{Tv, Ti} - rowPtr::JLArray{Ti, 1} - colVal::JLArray{Ti, 1} - nzVal::JLArray{Tv, 1} - dims::NTuple{2,Int} - nnz::Ti - - function JLSparseMatrixCSR{Tv, Ti}(rowPtr::JLArray{<:Integer, 1}, colVal::JLArray{<:Integer, 1}, - nzVal::JLArray{Tv, 1}, dims::NTuple{2,<:Integer}) where {Tv, Ti<:Integer} - new{Tv, Ti}(rowPtr, colVal, nzVal, dims, length(nzVal)) - end -end -function JLSparseMatrixCSR(rowPtr::JLArray{Ti, 1}, colVal::JLArray{Ti, 1}, nzVal::JLArray{Tv, 1}, dims::NTuple{2,<:Integer}) where {Tv, Ti <: Integer} - return JLSparseMatrixCSR{Tv, Ti}(rowPtr, colVal, nzVal, dims) -end -function SparseArrays.SparseMatrixCSC(x::JLSparseMatrixCSR) - x_transpose = SparseMatrixCSC(size(x, 2), size(x, 1), Array(x.rowPtr), Array(x.colVal), Array(x.nzVal)) - return SparseMatrixCSC(transpose(x_transpose)) -end - -JLSparseMatrixCSC(Mat::Union{Transpose{Tv, <:SparseMatrixCSC}, Adjoint{Tv, <:SparseMatrixCSC}}) where {Tv} = JLSparseMatrixCSC(JLSparseMatrixCSR(Mat)) - -function Base.size(g::JLSparseMatrixCSR, d::Integer) - if 1 <= d <= 2 - return g.dims[d] - elseif d > 1 - return 1 - else - throw(ArgumentError("dimension must be ≥ 1, got $d")) - end -end - -JLSparseMatrixCSR(Mat::Transpose{Tv, <:SparseMatrixCSC}) where {Tv} = - JLSparseMatrixCSR(JLVector{Cint}(parent(Mat).colptr), JLVector{Cint}(parent(Mat).rowval), - JLVector(parent(Mat).nzval), size(Mat)) -JLSparseMatrixCSR(Mat::Adjoint{Tv, <:SparseMatrixCSC}) where {Tv} = - JLSparseMatrixCSR(JLVector{Cint}(parent(Mat).colptr), JLVector{Cint}(parent(Mat).rowval), - JLVector(conj.(parent(Mat).nzval)), size(Mat)) - -JLSparseMatrixCSR(A::JLSparseMatrixCSR) = A - -function Base.getindex(A::JLSparseMatrixCSR{Tv, Ti}, i0::Integer, i1::Integer) where {Tv, Ti} - @boundscheck checkbounds(A, i0, i1) - c1 = Int(A.rowPtr[i0]) - c2 = Int(A.rowPtr[i0+1]-1) - (c1 > c2) && return zero(Tv) - c1 = searchsortedfirst(A.colVal, i1, c1, c2, Base.Order.Forward) - (c1 > c2 || A.colVal[c1] != i1) && return zero(Tv) - nonzeros(A)[c1] -end - -GPUArrays.sparse_array_type(sa::JLSparseMatrixCSC) = JLSparseMatrixCSC -GPUArrays.sparse_array_type(::Type{<:JLSparseMatrixCSC}) = JLSparseMatrixCSC -GPUArrays.sparse_array_type(sa::JLSparseMatrixCSR) = JLSparseMatrixCSR -GPUArrays.sparse_array_type(::Type{<:JLSparseMatrixCSR}) = JLSparseMatrixCSR -GPUArrays.sparse_array_type(sa::JLSparseVector) = JLSparseVector -GPUArrays.sparse_array_type(::Type{<:JLSparseVector}) = JLSparseVector - -GPUArrays.dense_array_type(sa::JLSparseVector) = JLArray -GPUArrays.dense_array_type(::Type{<:JLSparseVector}) = JLArray -GPUArrays.dense_array_type(sa::JLSparseMatrixCSC) = JLArray -GPUArrays.dense_array_type(::Type{<:JLSparseMatrixCSC}) = JLArray -GPUArrays.dense_array_type(sa::JLSparseMatrixCSR) = JLArray -GPUArrays.dense_array_type(::Type{<:JLSparseMatrixCSR}) = JLArray - -GPUArrays.csc_type(::Type{<:JLSparseMatrixCSR}) = JLSparseMatrixCSC -GPUArrays.csr_type(::Type{<:JLSparseMatrixCSC}) = JLSparseMatrixCSR - -Base.similar(Mat::JLSparseMatrixCSR) = JLSparseMatrixCSR(copy(Mat.rowPtr), copy(Mat.colVal), similar(nonzeros(Mat)), size(Mat)) -Base.similar(Mat::JLSparseMatrixCSR, T::Type) = JLSparseMatrixCSR(copy(Mat.rowPtr), copy(Mat.colVal), similar(nonzeros(Mat), T), size(Mat)) - -Base.similar(Mat::JLSparseMatrixCSC, T::Type, N::Int, M::Int) = JLSparseMatrixCSC(JLVector([zero(Int32)]), JLVector{Int32}(undef, 0), JLVector{T}(undef, 0), (N, M)) -Base.similar(Mat::JLSparseMatrixCSR, T::Type, N::Int, M::Int) = JLSparseMatrixCSR(JLVector([zero(Int32)]), JLVector{Int32}(undef, 0), JLVector{T}(undef, 0), (N, M)) - -Base.similar(Mat::JLSparseMatrixCSC{Tv, Ti}, N::Int, M::Int) where {Tv, Ti} = similar(Mat, Tv, N, M) -Base.similar(Mat::JLSparseMatrixCSR{Tv, Ti}, N::Int, M::Int) where {Tv, Ti} = similar(Mat, Tv, N, M) - -Base.similar(Mat::JLSparseMatrixCSC, T::Type, dims::Tuple{Int, Int}) = similar(Mat, T, dims...) -Base.similar(Mat::JLSparseMatrixCSR, T::Type, dims::Tuple{Int, Int}) = similar(Mat, T, dims...) - -Base.similar(Mat::JLSparseMatrixCSC, dims::Tuple{Int, Int}) = similar(Mat, dims...) -Base.similar(Mat::JLSparseMatrixCSR, dims::Tuple{Int, Int}) = similar(Mat, dims...) - -JLArray(x::JLSparseVector) = JLArray(collect(SparseVector(x))) -JLArray(x::JLSparseMatrixCSC) = JLArray(collect(SparseMatrixCSC(x))) -JLArray(x::JLSparseMatrixCSR) = JLArray(collect(SparseMatrixCSC(x))) - - -## interop with SparseArrays - -function JLSparseVector(xs::SparseVector{Tv, Ti}) where {Ti, Tv} - iPtr = JLVector{Ti}(undef, length(xs.nzind)) - nzVal = JLVector{Tv}(undef, length(xs.nzval)) - copyto!(iPtr, convert(Vector{Ti}, xs.nzind)) - copyto!(nzVal, convert(Vector{Tv}, xs.nzval)) - return JLSparseVector{Tv, Ti}(iPtr, nzVal, length(xs),) -end -Base.length(x::JLSparseVector) = x.len -Base.size(x::JLSparseVector) = (x.len,) - -function JLSparseMatrixCSC(xs::SparseMatrixCSC{Tv, Ti}) where {Ti, Tv} - colPtr = JLVector{Ti}(undef, length(xs.colptr)) - rowVal = JLVector{Ti}(undef, length(xs.rowval)) - nzVal = JLVector{Tv}(undef, length(xs.nzval)) - copyto!(colPtr, convert(Vector{Ti}, xs.colptr)) - copyto!(rowVal, convert(Vector{Ti}, xs.rowval)) - copyto!(nzVal, convert(Vector{Tv}, xs.nzval)) - return JLSparseMatrixCSC{Tv, Ti}(colPtr, rowVal, nzVal, (xs.m, xs.n)) -end -JLSparseMatrixCSC(xs::SparseVector) = JLSparseMatrixCSC(SparseMatrixCSC(xs)) -Base.length(x::JLSparseMatrixCSC) = prod(x.dims) -Base.size(x::JLSparseMatrixCSC) = x.dims - -function JLSparseMatrixCSR(xs::SparseMatrixCSC{Tv, Ti}) where {Ti, Tv} - csr_xs = SparseMatrixCSC(transpose(xs)) - rowPtr = JLVector{Ti}(undef, length(csr_xs.colptr)) - colVal = JLVector{Ti}(undef, length(csr_xs.rowval)) - nzVal = JLVector{Tv}(undef, length(csr_xs.nzval)) - copyto!(rowPtr, convert(Vector{Ti}, csr_xs.colptr)) - copyto!(colVal, convert(Vector{Ti}, csr_xs.rowval)) - copyto!(nzVal, convert(Vector{Tv}, csr_xs.nzval)) - return JLSparseMatrixCSR{Tv, Ti}(rowPtr, colVal, nzVal, (xs.m, xs.n)) -end -JLSparseMatrixCSR(xs::SparseVector{Tv, Ti}) where {Ti, Tv} = JLSparseMatrixCSR(SparseMatrixCSC(xs)) -function JLSparseMatrixCSR(xs::JLSparseMatrixCSC{Tv, Ti}) where {Ti, Tv} - return JLSparseMatrixCSR(SparseMatrixCSC(xs)) -end -function JLSparseMatrixCSC(xs::JLSparseMatrixCSR{Tv, Ti}) where {Ti, Tv} - return JLSparseMatrixCSC(SparseMatrixCSC(xs)) -end -function Base.copyto!(dst::JLSparseMatrixCSR, src::JLSparseMatrixCSR) - if size(dst) != size(src) - throw(ArgumentError("Inconsistent Sparse Matrix size")) +# sparse arrays with JLArray storage: GPUArrays implements the formats, JLArrays only names +# them and adds constructors + +const JLSparseVector{Tv,Ti} = GPUSparseVector{Tv,Ti,<:JLVector{Ti},<:JLVector{Tv}} +const JLSparseMatrixCSR{Tv,Ti} = GPUSparseMatrixCSR{Tv,Ti,<:JLVector{<:Integer},<:JLVector{Ti},<:JLVector{Tv}} +const JLSparseMatrixCSC{Tv,Ti} = GPUSparseMatrixCSC{Tv,Ti,<:JLVector{<:Integer},<:JLVector{Ti},<:JLVector{Tv}} +const JLSparseMatrixCOO{Tv,Ti} = GPUSparseMatrixCOO{Tv,Ti,<:JLVector{Ti},<:JLVector{Tv}} +const JLSparseMatrix = Union{JLSparseMatrixCSR,JLSparseMatrixCSC,JLSparseMatrixCOO} + +# move the input to JLArray storage (keeping its format), then convert on the device +for (alias, S) in ((:JLSparseVector, :GPUSparseVector), (:JLSparseMatrixCSR, :GPUSparseMatrixCSR), + (:JLSparseMatrixCSC, :GPUSparseMatrixCSC), (:JLSparseMatrixCOO, :GPUSparseMatrixCOO)) + @eval begin + $alias(A::AbstractArray) = $S(adapt(JLArray, A)) + $alias{Tv}(A::AbstractArray) where {Tv} = $S{Tv}(adapt(JLArray, A)) + $alias{Tv,Ti}(A::AbstractArray) where {Tv,Ti} = $S{Tv,Ti}(adapt(JLArray, A)) end - resize!(dst.rowPtr, length(src.rowPtr)) - resize!(dst.colVal, length(src.colVal)) - resize!(SparseArrays.nonzeros(dst), length(SparseArrays.nonzeros(src))) - copyto!(dst.rowPtr, src.rowPtr) - copyto!(dst.colVal, src.colVal) - copyto!(SparseArrays.nonzeros(dst), SparseArrays.nonzeros(src)) - dst.nnz = src.nnz - dst end -Base.length(x::JLSparseMatrixCSR) = prod(x.dims) -Base.size(x::JLSparseMatrixCSR) = x.dims - -function GPUArrays._spadjoint(A::JLSparseMatrixCSR) - Aᴴ = JLSparseMatrixCSC(A.rowPtr, A.colVal, conj(A.nzVal), reverse(size(A))) - JLSparseMatrixCSR(Aᴴ) -end -function GPUArrays._sptranspose(A::JLSparseMatrixCSR) - Aᵀ = JLSparseMatrixCSC(A.rowPtr, A.colVal, A.nzVal, reverse(size(A))) - JLSparseMatrixCSR(Aᵀ) -end -function GPUArrays._spadjoint(A::JLSparseMatrixCSC) - Aᴴ = JLSparseMatrixCSR(A.colPtr, A.rowVal, conj(A.nzVal), reverse(size(A))) - JLSparseMatrixCSC(Aᴴ) -end -function GPUArrays._sptranspose(A::JLSparseMatrixCSC) - Aᵀ = JLSparseMatrixCSR(A.colPtr, A.rowVal, A.nzVal, reverse(size(A))) - JLSparseMatrixCSC(Aᵀ) -end - - -## adapt for the device -Adapt.adapt_structure(to::Adaptor, x::JLSparseMatrixCSC{Tv,Ti}) where {Tv,Ti} = -GPUSparseDeviceMatrixCSC{Tv,Ti,JLDeviceArray{Ti, 1}, JLDeviceArray{Tv, 1}, AS.Generic}(adapt(to, x.colPtr), adapt(to, x.rowVal), adapt(to, x.nzVal), x.dims, x.nnz) -Adapt.adapt_structure(to::Adaptor, x::JLSparseMatrixCSR{Tv,Ti}) where {Tv,Ti} = -GPUSparseDeviceMatrixCSR{Tv,Ti,JLDeviceArray{Ti, 1}, JLDeviceArray{Tv, 1}, AS.Generic}(adapt(to, x.rowPtr), adapt(to, x.colVal), adapt(to, x.nzVal), x.dims, x.nnz) -Adapt.adapt_structure(to::Adaptor, x::JLSparseVector{Tv,Ti}) where {Tv,Ti} = -GPUSparseDeviceVector{Tv,Ti,JLDeviceArray{Ti, 1}, JLDeviceArray{Tv, 1}, AS.Generic}(adapt(to, x.iPtr), adapt(to, x.nzVal), x.len, x.nnz) +# densify on the device +JLArray{T,1}(x::JLSparseVector) where {T} = copyto!(similar(nonzeros(x), T, size(x)), x) +JLArray{T,2}(A::JLSparseMatrix) where {T} = copyto!(similar(nonzeros(A), T, size(A)), A) diff --git a/perf/REPORT.sparse-v13.md b/perf/REPORT.sparse-v13.md new file mode 100644 index 000000000..2884b8234 --- /dev/null +++ b/perf/REPORT.sparse-v13.md @@ -0,0 +1,110 @@ +# Sparse v13 performance checkpoint + +Measured Oct21,2026 on Pegasus, RTX5080 (16GB), Ryzen9950X, Julia1.13.2, +CUDA runtime13.4.0, driver615.78.8, cuSPARSE12.8.6. This is a CUDA performance +checkpoint, not evidence for other hardware, Julia versions or older cuSPARSE. + +Baseline: clean CUDA44ac4ebb3794013be79b607aec0c8aee4321d71d (6.4.2), registry +GPUArrays12.1.1 (tree f72b034a94bf1578dcc6e5410990f257b2091ada). +Migrated: clean CUDA8a9379519f03586ca41d1a472d0ddf63eb9ba7be (7.0.0), +GPUArraysc8dd580afebcebf86d87b01243f9bb773176c371 (13.0.0), +AKa915b419728e3abd258bce78ea3d1d3255b88ed7, Atomix1ffd4ae4746fbc9463882c68563c6bdb81e0d9c0, +KA0.10.1. Both use the same installed CUDA libraries. Baseline and migrated grids +ran serially, without other test activity on the GPU. The harness checks every +result against CPU sparse mathematics. Raw measurements are in results/*.tsv. + +The grid includes diagonal, nine-diagonal banded, uniform-random, skewed and +empty-row matrices; HB1138_bus and bcsstk17; CSR/CSC; normal/transpose SpMV; +SpMM widths1/8/64; complex adjoints; duplicate assembly; and heavy-candidate SpGEMM. +Three warmups precede eleven synchronized samples; tables use warm medians. +`first_seconds` is the first call for that case in that process, not a fully cold +package/compiler measurement: earlier cases can already have compiled its methods. +Automatic precedes forced vendor and portable calls; do not compare their first +times as independent cold starts. CPU references and result downloads are outside +timing. Host and GPU cumulative requested allocations are recorded separately. + +## Results + +Times below are microseconds. Across115 matched native cases, migrated/old warm +time has median1.029 and maximum1.171 (uniform CSR transposed SpMV,16.50→19.33µs). +The small-operation validation/selection cost is visible; this is not a claim that +every case is faster. No hardware-specific automatic-selection threshold was added. + +| Case | Old vendor | Migrated vendor | Forced portable | +|---|---:|---:|---:| +| bcsstk17 CSR SpMV | 15.52 | 16.37 | 26.69 | +| bcsstk17 CSC SpMV | 17.00 | 17.40 | 271.62 | +| bcsstk17 CSR SpMM width64 | 86.64 | 85.57 | 710.97 | +| bcsstk17 CSC SpMM width64 | 152.82 | 146.07 | 953.48 | +| Duplicate-heavy CSR assembly | 1292.82 | 199.90 | same central assembly | +| Candidate-heavy CSR SpGEMM | 578.21 | 583.60 | 7369.04 | + +SpGEMM has8388608 candidate products but262144 final entries. Before CUDA8a9379519, +redundant canonical reassembly cost about0.85ms versus0.59ms baseline. Direct +publication after the vendor's completed canonical output restores parity without +weakening raw constructors or other vendor-output normalization. All four native +algorithms have dedicated canonicality/alias/empty/cancellation/GC tests. The older +library branch still assembles and has not been performance-tested on old hardware. +Automatic selection retains the supported native path. Portable transposed/CSC +regrouping and candidate expansion remain explicit optimization opportunities. + +## Observed device memory + +Separate Nsight system-wide CUDA/NVTX traces recorded device allocation/free events +inside one warmed, pool-purged call per range. These are observed incremental live +device peaks, including the new result, relative to range entry; not process RSS, +not solely temporary workspace, and not cumulative requested allocation. Inputs +and preallocated dense outputs already exist at range entry. They do not establish +a universal bound on driver-internal or future allocator behavior. + +| Case | Old peak bytes | Migrated vendor peak bytes | Portable peak bytes | +|---|---:|---:|---:| +| Banded CSR SpMM width8 | 1416 | 1416 | 0 | +| bcsstk17 CSC SpMV | 680 | 680 | 21861168 | +| Duplicate CSR assembly | 2723912 | 4320016 | same central assembly | +| Candidate-heavy CSR SpGEMM | 203605548 | 203605548 | 553648304 | + +SpGEMM cumulative GPU requests are203803492 bytes for either native stack and +569723060 portable. Portable assembly is much faster in the duplicate-heavy case +but has a larger measured peak. The portable1MiB candidate-estimate limit rejects +the heavy product before candidate materialization. This is an explicit estimate +gate, not a promise that arbitrary vendor workspace obeys that limit. + +## Reproduction + +Use separate task-owned environments under ~/Julia/tmp for the old and new stacks. +Develop the CUDA repository's CUDACore/CUDATools and library workspace packages +explicitly; use the exact revisions above, and develop GPUArrays plus its core, +AK and Atomix for the migrated environment. Add NVTX, SHA, Test, Statistics and +the script's standard-library imports as direct dependencies. Do not activate the +default environment. Preserve each resolved Manifest with the run logs. + +Cache these NIST copies of the SuiteSparse Harwell-Boeing workloads: + +- https://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/psadmit/1138_bus.mtx.gz + SHA2560544e52ed96893ae82c5a7f9051d662e04a1c27fe57768578de09932381fd963 +- https://math.nist.gov/pub/MatrixMarket2/Harwell-Boeing/bcsstruc2/bcsstk17.mtx.gz + SHA256105425dc922f5aaa4f2a7b52d707e946cbc3cc309accea11f5365a30528f49a5 + +Set SPARSE_BENCH_DATA to that cache. From each environment run: + +```sh +jld --name=unique-sparse-bench --idle-timeout=2h --no-revise --timeout=1800 run /path/to/perf/sparse-v13-cuda.jl +``` + +For memory profiling, start Nsight in system-wide mode before starting a fresh +daemon, then run the harness with CUDA_INJECTION_SHM_ALLOWED=TRUE, +SPARSE_BENCH_TRACE_MEMORY=1, SPARSE_BENCH_REPETITIONS=1 and a relevant +SPARSE_BENCH_FILTER regex. Profiling the jld launcher with fork tracing stalled +locally; the system-wide approach captures the actual daemon process. + +```sh +nsys profile --trace=cuda,nvtx --cuda-trace-scope=system-wide --cuda-memory-usage=true --sample=none --cpuctxsw=none --wait=primary --duration=50 --output=memory-final /bin/sleep 50 +nsys export --type=sqlite --output=memory-final.sqlite memory-final.nsys-rep +sqlite3 -header memory-final.sqlite < /path/to/perf/sparse-v13-memory.sql +``` + +No device-allocation events means zero incremental device allocation for that +range (the SQL has no row). Pegasus logs and reports remain under +ga13-cuda-{baseline,benchmark}-env/{benchmark-final-grid.log,memory-final*}. +These measurements do not replace functional backend suites or CI/release gates. diff --git a/perf/results/sparse-v13-cuda-baseline.tsv b/perf/results/sparse-v13-cuda-baseline.tsv new file mode 100644 index 000000000..4ffa13d88 --- /dev/null +++ b/perf/results/sparse-v13-cuda-baseline.tsv @@ -0,0 +1,134 @@ +STACK Julia=1.13.2 CUDA=6.4.2 cuSPARSE=6.4.2 GPUArrays=12.1.1 runtime=13.4.0 +COLUMNS case/format/operation/policy first_seconds warm_median_seconds host_alloc_bytes device_alloc_bytes +INPUT_SHA256 1138_bus.mtx.gz 0544e52ed96893ae82c5a7f9051d662e04a1c27fe57768578de09932381fd963 +INPUT_SHA256 bcsstk17.mtx.gz 105425dc922f5aaa4f2a7b52d707e946cbc3cc309accea11f5365a30528f49a5 +CASE diagonal csr (10000, 10000) 10000 +RESULT diagonal/csr/spmv-N/old-vendor 0.22180256 2.0128e-5 2192.0 24.0 +RESULT diagonal/csr/spmm-N-w1/old-vendor 0.068476181 1.8334e-5 2144.0 24.0 +RESULT diagonal/csr/spmm-N-w8/old-vendor 0.005661071 2.3324e-5 2144.0 168.0 +RESULT diagonal/csr/spmm-N-w64/old-vendor 2.8744e-5 1.7954e-5 2144.0 168.0 +RESULT diagonal/csr/spmv-T/old-vendor 0.000100808 1.6311e-5 2208.0 24.0 +RESULT diagonal/csr/spmm-T-w1/old-vendor 0.00166351 1.7212e-5 2160.0 24.0 +RESULT diagonal/csr/spmm-T-w8/old-vendor 5.9591e-5 1.5189e-5 2160.0 168.0 +RESULT diagonal/csr/spmm-T-w64/old-vendor 2.7031e-5 1.7894e-5 2160.0 168.0 +CASE diagonal csc (10000, 10000) 10000 +RESULT diagonal/csc/spmv-N/old-vendor 0.044621679 1.7663e-5 2192.0 24.0 +RESULT diagonal/csc/spmm-N-w1/old-vendor 0.000113722 1.7383e-5 2144.0 24.0 +RESULT diagonal/csc/spmm-N-w8/old-vendor 2.2943e-5 1.5218e-5 2144.0 168.0 +RESULT diagonal/csc/spmm-N-w64/old-vendor 2.5999e-5 1.7973e-5 2144.0 168.0 +RESULT diagonal/csc/spmv-T/old-vendor 3.2711e-5 1.5649e-5 2208.0 24.0 +RESULT diagonal/csc/spmm-T-w1/old-vendor 1.8635e-5 1.5619e-5 2160.0 24.0 +RESULT diagonal/csc/spmm-T-w8/old-vendor 1.8905e-5 1.4897e-5 2160.0 168.0 +RESULT diagonal/csc/spmm-T-w64/old-vendor 2.6991e-5 2.0198e-5 2160.0 168.0 +CASE banded csr (10000, 10000) 89980 +RESULT banded/csr/spmv-N/old-vendor 3.5977e-5 1.5298e-5 2192.0 152.0 +RESULT banded/csr/spmm-N-w1/old-vendor 1.9126e-5 1.5419e-5 2144.0 152.0 +RESULT banded/csr/spmm-N-w8/old-vendor 1.8965e-5 1.6681e-5 2144.0 1416.0 +RESULT banded/csr/spmm-N-w64/old-vendor 3.6598e-5 2.9956e-5 2144.0 1416.0 +RESULT banded/csr/spmv-T/old-vendor 2.8763e-5 1.5268e-5 2208.0 152.0 +RESULT banded/csr/spmm-T-w1/old-vendor 1.6961e-5 1.5388e-5 2160.0 152.0 +RESULT banded/csr/spmm-T-w8/old-vendor 2.0468e-5 1.6782e-5 2160.0 1416.0 +RESULT banded/csr/spmm-T-w64/old-vendor 0.016543325 3.1749e-5 2160.0 1416.0 +CASE banded csc (10000, 10000) 89980 +RESULT banded/csc/spmv-N/old-vendor 3.9564e-5 1.5259e-5 2192.0 152.0 +RESULT banded/csc/spmm-N-w1/old-vendor 1.9146e-5 1.5519e-5 2144.0 152.0 +RESULT banded/csc/spmm-N-w8/old-vendor 1.7533e-5 1.7292e-5 2144.0 1416.0 +RESULT banded/csc/spmm-N-w64/old-vendor 7.1003e-5 3.0657e-5 2144.0 1416.0 +RESULT banded/csc/spmv-T/old-vendor 2.5017e-5 1.4747e-5 2208.0 152.0 +RESULT banded/csc/spmm-T-w1/old-vendor 1.9276e-5 1.5018e-5 2160.0 152.0 +RESULT banded/csc/spmm-T-w8/old-vendor 1.8675e-5 1.6311e-5 2160.0 1416.0 +RESULT banded/csc/spmm-T-w64/old-vendor 4.2039e-5 5.1236e-5 2160.0 1416.0 +CASE uniform csr (10000, 10000) 79716 +RESULT uniform/csr/spmv-N/old-vendor 3.3492e-5 1.617e-5 2192.0 136.0 +RESULT uniform/csr/spmm-N-w1/old-vendor 1.8324e-5 1.6371e-5 2144.0 136.0 +RESULT uniform/csr/spmm-N-w8/old-vendor 2.6149e-5 2.3654e-5 2144.0 1256.0 +RESULT uniform/csr/spmm-N-w64/old-vendor 0.000100367 8.9868e-5 2736.0 1256.0 +RESULT uniform/csr/spmv-T/old-vendor 2.5388e-5 1.6501e-5 2208.0 136.0 +RESULT uniform/csr/spmm-T-w1/old-vendor 1.7984e-5 1.6661e-5 2160.0 136.0 +RESULT uniform/csr/spmm-T-w8/old-vendor 3.1479e-5 2.5377e-5 2160.0 1256.0 +RESULT uniform/csr/spmm-T-w64/old-vendor 0.000110176 0.00010174 2752.0 1256.0 +CASE uniform csc (10000, 10000) 79716 +RESULT uniform/csc/spmv-N/old-vendor 2.8312e-5 1.656e-5 2192.0 136.0 +RESULT uniform/csc/spmm-N-w1/old-vendor 1.8475e-5 1.6982e-5 2144.0 136.0 +RESULT uniform/csc/spmm-N-w8/old-vendor 2.6329e-5 2.5247e-5 2144.0 1256.0 +RESULT uniform/csc/spmm-N-w64/old-vendor 0.000113481 0.00010209 2736.0 1256.0 +RESULT uniform/csc/spmv-T/old-vendor 2.5788e-5 1.616e-5 2208.0 136.0 +RESULT uniform/csc/spmm-T-w1/old-vendor 1.7232e-5 1.64e-5 2160.0 136.0 +RESULT uniform/csc/spmm-T-w8/old-vendor 2.6961e-5 2.3554e-5 2160.0 1256.0 +RESULT uniform/csc/spmm-T-w64/old-vendor 0.000101319 9.2914e-5 2752.0 1256.0 +CASE skewed csr (10000, 10000) 76697 +RESULT skewed/csr/spmv-N/old-vendor 3.0967e-5 1.6752e-5 2192.0 128.0 +RESULT skewed/csr/spmm-N-w1/old-vendor 2.0899e-5 1.6961e-5 2144.0 128.0 +RESULT skewed/csr/spmm-N-w8/old-vendor 2.3173e-5 2.1389e-5 2144.0 1208.0 +RESULT skewed/csr/spmm-N-w64/old-vendor 8.3856e-5 7.1674e-5 2736.0 1208.0 +RESULT skewed/csr/spmv-T/old-vendor 2.4295e-5 1.6921e-5 2208.0 128.0 +RESULT skewed/csr/spmm-T-w1/old-vendor 1.7543e-5 1.7172e-5 2160.0 128.0 +RESULT skewed/csr/spmm-T-w8/old-vendor 2.668e-5 2.3584e-5 2160.0 1208.0 +RESULT skewed/csr/spmm-T-w64/old-vendor 9.7963e-5 8.7934e-5 2752.0 1208.0 +CASE skewed csc (10000, 10000) 76697 +RESULT skewed/csc/spmv-N/old-vendor 3.7499e-5 2.3865e-5 2192.0 128.0 +RESULT skewed/csc/spmm-N-w1/old-vendor 2.5087e-5 2.3975e-5 2144.0 128.0 +RESULT skewed/csc/spmm-N-w8/old-vendor 3.207e-5 3.0877e-5 2144.0 1208.0 +RESULT skewed/csc/spmm-N-w64/old-vendor 0.000141344 0.000131294 2736.0 1208.0 +RESULT skewed/csc/spmv-T/old-vendor 2.9005e-5 1.5119e-5 2208.0 128.0 +RESULT skewed/csc/spmm-T-w1/old-vendor 1.7693e-5 1.5409e-5 2160.0 128.0 +RESULT skewed/csc/spmm-T-w8/old-vendor 2.2843e-5 2.0929e-5 2160.0 1208.0 +RESULT skewed/csc/spmm-T-w64/old-vendor 7.3617e-5 6.1805e-5 2752.0 1208.0 +CASE empty-rows csr (10000, 10000) 79860 +RESULT empty-rows/csr/spmv-N/old-vendor 3.0837e-5 1.6901e-5 2192.0 136.0 +RESULT empty-rows/csr/spmm-N-w1/old-vendor 1.8455e-5 1.6551e-5 2144.0 136.0 +RESULT empty-rows/csr/spmm-N-w8/old-vendor 2.4125e-5 2.2953e-5 2144.0 1256.0 +RESULT empty-rows/csr/spmm-N-w64/old-vendor 9.5668e-5 8.5699e-5 2736.0 1256.0 +RESULT empty-rows/csr/spmv-T/old-vendor 2.698e-5 1.6732e-5 2208.0 136.0 +RESULT empty-rows/csr/spmm-T-w1/old-vendor 1.7793e-5 1.6901e-5 2160.0 136.0 +RESULT empty-rows/csr/spmm-T-w8/old-vendor 2.8473e-5 2.5337e-5 2160.0 1256.0 +RESULT empty-rows/csr/spmm-T-w64/old-vendor 0.000110316 0.000102361 2752.0 1256.0 +CASE empty-rows csc (10000, 10000) 79860 +RESULT empty-rows/csc/spmv-N/old-vendor 2.9445e-5 1.6601e-5 2192.0 136.0 +RESULT empty-rows/csc/spmm-N-w1/old-vendor 2.0779e-5 1.6841e-5 2144.0 136.0 +RESULT empty-rows/csc/spmm-N-w8/old-vendor 2.6569e-5 2.5328e-5 2144.0 1256.0 +RESULT empty-rows/csc/spmm-N-w64/old-vendor 0.00011259 0.0001022 2736.0 1256.0 +RESULT empty-rows/csc/spmv-T/old-vendor 2.7471e-5 1.605e-5 2208.0 136.0 +RESULT empty-rows/csc/spmm-T-w1/old-vendor 1.7232e-5 1.633e-5 2160.0 136.0 +RESULT empty-rows/csc/spmm-T-w8/old-vendor 2.643e-5 2.3374e-5 2160.0 1256.0 +RESULT empty-rows/csc/spmm-T-w64/old-vendor 0.000100817 9.2062e-5 2752.0 1256.0 +CASE 1138_bus csr (1138, 1138) 4054 +RESULT 1138_bus/csr/spmv-N/old-vendor 2.7982e-5 1.4497e-5 2192.0 16.0 +RESULT 1138_bus/csr/spmm-N-w1/old-vendor 1.5429e-5 1.4347e-5 2144.0 16.0 +RESULT 1138_bus/csr/spmm-N-w8/old-vendor 3.9504e-5 1.4126e-5 2144.0 72.0 +RESULT 1138_bus/csr/spmm-N-w64/old-vendor 1.5098e-5 1.5128e-5 2144.0 72.0 +RESULT 1138_bus/csr/spmv-T/old-vendor 2.139e-5 1.4907e-5 2208.0 16.0 +RESULT 1138_bus/csr/spmm-T-w1/old-vendor 1.4637e-5 1.5008e-5 2160.0 16.0 +RESULT 1138_bus/csr/spmm-T-w8/old-vendor 1.8855e-5 1.4687e-5 2160.0 72.0 +RESULT 1138_bus/csr/spmm-T-w64/old-vendor 1.5348e-5 1.5849e-5 2160.0 72.0 +CASE 1138_bus csc (1138, 1138) 4054 +RESULT 1138_bus/csc/spmv-N/old-vendor 2.0818e-5 1.4807e-5 2192.0 16.0 +RESULT 1138_bus/csc/spmm-N-w1/old-vendor 1.5189e-5 1.4888e-5 2144.0 16.0 +RESULT 1138_bus/csc/spmm-N-w8/old-vendor 1.9617e-5 1.4848e-5 2144.0 72.0 +RESULT 1138_bus/csc/spmm-N-w64/old-vendor 1.5579e-5 1.596e-5 2144.0 72.0 +RESULT 1138_bus/csc/spmv-T/old-vendor 1.9857e-5 1.4066e-5 2208.0 16.0 +RESULT 1138_bus/csc/spmm-T-w1/old-vendor 1.4016e-5 1.4287e-5 2160.0 16.0 +RESULT 1138_bus/csc/spmm-T-w8/old-vendor 1.7724e-5 1.4307e-5 2160.0 72.0 +RESULT 1138_bus/csc/spmm-T-w64/old-vendor 1.4857e-5 1.5179e-5 2160.0 72.0 +CASE bcsstk17 csr (10974, 10974) 428650 +RESULT bcsstk17/csr/spmv-N/old-vendor 2.2441e-5 1.5519e-5 2192.0 680.0 +RESULT bcsstk17/csr/spmm-N-w1/old-vendor 1.7612e-5 1.565e-5 2144.0 680.0 +RESULT bcsstk17/csr/spmm-N-w8/old-vendor 3.0086e-5 2.4275e-5 2144.0 6708.0 +RESULT bcsstk17/csr/spmm-N-w64/old-vendor 0.000109254 8.6642e-5 2736.0 6708.0 +RESULT bcsstk17/csr/spmv-T/old-vendor 3.5476e-5 1.7232e-5 2208.0 680.0 +RESULT bcsstk17/csr/spmm-T-w1/old-vendor 1.8514e-5 1.7242e-5 2160.0 680.0 +RESULT bcsstk17/csr/spmm-T-w8/old-vendor 3.766e-5 3.234e-5 2160.0 6708.0 +RESULT bcsstk17/csr/spmm-T-w64/old-vendor 0.000183152 0.000153265 2752.0 6708.0 +CASE bcsstk17 csc (10974, 10974) 428650 +RESULT bcsstk17/csc/spmv-N/old-vendor 3.8662e-5 1.7001e-5 2192.0 680.0 +RESULT bcsstk17/csc/spmm-N-w1/old-vendor 1.9306e-5 1.7593e-5 2144.0 680.0 +RESULT bcsstk17/csc/spmm-N-w8/old-vendor 3.7991e-5 3.3272e-5 2144.0 6708.0 +RESULT bcsstk17/csc/spmm-N-w64/old-vendor 0.000176959 0.000152815 2736.0 6708.0 +RESULT bcsstk17/csc/spmv-T/old-vendor 3.4344e-5 1.5609e-5 2208.0 680.0 +RESULT bcsstk17/csc/spmm-T-w1/old-vendor 1.9767e-5 1.5969e-5 2160.0 680.0 +RESULT bcsstk17/csc/spmm-T-w8/old-vendor 2.8804e-5 2.4446e-5 2160.0 6708.0 +RESULT bcsstk17/csc/spmm-T-w64/old-vendor 0.000114994 8.7222e-5 2752.0 6708.0 +RESULT complex-banded/csr/spmm-C-w8/old-vendor 0.096398553 1.9476e-5 2160.0 1416.0 +RESULT complex-banded/csc/spmm-C-w8/old-vendor 0.030163 2.3494e-5 2160.0 1416.0 +RESULT duplicates/assembly/csr 3.249939846 0.00129282 23280.0 4.37522e6 +RESULT candidate-expansion/csr/spgemm/old-vendor 0.429357572 0.000578208 4736.0 2.03803492e8 diff --git a/perf/results/sparse-v13-cuda-migrated.tsv b/perf/results/sparse-v13-cuda-migrated.tsv new file mode 100644 index 000000000..d503c4d85 --- /dev/null +++ b/perf/results/sparse-v13-cuda-migrated.tsv @@ -0,0 +1,365 @@ +STACK Julia=1.13.2 CUDA=7.0.0 cuSPARSE=7.0.0 GPUArrays=13.0.0 runtime=13.4.0 +COLUMNS case/format/operation/policy first_seconds warm_median_seconds host_alloc_bytes device_alloc_bytes +INPUT_SHA256 1138_bus.mtx.gz 0544e52ed96893ae82c5a7f9051d662e04a1c27fe57768578de09932381fd963 +INPUT_SHA256 bcsstk17.mtx.gz 105425dc922f5aaa4f2a7b52d707e946cbc3cc309accea11f5365a30528f49a5 +CASE diagonal csr (10000, 10000) 10000 +RESULT diagonal/csr/spmv-N/automatic 0.126750304 1.6561e-5 2464.0 24.0 +RESULT diagonal/csr/spmv-N/vendor 8.8795e-5 1.633e-5 2464.0 24.0 +RESULT diagonal/csr/spmv-N/portable 0.135845656 1.0369e-5 2320.0 0.0 +RESULT diagonal/csr/spmm-N-w1/automatic 0.079291177 1.7011e-5 2336.0 24.0 +RESULT diagonal/csr/spmm-N-w1/vendor 9.661e-5 1.6651e-5 2336.0 24.0 +RESULT diagonal/csr/spmm-N-w1/portable 0.151696039 1.0459e-5 2288.0 0.0 +RESULT diagonal/csr/spmm-N-w8/automatic 0.004287751 1.584e-5 2336.0 168.0 +RESULT diagonal/csr/spmm-N-w8/vendor 1.9356e-5 1.5779e-5 2336.0 168.0 +RESULT diagonal/csr/spmm-N-w8/portable 1.8965e-5 1.045e-5 2288.0 0.0 +RESULT diagonal/csr/spmm-N-w64/automatic 3.2551e-5 1.8935e-5 2336.0 168.0 +RESULT diagonal/csr/spmm-N-w64/vendor 1.8374e-5 1.9145e-5 2336.0 168.0 +RESULT diagonal/csr/spmm-N-w64/portable 3.9223e-5 1.4587e-5 2288.0 0.0 +RESULT diagonal/csr/spmv-T/automatic 0.000117128 1.6751e-5 2464.0 24.0 +RESULT diagonal/csr/spmv-T/vendor 1.5709e-5 1.6541e-5 2464.0 24.0 +RESULT diagonal/csr/spmv-T/portable 0.000256317 0.000116447 24880.0 630016.0 +RESULT diagonal/csr/spmm-T-w1/automatic 0.001757866 1.6921e-5 2336.0 24.0 +RESULT diagonal/csr/spmm-T-w1/vendor 1.575e-5 1.6801e-5 2336.0 24.0 +RESULT diagonal/csr/spmm-T-w1/portable 0.000253994 0.000116538 24848.0 630016.0 +RESULT diagonal/csr/spmm-T-w8/automatic 5.881e-5 1.597e-5 2336.0 168.0 +RESULT diagonal/csr/spmm-T-w8/vendor 1.4157e-5 1.5789e-5 2336.0 168.0 +RESULT diagonal/csr/spmm-T-w8/portable 0.00015042 0.000116137 24848.0 630016.0 +RESULT diagonal/csr/spmm-T-w64/automatic 3.1499e-5 1.9787e-5 2336.0 168.0 +RESULT diagonal/csr/spmm-T-w64/vendor 1.8255e-5 1.8836e-5 2336.0 168.0 +RESULT diagonal/csr/spmm-T-w64/portable 0.000259844 0.000120194 24848.0 630016.0 +CASE diagonal csc (10000, 10000) 10000 +RESULT diagonal/csc/spmv-N/automatic 0.000127177 1.6811e-5 2464.0 24.0 +RESULT diagonal/csc/spmv-N/vendor 7.7174e-5 1.6801e-5 2464.0 24.0 +RESULT diagonal/csc/spmv-N/portable 0.004253509 0.000116117 24784.0 630016.0 +RESULT diagonal/csc/spmm-N-w1/automatic 0.000115455 1.6711e-5 2336.0 24.0 +RESULT diagonal/csc/spmm-N-w1/vendor 9.3705e-5 1.6731e-5 2336.0 24.0 +RESULT diagonal/csc/spmm-N-w1/portable 0.000361203 0.000117028 24752.0 630016.0 +RESULT diagonal/csc/spmm-N-w8/automatic 4.5976e-5 1.604e-5 2336.0 168.0 +RESULT diagonal/csc/spmm-N-w8/vendor 1.4497e-5 1.5639e-5 2336.0 168.0 +RESULT diagonal/csc/spmm-N-w8/portable 0.000145972 0.000116367 24752.0 630016.0 +RESULT diagonal/csc/spmm-N-w64/automatic 2.8924e-5 1.8564e-5 2336.0 168.0 +RESULT diagonal/csc/spmm-N-w64/vendor 1.7052e-5 1.8184e-5 2336.0 168.0 +RESULT diagonal/csc/spmm-N-w64/portable 0.000146083 0.000119924 24752.0 630016.0 +RESULT diagonal/csc/spmv-T/automatic 4.0035e-5 1.6531e-5 2464.0 24.0 +RESULT diagonal/csc/spmv-T/vendor 1.5008e-5 1.614e-5 2464.0 24.0 +RESULT diagonal/csc/spmv-T/portable 2.0919e-5 1.042e-5 2416.0 0.0 +RESULT diagonal/csc/spmm-T-w1/automatic 2.4356e-5 1.6341e-5 2336.0 24.0 +RESULT diagonal/csc/spmm-T-w1/vendor 1.5088e-5 1.631e-5 2336.0 24.0 +RESULT diagonal/csc/spmm-T-w1/portable 1.5379e-5 1.0259e-5 2384.0 0.0 +RESULT diagonal/csc/spmm-T-w8/automatic 1.9015e-5 1.5489e-5 2336.0 168.0 +RESULT diagonal/csc/spmm-T-w8/vendor 1.4136e-5 1.5369e-5 2336.0 168.0 +RESULT diagonal/csc/spmm-T-w8/portable 1.4126e-5 1.0389e-5 2384.0 0.0 +RESULT diagonal/csc/spmm-T-w64/automatic 2.8954e-5 1.8454e-5 2336.0 168.0 +RESULT diagonal/csc/spmm-T-w64/vendor 1.7292e-5 1.8495e-5 2336.0 168.0 +RESULT diagonal/csc/spmm-T-w64/portable 2.0218e-5 1.4377e-5 2384.0 0.0 +CASE banded csr (10000, 10000) 89980 +RESULT banded/csr/spmv-N/automatic 4.5916e-5 1.5498e-5 2464.0 152.0 +RESULT banded/csr/spmv-N/vendor 1.9626e-5 1.5469e-5 2464.0 152.0 +RESULT banded/csr/spmv-N/portable 2.2652e-5 1.1191e-5 2320.0 0.0 +RESULT banded/csr/spmm-N-w1/automatic 2.4085e-5 1.5489e-5 2336.0 152.0 +RESULT banded/csr/spmm-N-w1/vendor 1.4156e-5 1.5378e-5 2336.0 152.0 +RESULT banded/csr/spmm-N-w1/portable 1.6781e-5 1.1501e-5 2288.0 0.0 +RESULT banded/csr/spmm-N-w8/automatic 2.3424e-5 1.6921e-5 2336.0 1416.0 +RESULT banded/csr/spmm-N-w8/vendor 1.5369e-5 1.7011e-5 2336.0 1416.0 +RESULT banded/csr/spmm-N-w8/portable 1.7212e-5 1.4116e-5 2288.0 0.0 +RESULT banded/csr/spmm-N-w64/automatic 4.2228e-5 3.1309e-5 2336.0 1416.0 +RESULT banded/csr/spmm-N-w64/vendor 3.0437e-5 3.0717e-5 2336.0 1416.0 +RESULT banded/csr/spmm-N-w64/portable 0.018006982 4.0496e-5 2880.0 0.0 +RESULT banded/csr/spmv-T/automatic 5.8429e-5 1.5729e-5 2464.0 152.0 +RESULT banded/csr/spmv-T/vendor 1.4397e-5 1.5739e-5 2464.0 152.0 +RESULT banded/csr/spmv-T/portable 0.000228897 0.000146393 27696.0 5.348836e6 +RESULT banded/csr/spmm-T-w1/automatic 3.4555e-5 1.5909e-5 2336.0 152.0 +RESULT banded/csr/spmm-T-w1/vendor 1.4397e-5 1.5799e-5 2336.0 152.0 +RESULT banded/csr/spmm-T-w1/portable 0.000191116 0.000149158 27664.0 5.348836e6 +RESULT banded/csr/spmm-T-w8/automatic 3.754e-5 1.7392e-5 2336.0 1416.0 +RESULT banded/csr/spmm-T-w8/vendor 1.593e-5 1.7312e-5 2336.0 1416.0 +RESULT banded/csr/spmm-T-w8/portable 0.000233355 0.000149679 27664.0 5.348836e6 +RESULT banded/csr/spmm-T-w64/automatic 5.7016e-5 3.1118e-5 2336.0 1416.0 +RESULT banded/csr/spmm-T-w64/vendor 4.3081e-5 3.0817e-5 2336.0 1416.0 +RESULT banded/csr/spmm-T-w64/portable 0.000208488 0.000177481 28256.0 5.348836e6 +CASE banded csc (10000, 10000) 89980 +RESULT banded/csc/spmv-N/automatic 4.0566e-5 1.585e-5 2464.0 152.0 +RESULT banded/csc/spmv-N/vendor 1.4446e-5 1.5689e-5 2464.0 152.0 +RESULT banded/csc/spmv-N/portable 0.000185236 0.000143227 27600.0 5.348836e6 +RESULT banded/csc/spmm-N-w1/automatic 2.2441e-5 1.595e-5 2336.0 152.0 +RESULT banded/csc/spmm-N-w1/vendor 1.4357e-5 1.596e-5 2336.0 152.0 +RESULT banded/csc/spmm-N-w1/portable 0.000175858 0.000142807 27568.0 5.348836e6 +RESULT banded/csc/spmm-N-w8/automatic 2.2702e-5 1.7232e-5 2336.0 1416.0 +RESULT banded/csc/spmm-N-w8/vendor 2.4365e-5 1.7142e-5 2336.0 1416.0 +RESULT banded/csc/spmm-N-w8/portable 0.000187629 0.00014527 27568.0 5.348836e6 +RESULT banded/csc/spmm-N-w64/automatic 5.1626e-5 3.1118e-5 2336.0 1416.0 +RESULT banded/csc/spmm-N-w64/vendor 2.9776e-5 3.1098e-5 2336.0 1416.0 +RESULT banded/csc/spmm-N-w64/portable 0.00022004 0.000175046 28160.0 5.348836e6 +RESULT banded/csc/spmv-T/automatic 2.9374e-5 1.5719e-5 2464.0 152.0 +RESULT banded/csc/spmv-T/vendor 1.4166e-5 1.5379e-5 2464.0 152.0 +RESULT banded/csc/spmv-T/portable 1.8635e-5 1.1462e-5 2416.0 0.0 +RESULT banded/csc/spmm-T-w1/automatic 1.9606e-5 1.5358e-5 2336.0 152.0 +RESULT banded/csc/spmm-T-w1/vendor 1.3876e-5 1.5239e-5 2336.0 152.0 +RESULT banded/csc/spmm-T-w1/portable 1.5369e-5 1.1732e-5 2384.0 0.0 +RESULT banded/csc/spmm-T-w8/automatic 2.0378e-5 1.6681e-5 2336.0 1416.0 +RESULT banded/csc/spmm-T-w8/vendor 1.5599e-5 1.6701e-5 2336.0 1416.0 +RESULT banded/csc/spmm-T-w8/portable 1.6721e-5 1.6661e-5 2384.0 0.0 +RESULT banded/csc/spmm-T-w64/automatic 8.1302e-5 3.8632e-5 2336.0 1416.0 +RESULT banded/csc/spmm-T-w64/vendor 3.208e-5 3.3502e-5 2336.0 1416.0 +RESULT banded/csc/spmm-T-w64/portable 7.6853e-5 4.4223e-5 2976.0 0.0 +CASE uniform csr (10000, 10000) 79716 +RESULT uniform/csr/spmv-N/automatic 5.373e-5 1.8705e-5 2464.0 136.0 +RESULT uniform/csr/spmv-N/vendor 1.9677e-5 1.8214e-5 2464.0 136.0 +RESULT uniform/csr/spmv-N/portable 3.5206e-5 1.4738e-5 2320.0 0.0 +RESULT uniform/csr/spmm-N-w1/automatic 4.7168e-5 1.9546e-5 2336.0 136.0 +RESULT uniform/csr/spmm-N-w1/vendor 1.8113e-5 1.8585e-5 2336.0 136.0 +RESULT uniform/csr/spmm-N-w1/portable 4.1928e-5 1.5058e-5 2288.0 0.0 +RESULT uniform/csr/spmm-N-w8/automatic 4.0215e-5 2.6089e-5 2336.0 1256.0 +RESULT uniform/csr/spmm-N-w8/vendor 2.4295e-5 2.6059e-5 2336.0 1256.0 +RESULT uniform/csr/spmm-N-w8/portable 4.9552e-5 2.169e-5 2288.0 0.0 +RESULT uniform/csr/spmm-N-w64/automatic 0.000109003 9.1761e-5 2928.0 1256.0 +RESULT uniform/csr/spmm-N-w64/vendor 9.5247e-5 9.0068e-5 2928.0 1256.0 +RESULT uniform/csr/spmm-N-w64/portable 9.8113e-5 7.1183e-5 2880.0 0.0 +RESULT uniform/csr/spmv-T/automatic 5.7787e-5 1.9416e-5 2464.0 136.0 +RESULT uniform/csr/spmv-T/vendor 1.6901e-5 1.9326e-5 2464.0 136.0 +RESULT uniform/csr/spmv-T/portable 0.000303205 0.000159477 27696.0 4.74326e6 +RESULT uniform/csr/spmm-T-w1/automatic 3.6448e-5 1.9416e-5 2336.0 136.0 +RESULT uniform/csr/spmm-T-w1/vendor 1.7613e-5 1.9136e-5 2336.0 136.0 +RESULT uniform/csr/spmm-T-w1/portable 0.000177661 0.000145461 27664.0 4.74326e6 +RESULT uniform/csr/spmm-T-w8/automatic 3.6799e-5 2.6329e-5 2336.0 1256.0 +RESULT uniform/csr/spmm-T-w8/vendor 3.0397e-5 2.6058e-5 2336.0 1256.0 +RESULT uniform/csr/spmm-T-w8/portable 0.000193761 0.000150711 27664.0 4.74326e6 +RESULT uniform/csr/spmm-T-w64/automatic 0.000114393 0.000100768 2928.0 1256.0 +RESULT uniform/csr/spmm-T-w64/vendor 9.9225e-5 0.000100217 2928.0 1256.0 +RESULT uniform/csr/spmm-T-w64/portable 0.00023619 0.000202868 28256.0 4.74326e6 +CASE uniform csc (10000, 10000) 79716 +RESULT uniform/csc/spmv-N/automatic 3.8722e-5 1.7663e-5 2464.0 136.0 +RESULT uniform/csc/spmv-N/vendor 1.605e-5 1.7583e-5 2464.0 136.0 +RESULT uniform/csc/spmv-N/portable 0.000175317 0.000144079 27600.0 4.74326e6 +RESULT uniform/csc/spmm-N-w1/automatic 2.3424e-5 1.7693e-5 2336.0 136.0 +RESULT uniform/csc/spmm-N-w1/vendor 1.615e-5 1.7703e-5 2336.0 136.0 +RESULT uniform/csc/spmm-N-w1/portable 0.000167402 0.000146313 27568.0 4.74326e6 +RESULT uniform/csc/spmm-N-w8/automatic 3.0647e-5 2.6209e-5 2336.0 1256.0 +RESULT uniform/csc/spmm-N-w8/vendor 2.4596e-5 2.6289e-5 2336.0 1256.0 +RESULT uniform/csc/spmm-N-w8/portable 0.000190976 0.000153767 27568.0 4.74326e6 +RESULT uniform/csc/spmm-N-w64/automatic 0.000113813 0.000100407 2928.0 1256.0 +RESULT uniform/csc/spmm-N-w64/vendor 9.9536e-5 9.9535e-5 2928.0 1256.0 +RESULT uniform/csc/spmm-N-w64/portable 0.000304789 0.000201515 28160.0 4.74326e6 +RESULT uniform/csc/spmv-T/automatic 3.9493e-5 1.6992e-5 2464.0 136.0 +RESULT uniform/csc/spmv-T/vendor 1.5348e-5 1.6791e-5 2464.0 136.0 +RESULT uniform/csc/spmv-T/portable 2.3053e-5 1.2743e-5 2416.0 0.0 +RESULT uniform/csc/spmm-T-w1/automatic 2.143e-5 1.7042e-5 2336.0 136.0 +RESULT uniform/csc/spmm-T-w1/vendor 1.5449e-5 1.6871e-5 2336.0 136.0 +RESULT uniform/csc/spmm-T-w1/portable 1.7743e-5 1.3705e-5 2384.0 0.0 +RESULT uniform/csc/spmm-T-w8/automatic 2.8483e-5 2.4015e-5 2336.0 1256.0 +RESULT uniform/csc/spmm-T-w8/vendor 2.2702e-5 2.3945e-5 2336.0 1256.0 +RESULT uniform/csc/spmm-T-w8/portable 2.3945e-5 2.0308e-5 2384.0 0.0 +RESULT uniform/csc/spmm-T-w64/automatic 0.000100757 8.7844e-5 2928.0 1256.0 +RESULT uniform/csc/spmm-T-w64/vendor 9.3333e-5 8.7864e-5 2928.0 1256.0 +RESULT uniform/csc/spmm-T-w64/portable 7.6142e-5 6.9089e-5 2976.0 0.0 +CASE skewed csr (10000, 10000) 76697 +RESULT skewed/csr/spmv-N/automatic 3.1458e-5 1.7413e-5 2464.0 128.0 +RESULT skewed/csr/spmv-N/vendor 1.584e-5 1.7352e-5 2464.0 128.0 +RESULT skewed/csr/spmv-N/portable 0.000222966 0.000206284 2912.0 0.0 +RESULT skewed/csr/spmm-N-w1/automatic 2.9405e-5 1.7382e-5 2336.0 128.0 +RESULT skewed/csr/spmm-N-w1/vendor 1.5879e-5 1.7192e-5 2336.0 128.0 +RESULT skewed/csr/spmm-N-w1/portable 0.000675919 0.000669899 2880.0 0.0 +RESULT skewed/csr/spmm-N-w8/automatic 2.4817e-5 2.1821e-5 2336.0 1208.0 +RESULT skewed/csr/spmm-N-w8/vendor 2.0127e-5 2.1781e-5 2336.0 1208.0 +RESULT skewed/csr/spmm-N-w8/portable 0.000688052 0.000683925 2880.0 0.0 +RESULT skewed/csr/spmm-N-w64/automatic 8.6231e-5 7.0131e-5 2928.0 1208.0 +RESULT skewed/csr/spmm-N-w64/vendor 6.6965e-5 6.991e-5 2928.0 1208.0 +RESULT skewed/csr/spmm-N-w64/portable 0.00072939 0.000704713 2880.0 0.0 +RESULT skewed/csr/spmv-T/automatic 5.344e-5 1.7813e-5 2464.0 128.0 +RESULT skewed/csr/spmv-T/vendor 1.617e-5 1.7582e-5 2464.0 128.0 +RESULT skewed/csr/spmv-T/portable 0.000229588 0.000144229 27696.0 4.565148e6 +RESULT skewed/csr/spmm-T-w1/automatic 2.3985e-5 1.8014e-5 2336.0 128.0 +RESULT skewed/csr/spmm-T-w1/vendor 1.6731e-5 1.7843e-5 2336.0 128.0 +RESULT skewed/csr/spmm-T-w1/portable 0.000178472 0.000144099 27664.0 4.565148e6 +RESULT skewed/csr/spmm-T-w8/automatic 3.0256e-5 2.4385e-5 2336.0 1208.0 +RESULT skewed/csr/spmm-T-w8/vendor 2.2783e-5 2.4315e-5 2336.0 1208.0 +RESULT skewed/csr/spmm-T-w8/portable 0.000189683 0.000147936 27664.0 4.565148e6 +RESULT skewed/csr/spmm-T-w64/automatic 0.000100438 8.7594e-5 2928.0 1208.0 +RESULT skewed/csr/spmm-T-w64/vendor 8.6521e-5 8.7253e-5 2928.0 1208.0 +RESULT skewed/csr/spmm-T-w64/portable 0.000286083 0.000181879 28256.0 4.565148e6 +CASE skewed csc (10000, 10000) 76697 +RESULT skewed/csc/spmv-N/automatic 4.9703e-5 2.5056e-5 2464.0 128.0 +RESULT skewed/csc/spmv-N/vendor 2.3624e-5 2.4897e-5 2464.0 128.0 +RESULT skewed/csc/spmv-N/portable 0.000390989 0.000341476 28192.0 4.565148e6 +RESULT skewed/csc/spmm-N-w1/automatic 3.7109e-5 2.4536e-5 2336.0 128.0 +RESULT skewed/csc/spmm-N-w1/vendor 2.2872e-5 2.4575e-5 2336.0 128.0 +RESULT skewed/csc/spmm-N-w1/portable 0.000822363 0.000803899 28160.0 4.565148e6 +RESULT skewed/csc/spmm-N-w8/automatic 5.0574e-5 3.3252e-5 2336.0 1208.0 +RESULT skewed/csc/spmm-N-w8/vendor 3.4695e-5 3.3142e-5 2336.0 1208.0 +RESULT skewed/csc/spmm-N-w8/portable 0.000837652 0.00082612 28160.0 4.565148e6 +RESULT skewed/csc/spmm-N-w64/automatic 0.000147815 0.000132267 2928.0 1208.0 +RESULT skewed/csc/spmm-N-w64/vendor 0.000132296 0.000132467 2928.0 1208.0 +RESULT skewed/csc/spmm-N-w64/portable 0.000873559 0.000835488 28160.0 4.565148e6 +RESULT skewed/csc/spmv-T/automatic 4.0275e-5 1.595e-5 2464.0 128.0 +RESULT skewed/csc/spmv-T/vendor 1.4898e-5 1.587e-5 2464.0 128.0 +RESULT skewed/csc/spmv-T/portable 2.3043e-5 1.2493e-5 2416.0 0.0 +RESULT skewed/csc/spmm-T-w1/automatic 2.8603e-5 1.602e-5 2336.0 128.0 +RESULT skewed/csc/spmm-T-w1/vendor 1.4618e-5 1.599e-5 2336.0 128.0 +RESULT skewed/csc/spmm-T-w1/portable 1.7623e-5 1.3495e-5 2384.0 0.0 +RESULT skewed/csc/spmm-T-w8/automatic 2.8243e-5 2.0528e-5 2336.0 1208.0 +RESULT skewed/csc/spmm-T-w8/vendor 1.8896e-5 2.0378e-5 2336.0 1208.0 +RESULT skewed/csc/spmm-T-w8/portable 2.0418e-5 1.6591e-5 2384.0 0.0 +RESULT skewed/csc/spmm-T-w64/automatic 7.3707e-5 6.1535e-5 2928.0 1208.0 +RESULT skewed/csc/spmm-T-w64/vendor 5.9842e-5 6.1474e-5 2928.0 1208.0 +RESULT skewed/csc/spmm-T-w64/portable 5.41e-5 4.793e-5 2976.0 0.0 +CASE empty-rows csr (10000, 10000) 79860 +RESULT empty-rows/csr/spmv-N/automatic 3.19e-5 1.6932e-5 2464.0 136.0 +RESULT empty-rows/csr/spmv-N/vendor 1.5589e-5 1.662e-5 2464.0 136.0 +RESULT empty-rows/csr/spmv-N/portable 2.2442e-5 1.3976e-5 2320.0 0.0 +RESULT empty-rows/csr/spmm-N-w1/automatic 2.2252e-5 1.6872e-5 2336.0 136.0 +RESULT empty-rows/csr/spmm-N-w1/vendor 1.5609e-5 1.6821e-5 2336.0 136.0 +RESULT empty-rows/csr/spmm-N-w1/portable 2.5818e-5 1.8204e-5 2288.0 0.0 +RESULT empty-rows/csr/spmm-N-w8/automatic 2.9355e-5 2.3534e-5 2336.0 1256.0 +RESULT empty-rows/csr/spmm-N-w8/vendor 3.0376e-5 2.3444e-5 2336.0 1256.0 +RESULT empty-rows/csr/spmm-N-w8/portable 2.9816e-5 2.2452e-5 2288.0 0.0 +RESULT empty-rows/csr/spmm-N-w64/automatic 9.9666e-5 8.9216e-5 2928.0 1256.0 +RESULT empty-rows/csr/spmm-N-w64/vendor 8.5098e-5 8.4056e-5 2928.0 1256.0 +RESULT empty-rows/csr/spmm-N-w64/portable 0.000100387 7.7144e-5 2880.0 0.0 +RESULT empty-rows/csr/spmv-T/automatic 4.4573e-5 1.7362e-5 2464.0 136.0 +RESULT empty-rows/csr/spmv-T/vendor 1.5969e-5 1.7433e-5 2464.0 136.0 +RESULT empty-rows/csr/spmv-T/portable 0.000226753 0.000141864 27696.0 4.751756e6 +RESULT empty-rows/csr/spmm-T-w1/automatic 2.7602e-5 1.7392e-5 2336.0 136.0 +RESULT empty-rows/csr/spmm-T-w1/vendor 1.5849e-5 1.7343e-5 2336.0 136.0 +RESULT empty-rows/csr/spmm-T-w1/portable 0.00017207 0.000142565 27664.0 4.751756e6 +RESULT empty-rows/csr/spmm-T-w8/automatic 3.2621e-5 2.6019e-5 2336.0 1256.0 +RESULT empty-rows/csr/spmm-T-w8/vendor 2.4386e-5 2.6029e-5 2336.0 1256.0 +RESULT empty-rows/csr/spmm-T-w8/portable 0.000189043 0.00014985 27664.0 4.751756e6 +RESULT empty-rows/csr/spmm-T-w64/automatic 0.000112499 0.000100458 2928.0 1256.0 +RESULT empty-rows/csr/spmm-T-w64/vendor 9.8904e-5 0.000100347 2928.0 1256.0 +RESULT empty-rows/csr/spmm-T-w64/portable 0.000249495 0.000202157 28256.0 4.751756e6 +CASE empty-rows csc (10000, 10000) 79860 +RESULT empty-rows/csc/spmv-N/automatic 3.3543e-5 1.7432e-5 2464.0 136.0 +RESULT empty-rows/csc/spmv-N/vendor 2.4646e-5 1.7353e-5 2464.0 136.0 +RESULT empty-rows/csc/spmv-N/portable 0.000178353 0.00014496 27600.0 4.751756e6 +RESULT empty-rows/csc/spmm-N-w1/automatic 2.3534e-5 1.7643e-5 2336.0 136.0 +RESULT empty-rows/csc/spmm-N-w1/vendor 1.64e-5 1.7423e-5 2336.0 136.0 +RESULT empty-rows/csc/spmm-N-w1/portable 0.000182851 0.0001501 27568.0 4.751756e6 +RESULT empty-rows/csc/spmm-N-w8/automatic 3.5847e-5 2.5978e-5 2336.0 1256.0 +RESULT empty-rows/csc/spmm-N-w8/vendor 2.4505e-5 2.6038e-5 2336.0 1256.0 +RESULT empty-rows/csc/spmm-N-w8/portable 0.000179274 0.000153336 27568.0 4.751756e6 +RESULT empty-rows/csc/spmm-N-w64/automatic 0.000114673 0.000100507 2928.0 1256.0 +RESULT empty-rows/csc/spmm-N-w64/vendor 9.8383e-5 0.000100688 2928.0 1256.0 +RESULT empty-rows/csc/spmm-N-w64/portable 0.000239627 0.000207397 28160.0 4.751756e6 +RESULT empty-rows/csc/spmv-T/automatic 3.1568e-5 1.7042e-5 2464.0 136.0 +RESULT empty-rows/csc/spmv-T/vendor 1.5489e-5 1.6992e-5 2464.0 136.0 +RESULT empty-rows/csc/spmv-T/portable 2.0117e-5 1.2874e-5 2416.0 0.0 +RESULT empty-rows/csc/spmm-T-w1/automatic 2.3033e-5 1.7132e-5 2336.0 136.0 +RESULT empty-rows/csc/spmm-T-w1/vendor 1.5789e-5 1.6992e-5 2336.0 136.0 +RESULT empty-rows/csc/spmm-T-w1/portable 2.8594e-5 1.3615e-5 2384.0 0.0 +RESULT empty-rows/csc/spmm-T-w8/automatic 3.8632e-5 2.4125e-5 2336.0 1256.0 +RESULT empty-rows/csc/spmm-T-w8/vendor 2.2923e-5 2.4145e-5 2336.0 1256.0 +RESULT empty-rows/csc/spmm-T-w8/portable 4.3822e-5 2.0458e-5 2384.0 0.0 +RESULT empty-rows/csc/spmm-T-w64/automatic 0.000119312 8.9927e-5 2928.0 1256.0 +RESULT empty-rows/csc/spmm-T-w64/vendor 8.8806e-5 8.8285e-5 2928.0 1256.0 +RESULT empty-rows/csc/spmm-T-w64/portable 9.3715e-5 6.9921e-5 2976.0 0.0 +CASE 1138_bus csr (1138, 1138) 4054 +RESULT 1138_bus/csr/spmv-N/automatic 3.7961e-5 1.5318e-5 2464.0 16.0 +RESULT 1138_bus/csr/spmv-N/vendor 1.3806e-5 1.5128e-5 2464.0 16.0 +RESULT 1138_bus/csr/spmv-N/portable 2.2642e-5 1.1762e-5 2320.0 0.0 +RESULT 1138_bus/csr/spmm-N-w1/automatic 2.3043e-5 1.5439e-5 2336.0 16.0 +RESULT 1138_bus/csr/spmm-N-w1/vendor 1.4197e-5 1.5379e-5 2336.0 16.0 +RESULT 1138_bus/csr/spmm-N-w1/portable 1.6881e-5 1.1651e-5 2288.0 0.0 +RESULT 1138_bus/csr/spmm-N-w8/automatic 4.9502e-5 1.5028e-5 2336.0 72.0 +RESULT 1138_bus/csr/spmm-N-w8/vendor 1.8294e-5 1.5058e-5 2336.0 72.0 +RESULT 1138_bus/csr/spmm-N-w8/portable 1.4657e-5 1.1742e-5 2288.0 0.0 +RESULT 1138_bus/csr/spmm-N-w64/automatic 2.3353e-5 1.601e-5 2336.0 72.0 +RESULT 1138_bus/csr/spmm-N-w64/vendor 1.4517e-5 1.611e-5 2336.0 72.0 +RESULT 1138_bus/csr/spmm-N-w64/portable 1.4968e-5 1.2203e-5 2288.0 0.0 +RESULT 1138_bus/csr/spmv-T/automatic 2.3844e-5 1.5639e-5 2464.0 16.0 +RESULT 1138_bus/csr/spmv-T/vendor 1.4136e-5 1.565e-5 2464.0 16.0 +RESULT 1138_bus/csr/spmv-T/portable 0.000210422 0.000108322 23344.0 243760.0 +RESULT 1138_bus/csr/spmm-T-w1/automatic 2.0318e-5 1.612e-5 2336.0 16.0 +RESULT 1138_bus/csr/spmm-T-w1/vendor 1.8896e-5 1.5829e-5 2336.0 16.0 +RESULT 1138_bus/csr/spmm-T-w1/portable 0.000129742 0.000110776 23312.0 243760.0 +RESULT 1138_bus/csr/spmm-T-w8/automatic 2.2542e-5 1.5719e-5 2336.0 72.0 +RESULT 1138_bus/csr/spmm-T-w8/vendor 1.4127e-5 1.571e-5 2336.0 72.0 +RESULT 1138_bus/csr/spmm-T-w8/portable 0.000125144 0.000108201 23312.0 243760.0 +RESULT 1138_bus/csr/spmm-T-w64/automatic 2.0178e-5 1.7022e-5 2336.0 72.0 +RESULT 1138_bus/csr/spmm-T-w64/vendor 1.5479e-5 1.6751e-5 2336.0 72.0 +RESULT 1138_bus/csr/spmm-T-w64/portable 0.000133148 0.000109393 23312.0 243760.0 +CASE 1138_bus csc (1138, 1138) 4054 +RESULT 1138_bus/csc/spmv-N/automatic 2.2643e-5 1.5789e-5 2464.0 16.0 +RESULT 1138_bus/csc/spmv-N/vendor 1.4096e-5 1.6049e-5 2464.0 16.0 +RESULT 1138_bus/csc/spmv-N/portable 0.000130443 0.000108152 23248.0 243760.0 +RESULT 1138_bus/csc/spmm-N-w1/automatic 2.0058e-5 1.584e-5 2336.0 16.0 +RESULT 1138_bus/csc/spmm-N-w1/vendor 1.9977e-5 1.5709e-5 2336.0 16.0 +RESULT 1138_bus/csc/spmm-N-w1/portable 0.000127819 0.000109103 23216.0 243760.0 +RESULT 1138_bus/csc/spmm-N-w8/automatic 2.0067e-5 1.5299e-5 2336.0 72.0 +RESULT 1138_bus/csc/spmm-N-w8/vendor 1.3916e-5 1.5358e-5 2336.0 72.0 +RESULT 1138_bus/csc/spmm-N-w8/portable 0.000124903 0.00010704 23216.0 243760.0 +RESULT 1138_bus/csc/spmm-N-w64/automatic 2.0869e-5 1.6731e-5 2336.0 72.0 +RESULT 1138_bus/csc/spmm-N-w64/vendor 1.5259e-5 1.6551e-5 2336.0 72.0 +RESULT 1138_bus/csc/spmm-N-w64/portable 0.000130323 0.000107761 23216.0 243760.0 +RESULT 1138_bus/csc/spmv-T/automatic 2.8283e-5 1.5018e-5 2464.0 16.0 +RESULT 1138_bus/csc/spmv-T/vendor 1.3626e-5 1.5078e-5 2464.0 16.0 +RESULT 1138_bus/csc/spmv-T/portable 1.9507e-5 1.1652e-5 2416.0 0.0 +RESULT 1138_bus/csc/spmm-T-w1/automatic 1.7914e-5 1.7172e-5 2336.0 16.0 +RESULT 1138_bus/csc/spmm-T-w1/vendor 1.4557e-5 1.5199e-5 2336.0 16.0 +RESULT 1138_bus/csc/spmm-T-w1/portable 2.0088e-5 1.1762e-5 2384.0 0.0 +RESULT 1138_bus/csc/spmm-T-w8/automatic 1.8334e-5 1.4948e-5 2336.0 72.0 +RESULT 1138_bus/csc/spmm-T-w8/vendor 1.3686e-5 1.5048e-5 2336.0 72.0 +RESULT 1138_bus/csc/spmm-T-w8/portable 1.596e-5 1.1852e-5 2384.0 0.0 +RESULT 1138_bus/csc/spmm-T-w64/automatic 1.9286e-5 1.576e-5 2336.0 72.0 +RESULT 1138_bus/csc/spmm-T-w64/vendor 1.4457e-5 1.5929e-5 2336.0 72.0 +RESULT 1138_bus/csc/spmm-T-w64/portable 1.5489e-5 1.2163e-5 2384.0 0.0 +CASE bcsstk17 csr (10974, 10974) 428650 +RESULT bcsstk17/csr/spmv-N/automatic 2.6088e-5 1.6451e-5 2464.0 680.0 +RESULT bcsstk17/csr/spmv-N/vendor 1.5199e-5 1.6371e-5 2464.0 680.0 +RESULT bcsstk17/csr/spmv-N/portable 3.2691e-5 2.669e-5 2320.0 0.0 +RESULT bcsstk17/csr/spmm-N-w1/automatic 2.7611e-5 1.626e-5 2336.0 680.0 +RESULT bcsstk17/csr/spmm-N-w1/vendor 1.4908e-5 1.6401e-5 2336.0 680.0 +RESULT bcsstk17/csr/spmm-N-w1/portable 7.3827e-5 6.2046e-5 2880.0 0.0 +RESULT bcsstk17/csr/spmm-N-w8/automatic 2.7751e-5 2.5017e-5 2336.0 6708.0 +RESULT bcsstk17/csr/spmm-N-w8/vendor 2.3313e-5 2.4666e-5 2336.0 6708.0 +RESULT bcsstk17/csr/spmm-N-w8/portable 0.000127548 0.000121687 2880.0 0.0 +RESULT bcsstk17/csr/spmm-N-w64/automatic 0.000105166 8.5779e-5 2928.0 6708.0 +RESULT bcsstk17/csr/spmm-N-w64/vendor 8.4207e-5 8.557e-5 2928.0 6708.0 +RESULT bcsstk17/csr/spmm-N-w64/portable 0.000733527 0.000710965 2880.0 0.0 +RESULT bcsstk17/csr/spmv-T/automatic 4.4803e-5 1.7973e-5 2464.0 680.0 +RESULT bcsstk17/csr/spmv-T/vendor 1.8815e-5 1.8394e-5 2464.0 680.0 +RESULT bcsstk17/csr/spmv-T/portable 0.000390377 0.00027369 29584.0 2.5334268e7 +RESULT bcsstk17/csr/spmm-T-w1/automatic 3.5777e-5 1.8364e-5 2336.0 680.0 +RESULT bcsstk17/csr/spmm-T-w1/vendor 1.6982e-5 1.8124e-5 2336.0 680.0 +RESULT bcsstk17/csr/spmm-T-w1/portable 0.000336387 0.000315188 30144.0 2.5334268e7 +RESULT bcsstk17/csr/spmm-T-w8/automatic 4.7619e-5 3.2842e-5 2336.0 6708.0 +RESULT bcsstk17/csr/spmm-T-w8/vendor 3.9754e-5 3.3282e-5 2336.0 6708.0 +RESULT bcsstk17/csr/spmm-T-w8/portable 0.000408632 0.000373787 30144.0 2.5334268e7 +RESULT bcsstk17/csr/spmm-T-w64/automatic 0.000166991 0.000147104 2928.0 6708.0 +RESULT bcsstk17/csr/spmm-T-w64/vendor 0.000146272 0.000146052 2928.0 6708.0 +RESULT bcsstk17/csr/spmm-T-w64/portable 0.001063682 0.000962033 30144.0 2.5334268e7 +CASE bcsstk17 csc (10974, 10974) 428650 +RESULT bcsstk17/csc/spmv-N/automatic 4.1708e-5 1.7633e-5 2464.0 680.0 +RESULT bcsstk17/csc/spmv-N/vendor 1.64e-5 1.7402e-5 2464.0 680.0 +RESULT bcsstk17/csc/spmv-N/portable 0.000299689 0.000271616 29488.0 2.5334268e7 +RESULT bcsstk17/csc/spmm-N-w1/automatic 2.5167e-5 1.8244e-5 2336.0 680.0 +RESULT bcsstk17/csc/spmm-N-w1/vendor 1.6901e-5 1.8164e-5 2336.0 680.0 +RESULT bcsstk17/csc/spmm-N-w1/portable 0.000334133 0.000308475 30048.0 2.5334268e7 +RESULT bcsstk17/csc/spmm-N-w8/automatic 4.3401e-5 3.4505e-5 2336.0 6708.0 +RESULT bcsstk17/csc/spmm-N-w8/vendor 4.2309e-5 3.4504e-5 2336.0 6708.0 +RESULT bcsstk17/csc/spmm-N-w8/portable 0.000391439 0.000367986 30048.0 2.5334268e7 +RESULT bcsstk17/csc/spmm-N-w64/automatic 0.000172111 0.000146073 2928.0 6708.0 +RESULT bcsstk17/csc/spmm-N-w64/vendor 0.000145341 0.000146072 2928.0 6708.0 +RESULT bcsstk17/csc/spmm-N-w64/portable 0.001057882 0.000953477 30048.0 2.5334268e7 +RESULT bcsstk17/csc/spmv-T/automatic 5.3279e-5 1.6882e-5 2464.0 680.0 +RESULT bcsstk17/csc/spmv-T/vendor 1.4998e-5 1.6691e-5 2464.0 680.0 +RESULT bcsstk17/csc/spmv-T/portable 3.9653e-5 2.695e-5 2416.0 0.0 +RESULT bcsstk17/csc/spmm-T-w1/automatic 3.1008e-5 1.6711e-5 2336.0 680.0 +RESULT bcsstk17/csc/spmm-T-w1/vendor 1.5229e-5 1.6611e-5 2336.0 680.0 +RESULT bcsstk17/csc/spmm-T-w1/portable 7.2314e-5 6.3008e-5 2976.0 0.0 +RESULT bcsstk17/csc/spmm-T-w8/automatic 3.3843e-5 2.4967e-5 2336.0 6708.0 +RESULT bcsstk17/csc/spmm-T-w8/vendor 2.3524e-5 2.4946e-5 2336.0 6708.0 +RESULT bcsstk17/csc/spmm-T-w8/portable 0.000128569 0.00012361 2976.0 0.0 +RESULT bcsstk17/csc/spmm-T-w64/automatic 0.000102591 8.577e-5 2928.0 6708.0 +RESULT bcsstk17/csc/spmm-T-w64/vendor 8.9787e-5 8.5449e-5 2928.0 6708.0 +RESULT bcsstk17/csc/spmm-T-w64/portable 0.000755719 0.00070827 2976.0 0.0 +RESULT complex-banded/csr/spmm-C-w8/automatic 0.082285307 1.9977e-5 2336.0 1416.0 +RESULT complex-banded/csr/spmm-C-w8/vendor 8.4157e-5 1.9647e-5 2336.0 1416.0 +RESULT complex-banded/csr/spmm-C-w8/portable 0.147671097 0.000157303 27728.0 5.708756e6 +RESULT complex-banded/csc/spmm-C-w8/automatic 0.000151081 2.0779e-5 2384.0 1416.0 +RESULT complex-banded/csc/spmm-C-w8/vendor 8.9777e-5 1.9366e-5 2384.0 1416.0 +RESULT complex-banded/csc/spmm-C-w8/portable 0.004121061 2.1611e-5 2384.0 0.0 +RESULT duplicates/assembly/csr 1.58440911 0.000199903 32832.0 4.686868e6 +RESULT candidate-expansion/csr/spgemm/automatic 0.779279949 0.000585862 4784.0 2.03803492e8 +RESULT candidate-expansion/csr/spgemm/vendor 0.011787309 0.000583598 4784.0 2.03803492e8 +RESULT candidate-expansion/csr/spgemm/portable 0.86383144 0.007369035 50800.0 5.6972306e8 +RESOURCE_LIMIT candidate-expansion 1048576 rejected diff --git a/perf/sparse-v13-cuda.jl b/perf/sparse-v13-cuda.jl new file mode 100644 index 000000000..74b07dfaf --- /dev/null +++ b/perf/sparse-v13-cuda.jl @@ -0,0 +1,171 @@ +# Run through jld in the migrated CUDA environment and again against CUDA 6.4.2. +# SPARSE_BENCH_DATA points to cached NIST Matrix Market .mtx.gz files. +# SPARSE_BENCH_FILTER optionally restricts labels; SPARSE_BENCH_TRACE_MEMORY=1 +# adds one warmed, pool-purged NVTX range per operation for Nsight memory tracing. +using CUDA,CUDACore,cuSPARSE,GPUArrays,NVTX +using SparseArrays,LinearAlgebra,Random,Statistics,SHA,Test + +GPUArrays.allowscalar(false) +const modern_sparse=isdefined(GPUArrays,:Portable) +const bench_filter=Regex(get(ENV,"SPARSE_BENCH_FILTER","")) +const trace_memory=get(ENV,"SPARSE_BENCH_TRACE_MEMORY","0")=="1" +const repetitions=parse(Int,get(ENV,"SPARSE_BENCH_REPETITIONS","11")) + +function read_real_symmetric(path) + println("INPUT_SHA256\t",basename(path),'\t',bytes2hex(sha256(read(path)))) + open(`gzip -dc $path`) do io + header=lowercase(readline(io)) + occursin("coordinate real symmetric",header) || error("unsupported Matrix Market header") + line=readline(io) + while startswith(line,"%") || isempty(strip(line)) + line=readline(io) + end + m,n,entries=parse.(Int,split(line)) + rows=Int32[];cols=Int32[];values=Float32[] + sizehint!(rows,2entries);sizehint!(cols,2entries);sizehint!(values,2entries) + for line in eachline(io) + isempty(strip(line)) && continue + i,j,v=split(line) + row=parse(Int32,i);col=parse(Int32,j);value=parse(Float32,v) + push!(rows,row);push!(cols,col);push!(values,value) + if row!=col + push!(rows,col);push!(cols,row);push!(values,value) + end + end + sparse(rows,cols,values,m,n) + end +end + +function measure(label,f,reference) + occursin(bench_filter,label) || return + CUDACore.synchronize() + first=NVTX.@range "first:"*label CUDACore.@timed f() + actual=Array(first.value) + @test actual ≈ reference rtol=2f-4 atol=1f-5 + first_seconds=first.time + first=nothing + for warmup in 1:3 + f() + CUDACore.synchronize() + GC.gc(false) + end + times=Float64[];host_bytes=Int[];device_bytes=Int[] + for trial in 1:repetitions + GC.gc(false) + CUDACore.synchronize() + sample=NVTX.@range "warm:"*label CUDACore.@timed f() + push!(times,sample.time) + push!(host_bytes,sample.cpu_bytes) + push!(device_bytes,sample.gpu_bytes) + end + # gpu_bytes is cumulative requested allocation, NOT observed peak live memory. + # It is an upper bound on incremental peak, excluding pre-existing inputs. + println("RESULT\t",label,'\t',first_seconds,'\t',median(times),'\t', + median(host_bytes),'\t',median(device_bytes)) + if trace_memory + GC.gc(true) + CUDACore.reclaim() + CUDACore.synchronize() + result=NVTX.@range "memory:"*label begin + value=f() + CUDACore.synchronize() + value + end + result=nothing + end + nothing +end + +function apply_mv!(policy,y,t,A,x) + if modern_sparse + GPUArrays.spmv!(y,t,A,x,one(eltype(y)),zero(eltype(y));algorithm=policy) + else + mul!(y,LinearAlgebra.wrap(A,t),x) + end +end +function apply_mm!(policy,C,t,A,B) + if modern_sparse + GPUArrays.spmm!(C,t,'N',A,B,one(eltype(C)),zero(eltype(C));algorithm=policy) + else + mul!(C,LinearAlgebra.wrap(A,t),B) + end +end +function policies() + modern_sparse ? (("automatic",GPUArrays.Automatic()),("vendor",GPUArrays.Vendor()), + ("portable",GPUArrays.Portable())) : (("old-vendor",nothing),) +end + +println("STACK\tJulia=",VERSION,"\tCUDA=",pkgversion(CUDA),"\tcuSPARSE=",pkgversion(cuSPARSE), + "\tGPUArrays=",pkgversion(GPUArrays),"\truntime=",CUDA.runtime_version()) +CUDA.versioninfo() +println("COLUMNS\tcase/format/operation/policy\tfirst_seconds\twarm_median_seconds\thost_alloc_bytes\tdevice_alloc_bytes") +Random.seed!(0x74513) +n=10000 +cases=Pair{String,SparseMatrixCSC{Float32,Int32}}[] +push!(cases,"diagonal"=>SparseMatrixCSC{Float32,Int32}(spdiagm(0=>rand(Float32,n)))) +push!(cases,"banded"=>SparseMatrixCSC{Float32,Int32}(spdiagm( + [k=>rand(Float32,n-abs(k)) for k in -4:4]...))) +push!(cases,"uniform"=>SparseMatrixCSC{Float32,Int32}(sprand(Float32,n,n,8/n))) +rows=clamp.(floor.(Int,exp.(rand(80000).*log(n))),1,n) +push!(cases,"skewed"=>SparseMatrixCSC{Float32,Int32}(sparse(rows,rand(1:n,80000),rand(Float32,80000),n,n))) +push!(cases,"empty-rows"=>SparseMatrixCSC{Float32,Int32}(sparse(4 .* rand(1:div(n,4),80000), + rand(1:n,80000),rand(Float32,80000),n,n))) +data=get(ENV,"SPARSE_BENCH_DATA",joinpath(homedir(),"Julia/tmp/ga13-benchmark-data")) +for name in ("1138_bus","bcsstk17") + path=joinpath(data,name*".mtx.gz") + isfile(path) || error("Cache $path from math.nist.gov before running") + push!(cases,name=>SparseMatrixCSC{Float32,Int32}(read_real_symmetric(path))) +end + +for (name,host) in cases, (format,S) in (("csr",CuSparseMatrixCSR),("csc",CuSparseMatrixCSC)) + println("CASE\t",name,'\t',format,'\t',size(host),'\t',nnz(host)) + A=S(host) + for trans in ('N','T') + H=LinearAlgebra.wrap(host,trans) + x=CuArray(rand(Float32,size(H,2)));y=CuArray{Float32}(undef,size(H,1)) + reference=H*Array(x) + for (policy_name,policy) in policies() + label="$name/$format/spmv-$trans/$policy_name" + measure(label,()->apply_mv!(policy,y,trans,A,x),reference) + end + for width in (1,8,64) + B=CuArray(rand(Float32,size(H,2),width));C=CuArray{Float32}(undef,size(H,1),width) + reference=H*Array(B) + for (policy_name,policy) in policies() + label="$name/$format/spmm-$trans-w$width/$policy_name" + measure(label,()->apply_mm!(policy,C,trans,A,B),reference) + end + end + end +end + +host=ComplexF32.(last(cases[2]),0.25f0 .* last(cases[2])) +for (format,S) in (("csr",CuSparseMatrixCSR),("csc",CuSparseMatrixCSC)) + A=S(host);B=CuArray(rand(ComplexF32,size(host,1),8));C=similar(B) + reference=adjoint(host)*Array(B) + for (policy_name,policy) in policies() + measure("complex-banded/$format/spmm-C-w8/$policy_name", + ()->apply_mm!(policy,C,'C',A,B),reference) + end +end + +# Duplicate-heavy assembly uses the same raw coordinate/value inputs in both stacks. +I=Int32.(rand(1:1000,10000));J=Int32.(rand(1:1000,10000));V=rand(Float32,10000) +I=repeat(I,8);J=repeat(J,8);V=repeat(V,8) +dI=CuArray(I);dJ=CuArray(J);dV=CuArray(V) +reference=Matrix(sparse(I,J,V,1000,1000)) +measure("duplicates/assembly/csr",()->sparse(dI,dJ,dV,1000,1000;fmt=:csr),reference) + +# 8,388,608 candidate products but only 262,144 final entries. +left=sparse(repeat(1:512,inner=32),repeat(1:32,512),ones(Float32,512*32),512,512) +right=sparse(repeat(1:32,inner=512),repeat(1:512,32),ones(Float32,512*32),512,512) +A=CuSparseMatrixCSR(left);B=CuSparseMatrixCSR(right) +reference=Matrix(left*right) +for (policy_name,policy) in policies() + product=modern_sparse ? ()->GPUArrays.spgemm(A,B;algorithm=policy) : ()->A*B + measure("candidate-expansion/csr/spgemm/$policy_name",product,reference) +end +if modern_sparse + @test_throws ArgumentError GPUArrays.spgemm(A,B;algorithm=GPUArrays.Portable(),max_temp_bytes=1<<20) + println("RESOURCE_LIMIT\tcandidate-expansion\t1048576\trejected") +end diff --git a/perf/sparse-v13-memory.sql b/perf/sparse-v13-memory.sql new file mode 100644 index 000000000..8676ba8d0 --- /dev/null +++ b/perf/sparse-v13-memory.sql @@ -0,0 +1,19 @@ +-- Nsight Systems sqlite export, device allocations in each pool-purged NVTX range. +-- Incremental live memory includes the result and is relative to the range start. +WITH ranges AS ( + SELECT n.start AS lo,n.end AS hi,coalesce(n.text,s.value) AS label + FROM NVTX_EVENTS n LEFT JOIN StringIds s ON n.textId=s.id + WHERE coalesce(n.text,s.value) LIKE 'memory:%' +), events AS ( + SELECT r.label,m.start,m.rowid AS ord, + CASE WHEN m.memoryOperationType=0 THEN m.bytes ELSE -m.bytes END AS delta, + CASE WHEN m.memoryOperationType=0 THEN m.bytes ELSE 0 END AS allocated + FROM ranges r JOIN CUDA_GPU_MEMORY_USAGE_EVENTS m ON m.start BETWEEN r.lo AND r.hi + WHERE m.memKind=2 +), live AS ( + SELECT *,sum(delta) OVER(PARTITION BY label ORDER BY start,ord ROWS UNBOUNDED PRECEDING) + AS current FROM events +) +SELECT label,sum(allocated) AS device_allocated,max(current) AS peak_increment, + sum(delta) AS end_increment +FROM live GROUP BY label; diff --git a/src/GPUArrays.jl b/src/GPUArrays.jl index d12d6762b..b3ed5b508 100644 --- a/src/GPUArrays.jl +++ b/src/GPUArrays.jl @@ -4,6 +4,7 @@ using KernelAbstractions using Serialization using Random using LinearAlgebra +using SparseArrays using Printf using LinearAlgebra.BLAS @@ -21,7 +22,6 @@ import AcceleratedKernels as AK # device functionality include("device/abstractarray.jl") -include("device/sparse.jl") # host abstractions include("host/abstractarray.jl") @@ -40,7 +40,18 @@ include("host/random.jl") include("host/quirks.jl") include("host/uniformscaling.jl") include("host/statistics.jl") -include("host/sparse.jl") +include("host/sparse/types.jl") +include("host/sparse/primitives.jl") +include("host/sparse/conversions.jl") +include("host/sparse/assembly.jl") +include("host/sparse/show.jl") +include("host/sparse/broadcast.jl") +include("host/sparse/reductions.jl") +include("host/sparse/linalg.jl") +include("host/sparse/selection.jl") +include("host/sparse/structure.jl") +include("host/sparse/indexing.jl") +include("host/sparse/array.jl") include("host/alloc_cache.jl") diff --git a/src/device/sparse.jl b/src/device/sparse.jl deleted file mode 100644 index b8346eafe..000000000 --- a/src/device/sparse.jl +++ /dev/null @@ -1,171 +0,0 @@ -# on-device sparse array types -# should be excluded from coverage counts -# COV_EXCL_START -using SparseArrays - -# NOTE: this functionality is currently very bare-bones, only defining the array types -# without any device-compatible sparse array functionality - - -# core types - -export GPUSparseDeviceVector, GPUSparseDeviceMatrixCSC, GPUSparseDeviceMatrixCSR, - GPUSparseDeviceMatrixBSR, GPUSparseDeviceMatrixCOO - -abstract type AbstractGPUSparseDeviceMatrix{Tv, Ti} <: AbstractSparseMatrix{Tv, Ti} end - - -struct GPUSparseDeviceVector{Tv,Ti,Vi,Vv, A} <: AbstractSparseVector{Tv,Ti} - iPtr::Vi - nzVal::Vv - len::Int - nnz::Ti -end - -Base.length(g::GPUSparseDeviceVector) = g.len -Base.size(g::GPUSparseDeviceVector) = (g.len,) -SparseArrays.nnz(g::GPUSparseDeviceVector) = g.nnz -SparseArrays.nonzeroinds(g::GPUSparseDeviceVector) = g.iPtr -SparseArrays.nonzeros(g::GPUSparseDeviceVector) = g.nzVal - -struct GPUSparseDeviceMatrixCSC{Tv,Ti,Vi,Vv,A} <: AbstractGPUSparseDeviceMatrix{Tv, Ti} - colPtr::Vi - rowVal::Vi - nzVal::Vv - dims::NTuple{2,Int} - nnz::Ti -end - -Base.length(g::GPUSparseDeviceMatrixCSC) = prod(g.dims) -Base.size(g::GPUSparseDeviceMatrixCSC) = g.dims -SparseArrays.nnz(g::GPUSparseDeviceMatrixCSC) = g.nnz -SparseArrays.rowvals(g::GPUSparseDeviceMatrixCSC) = g.rowVal -SparseArrays.getcolptr(g::GPUSparseDeviceMatrixCSC) = g.colPtr -SparseArrays.nzrange(g::GPUSparseDeviceMatrixCSC, col::Integer) = @inbounds SparseArrays.getcolptr(g)[col]:(SparseArrays.getcolptr(g)[col+1]-1) -SparseArrays.nonzeros(g::GPUSparseDeviceMatrixCSC) = g.nzVal - -const GPUSparseDeviceColumnView{Tv, Ti, Vi, Vv, A} = SubArray{Tv, 1, GPUSparseDeviceMatrixCSC{Tv, Ti, Vi, Vv, A}, Tuple{Base.Slice{Base.OneTo{Int}}, Int}} -function SparseArrays.nonzeros(x::GPUSparseDeviceColumnView) - rowidx, colidx = parentindices(x) - A = parent(x) - @inbounds y = view(SparseArrays.nonzeros(A), SparseArrays.nzrange(A, colidx)) - return y -end - -function SparseArrays.nonzeroinds(x::GPUSparseDeviceColumnView) - rowidx, colidx = parentindices(x) - A = parent(x) - @inbounds y = view(SparseArrays.rowvals(A), SparseArrays.nzrange(A, colidx)) - return y -end -SparseArrays.rowvals(x::GPUSparseDeviceColumnView) = SparseArrays.nonzeroinds(x) - -function SparseArrays.nnz(x::GPUSparseDeviceColumnView) - rowidx, colidx = parentindices(x) - A = parent(x) - return length(SparseArrays.nzrange(A, colidx)) -end - -struct GPUSparseDeviceMatrixCSR{Tv,Ti,Vi,Vv,A} <: AbstractGPUSparseDeviceMatrix{Tv,Ti} - rowPtr::Vi - colVal::Vi - nzVal::Vv - dims::NTuple{2, Int} - nnz::Ti -end - -@inline function _getindex(arg::Union{GPUSparseDeviceMatrixCSR{Tv}, - GPUSparseDeviceMatrixCSC{Tv}, - GPUSparseDeviceVector{Tv}}, I, ptr)::Tv where {Tv} - if ptr == 0 - return zero(Tv) - else - return @inbounds arg.nzVal[ptr]::Tv - end -end - -struct GPUSparseDeviceMatrixBSR{Tv,Ti,Vi,Vv,A} <: AbstractGPUSparseDeviceMatrix{Tv,Ti} - rowPtr::Vi - colVal::Vi - nzVal::Vv - dims::NTuple{2,Int} - blockDim::Ti - dir::Char - nnz::Ti -end - -struct GPUSparseDeviceMatrixCOO{Tv,Ti,Vi,Vv, A} <: AbstractGPUSparseDeviceMatrix{Tv,Ti} - rowInd::Vi - colInd::Vi - nzVal::Vv - dims::NTuple{2,Int} - nnz::Ti -end - -Base.length(g::AbstractGPUSparseDeviceMatrix) = prod(g.dims) -Base.size(g::AbstractGPUSparseDeviceMatrix) = g.dims -SparseArrays.nnz(g::AbstractGPUSparseDeviceMatrix) = g.nnz -SparseArrays.getnzval(g::AbstractGPUSparseDeviceMatrix) = g.nzVal - -struct GPUSparseDeviceArrayCSR{Tv, Ti, Vi, Vv, N, M, A} <: AbstractSparseArray{Tv, Ti, N} - rowPtr::Vi - colVal::Vi - nzVal::Vv - dims::NTuple{N, Int} - nnz::Ti -end - -function GPUSparseDeviceArrayCSR{Tv, Ti, Vi, Vv, N}(rowPtr::Vi, colVal::Vi, nzVal::Vv, dims::NTuple{N,<:Integer}) where {Tv, Ti<:Integer, M, Vi<:AbstractDeviceArray{<:Integer,M}, Vv<:AbstractDeviceArray{Tv, M}, N} - @assert M == N - 1 "GPUSparseDeviceArrayCSR requires ndims(rowPtr) == ndims(colVal) == ndims(nzVal) == length(dims) - 1" - GPUSparseDeviceArrayCSR{Tv, Ti, Vi, Vv, N, M}(rowPtr, colVal, nzVal, dims, length(nzVal)) -end - -Base.length(g::GPUSparseDeviceArrayCSR) = prod(g.dims) -Base.size(g::GPUSparseDeviceArrayCSR) = g.dims -SparseArrays.nnz(g::GPUSparseDeviceArrayCSR) = g.nnz -SparseArrays.getnzval(g::GPUSparseDeviceArrayCSR) = g.nzVal - -# input/output - -function Base.show(io::IO, ::MIME"text/plain", A::GPUSparseDeviceVector) - println(io, "$(length(A))-element device sparse vector at:") - println(io, " iPtr: $(A.iPtr)") - print(io, " nzVal: $(A.nzVal)") -end - -function Base.show(io::IO, ::MIME"text/plain", A::GPUSparseDeviceMatrixCSR) - println(io, "$(length(A))-element device sparse matrix CSR at:") - println(io, " rowPtr: $(A.rowPtr)") - println(io, " colVal: $(A.colVal)") - print(io, " nzVal: $(A.nzVal)") -end - -function Base.show(io::IO, ::MIME"text/plain", A::GPUSparseDeviceMatrixCSC) - println(io, "$(length(A))-element device sparse matrix CSC at:") - println(io, " colPtr: $(A.colPtr)") - println(io, " rowVal: $(A.rowVal)") - print(io, " nzVal: $(A.nzVal)") -end - -function Base.show(io::IO, ::MIME"text/plain", A::GPUSparseDeviceMatrixBSR) - println(io, "$(length(A))-element device sparse matrix BSR at:") - println(io, " rowPtr: $(A.rowPtr)") - println(io, " colVal: $(A.colVal)") - print(io, " nzVal: $(A.nzVal)") -end - -function Base.show(io::IO, ::MIME"text/plain", A::GPUSparseDeviceMatrixCOO) - println(io, "$(length(A))-element device sparse matrix COO at:") - println(io, " rowPtr: $(A.rowPtr)") - println(io, " colInd: $(A.colInd)") - print(io, " nzVal: $(A.nzVal)") -end - -function Base.show(io::IO, ::MIME"text/plain", A::GPUSparseDeviceArrayCSR) - println(io, "$(length(A))-element device sparse array CSR at:") - println(io, " rowPtr: $(A.rowPtr)") - println(io, " colVal: $(A.colVal)") - print(io, " nzVal: $(A.nzVal)") -end - -# COV_EXCL_STOP diff --git a/src/host/abstractarray.jl b/src/host/abstractarray.jl index 6e54ae7e1..f45fe8fd2 100644 --- a/src/host/abstractarray.jl +++ b/src/host/abstractarray.jl @@ -139,6 +139,31 @@ when they share the `DataRef` returned by `GPUArrays.storage`. """ memory_location(::AbstractGPUArray) = nothing +""" + GPUArrays.supports_resize(A::AbstractVector) + +Whether this storage instance supports structural resizing. The default is false; +ordinary host vectors return true. Dense backends that implement `resize!` should +specialize this predicate, including any memory-mode or instance restrictions. Views +and device-side representations are not resizable structural storage. + +This capability does not establish ownership: callers must guarantee exclusive +structural access, and allocation failure need not leave a collection unchanged. +""" +supports_resize(::AbstractVector) = false +supports_resize(::Vector) = true + +# Host wrappers may report physical placement in backend extensions. Device structs +# must never invoke this host boundary. Unwrap only known host GPU wrappers. +storage_parent(A::AbstractArray) = A +storage_parent(A::WrappedGPUArray) = storage_parent(parent(A)) +function check_storage_compatibility(arrays::AbstractArray...) + backend = get_backend(first(arrays)) + all(A -> get_backend(A) == backend, arrays) || + throw(ArgumentError("array buffers must use compatible backends")) + return nothing +end + # `copy(::DataRef)` shares `rc`. Views and reshapes are built that way, so # alias detection has to key off the shared record rather than the wrapper. Base.dataids(ref::DataRef) = (objectid(ref.rc),) @@ -455,3 +480,11 @@ function Base.append!(a::AbstractGPUVector, @nospecialize(items::Tuple)) append!(a, collect(items)) return a end + +# Exact numerical snapshots for retained vendor resources, including signed zero and +# NaN payloads. This uses dense operations and does not require sparse backend hooks. +function same_value_bits(A::AbstractVector{T},B::AbstractVector{T}) where {T} + length(A)==length(B) || return false + isbitstype(T) || throw(ArgumentError("bitwise comparison requires isbits values")) + mapreduce(==, &, reinterpret(UInt8,A), reinterpret(UInt8,B); init=true) +end diff --git a/src/host/linalg.jl b/src/host/linalg.jl index c9bbadbdd..48239ee30 100644 --- a/src/host/linalg.jl +++ b/src/host/linalg.jl @@ -325,6 +325,9 @@ end ## diagonal Base.copy(D::Diagonal{T, <:AbstractGPUArray{T, N}}) where {T, N} = Diagonal(copy(D.diag)) +Base.Array(D::Diagonal{<:Any,<:AnyGPUVector}) = Array(Diagonal(Array(D.diag))) +Base.Array{T}(D::Diagonal{<:Any,<:AnyGPUVector}) where {T} = Array{T}(Diagonal(Array(D.diag))) +Base.Array{T,2}(D::Diagonal{<:Any,<:AnyGPUVector}) where {T} = Matrix{T}(Diagonal(Array(D.diag))) _isrealandpositive(x) = isreal(x) && real(x) > 0 @@ -1181,7 +1184,10 @@ end ## dot -LinearAlgebra.dot(x::AbstractGPUArray, y::AbstractGPUArray) = mapreduce(dot, +, x, y) +function LinearAlgebra.dot(x::AbstractGPUArray,y::AbstractGPUArray) + length(x)==length(y) || throw(DimensionMismatch("dot product arguments must have equal lengths")) + mapreduce(dot,+,x,y) +end ## axp{b}y diff --git a/src/host/sparse.jl b/src/host/sparse.jl deleted file mode 100644 index 56c3599b0..000000000 --- a/src/host/sparse.jl +++ /dev/null @@ -1,1046 +0,0 @@ -using LinearAlgebra -using LinearAlgebra: BlasFloat - -abstract type AbstractGPUSparseArray{Tv, Ti, N} <: AbstractSparseArray{Tv, Ti, N} end -const AbstractGPUSparseVector{Tv, Ti} = AbstractGPUSparseArray{Tv, Ti, 1} -const AbstractGPUSparseMatrix{Tv, Ti} = AbstractGPUSparseArray{Tv, Ti, 2} - -abstract type AbstractGPUSparseMatrixCSC{Tv, Ti} <: AbstractGPUSparseArray{Tv, Ti, 2} end -abstract type AbstractGPUSparseMatrixCSR{Tv, Ti} <: AbstractGPUSparseArray{Tv, Ti, 2} end -abstract type AbstractGPUSparseMatrixCOO{Tv, Ti} <: AbstractGPUSparseArray{Tv, Ti, 2} end -abstract type AbstractGPUSparseMatrixBSR{Tv, Ti} <: AbstractGPUSparseArray{Tv, Ti, 2} end - -const AbstractGPUSparseVecOrMat = Union{AbstractGPUSparseVector,AbstractGPUSparseMatrix} - -SparseArrays.nnz(g::T) where {T<:AbstractGPUSparseArray} = g.nnz -SparseArrays.nonzeros(g::T) where {T<:AbstractGPUSparseArray} = g.nzVal - -SparseArrays.nonzeroinds(g::T) where {T<:AbstractGPUSparseVector} = g.iPtr -SparseArrays.rowvals(g::T) where {T<:AbstractGPUSparseVector} = SparseArrays.nonzeroinds(g) - -SparseArrays.rowvals(g::AbstractGPUSparseMatrixCSC) = g.rowVal -SparseArrays.getcolptr(S::AbstractGPUSparseMatrixCSC) = S.colPtr - -Base.convert(T::Type{<:AbstractGPUSparseArray}, m::AbstractArray) = m isa T ? m : T(m) - -# collect to Array -Base.collect(x::AbstractGPUSparseVector) = collect(SparseVector(x)) -Base.collect(x::AbstractGPUSparseMatrixCSC) = collect(SparseMatrixCSC(x)) -Base.collect(x::AbstractGPUSparseMatrixCSR) = collect(SparseMatrixCSC(x)) -Base.collect(x::AbstractGPUSparseMatrixBSR) = collect(SparseMatrixCSC(x)) -Base.collect(x::AbstractGPUSparseMatrixCOO) = collect(SparseMatrixCSC(x)) - -Base.Array(x::AbstractGPUSparseVector) = collect(SparseVector(x)) -Base.Array(x::AbstractGPUSparseMatrixCSC) = collect(SparseMatrixCSC(x)) -Base.Array(x::AbstractGPUSparseMatrixCSR) = collect(SparseMatrixCSC(x)) -Base.Array(x::AbstractGPUSparseMatrixBSR) = collect(SparseMatrixCSC(x)) -Base.Array(x::AbstractGPUSparseMatrixCOO) = collect(SparseMatrixCSC(x)) - -# iszero that avoids scalar indexing by using GPU-compatible reduction -# nnz(A) == 0 means no stored elements, so it's definitely zero -# all(iszero, nonzeros(A)) uses GPU reduction to check stored values -Base.iszero(A::AbstractGPUSparseArray) = SparseArrays.nnz(A) == 0 || all(iszero, SparseArrays.nonzeros(A)) - -SparseArrays.SparseVector(x::AbstractGPUSparseVector) = SparseVector(length(x), Array(SparseArrays.nonzeroinds(x)), Array(SparseArrays.nonzeros(x))) -SparseArrays.SparseMatrixCSC(x::AbstractGPUSparseMatrixCSC) = SparseMatrixCSC(size(x)..., Array(SparseArrays.getcolptr(x)), Array(SparseArrays.rowvals(x)), Array(SparseArrays.nonzeros(x))) - -# similar -Base.similar(Vec::V) where {V<:AbstractGPUSparseVector} = V(copy(SparseArrays.nonzeroinds(Vec)), similar(SparseArrays.nonzeros(Vec)), length(Vec)) -Base.similar(Mat::M) where {M<:AbstractGPUSparseMatrixCSC} = M(copy(SparseArrays.getcolptr(Mat)), copy(SparseArrays.rowvals(Mat)), similar(SparseArrays.nonzeros(Mat)), size(Mat)) - -Base.similar(Vec::V, T::Type) where {Tv, Ti, V<:AbstractGPUSparseVector{Tv, Ti}} = sparse_array_type(V){T, Ti}(copy(SparseArrays.nonzeroinds(Vec)), similar(SparseArrays.nonzeros(Vec), T), length(Vec)) -Base.similar(Mat::M, T::Type) where {M<:AbstractGPUSparseMatrixCSC} = sparse_array_type(M)(copy(SparseArrays.getcolptr(Mat)), copy(SparseArrays.rowvals(Mat)), similar(SparseArrays.nonzeros(Mat), T), size(Mat)) - -dense_array_type(a::AbstractArray) = dense_array_type(typeof(a)) -sparse_array_type(a::AbstractArray) = sparse_array_type(typeof(a)) -dense_vector_type(a::AbstractArray) = dense_vector_type(typeof(a)) - -dense_array_type(::Type{<:SparseVector}) = SparseVector -sparse_array_type(::Type{<:SparseVector}) = SparseVector -dense_vector_type(::Type{<:AbstractSparseArray}) = Vector -dense_vector_type(::Type{<:AbstractArray}) = Vector -dense_array_type(::Type{<:SparseMatrixCSC}) = SparseMatrixCSC -sparse_array_type(::Type{<:SparseMatrixCSC}) = SparseMatrixCSC - -coo_type(sa::AbstractArray) = coo_type(typeof(sa)) - -function _spadjoint end -function _sptranspose end - -function LinearAlgebra.opnorm(A::AbstractGPUSparseMatrixCSR, p::Real=2) - if p == Inf - return maximum(sum(abs, A; dims=2)) - elseif p == 1 - return maximum(sum(abs, A; dims=1)) - else - throw(ArgumentError("p=$p is not supported")) - end -end - -LinearAlgebra.opnorm(A::AbstractGPUSparseMatrixCSC, p::Real=2) = opnorm(csr_type(A)(A), p) - -function LinearAlgebra.norm(A::AbstractGPUSparseMatrix{T}, p::Real=2) where T - if p == Inf - return maximum(abs.(SparseArrays.nonzeros(A))) - elseif p == -Inf - return minimum(abs.(SparseArrays.nonzeros(A))) - elseif p == 0 - return Float64(SparseArrays.nnz(A)) - else - return sum(abs.(SparseArrays.nonzeros(A)).^p)^(1/p) - end -end - -function SparseArrays.findnz(S::MT) where {MT <: AbstractGPUSparseMatrix} - S2 = coo_type(MT)(S) - I = S2.rowInd - J = S2.colInd - V = S2.nzVal - - # To make it compatible with the SparseArrays.jl version - idxs = sortperm(J) - I = I[idxs] - J = J[idxs] - V = V[idxs] - - return (I, J, V) -end - -### WRAPPED ARRAYS -# Without these, `copy` of a lazy wrapper falls back to the generic `AbstractArray` method, -# which reads the parent one element at a time and so hits scalar indexing. -Base.copy(A::Transpose{<:Any, <:AbstractGPUSparseMatrix}) = _sptranspose(parent(A)) -Base.copy(A::Adjoint{<:Any, <:AbstractGPUSparseMatrix}) = _spadjoint(parent(A)) - -function Base.permutedims(A::AbstractGPUSparseMatrix, (a, b)) - (a, b) == (2, 1) && return _sptranspose(A) - (a, b) == (1, 2) && return copy(A) - throw(ArgumentError("no valid permutation of dimensions")) -end - -LinearAlgebra.issymmetric(M::Union{AbstractGPUSparseMatrixCSC,AbstractGPUSparseMatrixCSR}) = size(M, 1) == size(M, 2) ? norm(M - transpose(M), Inf) == 0 : false -LinearAlgebra.ishermitian(M::Union{AbstractGPUSparseMatrixCSC,AbstractGPUSparseMatrixCSR}) = size(M, 1) == size(M, 2) ? norm(M - adjoint(M), Inf) == 0 : false - -LinearAlgebra.istriu(M::UpperTriangular{T,S}) where {T<:BlasFloat, S<:Union{<:AbstractGPUSparseMatrix, Adjoint{<:Any, <:AbstractGPUSparseMatrix}, Transpose{<:Any, <:AbstractGPUSparseMatrix}}} = true -LinearAlgebra.istril(M::UpperTriangular{T,S}) where {T<:BlasFloat, S<:Union{<:AbstractGPUSparseMatrix, Adjoint{<:Any, <:AbstractGPUSparseMatrix}, Transpose{<:Any, <:AbstractGPUSparseMatrix}}} = false -LinearAlgebra.istriu(M::LowerTriangular{T,S}) where {T<:BlasFloat, S<:Union{<:AbstractGPUSparseMatrix, Adjoint{<:Any, <:AbstractGPUSparseMatrix}, Transpose{<:Any, <:AbstractGPUSparseMatrix}}} = false -LinearAlgebra.istril(M::LowerTriangular{T,S}) where {T<:BlasFloat, S<:Union{<:AbstractGPUSparseMatrix, Adjoint{<:Any, <:AbstractGPUSparseMatrix}, Transpose{<:Any, <:AbstractGPUSparseMatrix}}} = true - -Hermitian{T}(Mat::AbstractGPUSparseMatrix{T}) where {T} = Hermitian{eltype(Mat),typeof(Mat)}(Mat,'U') - -# work around upstream breakage from JuliaLang/julia#55547 -@static if VERSION >= v"1.11.2" - const GPUSparseUpperOrUnitUpperTriangular = LinearAlgebra.UpperOrUnitUpperTriangular{ - <:Any,<:Union{<:AbstractGPUSparseMatrix, Adjoint{<:Any, <:AbstractGPUSparseMatrix}, Transpose{<:Any, <:AbstractGPUSparseMatrix}}} - const GPUSparseLowerOrUnitLowerTriangular = LinearAlgebra.LowerOrUnitLowerTriangular{ - <:Any,<:Union{<:AbstractGPUSparseMatrix, Adjoint{<:Any, <:AbstractGPUSparseMatrix}, Transpose{<:Any, <:AbstractGPUSparseMatrix}}} - LinearAlgebra.istriu(::GPUSparseUpperOrUnitUpperTriangular) = true - LinearAlgebra.istril(::GPUSparseUpperOrUnitUpperTriangular) = false - LinearAlgebra.istriu(::GPUSparseLowerOrUnitLowerTriangular) = false - LinearAlgebra.istril(::GPUSparseLowerOrUnitLowerTriangular) = true -end - -for SparseMatrixType in [:AbstractGPUSparseMatrixCSC, :AbstractGPUSparseMatrixCSR] - @eval begin - LinearAlgebra.triu(A::ST, k::Integer) where {T, ST<:$SparseMatrixType{T}} = - ST( triu(coo_type(A)(A), k) ) - LinearAlgebra.triu(A::Transpose{T,<:ST}, k::Integer) where {T, ST<:$SparseMatrixType{T}} = - ST( triu(coo_type(A)(_sptranspose(parent(A))), k) ) - LinearAlgebra.triu(A::Adjoint{T,<:ST}, k::Integer) where {T, ST<:$SparseMatrixType{T}} = - ST( triu(coo_type(A)(_spadjoint(parent(A))), k) ) - - LinearAlgebra.tril(A::ST, k::Integer) where {T, ST<:$SparseMatrixType{T}} = - ST( tril(coo_type(A)(A), k) ) - LinearAlgebra.tril(A::Transpose{T,<:ST}, k::Integer) where {T, ST<:$SparseMatrixType{T}} = - ST( tril(coo_type(A)(_sptranspose(parent(A))), k) ) - LinearAlgebra.tril(A::Adjoint{T,<:ST}, k::Integer) where {T, ST<:$SparseMatrixType{T}} = - ST( tril(coo_type(A)(_spadjoint(parent(A))), k) ) - - LinearAlgebra.triu(A::Union{ST, Transpose{T,<:ST}, Adjoint{T,<:ST}}) where {T, ST<:$SparseMatrixType{T}} = - ST( triu(coo_type(A)(A), 0) ) - LinearAlgebra.tril(A::Union{ST,Transpose{T,<:ST}, Adjoint{T,<:ST}}) where {T, ST<:$SparseMatrixType{T}} = - ST( tril(coo_type(A)(A), 0) ) - - LinearAlgebra.kron(A::ST, B::ST) where {T, ST<:$SparseMatrixType{T}} = - ST( kron(coo_type(A)(A), coo_type(B)(B)) ) - LinearAlgebra.kron(A::ST, B::Diagonal) where {T, ST<:$SparseMatrixType{T}} = - ST( kron(coo_type(A)(A), B) ) - LinearAlgebra.kron(A::Diagonal, B::ST) where {T, ST<:$SparseMatrixType{T}} = - ST( kron(A, coo_type(B)(B)) ) - - LinearAlgebra.kron(A::Transpose{T,<:ST}, B::ST) where {T, ST<:$SparseMatrixType{T}} = - ST( kron(coo_type(A)(_sptranspose(parent(A))), coo_type(B)(B)) ) - LinearAlgebra.kron(A::ST, B::Transpose{T,<:ST}) where {T, ST<:$SparseMatrixType{T}} = - ST( kron(coo_type(A)(A), coo_type(parent(B))(_sptranspose(parent(B)))) ) - LinearAlgebra.kron(A::Transpose{T,<:ST}, B::Transpose{T,<:ST}) where {T, ST<:$SparseMatrixType{T}} = - ST( kron(coo_type(A)(_sptranspose(parent(A))), coo_type(parent(B))(_sptranspose(parent(B)))) ) - LinearAlgebra.kron(A::Transpose{T,<:ST}, B::Diagonal) where {T, ST<:$SparseMatrixType{T}} = - ST( kron(coo_type(A)(_sptranspose(parent(A))), B) ) - LinearAlgebra.kron(A::Diagonal, B::Transpose{T,<:ST}) where {T, ST<:$SparseMatrixType{T}} = - ST( kron(A, coo_type(B)(_sptranspose(parent(B)))) ) - - LinearAlgebra.kron(A::Adjoint{T,<:ST}, B::ST) where {T, ST<:$SparseMatrixType{T}} = - ST( kron(coo_type(A)(_spadjoint(parent(A))), coo_type(B)(B)) ) - LinearAlgebra.kron(A::ST, B::Adjoint{T,<:ST}) where {T, ST<:$SparseMatrixType{T}} = - ST( kron(coo_type(A)(A), coo_type(parent(B))(_spadjoint(parent(B)))) ) - LinearAlgebra.kron(A::Adjoint{T,<:ST}, B::Adjoint{T,<:ST}) where {T, ST<:$SparseMatrixType{T}} = - ST( kron(coo_type(parent(A))(_spadjoint(parent(A))), coo_type(parent(B))(_spadjoint(parent(B)))) ) - LinearAlgebra.kron(A::Adjoint{T,<:ST}, B::Diagonal) where {T, ST<:$SparseMatrixType{T}} = - ST( kron(coo_type(parent(A))(_spadjoint(parent(A))), B) ) - LinearAlgebra.kron(A::Diagonal, B::Adjoint{T,<:ST}) where {T, ST<:$SparseMatrixType{T}} = - ST( kron(A, coo_type(parent(B))(_spadjoint(parent(B)))) ) - - - function Base.reshape(A::ST, dims::Dims) where {ST<:$SparseMatrixType} - B = coo_type(A)(A) - ST(reshape(B, dims)) - end - - function SparseArrays.droptol!(A::ST, tol::Real) where {ST<:$SparseMatrixType} - B = coo_type(A)(A) - droptol!(B, tol) - copyto!(A, ST(B)) - end - - function LinearAlgebra.exp(A::ST; threshold = 1e-7, nonzero_tol = 1e-14) where {ST<:$SparseMatrixType} - rows = LinearAlgebra.checksquare(A) # Throws exception if not square - typeA = eltype(A) - - mat_norm = norm(A, Inf) - scaling_factor = nextpow(2, mat_norm) # Native routine, faster - A = A ./ scaling_factor - delta = 1 - - diag_vec = similar(SparseArrays.nonzeros(A), rows) - fill!(diag_vec, one(eltype(A))) - P = ST(spdiagm(0 => diag_vec)) - next_term = P - n = 1 - - while delta > threshold - next_term = typeA(1 / n) * A * next_term - droptol!(next_term, nonzero_tol) - delta = norm(next_term, Inf) - copyto!(P, P + next_term) - n = n + 1 - end - for n = 1:log2(scaling_factor) - P = P * P; - if nnz(P) / length(P) < 0.25 - droptol!(P, nonzero_tol) - end - end - P - end - end -end - - -### INDEXING -Base.getindex(A::AbstractGPUSparseVector, ::Colon) = copy(A) -Base.getindex(A::AbstractGPUSparseMatrix, ::Colon, ::Colon) = copy(A) -Base.getindex(A::AbstractGPUSparseMatrix, i, ::Colon) = getindex(A, i, 1:size(A, 2)) -Base.getindex(A::AbstractGPUSparseMatrix, ::Colon, i) = getindex(A, 1:size(A, 1), i) -Base.getindex(A::AbstractGPUSparseMatrix, I::Tuple{Integer,Integer}) = getindex(A, I[1], I[2]) - -function Base.getindex(A::AbstractGPUSparseVector{Tv, Ti}, i::Integer) where {Tv, Ti} - @boundscheck checkbounds(A, i) - ii = searchsortedfirst(SparseArrays.nonzeroinds(A), convert(Ti, i)) - (ii > SparseArrays.nnz(A) || SparseArrays.nonzeroinds(A)[ii] != i) && return zero(Tv) - SparseArrays.nonzeros(A)[ii] -end - -function Base.getindex(A::AbstractGPUSparseMatrixCSC{T}, i0::Integer, i1::Integer) where T - @boundscheck checkbounds(A, i0, i1) - r1 = Int(SparseArrays.getcolptr(A)[i1]) - r2 = Int(SparseArrays.getcolptr(A)[i1+1]-1) - (r1 > r2) && return zero(T) - r1 = searchsortedfirst(SparseArrays.rowvals(A), i0, r1, r2, Base.Order.Forward) - (r1 > r2 || SparseArrays.rowvals(A)[r1] != i0) && return zero(T) - SparseArrays.nonzeros(A)[r1] -end - -## copying between sparse GPU arrays -Base.copy(Vec::AbstractGPUSparseVector) = copyto!(similar(Vec), Vec) - -function Base.copyto!(dst::AbstractGPUSparseVector, src::AbstractGPUSparseVector) - if length(dst) != length(src) - throw(ArgumentError("Inconsistent Sparse Vector size")) - end - resize!(SparseArrays.nonzeroinds(dst), length(SparseArrays.nonzeroinds(src))) - resize!(SparseArrays.nonzeros(dst), length(SparseArrays.nonzeros(src))) - copyto!(SparseArrays.nonzeroinds(dst), SparseArrays.nonzeroinds(src)) - copyto!(SparseArrays.nonzeros(dst), SparseArrays.nonzeros(src)) - dst.nnz = src.nnz - dst -end - -function Base.copyto!(dst::AbstractGPUSparseMatrixCSC, src::AbstractGPUSparseMatrixCSC) - if size(dst) != size(src) - throw(ArgumentError("Inconsistent Sparse Matrix size")) - end - resize!(SparseArrays.getcolptr(dst), length(SparseArrays.getcolptr(src))) - resize!(SparseArrays.rowvals(dst), length(SparseArrays.rowvals(src))) - resize!(SparseArrays.nonzeros(dst), length(SparseArrays.nonzeros(src))) - copyto!(SparseArrays.getcolptr(dst), SparseArrays.getcolptr(src)) - copyto!(SparseArrays.rowvals(dst), SparseArrays.rowvals(src)) - copyto!(SparseArrays.nonzeros(dst), SparseArrays.nonzeros(src)) - dst.nnz = src.nnz - dst -end - -### BROADCAST - -# broadcast container type promotion for combinations of sparse arrays and other types -struct GPUSparseVecStyle <: Broadcast.AbstractArrayStyle{1} end -struct GPUSparseMatStyle <: Broadcast.AbstractArrayStyle{2} end -Broadcast.BroadcastStyle(::Type{<:AbstractGPUSparseVector}) = GPUSparseVecStyle() -Broadcast.BroadcastStyle(::Type{<:AbstractGPUSparseMatrix}) = GPUSparseMatStyle() -const SPVM = Union{GPUSparseVecStyle,GPUSparseMatStyle} - -# GPUSparseVecStyle handles 0-1 dimensions, GPUSparseMatStyle 0-2 dimensions. -# GPUSparseVecStyle promotes to GPUSparseMatStyle for 2 dimensions. -# Fall back to DefaultArrayStyle for higher dimensionality. -GPUSparseVecStyle(::Val{0}) = GPUSparseVecStyle() -GPUSparseVecStyle(::Val{1}) = GPUSparseVecStyle() -GPUSparseVecStyle(::Val{2}) = GPUSparseMatStyle() -GPUSparseVecStyle(::Val{N}) where N = Broadcast.DefaultArrayStyle{N}() -GPUSparseMatStyle(::Val{0}) = GPUSparseMatStyle() -GPUSparseMatStyle(::Val{1}) = GPUSparseMatStyle() -GPUSparseMatStyle(::Val{2}) = GPUSparseMatStyle() -GPUSparseMatStyle(::Val{N}) where N = Broadcast.DefaultArrayStyle{N}() - -Broadcast.BroadcastStyle(::GPUSparseVecStyle, ::AbstractGPUArrayStyle{1}) = GPUSparseVecStyle() -Broadcast.BroadcastStyle(::GPUSparseVecStyle, ::AbstractGPUArrayStyle{2}) = GPUSparseMatStyle() -Broadcast.BroadcastStyle(::GPUSparseMatStyle, ::AbstractGPUArrayStyle{2}) = GPUSparseMatStyle() - -# don't wrap sparse arrays with Extruded -Broadcast.extrude(x::AbstractGPUSparseVecOrMat) = x - -## detection of zero-preserving functions - -# modified from SparseArrays.jl - -# capturescalars takes a function (f) and a tuple of broadcast arguments, and returns a -# partially-evaluated function and a reduced argument tuple where all scalar operations have -# been applied already. -@inline function capturescalars(f, mixedargs) - let (passedsrcargstup, makeargs) = _capturescalars(mixedargs...) - parevalf = (passed...) -> f(makeargs(passed...)...) - return (parevalf, passedsrcargstup) - end -end - -## sparse broadcast style - -# Work around losing Type{T}s as DataTypes within the tuple that makeargs creates -@inline capturescalars(f, mixedargs::Tuple{Ref{Type{T}}, Vararg{Any}}) where {T} = - capturescalars((args...)->f(T, args...), Base.tail(mixedargs)) -@inline capturescalars(f, mixedargs::Tuple{Ref{Type{T}}, Ref{Type{S}}, Vararg{Any}}) where {T, S} = - # This definition is identical to the one above and necessary only for - # avoiding method ambiguity. - capturescalars((args...)->f(T, args...), Base.tail(mixedargs)) -@inline capturescalars(f, mixedargs::Tuple{AbstractGPUSparseVecOrMat, Ref{Type{T}}, Vararg{Any}}) where {T} = - capturescalars((a1, args...)->f(a1, T, args...), (mixedargs[1], Base.tail(Base.tail(mixedargs))...)) -@inline capturescalars(f, mixedargs::Tuple{Union{Ref,AbstractArray{<:Any,0}}, Ref{Type{T}}, Vararg{Any}}) where {T} = - capturescalars((args...)->f(mixedargs[1], T, args...), Base.tail(Base.tail(mixedargs))) - -scalararg(::Number) = true -scalararg(::Any) = false -scalarwrappedarg(::Union{AbstractArray{<:Any,0},Ref}) = true -scalarwrappedarg(::Any) = false - -@inline function _capturescalars() - return (), () -> () -end -@inline function _capturescalars(arg, mixedargs...) - let (rest, f) = _capturescalars(mixedargs...) - if scalararg(arg) - return rest, @inline function(tail...) - (arg, f(tail...)...) - end # add back scalararg after (in makeargs) - elseif scalarwrappedarg(arg) - return rest, @inline function(tail...) - (arg[], f(tail...)...) # TODO: This can put a Type{T} in a tuple - end # unwrap and add back scalararg after (in makeargs) - else - return (arg, rest...), @inline function(head, tail...) - (head, f(tail...)...) - end # pass-through to broadcast - end - end -end -@inline function _capturescalars(arg) # this definition is just an optimization (to bottom out the recursion slightly sooner) - if scalararg(arg) - return (), () -> (arg,) # add scalararg - elseif scalarwrappedarg(arg) - return (), () -> (arg[],) # unwrap - else - return (arg,), (head,) -> (head,) # pass-through - end -end - -@inline _iszero(x) = x == 0 -@inline _iszero(x::Number) = Base.iszero(x) -@inline _iszero(x::AbstractArray) = Base.iszero(x) -@inline _zeros_eltypes(A) = (zero(eltype(A)),) -@inline _zeros_eltypes(A, Bs...) = (zero(eltype(A)), _zeros_eltypes(Bs...)...) - -## COV_EXCL_START -## iteration helpers - -""" - CSRIterator{Ti}(row, args...) - -A GPU-compatible iterator for accessing the elements of a single row `row` of several CSR -matrices `args` in one go. The row should be in-bounds for every sparse argument. Each -iteration returns a 2-element tuple: The current column, and each arguments' pointer index -(or 0 if that input didn't have an element at that column). The pointers can then be used to -access the elements themselves. - -For convenience, this iterator can be passed non-sparse arguments as well, which will be -ignored (with the returned `col`/`ptr` values set to 0). -""" -struct CSRIterator{Ti,N,ATs} - row::Ti - col_ends::NTuple{N, Ti} - args::ATs -end - -function CSRIterator{Ti}(row, args::Vararg{Any, N}) where {Ti,N} - # check that `row` is valid for all arguments - @boundscheck begin - ntuple(Val(N)) do i - arg = @inbounds args[i] - arg isa GPUSparseDeviceMatrixCSR && checkbounds(arg, row, 1) - end - end - - col_ends = ntuple(Val(N)) do i - arg = @inbounds args[i] - if arg isa GPUSparseDeviceMatrixCSR - @inbounds(arg.rowPtr[row+1]) - else - zero(Ti) - end - end - - CSRIterator{Ti, N, typeof(args)}(row, col_ends, args) -end - -@inline function Base.iterate(iter::CSRIterator{Ti,N}, state=nothing) where {Ti,N} - # helper function to get the column of a sparse array at a specific pointer - @inline function get_col(i, ptr) - arg = @inbounds iter.args[i] - if arg isa GPUSparseDeviceMatrixCSR - col_end = @inbounds iter.col_ends[i] - if ptr < col_end - return @inbounds arg.colVal[ptr] % Ti - end - end - typemax(Ti) - end - - # initialize the state - # - ptr: the current index into the colVal/nzVal arrays - # - col: the current column index (cached so that we don't have to re-read each time) - state = something(state, - ntuple(Val(N)) do i - arg = @inbounds iter.args[i] - if arg isa GPUSparseDeviceMatrixCSR - ptr = @inbounds iter.args[i].rowPtr[iter.row] % Ti - col = @inbounds get_col(i, ptr) - else - ptr = typemax(Ti) - col = typemax(Ti) - end - (; ptr, col) - end - ) - - # determine the column we're currently processing - cols = ntuple(i -> @inbounds(state[i].col), Val(N)) - cur_col = min(cols...) - cur_col == typemax(Ti) && return - - # fetch the pointers (we don't look up the values, as the caller might want to index - # the sparse array directly, e.g., to mutate it). we don't return `ptrs` from the state - # directly, but first convert the `typemax(Ti)` to a more convenient zero value. - # NOTE: these values may end up unused by the caller (e.g. in the count_nnzs kernels), - # but LLVM appears smart enough to filter them away. - ptrs = ntuple(Val(N)) do i - ptr, col = @inbounds state[i] - col == cur_col ? ptr : zero(Ti) - end - - # advance the state - new_state = ntuple(Val(N)) do i - ptr, col = @inbounds state[i] - if col == cur_col - ptr += one(Ti) - col = get_col(i, ptr) - end - (; ptr, col) - end - - return (cur_col, ptrs), new_state -end - -struct CSCIterator{Ti,N,ATs} - col::Ti - row_ends::NTuple{N, Ti} - args::ATs -end - -function CSCIterator{Ti}(col, args::Vararg{Any, N}) where {Ti,N} - # check that `col` is valid for all arguments - @boundscheck begin - ntuple(Val(N)) do i - arg = @inbounds args[i] - arg isa GPUSparseDeviceMatrixCSR && checkbounds(arg, 1, col) - end - end - - row_ends = ntuple(Val(N)) do i - arg = @inbounds args[i] - x = if arg isa GPUSparseDeviceMatrixCSC - @inbounds(arg.colPtr[col+1]) - else - zero(Ti) - end - x - end - - CSCIterator{Ti, N, typeof(args)}(col, row_ends, args) -end - -@inline function Base.iterate(iter::CSCIterator{Ti,N}, state=nothing) where {Ti,N} - # helper function to get the column of a sparse array at a specific pointer - @inline function get_col(i, ptr) - arg = @inbounds iter.args[i] - if arg isa GPUSparseDeviceMatrixCSC - col_end = @inbounds iter.row_ends[i] - if ptr < col_end - return @inbounds arg.rowVal[ptr] % Ti - end - end - typemax(Ti) - end - - # initialize the state - # - ptr: the current index into the rowVal/nzVal arrays - # - row: the current row index (cached so that we don't have to re-read each time) - state = something(state, - ntuple(Val(N)) do i - arg = @inbounds iter.args[i] - if arg isa GPUSparseDeviceMatrixCSC - ptr = @inbounds iter.args[i].colPtr[iter.col] % Ti - row = @inbounds get_col(i, ptr) - else - ptr = typemax(Ti) - row = typemax(Ti) - end - (; ptr, row) - end - ) - - # determine the row we're currently processing - rows = ntuple(i -> @inbounds(state[i].row), Val(N)) - cur_row = min(rows...) - cur_row == typemax(Ti) && return - - # fetch the pointers (we don't look up the values, as the caller might want to index - # the sparse array directly, e.g., to mutate it). we don't return `ptrs` from the state - # directly, but first convert the `typemax(Ti)` to a more convenient zero value. - # NOTE: these values may end up unused by the caller (e.g. in the count_nnzs kernels), - # but LLVM appears smart enough to filter them away. - ptrs = ntuple(Val(N)) do i - ptr, row = @inbounds state[i] - row == cur_row ? ptr : zero(Ti) - end - - # advance the state - new_state = ntuple(Val(N)) do i - ptr, row = @inbounds state[i] - if row == cur_row - ptr += one(Ti) - row = get_col(i, ptr) - end - (; ptr, row) - end - - return (cur_row, ptrs), new_state -end - -# helpers to index a sparse or dense array -function _getindex(arg::Union{<:GPUSparseDeviceMatrixCSR,GPUSparseDeviceMatrixCSC}, I, ptr) - if ptr == 0 - zero(eltype(arg)) - else - @inbounds arg.nzVal[ptr] - end -end -@inline function _getindex(arg::DenseArray{Tv}, I, ptr)::Tv where {Tv} - return @inbounds arg[I]::Tv -end -@inline _getindex(arg, I, ptr) = Broadcast._broadcast_getindex(arg, I) - -## sparse broadcast implementation -iter_type(::Type{<:AbstractGPUSparseMatrixCSC}, ::Type{Ti}) where {Ti} = CSCIterator{Ti} -iter_type(::Type{<:AbstractGPUSparseMatrixCSR}, ::Type{Ti}) where {Ti} = CSRIterator{Ti} -iter_type(::Type{<:GPUSparseDeviceMatrixCSC}, ::Type{Ti}) where {Ti} = CSCIterator{Ti} -iter_type(::Type{<:GPUSparseDeviceMatrixCSR}, ::Type{Ti}) where {Ti} = CSRIterator{Ti} - -_has_row(A, offsets, row, fpreszeros::Bool) = fpreszeros ? 0 : row -_has_row(A::AbstractDeviceArray, offsets, row, ::Bool) = row -function _has_row(A::GPUSparseDeviceVector, offsets, row, ::Bool) - for row_ix in 1:length(A.iPtr) - arg_row = @inbounds A.iPtr[row_ix] - arg_row == row && return row_ix - arg_row > row && break - end - return 0 -end - -@kernel function compute_offsets_kernel(::Type{<:AbstractGPUSparseVector}, first_row::Ti, last_row::Ti, - fpreszeros::Bool, offsets::AbstractVector{Pair{Ti, NTuple{N, Ti}}}, - args...) where {Ti, N} - my_ix = @index(Global, Linear) - row = my_ix + first_row - one(eltype(my_ix)) - if row ≤ last_row - # TODO load arg.iPtr slices into shared memory - arg_row_is_nnz = ntuple(Val(N)) do i - arg = @inbounds args[i] - _has_row(arg, offsets, row, fpreszeros) - end - row_is_nnz = 0 - for i in 1:N - row_is_nnz |= @inbounds arg_row_is_nnz[i] - end - key = (row_is_nnz == 0) ? typemax(Ti) : row - @inbounds offsets[my_ix] = key => arg_row_is_nnz - end -end - -# kernel to count the number of non-zeros in a row, to determine the row offsets -@kernel function compute_offsets_kernel(T::Type{<:Union{AbstractGPUSparseMatrixCSR, AbstractGPUSparseMatrixCSC}}, - offsets::AbstractVector{Ti}, args...) where Ti - # every thread processes an entire row - leading_dim = @index(Global, Linear) - if leading_dim ≤ length(offsets)-1 - iter = @inbounds iter_type(T, Ti)(leading_dim, args...) - - # count the nonzero leading_dims of all inputs - accum = zero(Ti) - for (leading_dim, vals) in iter - accum += one(Ti) - end - - # the way we write the nnz counts is a bit strange, but done so that the result - # after accumulation can be directly used as the rowPtr/colPtr array of a CSR/CSC matrix. - @inbounds begin - if leading_dim == 1 - offsets[1] = 1 - end - offsets[leading_dim+1] = accum - end - end -end - -@kernel function sparse_to_sparse_broadcast_kernel(f::F, output::GPUSparseDeviceVector{Tv,Ti}, - offsets::AbstractVector{Pair{Ti, NTuple{N, Ti}}}, - args...) where {Tv, Ti, N, F} - row_ix = @index(Global, Linear) - if row_ix ≤ output.nnz - row_and_ptrs = @inbounds offsets[row_ix] - row = @inbounds row_and_ptrs[1] - arg_ptrs = @inbounds row_and_ptrs[2] - vals = ntuple(Val(N)) do i - @inline - arg = @inbounds args[i] - # ptr is 0 if the sparse vector doesn't have an element at this row - # ptr is 0 if the arg is a scalar AND f preserves zeros - ptr = @inbounds arg_ptrs[i] - _getindex(arg, row, ptr) - end - output_val = f(vals...) - @inbounds output.iPtr[row_ix] = row - @inbounds output.nzVal[row_ix] = output_val - end -end - -@kernel function sparse_to_sparse_broadcast_kernel(f, output::T, offsets::Union{<:AbstractArray,Nothing}, - args...) where {Ti, T<:Union{GPUSparseDeviceMatrixCSR{<:Any,Ti}, - GPUSparseDeviceMatrixCSC{<:Any,Ti}}} - # every thread processes an entire row - leading_dim = @index(Global, Linear) - leading_dim_size = output isa GPUSparseDeviceMatrixCSR ? size(output, 1) : size(output, 2) - if leading_dim ≤ leading_dim_size - iter = @inbounds iter_type(T, Ti)(leading_dim, args...) - - output_ptrs = output isa GPUSparseDeviceMatrixCSR ? output.rowPtr : output.colPtr - output_ivals = output isa GPUSparseDeviceMatrixCSR ? output.colVal : output.rowVal - # fetch the row offset, and write it to the output - @inbounds begin - output_ptr = output_ptrs[leading_dim] = offsets[leading_dim] - if leading_dim == leading_dim_size - output_ptrs[leading_dim+one(eltype(leading_dim))] = offsets[leading_dim+one(eltype(leading_dim))] - end - end - - # set the values for this row - for (sub_leading_dim, ptrs) in iter - index_first = output isa GPUSparseDeviceMatrixCSR ? leading_dim : sub_leading_dim - index_second = output isa GPUSparseDeviceMatrixCSR ? sub_leading_dim : leading_dim - I = CartesianIndex(index_first, index_second) - vals = ntuple(Val(length(args))) do i - arg = @inbounds args[i] - ptr = @inbounds ptrs[i] - _getindex(arg, I, ptr) - end - @inbounds output_ivals[output_ptr] = sub_leading_dim - @inbounds output.nzVal[output_ptr] = f(vals...) - output_ptr += one(Ti) - end - end -end -@kernel function sparse_to_dense_broadcast_kernel(T::Type{<:Union{AbstractGPUSparseMatrixCSR{Tv, Ti}, - AbstractGPUSparseMatrixCSC{Tv, Ti}}}, - f, output::AbstractArray, args...) where {Tv, Ti} - # every thread processes an entire row - leading_dim = @index(Global, Linear) - leading_dim_size = T <: AbstractGPUSparseMatrixCSR ? size(output, 1) : size(output, 2) - if leading_dim ≤ leading_dim_size - iter = @inbounds iter_type(T, Ti)(leading_dim, args...) - - # set the values for this row - for (sub_leading_dim, ptrs) in iter - index_first = T <: AbstractGPUSparseMatrixCSR ? leading_dim : sub_leading_dim - index_second = T <: AbstractGPUSparseMatrixCSR ? sub_leading_dim : leading_dim - I = CartesianIndex(index_first, index_second) - vals = ntuple(Val(length(args))) do i - arg = @inbounds args[i] - ptr = @inbounds ptrs[i] - _getindex(arg, I, ptr) - end - - @inbounds output[I] = f(vals...) - end - end -end - -@kernel function sparse_to_dense_broadcast_kernel(::Type{<:AbstractGPUSparseVector}, f::F, - output::AbstractArray{Tv}, - offsets::AbstractVector{Pair{Ti, NTuple{N, Ti}}}, - args...) where {Tv, F, N, Ti} - # every thread processes an entire row - row_ix = @index(Global, Linear) - if row_ix ≤ length(output) - row_and_ptrs = @inbounds offsets[row_ix] - row = @inbounds row_and_ptrs[1] - arg_ptrs = @inbounds row_and_ptrs[2] - vals = ntuple(Val(length(args))) do i - @inline - arg = @inbounds args[i] - # ptr is 0 if the sparse vector doesn't have an element at this row - # ptr is row if the arg is dense OR a scalar with non-zero-preserving f - # ptr is 0 if the arg is a scalar AND f preserves zeros - ptr = @inbounds arg_ptrs[i] - _getindex(arg, row, ptr) - end - @inbounds output[row] = f(vals...) - end -end -## COV_EXCL_STOP - -function Broadcast.copy(bc::Broadcasted{<:Union{GPUSparseVecStyle,GPUSparseMatStyle}}) - # find the sparse inputs - bc = Broadcast.flatten(bc) - sparse_args = findall(bc.args) do arg - arg isa AbstractGPUSparseArray - end - sparse_types = unique(map(i->nameof(typeof(bc.args[i])), sparse_args)) - if length(sparse_types) > 1 - error("broadcast with multiple types of sparse arrays ($(join(sparse_types, ", "))) is not supported") - end - sparse_typ = typeof(bc.args[first(sparse_args)]) - sparse_typ <: Union{AbstractGPUSparseMatrixCSR,AbstractGPUSparseMatrixCSC,AbstractGPUSparseVector} || - error("broadcast with sparse arrays is currently only implemented for vectors and CSR and CSC matrices") - Ti = if sparse_typ <: AbstractGPUSparseMatrixCSR - reduce(promote_type, map(i->eltype(bc.args[i].rowPtr), sparse_args)) - elseif sparse_typ <: AbstractGPUSparseMatrixCSC - reduce(promote_type, map(i->eltype(bc.args[i].colPtr), sparse_args)) - elseif sparse_typ <: AbstractGPUSparseVector - reduce(promote_type, map(i->eltype(bc.args[i].iPtr), sparse_args)) - end - - # determine the output type - Tv = Broadcast.combine_eltypes(bc.f, eltype.(bc.args)) - if !Base.isconcretetype(Tv) - error("""GPU sparse broadcast resulted in non-concrete element type $Tv. - This probably means that the function you are broadcasting contains an error or type instability.""") - end - - # partially-evaluate the function, removing scalars. - parevalf, passedsrcargstup = capturescalars(bc.f, bc.args) - # check if the partially-evaluated function preserves zeros. if so, we'll only need to - # apply it to the sparse input arguments, preserving the sparse structure. - if all(arg->isa(arg, AbstractSparseArray), passedsrcargstup) - fofzeros = parevalf(_zeros_eltypes(passedsrcargstup...)...) - fpreszeros = _iszero(fofzeros) - else - fpreszeros = false - end - - # the kernels below parallelize across rows or cols, not elements, so it's unlikely - # we'll launch many threads. to maximize utilization, parallelize across blocks first. - rows, cols = get(size(bc), 1, 1), get(size(bc), 2, 1) - # `size(bc, ::Int)` is missing - # for AbstractGPUSparseVec, figure out the actual row range we need to address, e.g. if m = 2^20 - # but the only rows present in any sparse vector input are between 2 and 128, no need to - # launch massive threads. - # TODO: use the difference here to set the thread count - overall_first_row = one(Ti) - overall_last_row = Ti(rows) - offsets = nothing - # allocate the output container - sparse_arg = bc.args[first(sparse_args)] - if !fpreszeros && sparse_typ <: Union{AbstractGPUSparseMatrixCSR, AbstractGPUSparseMatrixCSC} - # either we have dense inputs, or the function isn't preserving zeros, - # so use a dense output to broadcast into. - val_array = nonzeros(sparse_arg) - output = similar(val_array, Tv, size(bc)) - # since we'll be iterating the sparse inputs, we need to pre-fill the dense output - # with appropriate values (while setting the sparse inputs to zero). we do this by - # re-using the dense broadcast implementation. - nonsparse_args = map(bc.args) do arg - # NOTE: this assumes the broadcast is flattened, but not yet preprocessed - if arg isa AbstractGPUSparseArray - zero(eltype(arg)) - else - arg - end - end - broadcast!(bc.f, output, nonsparse_args...) - elseif length(sparse_args) == 1 && sparse_typ <: Union{AbstractGPUSparseMatrixCSR, AbstractGPUSparseMatrixCSC} - # we only have a single sparse input, so we can reuse its structure for the output. - # this avoids a kernel launch and costly synchronization. - if sparse_typ <: AbstractGPUSparseMatrixCSR - offsets = rowPtr = sparse_arg.rowPtr - colVal = similar(sparse_arg.colVal) - nzVal = similar(sparse_arg.nzVal, Tv) - output = sparse_array_type(sparse_typ)(rowPtr, colVal, nzVal, size(bc)) - elseif sparse_typ <: AbstractGPUSparseMatrixCSC - offsets = colPtr = sparse_arg.colPtr - rowVal = similar(sparse_arg.rowVal) - nzVal = similar(sparse_arg.nzVal, Tv) - output = sparse_array_type(sparse_typ)(colPtr, rowVal, nzVal, size(bc)) - end - else - # determine the number of non-zero elements per row so that we can create an - # appropriately-structured output container - offsets = if sparse_typ <: AbstractGPUSparseMatrixCSR - ptr_array = sparse_arg.rowPtr - similar(ptr_array, Ti, rows+1) - elseif sparse_typ <: AbstractGPUSparseMatrixCSC - ptr_array = sparse_arg.colPtr - similar(ptr_array, Ti, cols+1) - elseif sparse_typ <: AbstractGPUSparseVector - ptr_array = sparse_arg.iPtr - @allowscalar begin - arg_first_rows = ntuple(Val(length(bc.args))) do i - bc.args[i] isa AbstractGPUSparseVector && return bc.args[i].iPtr[1] - return one(Ti) - end - arg_last_rows = ntuple(Val(length(bc.args))) do i - bc.args[i] isa AbstractGPUSparseVector && return bc.args[i].iPtr[end] - return Ti(rows) - end - end - overall_first_row = min(arg_first_rows...) - overall_last_row = max(arg_last_rows...) - similar(ptr_array, Pair{Ti, NTuple{length(bc.args), Ti}}, overall_last_row - overall_first_row + 1) - end - let - args = if sparse_typ <: AbstractGPUSparseVector - (sparse_typ, overall_first_row, overall_last_row, fpreszeros, offsets, bc.args...) - else - (sparse_typ, offsets, bc.args...) - end - kernel = compute_offsets_kernel(get_backend(bc.args[first(sparse_args)])) - kernel(args...; ndrange=length(offsets)) - end - # accumulate these values so that we can use them directly as row pointer offsets, - # as well as to get the total nnz count to allocate the sparse output array. - # cusparseXcsrgeam2Nnz computes this in one go, but it doesn't seem worth the effort - if !(sparse_typ <: AbstractGPUSparseVector) - @allowscalar accumulate!(Base.add_sum, offsets, offsets) - total_nnz = @allowscalar last(offsets[end]) - 1 - else - @allowscalar sort!(offsets; by=first) - total_nnz = mapreduce(x->first(x) != typemax(first(x)), +, offsets) - end - output = if sparse_typ <: Union{AbstractGPUSparseMatrixCSR,AbstractGPUSparseMatrixCSC} - ixVal = similar(offsets, Ti, total_nnz) - nzVal = similar(offsets, Tv, total_nnz) - output_sparse_typ = sparse_array_type(sparse_typ) - output_sparse_typ(offsets, ixVal, nzVal, size(bc)) - elseif sparse_typ <: AbstractGPUSparseVector && !fpreszeros - val_array = bc.args[first(sparse_args)].nzVal - similar(val_array, Tv, size(bc)) - elseif sparse_typ <: AbstractGPUSparseVector && fpreszeros - iPtr = similar(offsets, Ti, total_nnz) - nzVal = similar(offsets, Tv, total_nnz) - sparse_array_type(sparse_arg){Tv, Ti}(iPtr, nzVal, rows) - end - if sparse_typ <: AbstractGPUSparseVector && !fpreszeros - nonsparse_args = map(bc.args) do arg - # NOTE: this assumes the broadcst is flattened, but not yet preprocessed - if arg isa AbstractGPUSparseArray - zero(eltype(arg)) - else - arg - end - end - broadcast!(bc.f, output, nonsparse_args...) - end - end - # perform the actual broadcast - if output isa AbstractGPUSparseArray - args = (bc.f, output, offsets, bc.args...) - kernel = sparse_to_sparse_broadcast_kernel(get_backend(bc.args[first(sparse_args)])) - ndrange = if sparse_typ <: AbstractGPUSparseVector - output.nnz - elseif sparse_typ <: AbstractGPUSparseMatrixCSC - size(output, 2) - else - size(output, 1) - end - else - args = sparse_typ <: AbstractGPUSparseVector ? (sparse_typ, bc.f, output, offsets, bc.args...) : - (sparse_typ, bc.f, output, bc.args...) - kernel = sparse_to_dense_broadcast_kernel(get_backend(bc.args[first(sparse_args)])) - ndrange = sparse_typ <: AbstractGPUSparseMatrixCSC ? size(output, 2) : size(output, 1) - end - kernel(args...; ndrange) - return output -end -## COV_EXCL_START -@kernel function csr_reduce_kernel(f::F, op::OP, neutral, zeros_preserved::Bool, output::DenseArray, args...) where {F, OP} - # every thread processes an entire row - row = @index(Global, Linear) - if row ≤ size(output, 1) - iter = @inbounds CSRIterator{Int}(row, args...) - - val = op(neutral, neutral) - - # reduce the values for this row - for (col, ptrs) in iter - I = CartesianIndex(row, col) - vals = ntuple(Val(length(args))) do i - arg = @inbounds args[i] - ptr = @inbounds ptrs[i] - _getindex(arg, I, ptr) - end - val = op(val, f(vals...)) - end - if !zeros_preserved - f_zero_val = f(zero(neutral)) - next_row_ind = row+1 - nzs_this_row = ntuple(Val(length(args))) do i - max_n_zeros = size(args[i], 2) - arg_row_ptr = args[i].rowPtr - nz_this_row = max_n_zeros - (@inbounds(arg_row_ptr[next_row_ind]) - @inbounds(arg_row_ptr[row])) - nz_this_row * f_zero_val - end - val = op(val, nzs_this_row...) - end - - @inbounds output[row] = val - end -end - -@kernel function csc_reduce_kernel(f::F, op::OP, neutral, zeros_preserved::Bool, output::DenseArray, args...) where {F, OP} - # every thread processes an entire column - col = @index(Global, Linear) - if col ≤ size(output, 2) - iter = @inbounds CSCIterator{Int}(col, args...) - - val = op(neutral, neutral) - - # reduce the values for this col - for (row, ptrs) in iter - I = CartesianIndex(row, col) - vals = ntuple(Val(length(args))) do i - arg = @inbounds args[i] - ptr = @inbounds ptrs[i] - _getindex(arg, I, ptr) - end - val = op(val, f(vals...)) - end - if !zeros_preserved - f_zero_val = f(zero(neutral)) - next_col_ind = col+1 - nzs_this_col = ntuple(Val(length(args))) do i - max_n_zeros = size(args[i], 1) - arg_col_ptr = args[i].colPtr - nz_this_col = max_n_zeros - (@inbounds(arg_col_ptr[next_col_ind]) - @inbounds(arg_col_ptr[col])) - nz_this_col * f_zero_val - end - val = op(val, nzs_this_col...) - end - @inbounds output[col] = val - end -end -## COV_EXCL_STOP - -csc_type(A::AbstractGPUSparseMatrix) = csc_type(typeof(A)) -csr_type(A::AbstractGPUSparseMatrix) = csr_type(typeof(A)) - -# TODO: implement mapreducedim! -function Base.mapreduce(f, op, A::AbstractGPUSparseMatrix; dims=:, init=nothing) - # figure out the destination container type by looking at the initializer element, - # or by relying on inference to reason through the map and reduce functions - if init === nothing - ET = Broadcast.combine_eltypes(f, (A,)) - ET = Base.promote_op(op, ET, ET) - (ET === Union{} || ET === Any) && - error("mapreduce cannot figure the output element type, please pass an explicit init value") - - init = zero(ET) - else - ET = typeof(init) - end - - f_preserves_zeros = ( f(zero(ET)) == zero(ET) ) - # we only handle reducing along one of the two dimensions, - # or a complete reduction (requiring an additional pass) - in(dims, [Colon(), 1, 2]) || error("only dims=:, dims=1 or dims=2 is supported") - - if A isa AbstractGPUSparseMatrixCSR && dims == 1 - A = csc_type(A)(A) - elseif A isa AbstractGPUSparseMatrixCSC && dims == 2 - A = csr_type(A)(A) - end - m, n = size(A) - val_array = nonzeros(A) - backend = get_backend(A) - output_dim = 0 - ndrange = 0 - if A isa AbstractGPUSparseMatrixCSR - output_dim = m - ndrange = m - kernel = csr_reduce_kernel(backend) - elseif A isa AbstractGPUSparseMatrixCSC - output_dim = (1, n) - ndrange = n - kernel = csc_reduce_kernel(backend) - end - output = similar(val_array, ET, output_dim) - kernel(f, op, init, f_preserves_zeros, output, A; ndrange = ndrange) - if dims == Colon() - return mapreduce(identity, op, output; init) - else - return output - end -end diff --git a/src/host/sparse/array.jl b/src/host/sparse/array.jl new file mode 100644 index 000000000..153185811 --- /dev/null +++ b/src/host/sparse/array.jl @@ -0,0 +1,188 @@ +# Array semantics that SparseArrays implements through host scalar loops or downloads. + +function Base.:(==)(A::GPUSparseArray, B::GPUSparseArray) + size(A) == size(B) || return false + # Stored zeros need not have identical patterns. + return !any(comparison_operand(A) .!= comparison_operand(B)) +end + +comparison_operand(A::Union{GPUSparseVector,GPUSparseMatrixCOO}) = A +function comparison_operand(A::GPUSparseMatrixCompressed) + # A scalar comparison does not create a public sparse output. Its structural union + # can exceed either input's pointer width, including when both store only zeros. + ptr = compressed_ptr(A) + eltype(ptr) === Int && return A + sparse_format_type(A)(convert_index_buffer(Int, ptr), compressed_ind(A), A.nzVal, size(A)) +end + +function sparse_broadcast!(dest::GPUSparseArray, bc::Broadcasted) + check_sparse_mutation(dest) + bc = Broadcast.instantiate(Broadcast.broadcasted(bc.f, bc.args...)) + if isempty(axes(bc)) + value = bc[] + iszero(value) || throw(ArgumentError("nonzero scalar fill of a GPU sparse destination is unsupported")) + fill!(nonzeros(dest), value) + return dest + end + axes(dest) == axes(bc) || throw(DimensionMismatch("sparse broadcast destination axes must match the result")) + # Materializing before touching dest makes structural growth and overlapping arguments + # safe. The temporary owns its buffers and can be copied after complete preflight. + tmp = copy(bc) + tmp isa GPUSparseArray || + throw(ArgumentError("dense or non-zero-preserving broadcast into a GPU sparse destination is unsupported")) + if dest isa GPUSparseMatrix && tmp isa GPUSparseMatrix + return copyto!(dest, convert(matrix_format(dest), tmp)) + elseif dest isa GPUSparseVector && tmp isa GPUSparseVector + return copyto!(dest, tmp) + end + throw(ArgumentError("broadcast cannot change the sparse destination's dimensionality")) +end +Broadcast.materialize!(dest::GPUSparseArray, bc::Broadcasted) = sparse_broadcast!(dest, bc) +Base.copyto!(dest::GPUSparseArray, bc::Broadcasted{<:SPVM}) = sparse_broadcast!(dest, bc) +Base.copyto!(dest::GPUSparseArray, bc::Broadcasted{Nothing}) = sparse_broadcast!(dest, bc) +Base.copyto!(dest::GPUSparseArray, bc::Broadcasted{<:AbstractGPUArrayStyle}) = sparse_broadcast!(dest, bc) +Base.copyto!(dest::GPUSparseArray, bc::Broadcasted{Broadcast.DefaultArrayStyle{0}}) = sparse_broadcast!(dest, bc) + +function Base.fill!(A::GPUSparseArray, value) + iszero(value) || throw(ArgumentError("nonzero fill of a GPU sparse destination is unsupported")) + fill!(nonzeros(A), value) + return A +end + +Base.getindex(A::GPUSparseMatrix, ::Colon) = vec(A) +Base.getindex(A::GPUSparseMatrix,I::AbstractRange{<:Integer}) = vec(A)[I] + +const GPUSparseTransposed = Union{Transpose{<:Any,<:GPUSparseMatrix},Adjoint{<:Any,<:GPUSparseMatrix}} +Base.:(*)(a::Number,A::GPUSparseTransposed)=a.*copy(A) +Base.:(*)(A::GPUSparseTransposed,a::Number)=copy(A).*a +Base.:(^)(A::GPUSparseTransposed,p::Integer)=copy(A)^p +# Base's fused three-argument product otherwise scales after accumulating in the +# unscaled element type. Match left association, including scalar promotion. +Base.:(*)(a::Number,A::Union{GPUSparseMatrix,GPUSparseTransposed}, + B::Union{GPUSparseMatrix,GPUSparseTransposed})=(a*materialize(A))*B +const GPUSparseRealComplexMatrix = Union{GPUSparseMatrixCSR{<:Union{Real,Complex}}, + GPUSparseMatrixCSC{<:Union{Real,Complex}},GPUSparseMatrixCOO{<:Union{Real,Complex}}, + Transpose{<:Union{Real,Complex},<:GPUSparseMatrix},Adjoint{<:Union{Real,Complex},<:GPUSparseMatrix}} +Base.:(*)(a::Union{Real,Complex},A::GPUSparseRealComplexMatrix, + B::GPUSparseRealComplexMatrix)=(a*materialize(A))*B +Base.:(*)(a::Number,A::Union{GPUSparseMatrix,GPUSparseTransposed}, + B::AnyGPUMatrix)=(a*materialize(A))*B +Base.:(*)(a::Union{Real,Complex},A::GPUSparseRealComplexMatrix, + B::AnyGPUMatrix{<:Union{Real,Complex}})=(a*materialize(A))*B +const GPUSparseStructured = Union{Symmetric{<:Any,<:GPUSparseMatrix},Hermitian{<:Any,<:GPUSparseMatrix}} +for op in (:+,:-) + @eval begin + Base.$op(A::Union{GPUSparseMatrix,GPUSparseTransposed},B::GPUSparseStructured)= + sparse_map($op,materialize(A),materialize(B)) + Base.$op(A::GPUSparseStructured,B::Union{GPUSparseMatrix,GPUSparseTransposed})= + sparse_map($op,materialize(A),materialize(B)) + end +end +Base.:(*)(a::Number,A::GPUSparseStructured)=a.*materialize(A) +Base.:(*)(A::GPUSparseStructured,a::Number)=materialize(A).*a +Base.:(*)(a::Real,A::Hermitian{<:Any,<:GPUSparseMatrix})=a.*materialize(A) +Base.:(*)(A::Hermitian{<:Any,<:GPUSparseMatrix},a::Real)=materialize(A).*a +SparseArrays.sparse(A::GPUSparseStructured)=materialize(A) +SparseArrays.SparseMatrixCSC(A::GPUSparseStructured)=SparseMatrixCSC(materialize(A)) +SparseArrays.SparseMatrixCSC{T}(A::Union{GPUSparseStructured,GPUSparseTransposed}) where {T} = + SparseMatrixCSC{T}(SparseMatrixCSC(A)) +SparseArrays.SparseMatrixCSC{T,Ti}(A::Union{GPUSparseStructured,GPUSparseTransposed}) where {T,Ti} = + SparseMatrixCSC{T,Ti}(SparseMatrixCSC(A)) +Base.Array(A::GPUSparseStructured)=Array(materialize(A)) +Base.collect(A::GPUSparseStructured)=Array(A) +LinearAlgebra.Hermitian{T}(A::GPUSparseMatrix,uplo::Symbol=:U) where {T} = + Hermitian(with_values(A,T),uplo) +LinearAlgebra.Symmetric{T}(A::GPUSparseMatrix,uplo::Symbol=:U) where {T} = + Symmetric(with_values(A,T),uplo) +for op in (:+,:-) + @eval begin + Base.$op(A::GPUSparseMatrix,B::GPUSparseTransposed) = sparse_map($op,A,copy(B)) + Base.$op(A::GPUSparseTransposed,B::GPUSparseMatrix) = sparse_map($op,copy(A),B) + Base.$op(A::GPUSparseTransposed,B::GPUSparseTransposed) = sparse_map($op,copy(A),copy(B)) + end +end +SparseArrays.nnz(A::GPUSparseTransposed) = nnz(parent(A)) +SparseArrays.nonzeros(A::Transpose{<:Any,<:GPUSparseMatrix}) = nonzeros(parent(A)) +SparseArrays.nonzeros(A::Adjoint{<:Any,<:GPUSparseMatrix}) = conj.(nonzeros(parent(A))) +SparseArrays.SparseMatrixCSC(A::GPUSparseTransposed) = SparseMatrixCSC(copy(A)) +Base.Array(A::GPUSparseTransposed) = Array(copy(A)) +Base.collect(A::GPUSparseTransposed) = Array(A) +Base.mapreduce(f, op, A::GPUSparseTransposed; kwargs...) = mapreduce(f, op, copy(A); kwargs...) +Base.getindex(A::GPUSparseTransposed, I::SparseRangeIndex, J::SparseRangeIndex) = copy(A)[I, J] + +function sparse_diagonal_like(A::GPUSparseMatrix,values::AbstractVector,dims::Dims{2}=size(A)) + n=min(dims...) + length(values)==n || throw(DimensionMismatch("diagonal length must match matrix dimensions")) + check_sparse_dims(indtype(A),dims) + inds=sparse_similar(nonzeros(A),indtype(A),n) + inds .= 1:n + Tp=A isa GPUSparseMatrixCompressed ? eltype(compressed_ptr(A)) : promote_type(Int,indtype(A)) + assemble_matrix(inds,copy(inds),move_buffer(nonzeros(A),values),dims..., + default_combine(eltype(values)),matrix_format_symbol(A);pointer_type=Tp) +end +matrix_format_symbol(::GPUSparseMatrixCSR)=:csr +matrix_format_symbol(::GPUSparseMatrixCSC)=:csc +matrix_format_symbol(::GPUSparseMatrixCOO)=:coo +function sparse_identity_like(A::GPUSparseMatrix,value) + n=min(size(A)...) + sparse_diagonal_like(A,fill!(sparse_similar(nonzeros(A),typeof(value),n),value)) +end +Base.one(A::GPUSparseMatrix)=(LinearAlgebra.checksquare(A);sparse_identity_like(A,one(eltype(A)))) +Base.oneunit(A::GPUSparseMatrix)=(LinearAlgebra.checksquare(A);sparse_identity_like(A,oneunit(eltype(A)))) +Base.:(*)(A::Union{GPUSparseMatrix,GPUSparseTransposed},J::UniformScaling)=materialize(A).*J.λ +Base.:(*)(J::UniformScaling,A::Union{GPUSparseMatrix,GPUSparseTransposed})=J.λ.*materialize(A) +function scale_sparse_diagonal(A,D,side) + B=materialize(A) + side===:right ? size(B,2)==size(D,1) || throw(DimensionMismatch("diagonal dimensions differ")) : + size(D,2)==size(B,1) || throw(DimensionMismatch("diagonal dimensions differ")) + d=move_buffer(nonzeros(B),D.diag) + rows,cols=entry_coordinates(B) + values=side===:right ? nonzeros(B).*d[cols] : d[rows].*nonzeros(B) + sparse_format_type(B)(map(copy,Base.front(sparse_buffers(B)))...,values,size(B)) +end +Base.:(*)(A::Union{GPUSparseMatrix,GPUSparseTransposed},D::Diagonal)=scale_sparse_diagonal(A,D,:right) +Base.:(*)(D::Diagonal,A::Union{GPUSparseMatrix,GPUSparseTransposed})=scale_sparse_diagonal(A,D,:left) +for op in (:+,:-) + @eval begin + function Base.$op(A::Union{GPUSparseMatrix,GPUSparseTransposed},J::UniformScaling) + B=materialize(A) + sparse_map($op,B,sparse_identity_like(B,J.λ)) + end + function Base.$op(J::UniformScaling,A::Union{GPUSparseMatrix,GPUSparseTransposed}) + B=materialize(A) + sparse_map($op,sparse_identity_like(B,J.λ),B) + end + function Base.$op(A::Union{GPUSparseMatrix,GPUSparseTransposed},D::Diagonal) + B=materialize(A) + size(B)==size(D) || throw(DimensionMismatch("matrix dimensions must match")) + sparse_map($op,B,sparse_diagonal_like(B,D.diag)) + end + function Base.$op(D::Diagonal,A::Union{GPUSparseMatrix,GPUSparseTransposed}) + B=materialize(A) + size(B)==size(D) || throw(DimensionMismatch("matrix dimensions must match")) + sparse_map($op,sparse_diagonal_like(B,D.diag),B) + end + end +end + +# Explicit errors prevent SparseArrays' concatenation fallbacks from downloading through +# SparseMatrixCSC. These operations can be implemented centrally without backend changes. +const GPUSparseConcatOperand = Union{GPUSparseArray,GPUSparseTransposed} +const GPUSparseConcatInput = Union{GPUSparseConcatOperand,AnyGPUArray} +# SparseArrays uses this helper for every position in hcat/vcat/cat/block syntax. +# Reject before its conversion of the first GPU operand can download any buffers. +SparseArrays._makesparse(::GPUSparseVector) = throw(ArgumentError("GPU sparse concatenation is not implemented")) +SparseArrays._makesparse(::Union{GPUSparseMatrix,GPUSparseTransposed}) = + throw(ArgumentError("GPU sparse concatenation is not implemented")) +SparseArrays._makesparse(::Union{AnyGPUVector,AnyGPUMatrix}) = + throw(ArgumentError("concatenation involving GPU arrays and sparse operands is not implemented")) +for f in (:hcat, :vcat) + @eval begin + Base.$f(A::GPUSparseConcatOperand, Bs::GPUSparseConcatInput...) = + throw(ArgumentError($(string(f, " of GPU sparse arrays is not implemented; concatenate coordinates and assemble explicitly")))) + Base.$f(A::AnyGPUArray, B::GPUSparseConcatOperand, Cs::GPUSparseConcatInput...) = + throw(ArgumentError($(string(f, " with GPU sparse arrays is not implemented")))) + end +end +SparseArrays.blockdiag(A::GPUSparseMatrix, Bs::GPUSparseMatrix...) = + throw(ArgumentError("blockdiag of GPU sparse arrays is not implemented")) diff --git a/src/host/sparse/assembly.jl b/src/host/sparse/assembly.jl new file mode 100644 index 000000000..3f65f2ed8 --- /dev/null +++ b/src/host/sparse/assembly.jl @@ -0,0 +1,213 @@ +# assembly of sparse arrays from coordinates + +## COV_EXCL_START +unpack_sparse_key(key::Integer, nminor) = ((key - 1) ÷ nminor + 1, (key - 1) % nminor + 1) +unpack_sparse_key(key::Tuple{Integer,Integer}, nminor) = key + +@kernel function unpack_key_kernel(major, minor, keys, nminor) + k = @index(Global, Linear) + if k <= length(keys) + i, j = unpack_sparse_key(@inbounds(keys[k]), nminor) + @inbounds major[k] = i % eltype(major) + @inbounds minor[k] = j % eltype(minor) + end +end +## COV_EXCL_STOP + +# AK's stable permutation sort preserves relative operand order within each run. +stable_sortperm(keys::AbstractVector) = sortperm(keys; alg=Base.Sort.MergeSort) + +function check_coordinates(I::AbstractVector, m::Integer, name) + isempty(I) || all(i -> 1 <= i <= m, I) || + throw(ArgumentError("all $name indices must be in 1:$m")) + return +end + +""" + GPUArrays.generic_assemble(I, J, V, m, n, combine; major=:col) + +Assemble the entries `V[k]` at coordinates `(I[k], J[k])` of an `m`×`n` sparse matrix on +the device, combining duplicate values with `combine` while preserving their relative input +order. Parenthesization is unspecified: custom combiners must be associative, +although commutativity is not required. Floating-point rounding may vary. Returns the sorted, unique +major and minor indices of the entries (rows and columns for `major=:row`, columns and +rows for `major=:col`) and their values, from which any format can be built. + +Coordinates use packed Int64 keys where representable, otherwise lexicographic tuple +keys. A single stable permutation sort preserves duplicate order. +""" +function generic_assemble(I::AbstractVector, J::AbstractVector, V::AbstractVector, + m::Integer, n::Integer, combine; major::Symbol=:col) + m,n=Int(m),Int(n) + length(I) == length(J) == length(V) || + throw(ArgumentError("the coordinate and value vectors must have the same length")) + check_coordinates(I, m, "row") + check_coordinates(J, n, "column") + Ti = promote_type(eltype(I), eltype(J)) + check_sparse_dims(Ti, (Int(m), Int(n))) + majors, minors, dims = major === :col ? (J, I, (n, m)) : + major === :row ? (I, J, (m, n)) : + throw(ArgumentError("major must be :row or :col")) + + keys = packed_keys(majors, minors, dims) + perm = stable_sortperm(keys) + sorted_keys = keys[perm] + sorted_vals = V[perm] + free_buffer!(keys) + free_buffer!(perm) + unique_keys, vals = reduce_by_key(combine, sorted_keys, sorted_vals) + free_buffer!(sorted_keys) + free_buffer!(sorted_vals) + + out_major = sparse_similar(I, Ti, length(unique_keys)) + out_minor = sparse_similar(I, Ti, length(unique_keys)) + if !isempty(unique_keys) + unpack_key_kernel(get_backend(unique_keys))(out_major, out_minor, unique_keys, + Int64(dims[2]); ndrange=length(unique_keys)) + end + free_buffer!(unique_keys) + return out_major, out_minor, vals +end + +default_combine(::Type{Bool}) = | +default_combine(::Type) = + + +const AnyGPUIndices = AnyGPUVector{<:Integer} + +# `combine` can be any callable, with separate methods for functions to be more specific +# than SparseArrays' own +function assemble_matrix(I, J, V, m, n, combine, fmt::Symbol; + pointer_type::Type=promote_type(Int, eltype(I), eltype(J))) + S = sparse_format(fmt) + major, minor, vals = generic_assemble(I, J, V, m, n, combine; + major=S <: GPUSparseMatrixCSC ? :col : :row) + if S <: GPUSparseMatrixCSC + GPUSparseMatrixCSC(compress(major, n, pointer_type), minor, vals, (m, n)) + elseif S <: GPUSparseMatrixCSR + GPUSparseMatrixCSR(compress(major, m, pointer_type), minor, vals, (m, n)) + else + GPUSparseMatrixCOO(major, minor, vals, (m, n)) + end +end +for C in (:Function, :Any) + @eval begin + SparseArrays.sparse(I::AnyGPUIndices, J::AnyGPUIndices, V::AnyGPUVector, m::Integer, + n::Integer, combine::$C; fmt::Symbol=:csc) = + assemble_matrix(I, J, V, m, n, combine, fmt) + SparseArrays.sparse(I::AnyGPUIndices, J::AnyGPUIndices, v::Number, m::Integer, + n::Integer, combine::$C; fmt::Symbol=:csc) = + assemble_matrix(I, J, fill!(sparse_similar(I, typeof(v)), v), m, n, combine, fmt) + end +end + +""" + sparse(I, J, V, [m, n, combine]; fmt=:csc) + +Assemble a GPU sparse matrix from GPU vectors of coordinates `I`, `J` and values `V` (or a +single value), like `SparseArrays.sparse`: repeated coordinates are combined with +`combine` (by default `+`, or `|` for `Bool` values) in input order, and explicit zeros are +kept. `fmt` selects the format: `:csc`, `:csr` or `:coo`. +""" +SparseArrays.sparse(::AnyGPUIndices, ::AnyGPUIndices, ::AnyGPUVector, ::Integer, ::Integer, ::Any) + +# the defaults: `+`, or `|` for Bool values, and the dimensions that hold the coordinates. +# (Separate methods for Bool values keep these more specific than SparseArrays' own.) +for V in (:AnyGPUVector, :(AnyGPUVector{Bool}), :Number, :Bool) + @eval begin + SparseArrays.sparse(I::AnyGPUIndices, J::AnyGPUIndices, V::$V, m::Integer, n::Integer; + fmt::Symbol=:csc) = + sparse(I, J, V, m, n, default_combine(eltype(V)); fmt) + SparseArrays.sparse(I::AnyGPUIndices, J::AnyGPUIndices, V::$V; fmt::Symbol=:csc) = + sparse(I, J, V, sparse_extent(I), sparse_extent(J); fmt) + end +end + +# the smallest dimension that holds the given indices +sparse_extent(I::AbstractVector) = isempty(I) ? 0 : Int(maximum(I)) + +function assemble_vector(I, V, n, combine) + J = fill!(sparse_similar(I), one(eltype(I))) + nzInd, _, vals = generic_assemble(I, J, V, n, 1, combine; major=:row) + GPUSparseVector(nzInd, vals, n) +end +for C in (:Function, :Any) + @eval begin + SparseArrays.sparsevec(I::AnyGPUIndices, V::AnyGPUVector, n::Integer, combine::$C) = + assemble_vector(I, V, n, combine) + SparseArrays.sparsevec(I::AnyGPUIndices, v::Number, n::Integer, combine::$C) = + assemble_vector(I, fill!(sparse_similar(I, typeof(v)), v), n, combine) + SparseArrays.sparsevec(I::AnyGPUIndices, V::Union{AnyGPUVector,Number}, combine::$C) = + sparsevec(I, V, sparse_extent(I), combine) + end +end +""" + sparsevec(I, V, [n, combine]) + +Assemble a GPU sparse vector from GPU vectors of indices `I` and values `V` (or a single +value), like `SparseArrays.sparsevec`. +""" +SparseArrays.sparsevec(::AnyGPUIndices, ::AnyGPUVector, ::Integer, ::Any) + +for V in (:AnyGPUVector, :(AnyGPUVector{Bool}), :Number, :Bool) + @eval begin + SparseArrays.sparsevec(I::AnyGPUIndices, V::$V, n::Integer) = + sparsevec(I, V, n, default_combine(eltype(V))) + SparseArrays.sparsevec(I::AnyGPUIndices, V::$V) = sparsevec(I, V, sparse_extent(I)) + end +end + +""" + spdiagm(kv::Pair{<:Integer,<:AbstractGPUVector}...) + spdiagm(m, n, kv...) + +A GPU sparse matrix in CSC format with the vectors `kv[i].second` on the diagonals +`kv[i].first`, like `SparseArrays.spdiagm`. Values on the same diagonal are added. +""" +const GPUDiagonal = Pair{<:Integer,<:AnyGPUVector} +SparseArrays.spdiagm(kv::GPUDiagonal, kvs::GPUDiagonal...) = spdiagm_gpu(nothing, kv, kvs...) +SparseArrays.spdiagm(m::Integer, n::Integer, kv::GPUDiagonal, kvs::GPUDiagonal...) = + spdiagm_gpu((Int(m), Int(n)), kv, kvs...) +SparseArrays.spdiagm(v::AnyGPUVector) = spdiagm(0 => v) +SparseArrays.spdiagm(m::Integer, n::Integer, v::AnyGPUVector) = spdiagm(m, n, 0 => v) + +## COV_EXCL_START +@kernel function diagonal_coordinates_kernel(I, J, k) + i = @index(Global, Linear) + if i <= length(I) + @inbounds I[i] = k >= 0 ? i : i - k + @inbounds J[i] = k >= 0 ? i + k : i + end +end +## COV_EXCL_STOP + +function spdiagm_gpu(dims, kv::Pair{<:Integer,<:AnyGPUVector}...) + # the extent that holds every diagonal, as in SparseArrays + m, n = if dims === nothing + extent = maximum(((k, v),) -> Base.checked_add(length(v), Base.Checked.checked_abs(k)), kv) + (extent, extent) + else + dims + end + Tv = mapreduce(((_, v),) -> eltype(v), promote_type, kv) + proto = last(first(kv)) + total = foldl((n, pair) -> Base.checked_add(n, length(last(pair))), kv; init=0) + I = sparse_similar(proto, Int, total) + J = sparse_similar(proto, Int, total) + V = sparse_similar(proto, Tv, total) + offset = 0 + for (k, v) in kv + len = length(v) + rows = k >= 0 ? len : len - k + cols = k >= 0 ? len + k : len + (rows <= m && cols <= n) || + throw(DimensionMismatch("diagonal $k of length $len does not fit in a $m×$n matrix")) + if len > 0 + range = offset+1:offset+len + diagonal_coordinates_kernel(get_backend(proto))(view(I, range), view(J, range), k; + ndrange=len) + view(V, range) .= v + end + offset += len + end + return sparse(I, J, V, m, n, +) +end diff --git a/src/host/sparse/broadcast.jl b/src/host/sparse/broadcast.jl new file mode 100644 index 000000000..b281f59ed --- /dev/null +++ b/src/host/sparse/broadcast.jl @@ -0,0 +1,675 @@ +# broadcasting + +# broadcast container type promotion for combinations of sparse arrays and other types +struct GPUSparseVecStyle <: Broadcast.AbstractArrayStyle{1} end +struct GPUSparseMatStyle <: Broadcast.AbstractArrayStyle{2} end +Broadcast.BroadcastStyle(::Type{<:GPUSparseVector}) = GPUSparseVecStyle() +Broadcast.BroadcastStyle(::Type{<:GPUSparseMatrix}) = GPUSparseMatStyle() +const SPVM = Union{GPUSparseVecStyle,GPUSparseMatStyle} + +# Keep sparse validation in control even for unsupported output dimensions. +# Falling back to DefaultArrayStyle would permit CPU execution on borrowed host buffers. +GPUSparseVecStyle(::Val{0}) = GPUSparseVecStyle() +GPUSparseVecStyle(::Val{1}) = GPUSparseVecStyle() +GPUSparseVecStyle(::Val) = GPUSparseMatStyle() +GPUSparseMatStyle(::Val) = GPUSparseMatStyle() + +Broadcast.BroadcastStyle(::GPUSparseVecStyle, ::AbstractGPUArrayStyle{N}) where N = + GPUSparseVecStyle(Val(N)) +Broadcast.BroadcastStyle(::GPUSparseVecStyle, ::GPUSparseMatStyle) = GPUSparseMatStyle() +Broadcast.BroadcastStyle(::GPUSparseMatStyle, ::GPUSparseVecStyle) = GPUSparseMatStyle() +Broadcast.BroadcastStyle(::GPUSparseMatStyle, ::AbstractGPUArrayStyle) = GPUSparseMatStyle() + +# don't wrap sparse arrays with Extruded +Broadcast.extrude(x::GPUSparseArray) = x + +## detection of zero-preserving functions + +# modified from SparseArrays.jl + +# capturescalars takes a function (f) and a tuple of broadcast arguments, and returns a +# partially-evaluated function and a reduced argument tuple where all scalar operations have +# been applied already. +@inline function capturescalars(f, mixedargs) + let (passedsrcargstup, makeargs) = _capturescalars(mixedargs...) + parevalf = (passed...) -> f(makeargs(passed...)...) + return (parevalf, passedsrcargstup) + end +end + +## sparse broadcast style + +# Work around losing Type{T}s as DataTypes within the tuple that makeargs creates +@inline capturescalars(f, mixedargs::Tuple{Ref{Type{T}}, Vararg{Any}}) where {T} = + capturescalars((args...)->f(T, args...), Base.tail(mixedargs)) +@inline capturescalars(f, mixedargs::Tuple{Ref{Type{T}}, Ref{Type{S}}, Vararg{Any}}) where {T, S} = + # This definition is identical to the one above and necessary only for + # avoiding method ambiguity. + capturescalars((args...)->f(T, args...), Base.tail(mixedargs)) +@inline capturescalars(f, mixedargs::Tuple{GPUSparseArray, Ref{Type{T}}, Vararg{Any}}) where {T} = + capturescalars((a1, args...)->f(a1, T, args...), (mixedargs[1], Base.tail(Base.tail(mixedargs))...)) +@inline capturescalars(f, mixedargs::Tuple{Union{Ref,AbstractArray{<:Any,0}}, Ref{Type{T}}, Vararg{Any}}) where {T} = + capturescalars((args...)->f(mixedargs[1], T, args...), Base.tail(Base.tail(mixedargs))) + +scalararg(::Number) = true +scalararg(::Any) = false +scalarwrappedarg(::Union{AbstractArray{<:Any,0},Ref}) = true +scalarwrappedarg(::Any) = false + +@inline function _capturescalars() + return (), () -> () +end +@inline function _capturescalars(arg, mixedargs...) + let (rest, f) = _capturescalars(mixedargs...) + if scalararg(arg) + return rest, @inline function(tail...) + (arg, f(tail...)...) + end # add back scalararg after (in makeargs) + elseif scalarwrappedarg(arg) + return rest, @inline function(tail...) + (arg[], f(tail...)...) # TODO: This can put a Type{T} in a tuple + end # unwrap and add back scalararg after (in makeargs) + else + return (arg, rest...), @inline function(head, tail...) + (head, f(tail...)...) + end # pass-through to broadcast + end + end +end +@inline function _capturescalars(arg) # this definition is just an optimization (to bottom out the recursion slightly sooner) + if scalararg(arg) + return (), () -> (arg,) # add scalararg + elseif scalarwrappedarg(arg) + return (), () -> (arg[],) # unwrap + else + return (arg,), (head,) -> (head,) # pass-through + end +end + +@inline _iszero(x) = x == 0 +@inline _iszero(x::Number) = Base.iszero(x) +@inline _iszero(x::AbstractArray) = Base.iszero(x) +@inline _zeros_eltypes(A) = (zero(eltype(A)),) +@inline _zeros_eltypes(A, Bs...) = (zero(eltype(A)), _zeros_eltypes(Bs...)...) + +## COV_EXCL_START +## iteration helpers + +""" + CSRIterator{Ti}(row, args...) + +A GPU-compatible iterator for accessing the elements of a single row `row` of several CSR +matrices `args` in one go. The row should be in-bounds for every sparse argument. Each +iteration returns a 2-element tuple: The current column, and each arguments' pointer index +(or 0 if that input didn't have an element at that column). The pointers can then be used to +access the elements themselves. + +For convenience, this iterator can be passed non-sparse arguments as well, which will be +ignored (with the returned `col`/`ptr` values set to 0). +""" +struct CSRIterator{Ti,N,ATs} + row::Ti + col_ends::NTuple{N, Ti} + args::ATs +end + +function CSRIterator{Ti}(row, args::Vararg{Any, N}) where {Ti,N} + # check that `row` is valid for all arguments + @boundscheck begin + ntuple(Val(N)) do i + arg = @inbounds args[i] + arg isa GPUSparseMatrixCSR && checkbounds(axes(arg, 1), row) + end + end + + col_ends = ntuple(Val(N)) do i + arg = @inbounds args[i] + if arg isa GPUSparseMatrixCSR + @inbounds(arg.rowPtr[row+1]) + else + zero(Ti) + end + end + + CSRIterator{Ti, N, typeof(args)}(row, col_ends, args) +end + +@inline function Base.iterate(iter::CSRIterator{Ti,N}, state=nothing) where {Ti,N} + # helper function to get the column of a sparse array at a specific pointer + @inline function get_col(i, ptr) + arg = @inbounds iter.args[i] + if arg isa GPUSparseMatrixCSR + col_end = @inbounds iter.col_ends[i] + if ptr < col_end + return @inbounds arg.colVal[ptr] % Ti + end + end + zero(Ti) + end + + # initialize the state + # - ptr: the current index into the colVal/nzVal arrays + # - col: the current column index (cached so that we don't have to re-read each time) + state = something(state, + ntuple(Val(N)) do i + arg = @inbounds iter.args[i] + if arg isa GPUSparseMatrixCSR + ptr = @inbounds iter.args[i].rowPtr[iter.row] % Ti + col = @inbounds get_col(i, ptr) + else + ptr = zero(Ti) + col = zero(Ti) + end + (; ptr, col) + end + ) + + # determine the column we're currently processing + cols = ntuple(i -> @inbounds(state[i].col), Val(N)) + cur_col = reduce((a, b) -> iszero(a) ? b : iszero(b) ? a : min(a, b), cols) + iszero(cur_col) && return + + # fetch the pointers (we don't look up the values, as the caller might want to index + # the sparse array directly, e.g., to mutate it). we don't return `ptrs` from the state + # directly, but first convert the `typemax(Ti)` to a more convenient zero value. + # NOTE: these values may end up unused by the caller (e.g. in the count_nnzs kernels), + # but LLVM appears smart enough to filter them away. + ptrs = ntuple(Val(N)) do i + ptr, col = @inbounds state[i] + col == cur_col ? ptr : zero(Ti) + end + + # advance the state + new_state = ntuple(Val(N)) do i + ptr, col = @inbounds state[i] + if col == cur_col + ptr += one(Ti) + col = get_col(i, ptr) + end + (; ptr, col) + end + + return (cur_col, ptrs), new_state +end + +struct CSCIterator{Ti,N,ATs} + col::Ti + row_ends::NTuple{N, Ti} + args::ATs +end + +function CSCIterator{Ti}(col, args::Vararg{Any, N}) where {Ti,N} + # check that `col` is valid for all arguments + @boundscheck begin + ntuple(Val(N)) do i + arg = @inbounds args[i] + arg isa GPUSparseMatrixCSC && checkbounds(axes(arg, 2), col) + end + end + + row_ends = ntuple(Val(N)) do i + arg = @inbounds args[i] + x = if arg isa GPUSparseMatrixCSC + @inbounds(arg.colPtr[col+1]) + else + zero(Ti) + end + x + end + + CSCIterator{Ti, N, typeof(args)}(col, row_ends, args) +end + +@inline function Base.iterate(iter::CSCIterator{Ti,N}, state=nothing) where {Ti,N} + # helper function to get the column of a sparse array at a specific pointer + @inline function get_col(i, ptr) + arg = @inbounds iter.args[i] + if arg isa GPUSparseMatrixCSC + col_end = @inbounds iter.row_ends[i] + if ptr < col_end + return @inbounds arg.rowVal[ptr] % Ti + end + end + zero(Ti) + end + + # initialize the state + # - ptr: the current index into the rowVal/nzVal arrays + # - row: the current row index (cached so that we don't have to re-read each time) + state = something(state, + ntuple(Val(N)) do i + arg = @inbounds iter.args[i] + if arg isa GPUSparseMatrixCSC + ptr = @inbounds iter.args[i].colPtr[iter.col] % Ti + row = @inbounds get_col(i, ptr) + else + ptr = zero(Ti) + row = zero(Ti) + end + (; ptr, row) + end + ) + + # determine the row we're currently processing + rows = ntuple(i -> @inbounds(state[i].row), Val(N)) + cur_row = reduce((a, b) -> iszero(a) ? b : iszero(b) ? a : min(a, b), rows) + iszero(cur_row) && return + + # fetch the pointers (we don't look up the values, as the caller might want to index + # the sparse array directly, e.g., to mutate it). we don't return `ptrs` from the state + # directly, but first convert the `typemax(Ti)` to a more convenient zero value. + # NOTE: these values may end up unused by the caller (e.g. in the count_nnzs kernels), + # but LLVM appears smart enough to filter them away. + ptrs = ntuple(Val(N)) do i + ptr, row = @inbounds state[i] + row == cur_row ? ptr : zero(Ti) + end + + # advance the state + new_state = ntuple(Val(N)) do i + ptr, row = @inbounds state[i] + if row == cur_row + ptr += one(Ti) + row = get_col(i, ptr) + end + (; ptr, row) + end + + return (cur_row, ptrs), new_state +end + +# helpers to index a sparse or dense array +function _getindex(arg::Union{GPUSparseMatrixCSR,GPUSparseMatrixCSC,GPUSparseVector}, I, ptr) + if ptr == 0 + zero(eltype(arg)) + else + @inbounds arg.nzVal[ptr] + end +end +@inline function _getindex(arg::DenseArray{Tv}, I, ptr)::Tv where {Tv} + return @inbounds Broadcast._broadcast_getindex(arg, I)::Tv +end +@inline _getindex(arg, I, ptr) = Broadcast._broadcast_getindex(arg, I) + +## sparse broadcast implementation +iter_type(::Type{<:GPUSparseMatrixCSC}, ::Type{Ti}) where {Ti} = CSCIterator{Ti} +iter_type(::Type{<:GPUSparseMatrixCSR}, ::Type{Ti}) where {Ti} = CSRIterator{Ti} + +sparse_iterator(::Val{:csr}, row, args...) = CSRIterator{Int}(row, args...) +sparse_iterator(::Val{:csc}, col, args...) = CSCIterator{Int}(col, args...) + +_has_row(A, offsets, row, fpreszeros::Bool) = fpreszeros ? 0 : row +_has_row(A::AbstractDeviceArray, offsets, row, ::Bool) = row +# the position of `row` among the stored indices of a sparse vector, or 0 if it isn't stored +function _has_row(A::GPUSparseVector, offsets, row, ::Bool) + ptr = searchsortedfirst(A.nzInd, row) + return (ptr <= length(A.nzInd) && @inbounds(A.nzInd[ptr]) == row) ? ptr : 0 +end + +# for every row, determine which arguments have an entry there (see `_has_row`), and whether +# the output has one (`key` is the row, or `typemax(Ti)` if not) +@kernel function compute_offsets_kernel(::Type{<:GPUSparseVector}, fpreszeros::Bool, + offsets::AbstractVector{Pair{Ti, NTuple{N, Ti}}}, + args...) where {Ti, N} + row = @index(Global, Linear) + if row ≤ length(offsets) + arg_row_is_nnz = ntuple(Val(N)) do i + arg = @inbounds args[i] + _has_row(arg, offsets, row, fpreszeros) % Ti + end + row_is_nnz = false + for i in 1:N + row_is_nnz |= @inbounds(arg_row_is_nnz[i]) != 0 + end + key = row_is_nnz ? row % Ti : zero(Ti) + @inbounds offsets[row] = key => arg_row_is_nnz + end +end + +# kernel to count the number of non-zeros in a row, to determine the row offsets +@kernel function compute_offsets_kernel(layout::Val, + offsets::AbstractVector{Ti}, args...) where Ti + # every thread processes an entire row + leading_dim = @index(Global, Linear) + if leading_dim ≤ length(offsets)-1 + iter = @inbounds sparse_iterator(layout, leading_dim, args...) + + # count the nonzero leading_dims of all inputs + accum = zero(Ti) + for (leading_dim, vals) in iter + accum += one(Ti) + end + + # the counts are shifted by one (`offsets[1]` is set to 1 by the caller) so that + # after accumulation they can be used as the rowPtr/colPtr array of a CSR/CSC matrix + # Keep this store bounds checked: Metal miscompiles the count-only iterator + # loop with narrow input pointers and wide counts without this branch. + offsets[leading_dim+1] = accum + end +end + +# `positions` is the inclusive prefix count of output entries, so an output entry for `row` +# goes to `positions[row]`. +@kernel function sparse_to_sparse_broadcast_kernel(f::F, output::GPUSparseVector{Tv,Ti}, + offsets::AbstractVector{Pair{Tp, NTuple{N, Tp}}}, + positions::AbstractVector{Int}, + args...) where {Tv, Ti, Tp, N, F} + row = @index(Global, Linear) + if row ≤ length(offsets) + key, arg_ptrs = @inbounds offsets[row] + if key != 0 + vals = ntuple(Val(N)) do i + @inline + arg = @inbounds args[i] + # ptr is 0 if the sparse vector doesn't have an element at this row + # ptr is 0 if the arg is a scalar AND f preserves zeros + ptr = @inbounds arg_ptrs[i] + _getindex(arg, row, ptr) + end + output_ix = @inbounds positions[row] + @inbounds output.nzInd[output_ix] = row + @inbounds output.nzVal[output_ix] = f(vals...) + end + end +end + +@kernel function sparse_to_sparse_broadcast_kernel(f, output::T, offsets::Union{<:AbstractArray,Nothing}, + args...) where {Ti, T<:Union{GPUSparseMatrixCSR{<:Any,Ti}, + GPUSparseMatrixCSC{<:Any,Ti}}} + # every thread processes an entire row + leading_dim = @index(Global, Linear) + leading_dim_size = output isa GPUSparseMatrixCSR ? size(output, 1) : size(output, 2) + if leading_dim ≤ leading_dim_size + iter = @inbounds iter_type(T, Int)(leading_dim, args...) + + output_ptrs = output isa GPUSparseMatrixCSR ? output.rowPtr : output.colPtr + output_ivals = output isa GPUSparseMatrixCSR ? output.colVal : output.rowVal + # fetch the row offset, and write it to the output + @inbounds begin + output_ptr = output_ptrs[leading_dim] = offsets[leading_dim] + if leading_dim == leading_dim_size + output_ptrs[leading_dim+one(eltype(leading_dim))] = offsets[leading_dim+one(eltype(leading_dim))] + end + end + + # set the values for this row + for (sub_leading_dim, ptrs) in iter + index_first = output isa GPUSparseMatrixCSR ? leading_dim : sub_leading_dim + index_second = output isa GPUSparseMatrixCSR ? sub_leading_dim : leading_dim + I = CartesianIndex(index_first, index_second) + vals = ntuple(Val(length(args))) do i + arg = @inbounds args[i] + ptr = @inbounds ptrs[i] + _getindex(arg, I, ptr) + end + @inbounds output_ivals[output_ptr] = sub_leading_dim + @inbounds output.nzVal[output_ptr] = f(vals...) + output_ptr += one(Ti) + end + end +end +@kernel function sparse_to_dense_broadcast_kernel(T::Type{<:Union{GPUSparseMatrixCSR{Tv, Ti}, + GPUSparseMatrixCSC{Tv, Ti}}}, + f, output::AbstractArray, args...) where {Tv, Ti} + # every thread processes an entire row + leading_dim = @index(Global, Linear) + leading_dim_size = T <: GPUSparseMatrixCSR ? size(output, 1) : size(output, 2) + if leading_dim ≤ leading_dim_size + iter = @inbounds iter_type(T, Int)(leading_dim, args...) + + # set the values for this row + for (sub_leading_dim, ptrs) in iter + index_first = T <: GPUSparseMatrixCSR ? leading_dim : sub_leading_dim + index_second = T <: GPUSparseMatrixCSR ? sub_leading_dim : leading_dim + I = CartesianIndex(index_first, index_second) + vals = ntuple(Val(length(args))) do i + arg = @inbounds args[i] + ptr = @inbounds ptrs[i] + _getindex(arg, I, ptr) + end + + @inbounds output[I] = f(vals...) + end + end +end + +@kernel function sparse_to_dense_broadcast_kernel(::Type{<:GPUSparseVector}, f::F, + output::AbstractArray{Tv}, + offsets::AbstractVector{Pair{Ti, NTuple{N, Ti}}}, + args...) where {Tv, F, N, Ti} + row = @index(Global, Linear) + if row ≤ length(output) + arg_ptrs = @inbounds offsets[row][2] + vals = ntuple(Val(length(args))) do i + @inline + arg = @inbounds args[i] + # ptr is 0 if the sparse vector doesn't have an element at this row + # ptr is row if the arg is dense OR a scalar with non-zero-preserving f + # ptr is 0 if the arg is a scalar AND f preserves zeros + ptr = @inbounds arg_ptrs[i] + _getindex(arg, row, ptr) + end + @inbounds output[row] = f(vals...) + end +end +## COV_EXCL_STOP + +# the CSR layout of a COO matrix, borrowing its buffers +csr_view(A::GPUSparseMatrixCOO) = + GPUSparseMatrixCSR(compress(A.rowInd, size(A, 1)), A.colInd, A.nzVal, size(A)) + +function Broadcast.copy(bc::Broadcasted{<:Union{GPUSparseVecStyle,GPUSparseMatStyle}}) + bc = Broadcast.flatten(bc) + sparse_inputs = filter(arg -> arg isa AbstractGPUSparseArray, bc.args) + all(arg -> arg isa GPUSparseArray, sparse_inputs) || + error("broadcast is only implemented for the sparse formats of GPUArrays") + (all(arg -> arg isa GPUSparseVector, sparse_inputs) || + all(arg -> arg isa GPUSparseMatrix, sparse_inputs)) || + error("broadcast with both sparse vectors and sparse matrices is not supported") + all(arg -> axes(arg) == axes(bc), sparse_inputs) || + throw(ArgumentError("broadcast expansion of sparse operands is not supported")) + # capturescalars evaluates wrapped scalars on the host to test zero preservation. + # GPU scalar arrays need device-side evaluation, which is not implemented here. + any(arg -> arg isa AnyGPUArray{<:Any,0}, bc.args) && + throw(ArgumentError("sparse broadcast with zero-dimensional GPU arrays is not supported")) + + # The kernels iterate a single compressed format: that of the first sparse matrix, with + # COO matrices read as CSR (which has the same order of entries). Other formats are + # converted, and a COO result is converted back. + lead = first(sparse_inputs) + if lead isa GPUSparseMatrix && !all(arg -> typeof(arg).name === typeof(lead).name && + !(arg isa GPUSparseMatrixCOO), sparse_inputs) + S = lead isa GPUSparseMatrixCSC ? GPUSparseMatrixCSC : GPUSparseMatrixCSR + args = map(bc.args) do arg + arg isa GPUSparseMatrixCOO && S === GPUSparseMatrixCSR ? csr_view(arg) : + arg isa GPUSparseMatrix ? convert(S, arg) : arg + end + out = copy(Broadcasted(bc.f, args, bc.axes)) + if lead isa GPUSparseMatrixCOO && out isa GPUSparseMatrixCSR + out = GPUSparseMatrixCOO(expand_ptr(out.rowPtr, nnz(out), indtype(out)), out.colVal, out.nzVal, + size(out)) + end + if out isa GPUSparseMatrixCompressed + prototypes = filter(A -> A isa GPUSparseMatrixCompressed, sparse_inputs) + Tp = isempty(prototypes) ? promote_type(Int, indtype(out)) : + mapreduce(A -> eltype(compressed_ptr(A)), promote_type, prototypes) + check_sparse_nnz(Tp, nnz(out)) + out = sparse_format_type(out)(convert_buffer(Tp, compressed_ptr(out)), + compressed_ind(out), out.nzVal, size(out)) + end + return out + end + + sparse_args = findall(bc.args) do arg + arg isa GPUSparseArray + end + sparse_typ = typeof(bc.args[first(sparse_args)]) + Ti = reduce(promote_type, map(i -> indtype(bc.args[i]), sparse_args)) + Tp = sparse_typ <: GPUSparseVector ? Int : + reduce(promote_type, map(i -> eltype(compressed_ptr(bc.args[i])), sparse_args)) + # A union cannot contain more entries than its inputs. Check the host-count bound + # before computing Int prefix sums; narrow the actual count only after the scan. + foldl(Base.checked_add, map(i -> nnz(bc.args[i]), sparse_args); init=1) + + # determine the output type + Tv = Broadcast.combine_eltypes(bc.f, eltype.(bc.args)) + if !Base.isconcretetype(Tv) + error("""GPU sparse broadcast resulted in non-concrete element type $Tv. + This probably means that the function you are broadcasting contains an error or type instability.""") + end + + # partially-evaluate the function, removing scalars. + parevalf, passedsrcargstup = capturescalars(bc.f, bc.args) + # check if the partially-evaluated function preserves zeros. if so, we'll only need to + # apply it to the sparse input arguments, preserving the sparse structure. + if all(arg->isa(arg, AbstractSparseArray), passedsrcargstup) + fofzeros = parevalf(_zeros_eltypes(passedsrcargstup...)...) + fpreszeros = _iszero(fofzeros) + else + fpreszeros = false + end + + # the kernels below parallelize across rows or cols, not elements, so it's unlikely + # we'll launch many threads. to maximize utilization, parallelize across blocks first. + rows, cols = get(size(bc), 1, 1), get(size(bc), 2, 1) + # `size(bc, ::Int)` is missing + offsets = nothing + positions = nothing + # allocate the output container + sparse_arg = bc.args[first(sparse_args)] + if !fpreszeros && sparse_typ <: Union{GPUSparseMatrixCSR, GPUSparseMatrixCSC} + # either we have dense inputs, or the function isn't preserving zeros, + # so use a dense output to broadcast into. + val_array = nonzeros(sparse_arg) + output = sparse_similar(val_array, Tv, size(bc)) + # since we'll be iterating the sparse inputs, we need to pre-fill the dense output + # with appropriate values (while setting the sparse inputs to zero). we do this by + # re-using the dense broadcast implementation. + nonsparse_args = map(bc.args) do arg + # NOTE: this assumes the broadcast is flattened, but not yet preprocessed + if arg isa GPUSparseArray + zero(eltype(arg)) + else + arg + end + end + broadcast!(bc.f, output, nonsparse_args...) + elseif length(sparse_args) == 1 && sparse_typ <: Union{GPUSparseMatrixCSR, GPUSparseMatrixCSC} + # we only have a single sparse input, so we can reuse its structure for the output. + # this avoids a kernel launch and costly synchronization. + if sparse_typ <: GPUSparseMatrixCSR + offsets = rowPtr = copy(sparse_arg.rowPtr) + colVal = sparse_similar(sparse_arg.colVal) + nzVal = sparse_similar(sparse_arg.nzVal, Tv) + output = GPUSparseMatrixCSR(rowPtr, colVal, nzVal, size(bc)) + elseif sparse_typ <: GPUSparseMatrixCSC + offsets = colPtr = copy(sparse_arg.colPtr) + rowVal = sparse_similar(sparse_arg.rowVal) + nzVal = sparse_similar(sparse_arg.nzVal, Tv) + output = GPUSparseMatrixCSC(colPtr, rowVal, nzVal, size(bc)) + end + else + # determine the number of non-zero elements per row so that we can create an + # appropriately-structured output container + offsets = if sparse_typ <: GPUSparseMatrixCSR + ptr_array = sparse_arg.rowPtr + fill!(sparse_similar(ptr_array, Int, Base.checked_add(rows, 1)), 1) + elseif sparse_typ <: GPUSparseMatrixCSC + ptr_array = sparse_arg.colPtr + fill!(sparse_similar(ptr_array, Int, Base.checked_add(cols, 1)), 1) + elseif sparse_typ <: GPUSparseVector + ptr_array = sparse_arg.nzInd + sparse_similar(ptr_array, Pair{Int, NTuple{length(bc.args), Int}}, rows) + end + let + args = if sparse_typ <: GPUSparseVector + (sparse_typ, fpreszeros, offsets, bc.args...) + else + (sparse_typ <: GPUSparseMatrixCSR ? Val(:csr) : Val(:csc), offsets, bc.args...) + end + kernel = compute_offsets_kernel(get_backend(bc.args[first(sparse_args)])) + # an empty launch fails on Metal (JuliaGPU/Metal.jl#980) + isempty(offsets) || kernel(args...; ndrange=length(offsets)) + end + # accumulate these values so that we can use them directly as row pointer offsets, + # as well as to get the total nnz count to allocate the sparse output array. + # cusparseXcsrgeam2Nnz computes this in one go, but it doesn't seem worth the effort + if !(sparse_typ <: GPUSparseVector) + accumulate!(Base.add_sum, offsets, offsets) + total_nnz = @allowscalar offsets[end] - 1 + elseif fpreszeros + # number the rows that have an output entry, which gives each its output position + positions = sparse_similar(offsets, Int) + positions .= first.(offsets) .!= 0 + rows == 0 || accumulate!(Base.add_sum, positions, positions) + total_nnz = rows == 0 ? 0 : Int(@allowscalar positions[end]) + end + output = if sparse_typ <: Union{GPUSparseMatrixCSR,GPUSparseMatrixCSC} + check_sparse_nnz(Tp, total_nnz) + offsets = convert_buffer(Tp, offsets) + ixVal = sparse_similar(offsets, Ti, total_nnz) + nzVal = sparse_similar(offsets, Tv, total_nnz) + sparse_format_type(sparse_arg)(offsets, ixVal, nzVal, size(bc)) + elseif sparse_typ <: GPUSparseVector && !fpreszeros + val_array = bc.args[first(sparse_args)].nzVal + sparse_similar(val_array, Tv, size(bc)) + elseif sparse_typ <: GPUSparseVector && fpreszeros + nzInd = sparse_similar(offsets, Ti, total_nnz) + nzVal = sparse_similar(offsets, Tv, total_nnz) + GPUSparseVector(nzInd, nzVal, rows) + end + if sparse_typ <: GPUSparseVector && !fpreszeros + nonsparse_args = map(bc.args) do arg + # NOTE: this assumes the broadcst is flattened, but not yet preprocessed + if arg isa GPUSparseArray + zero(eltype(arg)) + else + arg + end + end + broadcast!(bc.f, output, nonsparse_args...) + end + end + # perform the actual broadcast + if output isa GPUSparseArray + args = sparse_typ <: GPUSparseVector ? (bc.f, output, offsets, positions, bc.args...) : + (bc.f, output, offsets, bc.args...) + kernel = sparse_to_sparse_broadcast_kernel(get_backend(bc.args[first(sparse_args)])) + ndrange = if sparse_typ <: GPUSparseVector + rows + elseif sparse_typ <: GPUSparseMatrixCSC + size(output, 2) + else + size(output, 1) + end + else + args = sparse_typ <: GPUSparseVector ? (sparse_typ, bc.f, output, offsets, bc.args...) : + (sparse_typ, bc.f, output, bc.args...) + kernel = sparse_to_dense_broadcast_kernel(get_backend(bc.args[first(sparse_args)])) + ndrange = sparse_typ <: GPUSparseMatrixCSC ? size(output, 2) : size(output, 1) + end + # an empty launch fails on Metal (JuliaGPU/Metal.jl#980) + ndrange == 0 || kernel(args...; ndrange) + return output +end + +# like SparseArrays, `map` over sparse arrays of the same size is a broadcast +function sparse_map(f, A, Bs...) + all(B -> size(B) == size(A), Bs) || + throw(DimensionMismatch("map requires arrays of the same size")) + return broadcast(f, A, Bs...) +end +Base.map(f, A::GPUSparseArray, Bs::Union{GPUSparseArray,AnyGPUArray}...) = sparse_map(f, A, Bs...) +Base.map(f, A::AnyGPUArray, B::GPUSparseArray, Cs::Union{GPUSparseArray,AnyGPUArray}...) = + sparse_map(f, A, B, Cs...) + +# SparseArrays implements these for any sparse vector, through scalar indexing +for f in (:+, :-, :*, :min, :max) + @eval begin + Base.map(::typeof($f), x::GPUSparseVector, y::GPUSparseVector) = sparse_map($f, x, y) + Base.broadcast(::typeof($f), x::GPUSparseVector, y::GPUSparseVector) = + Broadcast.materialize(Broadcast.broadcasted($f, x, y)) + end +end +Base.:(+)(x::GPUSparseVector, y::GPUSparseVector) = sparse_map(+, x, y) +Base.:(-)(x::GPUSparseVector, y::GPUSparseVector) = sparse_map(-, x, y) diff --git a/src/host/sparse/conversions.jl b/src/host/sparse/conversions.jl new file mode 100644 index 000000000..e70ffa312 --- /dev/null +++ b/src/host/sparse/conversions.jl @@ -0,0 +1,494 @@ +# conversions between sparse formats, and to and from the host + +## to the host + +SparseArrays.SparseMatrixCSC(A::GPUSparseMatrixCSC) = + sparse_csc(Array(A.colPtr), Array(A.rowVal), Array(A.nzVal), size(A)) +# the CSR buffers are those of the transpose in CSC +SparseArrays.SparseMatrixCSC(A::GPUSparseMatrixCSR) = + copy(transpose(sparse_csc(Array(A.rowPtr), Array(A.colVal), Array(A.nzVal), reverse(size(A))))) +function SparseArrays.SparseMatrixCSC(A::GPUSparseMatrixCOO) + rowInd = Array(A.rowInd) + rowPtr = zeros(promote_type(Int, eltype(rowInd)), Base.checked_add(size(A, 1), 1)) + rowPtr[1] = 1 + for i in rowInd + rowPtr[i + 1] += one(eltype(rowPtr)) + end + cumsum!(rowPtr, rowPtr) + copy(transpose(SparseMatrixCSC(reverse(size(A))..., rowPtr, Vector{eltype(rowPtr)}(Array(A.colInd)), + Array(A.nzVal)))) +end +SparseArrays.SparseMatrixCSC(x::GPUSparseVector) = SparseMatrixCSC(SparseVector(x)) +SparseArrays.SparseMatrixCSC{Tv}(A::GPUSparseArray) where {Tv} = + SparseMatrixCSC{Tv}(SparseMatrixCSC(A)) +SparseArrays.SparseMatrixCSC{Tv,Ti}(A::GPUSparseArray) where {Tv,Ti} = + SparseMatrixCSC{Tv,Ti}(SparseMatrixCSC(A)) + +SparseArrays.SparseVector(x::GPUSparseVector) = + SparseVector(length(x), Array(x.nzInd), Array(x.nzVal)) +SparseArrays.SparseVector{Tv}(x::GPUSparseVector) where {Tv} = SparseVector{Tv}(SparseVector(x)) +SparseArrays.SparseVector{Tv,Ti}(x::GPUSparseVector) where {Tv,Ti} = + SparseVector{Tv,Ti}(SparseVector(x)) + +host_sparse(A::GPUSparseMatrix) = SparseMatrixCSC(A) +host_sparse(x::GPUSparseVector) = SparseVector(x) + +Base.Array(A::GPUSparseArray) = Array(host_sparse(A)) +Base.Array{T}(A::GPUSparseArray) where {T} = Array{T}(host_sparse(A)) +Base.Array{T,1}(x::GPUSparseVector) where {T} = Array{T,1}(host_sparse(x)) +Base.Array{T,2}(A::GPUSparseMatrix) where {T} = Array{T,2}(host_sparse(A)) +Base.collect(A::GPUSparseArray) = Array(A) +# (SparseArrays has a scalar loop for any sparse vector) +Base.Vector(x::GPUSparseVector) = Vector(SparseVector(x)) + + +## `adapt` + +# Kernel arguments and other adaptors: convert every buffer. With device vectors, the +# result is the isbits representation that kernels receive. Host buffers give SparseArrays' +# own types where there is one. +Adapt.adapt_structure(to, A::GPUSparseMatrixCSR) = + GPUSparseMatrixCSR(adapt(to, A.rowPtr), adapt(to, A.colVal), adapt(to, A.nzVal), size(A)) +Adapt.adapt_structure(to, A::GPUSparseMatrixCSC) = + sparse_csc(adapt(to, A.colPtr), adapt(to, A.rowVal), adapt(to, A.nzVal), size(A)) +Adapt.adapt_structure(to, A::GPUSparseMatrixCOO) = + GPUSparseMatrixCOO(adapt(to, A.rowInd), adapt(to, A.colInd), adapt(to, A.nzVal), size(A)) +Adapt.adapt_structure(to, x::GPUSparseVector) = + sparse_vector(adapt(to, x.nzInd), adapt(to, x.nzVal), length(x)) + +function sparse_csc(colPtr::Vector{Tp}, rowVal::Vector{Ti}, nzVal::Vector, dims) where {Tp,Ti} + T = promote_type(Tp, Ti) + SparseMatrixCSC(dims..., convert(Vector{T}, colPtr), convert(Vector{T}, rowVal), nzVal) +end +sparse_csc(colPtr, rowVal, nzVal, dims) = GPUSparseMatrixCSC(colPtr, rowVal, nzVal, dims) +sparse_vector(nzInd::Vector, nzVal::Vector, len) = SparseVector(len, nzInd, nzVal) +sparse_vector(nzInd, nzVal, len) = GPUSparseVector(nzInd, nzVal, len) + +# Storage types (`adapt(Array, A)`, `adapt(MtlArray, S)`): move the buffers, keeping the +# format, element and index types. The index buffers are allocated like the adapted +# values, so that a target with an element type (`Array{Float32}`, or the +# `CuArray{Float32,1,M}` that `cu` adapts to) only converts the values. +move_buffer(proto::AbstractVector, src::AbstractVector) = + copyto!(sparse_similar(proto, eltype(src), length(src)), src) + +function adapt_sparse_storage(to, A::GPUSparseMatrixCSR) + nzVal = adapt(to, A.nzVal) + nzVal === A.nzVal && return A + GPUSparseMatrixCSR(move_buffer(nzVal, A.rowPtr), move_buffer(nzVal, A.colVal), nzVal, + size(A)) +end +function adapt_sparse_storage(to, A::GPUSparseMatrixCSC) + nzVal = adapt(to, A.nzVal) + colPtr, rowVal = nzVal === A.nzVal ? (A.colPtr, A.rowVal) : + (move_buffer(nzVal, A.colPtr), move_buffer(nzVal, A.rowVal)) + sparse_csc(colPtr, rowVal, nzVal, size(A)) +end +function adapt_sparse_storage(to, A::GPUSparseMatrixCOO) + nzVal = adapt(to, A.nzVal) + nzVal === A.nzVal && return A + GPUSparseMatrixCOO(move_buffer(nzVal, A.rowInd), move_buffer(nzVal, A.colInd), nzVal, + size(A)) +end +function adapt_sparse_storage(to, x::GPUSparseVector) + nzVal = adapt(to, x.nzVal) + nzInd = nzVal === x.nzVal ? x.nzInd : move_buffer(nzVal, x.nzInd) + sparse_vector(nzInd, nzVal, length(x)) +end + +# host sparse arrays to GPU storage, likewise keeping format and types: a CSC matrix stays +# CSC. Adapting never densifies. +function adapt_sparse_storage(to, S::SparseMatrixCSC) + nzVal = adapt(to, nonzeros(S)) + GPUSparseMatrixCSC(move_buffer(nzVal, getcolptr(S)), move_buffer(nzVal, rowvals(S)), + nzVal, size(S)) +end +function adapt_sparse_storage(to, x::SparseVector) + nzVal = adapt(to, nonzeros(x)) + GPUSparseVector(move_buffer(nzVal, SparseArrays.nonzeroinds(x)), nzVal, length(x)) +end + +for S in (:GPUSparseMatrixCSR,:GPUSparseMatrixCSC,:GPUSparseMatrixCOO,:GPUSparseVector) + @eval Adapt.adapt_structure(to::Type{<:DenseArray},A::$S)=adapt_sparse_storage(to,A) +end +for S in (:SparseMatrixCSC,:SparseVector) + @eval Adapt.adapt_structure(to::Type{<:AbstractGPUArray},A::$S)=adapt_sparse_storage(to,A) +end + +# explicit sparse types (`adapt(MtlSparseMatrixCSR{Float32,Int32}, S)`) are constructors +for S in (:SparseMatrixCSC, :SparseVector, :GPUSparseMatrixCSR, :GPUSparseMatrixCSC, + :GPUSparseMatrixCOO, :GPUSparseVector) + @eval Adapt.adapt_structure(::Type{T}, A::$S) where {T<:AbstractGPUSparseArray} = T(A) +end + + +## between formats + +# the generic types do not determine a storage, so they cannot be constructed from host +# arrays: that needs a back-end alias, or `adapt` to a storage type +for S in (:GPUSparseMatrixCSR, :GPUSparseMatrixCSC, :GPUSparseMatrixCOO, :GPUSparseVector) + @eval function $S(A::Union{SparseMatrixCSC,SparseVector}) + throw(ArgumentError("""$($S)(::$(nameof(typeof(A)))) does not know where to store the result. + Use a back-end alias (e.g. `MtlSparseMatrixCSR(S)`), `adapt(MtlArray, S)` to keep the format, or `mtl(S)`/`cu(S)`.""")) + end +end + +# the same format: a copy +GPUSparseMatrixCSR(A::GPUSparseMatrixCSR) = copy(A) +GPUSparseMatrixCSC(A::GPUSparseMatrixCSC) = copy(A) +GPUSparseMatrixCOO(A::GPUSparseMatrixCOO) = copy(A) +GPUSparseVector(x::GPUSparseVector) = copy(x) + +function GPUSparseVector(A::GPUSparseMatrix) + size(A, 2) == 1 || throw(ArgumentError("sparse vector construction requires a single-column matrix")) + return vec(A) +end +function GPUSparseMatrixCSC(x::GPUSparseVector) + Tp = promote_type(Int, indtype(x)) + ptr = copyto!(sparse_similar(x.nzInd, Tp, 2), Tp[1, Base.checked_add(nnz(x), 1)]) + GPUSparseMatrixCSC(ptr, copy(x.nzInd), copy(x.nzVal), (length(x), 1)) +end +GPUSparseMatrixCSR(x::GPUSparseVector) = generic_regroup(GPUSparseMatrixCSC(x)) +GPUSparseMatrixCOO(x::GPUSparseVector) = generic_expand(GPUSparseMatrixCSR(x)) + +# Typed forms preflight the requested widths before format conversion allocates. +for S in (:GPUSparseMatrixCSR, :GPUSparseMatrixCSC, :GPUSparseMatrixCOO, :GPUSparseVector) + @eval begin + $S{Tv}(A::AbstractGPUSparseArray) where {Tv} = + (check_sparse_values(Tv,nonzeros(A));with_values($S(A),Tv)) + function $S{Tv,Ti}(A::AbstractGPUSparseArray) where {Tv,Ti} + check_sparse_dims(Ti,size(A)) + $S <: GPUSparseMatrixCompressed && check_sparse_nnz(Ti,nnz(A)) + check_sparse_values(Tv,nonzeros(A)) + with_eltypes($S(A),Tv,Ti) + end + $S{Tv,Ti}(A::$S) where {Tv,Ti} = with_eltypes(A, Tv, Ti; copy=true) + end +end + +# `A` with the given element and index types, reusing its buffers where they already match +# unless `copy` is set +function with_eltypes(A::GPUSparseArray, ::Type{Tv}, ::Type{Ti}; copy::Bool=false) where {Tv,Ti} + check_sparse_values(Tv, nonzeros(A)) + Tv == eltype(A) && Ti == indtype(A) && + (!(A isa GPUSparseMatrixCompressed) || Ti == eltype(compressed_ptr(A))) && + return copy ? Base.copy(A) : A + check_sparse_dims(Ti, size(A)) + A isa GPUSparseMatrixCompressed && check_sparse_nnz(Ti, nnz(A)) + buffers = map(fieldnames(typeof(A))) do field + buf = getfield(A, field) + if field === :nzVal + convert_buffer(Tv, buf) + elseif buf isa AbstractVector + convert_index_buffer(Ti, buf) + else + buf + end + end + return sparse_format_type(A)(buffers...) +end +function with_values(A::GPUSparseArray, ::Type{Tv}) where {Tv} + check_sparse_values(Tv, nonzeros(A)) + B = similar(A, Tv) + copyto!(nonzeros(B), nonzeros(A)) + return B +end +sparse_format_type(::GPUSparseMatrixCSR) = GPUSparseMatrixCSR +sparse_format_type(::GPUSparseMatrixCSC) = GPUSparseMatrixCSC +sparse_format_type(::GPUSparseMatrixCOO) = GPUSparseMatrixCOO +sparse_format_type(::GPUSparseVector) = GPUSparseVector + +""" + GPUArrays.generic_regroup(A::GPUSparseMatrixCSR)::GPUSparseMatrixCSC + GPUArrays.generic_regroup(A::GPUSparseMatrixCSC)::GPUSparseMatrixCSR + +Convert between CSR and CSC by regrouping the stored entries along the other dimension, on +the device. This is the generic implementation behind `GPUSparseMatrixCSC(::GPUSparseMatrixCSR)` +and vice versa, which back-ends can call when a vendor routine does not apply. +""" +function generic_regroup(A::GPUSparseMatrixCSR) + colPtr, rowVal, nzVal = regroup(A.rowPtr, A.colVal, A.nzVal, size(A)) + GPUSparseMatrixCSC(colPtr, rowVal, nzVal, size(A)) +end +function generic_regroup(A::GPUSparseMatrixCSC) + rowPtr, colVal, nzVal = regroup(A.colPtr, A.rowVal, A.nzVal, reverse(size(A))) + GPUSparseMatrixCSR(rowPtr, colVal, nzVal, size(A)) +end +GPUSparseMatrixCSC(A::GPUSparseMatrixCSR) = generic_regroup(A) +GPUSparseMatrixCSR(A::GPUSparseMatrixCSC) = generic_regroup(A) + +""" + GPUArrays.generic_expand(A::GPUSparseMatrixCSR)::GPUSparseMatrixCOO + GPUArrays.generic_compress(A::GPUSparseMatrixCOO)::GPUSparseMatrixCSR + +Convert between CSR and COO on the device, which have the same order of stored entries: +expand the row pointers into a row index per entry, or compress them back. +""" +generic_expand(A::GPUSparseMatrixCSR) = + GPUSparseMatrixCOO(expand_ptr(A.rowPtr, nnz(A), indtype(A)), copy(A.colVal), copy(A.nzVal), size(A)) +generic_compress(A::GPUSparseMatrixCOO) = + GPUSparseMatrixCSR(compress(A.rowInd, size(A, 1)), copy(A.colInd), copy(A.nzVal), size(A)) +GPUSparseMatrixCOO(A::GPUSparseMatrixCSR) = generic_expand(A) +GPUSparseMatrixCSR(A::GPUSparseMatrixCOO) = generic_compress(A) + +# CSC ↔ COO through CSR, borrowing the buffers of the intermediate +function GPUSparseMatrixCOO(A::GPUSparseMatrixCSC) + B = GPUSparseMatrixCSR(A) + GPUSparseMatrixCOO(expand_ptr(B.rowPtr, nnz(B), indtype(B)), B.colVal, B.nzVal, size(A)) +end +function GPUSparseMatrixCSC(A::GPUSparseMatrixCOO) + B = GPUSparseMatrixCSR(compress(A.rowInd, size(A, 1)), A.colInd, A.nzVal, size(A)) + generic_regroup(B) +end + +# The CSR buffers of `A` are the CSC buffers of `transpose(A)`, and vice versa. This +# borrowed view shares the buffers of `A`; it is only used internally, to share kernels +# between formats, and never outlives the operation that creates it. +transpose_view(A::GPUSparseMatrixCSR) = + GPUSparseMatrixCSC(A.rowPtr, A.colVal, A.nzVal, reverse(size(A))) +transpose_view(A::GPUSparseMatrixCSC) = + GPUSparseMatrixCSR(A.colPtr, A.rowVal, A.nzVal, reverse(size(A))) + +# materializing a transpose into the opposite format only copies the buffers +GPUSparseMatrixCSC(A::Transpose{<:Any,<:GPUSparseMatrixCSR}) = copy(transpose_view(parent(A))) +GPUSparseMatrixCSR(A::Transpose{<:Any,<:GPUSparseMatrixCSC}) = copy(transpose_view(parent(A))) +function GPUSparseMatrixCSC(A::Adjoint{<:Any,<:GPUSparseMatrixCSR}) + B = transpose_view(parent(A)) + GPUSparseMatrixCSC(copy(B.colPtr), copy(B.rowVal), conj.(B.nzVal), size(B)) +end +function GPUSparseMatrixCSR(A::Adjoint{<:Any,<:GPUSparseMatrixCSC}) + B = transpose_view(parent(A)) + GPUSparseMatrixCSR(copy(B.rowPtr), copy(B.colVal), conj.(B.nzVal), size(B)) +end + +Base.convert(::Type{T}, A::AbstractArray) where {T<:AbstractGPUSparseArray} = + A isa T ? A : T(A) + + +## to dense arrays + +## COV_EXCL_START +# every thread scatters one slice of a compressed matrix into `dst`, indexed linearly +# as an `m`-row matrix +@kernel function densify_compressed_kernel(dst, ptr, ind, nzVal, m, csr::Bool) + j = @index(Global, Linear) + if j < length(ptr) + for k in @inbounds(ptr[j]):@inbounds(ptr[j+1] - one(eltype(ptr))) + i = @inbounds ind[k] + row, col = csr ? (j, Int(i)) : (Int(i), j) + @inbounds dst[(col - 1) * m + row] = nzVal[k] + end + end +end + +@kernel function densify_coo_kernel(dst, rowInd, colInd, nzVal, m) + k = @index(Global, Linear) + if k <= length(nzVal) + @inbounds dst[(Int(colInd[k]) - 1) * m + rowInd[k]] = nzVal[k] + end +end + +@kernel function densify_vector_kernel(dst, nzInd, nzVal) + k = @index(Global, Linear) + if k <= length(nzVal) + @inbounds dst[nzInd[k]] = nzVal[k] + end +end +## COV_EXCL_STOP + +# densify on the device, into the first `length(src)` elements of `dst` (in column-major +# order, like `copyto!` between dense arrays). The formats are canonical, so every element +# is written at most once. +function Base.copyto!(dst::AbstractGPUArray, src::GPUSparseArray) + length(dst) >= length(src) || throw(BoundsError(dst, length(src))) + fill!(length(dst) == length(src) ? dst : view(dst, 1:length(src)), zero(eltype(dst))) + nnz(src) == 0 && return dst + backend = get_backend(dst) + if src isa GPUSparseMatrixCompressed + densify_compressed_kernel(backend)(dst, compressed_ptr(src), compressed_ind(src), + src.nzVal, size(src, 1), src isa GPUSparseMatrixCSR; + ndrange=major_dim(src)) + elseif src isa GPUSparseMatrixCOO + densify_coo_kernel(backend)(dst, src.rowInd, src.colInd, src.nzVal, size(src, 1); + ndrange=nnz(src)) + else + densify_vector_kernel(backend)(dst, src.nzInd, src.nzVal; ndrange=nnz(src)) + end + return dst +end + +Base.copyto!(dst::Array, src::GPUSparseArray) = copyto!(dst, Array(src)) +Base.copyto!(dst::Vector, src::GPUSparseVector) = copyto!(dst, Array(src)) + + +## from dense arrays + +## COV_EXCL_START +@kernel function split_linear_kernel(rows, cols, lin, m) + k = @index(Global, Linear) + if k <= length(lin) + l = @inbounds lin[k] - 1 + @inbounds rows[k] = (l % m + 1) % eltype(rows) + @inbounds cols[k] = (l ÷ m + 1) % eltype(cols) + end +end +## COV_EXCL_STOP + +# The nonzeros of a dense matrix, in column-major (`major = :col`) or row-major order: +# the sorted major and minor index of every entry, and its value. `findall` visits the +# elements in column-major order, so row-major order comes from a dense transpose. +function dense_entries(A::AbstractGPUMatrix, ::Type{Tv}, ::Type{Ti}, major::Symbol) where {Tv,Ti} + check_sparse_dims(Ti, size(A)) + B = major === :col ? A : permutedims(A) + lin = findall(!iszero, vec(B)) + n = length(lin) + minor_ind = sparse_similar(lin, Ti, n) + major_ind = sparse_similar(lin, Ti, n) + if n > 0 + split_linear_kernel(get_backend(lin))(minor_ind, major_ind, lin, size(B, 1); ndrange=n) + end + vals = vec(B)[lin] + B === A || free_buffer!(B) + free_buffer!(lin) + return major_ind, minor_ind, eltype(vals) == Tv ? vals : convert_buffer(Tv, vals) +end + +function GPUSparseMatrixCSC{Tv,Ti}(A::AbstractGPUMatrix) where {Tv,Ti} + cols, rows, vals = dense_entries(A, Tv, Ti, :col) + GPUSparseMatrixCSC(compress(cols, size(A, 2), Ti), rows, vals, size(A)) +end +function GPUSparseMatrixCSR{Tv,Ti}(A::AbstractGPUMatrix) where {Tv,Ti} + rows, cols, vals = dense_entries(A, Tv, Ti, :row) + GPUSparseMatrixCSR(compress(rows, size(A, 1), Ti), cols, vals, size(A)) +end +function GPUSparseMatrixCOO{Tv,Ti}(A::AbstractGPUMatrix) where {Tv,Ti} + rows, cols, vals = dense_entries(A, Tv, Ti, :row) + GPUSparseMatrixCOO(rows, cols, vals, size(A)) +end +function GPUSparseVector{Tv,Ti}(x::AbstractGPUVector) where {Tv,Ti} + check_sparse_dims(Ti, size(x)) + lin = findall(!iszero, x) + vals = x[lin] + GPUSparseVector(convert_buffer(Ti, lin), eltype(vals) == Tv ? vals : convert_buffer(Tv, vals), + length(x)) +end + +# like SparseArrays, the index type defaults to `Int` +for (S, D) in ((:GPUSparseMatrixCSR, :AbstractGPUMatrix), (:GPUSparseMatrixCSC, :AbstractGPUMatrix), + (:GPUSparseMatrixCOO, :AbstractGPUMatrix), (:GPUSparseVector, :AbstractGPUVector)) + @eval begin + $S(A::$D) = $S{eltype(A),Int}(A) + $S{Tv}(A::$D) where {Tv} = $S{Tv,Int}(A) + end +end + +# the format selected by a `fmt` keyword, as CUDA.jl and AMDGPU.jl accept it +sparse_format(fmt::Symbol) = + fmt === :csc ? GPUSparseMatrixCSC : + fmt === :csr ? GPUSparseMatrixCSR : + fmt === :coo ? GPUSparseMatrixCOO : + throw(ArgumentError("unknown sparse format :$fmt, expected :csc, :csr or :coo")) + +# Policy only for the dense `sparse(A)` convenience entrypoint. Explicit canonical +# constructors and coordinate assembly retain their own documented index policies. +default_sparse_indtype(A::AbstractGPUMatrix) = Int +sparse_format(fmt::Symbol,A::AbstractGPUMatrix) = + sparse_format(fmt){eltype(A),default_sparse_indtype(A)} + +""" + sparse(A::AbstractGPUMatrix; fmt=:csc) + sparse(x::AbstractGPUVector) + +The nonzero entries of a dense GPU array as a sparse array on the same device, in CSC +format (like SparseArrays) or the format selected by `fmt` (`:csc`, `:csr` or `:coo`). +The default index type is `Int`; GPUArrays' CUDA and AMDGPU integrations preserve +their existing uniform `Int32` default and additionally accept `fmt=:bsr`. +""" +SparseArrays.sparse(A::AbstractGPUMatrix; fmt::Symbol=:csc) = sparse_format(fmt,A)(A) +SparseArrays.sparse(x::AbstractGPUVector) = GPUSparseVector(x) +SparseArrays.sparsevec(x::AbstractGPUVector) = GPUSparseVector(x) +for S in (:GPUSparseMatrixCSR,:GPUSparseMatrixCSC,:GPUSparseMatrixCOO) + @eval begin + $S(D::Diagonal{<:Any,<:AnyGPUVector})=convert($S,spdiagm(D.diag)) + $S{T}(D::Diagonal{<:Any,<:AnyGPUVector}) where {T}= + (check_sparse_values(T,D.diag);with_values($S(D),T)) + function $S{T,Ti}(D::Diagonal{<:Any,<:AnyGPUVector}) where {T,Ti} + check_sparse_dims(Ti,size(D)) + # spdiagm retains every diagonal entry, including stored zeros. + $S <: GPUSparseMatrixCompressed && check_sparse_nnz(Ti,length(D.diag)) + check_sparse_values(T,D.diag) + with_eltypes($S(D),T,Ti) + end + end +end + +# Construct the sparse representation on-device before transferring its buffers. +SparseArrays.SparseMatrixCSC(A::AbstractGPUMatrix) = SparseMatrixCSC(sparse(A)) +SparseArrays.SparseMatrixCSC{T}(A::AbstractGPUMatrix) where {T} = + SparseMatrixCSC(GPUSparseMatrixCSC{T}(A)) +SparseArrays.SparseMatrixCSC{T,Ti}(A::AbstractGPUMatrix) where {T,Ti} = + SparseMatrixCSC(GPUSparseMatrixCSC{T,Ti}(A)) + + +## transposes + +# The transpose with new storage, in the same format: the other compressed layout of `A`, +# read with reversed dimensions. +materialize_transpose(A::GPUSparseMatrixCSR) = transpose_view(GPUSparseMatrixCSC(A)) +materialize_transpose(A::GPUSparseMatrixCSC) = transpose_view(GPUSparseMatrixCSR(A)) +materialize_transpose(A::GPUSparseMatrixCOO) = + GPUSparseMatrixCOO(transpose_view(GPUSparseMatrixCSC(A))) + +Base.copy(A::Transpose{<:Any,<:GPUSparseMatrix}) = materialize_transpose(parent(A)) +function Base.copy(A::Adjoint{<:Any,<:GPUSparseMatrix}) + B = materialize_transpose(parent(A)) + eltype(B) <: Real || (B.nzVal .= conj.(B.nzVal)) + return B +end + +# into any format; the cheap cases (CSR ↔ CSC) are defined above +for S in (:GPUSparseMatrixCSR, :GPUSparseMatrixCSC, :GPUSparseMatrixCOO) + @eval begin + $S(A::Union{Transpose{<:Any,<:GPUSparseMatrix},Adjoint{<:Any,<:GPUSparseMatrix}}) = + convert($S, copy(A)) + $S{T}(A::Union{Transpose{<:Any,<:GPUSparseMatrix},Adjoint{<:Any,<:GPUSparseMatrix}}) where {T} = + (check_sparse_values(T,nonzeros(parent(A)));with_values($S(A),T)) + function $S{T,Ti}(A::Union{Transpose{<:Any,<:GPUSparseMatrix},Adjoint{<:Any,<:GPUSparseMatrix}}) where {T,Ti} + check_sparse_dims(Ti,size(A)) + $S <: GPUSparseMatrixCompressed && check_sparse_nnz(Ti,nnz(parent(A))) + check_sparse_values(T,nonzeros(parent(A))) + with_eltypes($S(A),T,Ti) + end + end +end + +Base.permutedims(A::GPUSparseMatrix) = copy(transpose(A)) +function Base.permutedims(A::GPUSparseMatrix, perm) + Tuple(perm) == (2, 1) && return copy(transpose(A)) + Tuple(perm) == (1, 2) && return copy(A) + throw(ArgumentError("no valid permutation of dimensions")) +end + + +## coordinates + +# the coordinates and values of the stored entries, in column-major order like SparseArrays +function SparseArrays.findnz(A::GPUSparseMatrix) + C = A isa GPUSparseMatrixCSC ? copy(A) : GPUSparseMatrixCSC(A) + return C.rowVal, expand_ptr(C.colPtr, nnz(C), indtype(C)), C.nzVal +end +SparseArrays.findnz(x::GPUSparseVector) = (copy(x.nzInd), copy(x.nzVal)) + + +## element types + +Base.float(A::GPUSparseArray) = + eltype(A) <: AbstractFloat ? A : with_values(A, float(eltype(A))) +Base.complex(A::GPUSparseArray) = + eltype(A) <: Complex ? A : with_values(A, complex(eltype(A))) +Base.real(A::GPUSparseArray) = eltype(A) <: Real ? A : real.(A) +Base.imag(A::GPUSparseArray) = eltype(A) <: Real ? zero(A) : imag.(A) + +# a sparse array is already sparse; the matrix format can be chosen as for dense arrays +SparseArrays.sparse(A::GPUSparseMatrix; fmt::Symbol=:csc) = sparse_format(fmt)(A) +SparseArrays.sparse(x::GPUSparseVector) = copy(x) +SparseArrays.sparsevec(A::GPUSparseMatrix) = vec(A) diff --git a/src/host/sparse/indexing.jl b/src/host/sparse/indexing.jl new file mode 100644 index 000000000..8283a154f --- /dev/null +++ b/src/host/sparse/indexing.jl @@ -0,0 +1,174 @@ +# indexing + +# Scalar indexing reads a few entries of the buffers, so like dense GPU arrays it requires +# `@allowscalar`. The searches are binary searches over the stored indices of one slice. + +# the value stored at `ind[lo:hi] == i` if any, else zero +function stored_value(A, ind, lo::Integer, hi::Integer, i::Integer) + lo > hi && return zero(eltype(A)) + k = searchsortedfirst(ind, i, lo, hi, Base.Order.Forward) + (k > hi || ind[k] != i) && return zero(eltype(A)) + return nonzeros(A)[k] +end + +function Base.getindex(A::GPUSparseMatrixCSR, i::Integer, j::Integer) + @boundscheck checkbounds(A, i, j) + stored_value(A, A.colVal, Int(A.rowPtr[i]), Int(A.rowPtr[i+1]) - 1, j) +end + +function Base.getindex(A::GPUSparseMatrixCSC, i::Integer, j::Integer) + @boundscheck checkbounds(A, i, j) + stored_value(A, A.rowVal, Int(A.colPtr[j]), Int(A.colPtr[j+1]) - 1, i) +end + +function Base.getindex(A::GPUSparseMatrixCOO, i::Integer, j::Integer) + @boundscheck checkbounds(A, i, j) + # the entries of row `i` are contiguous, and sorted by column + lo = searchsortedfirst(A.rowInd, i) + hi = searchsortedlast(A.rowInd, i) + stored_value(A, A.colInd, lo, hi, j) +end + +function Base.getindex(x::GPUSparseVector, i::Integer) + @boundscheck checkbounds(x, i) + stored_value(x, x.nzInd, 1, nnz(x), i) +end + +Base.getindex(A::GPUSparseMatrix, I::Tuple{Integer,Integer}) = getindex(A, I...) +Base.getindex(x::GPUSparseVector, ::Colon) = copy(x) +Base.getindex(A::GPUSparseMatrix, ::Colon, ::Colon) = copy(A) + +function Base.setindex!(::GPUSparseArray, v, I...) + throw(ArgumentError("""GPU sparse arrays do not support setting individual entries. + Assemble a new array with `sparse(I, J, V, m, n)` or `sparsevec(I, V, n)`, use broadcasting, or update stored values through `nonzeros(A)`.""")) +end + + +## slices and ranges + +# Indexing with ranges (and colons) keeps the stored entries that fall in them and +# renumbers them. Ranges with a positive step preserve the order of the entries, so the +# result needs no sorting; other index vectors are not supported. +const SparseRangeIndex = Union{AbstractRange{<:Integer}, Colon} + +range_index(I::Colon, n) = Base.OneTo(n) +function range_index(I::AbstractRange{<:Integer}, n) + (length(I) <= 1 || step(I) > 0) || + throw(ArgumentError("indexing GPU sparse arrays requires ranges with a positive step")) + return I +end +range_index(i::Integer, n) = i:i + +# the position of `i` in the range `I` (from `first`, with `step`, of `len` elements), or 0 +@inline function range_position(i, first, step, len) + d = i - first + (d >= 0 && d % step == 0) || return 0 + p = d ÷ step + 1 + return 1 <= p <= len ? p : 0 +end + +## COV_EXCL_START +@kernel function select_entries_kernel(keep, new_rows, new_cols, rows, cols, I, J) + k = @index(Global, Linear) + if k <= length(keep) + i = range_position(Int(@inbounds rows[k]), I...) + j = range_position(Int(@inbounds cols[k]), J...) + @inbounds keep[k] = i != 0 && j != 0 + @inbounds new_rows[k] = i % eltype(new_rows) + @inbounds new_cols[k] = j % eltype(new_cols) + end +end + +@kernel function select_vector_entries_kernel(keep, new_inds, inds, I) + k = @index(Global, Linear) + if k <= length(keep) + i = range_position(Int(@inbounds inds[k]), I...) + @inbounds keep[k] = i != 0 + @inbounds new_inds[k] = i % eltype(new_inds) + end +end +## COV_EXCL_STOP + +# (a range of one element may have any step) +range_parameters(I::AbstractRange) = (Int(first(I)), length(I) > 1 ? Int(step(I)) : 1, length(I)) + +# the entries of `A[I, J]`, in the order they have in `A`: their new rows and columns, and +# their values +function select_entries(A::GPUSparseMatrix, I::AbstractRange, J::AbstractRange) + rows, cols = entry_coordinates(A) + n = nnz(A) + keep = similar(A.nzVal, Bool, n) + new_rows = similar(rows, n) + new_cols = similar(cols, n) + if n > 0 + select_entries_kernel(get_backend(A))(keep, new_rows, new_cols, rows, cols, + range_parameters(I), range_parameters(J); + ndrange=n) + end + kept = findall(keep) + return new_rows[kept], new_cols[kept], A.nzVal[kept] +end + +function sparse_getindex(A::GPUSparseMatrix, I, J) + @boundscheck checkbounds(A, I, J) + m, n = size(A) + rows, cols, vals = select_entries(A, range_index(I, m), range_index(J, n)) + if I isa Integer || J isa Integer + # entries of a single row or column are in order of the other index in every format + return GPUSparseVector(I isa Integer ? cols : rows, vals, + I isa Integer ? length(range_index(J, n)) : length(range_index(I, m))) + end + dims = (length(range_index(I, m)), length(range_index(J, n))) + if A isa GPUSparseMatrixCSR + GPUSparseMatrixCSR(compress(rows, dims[1], eltype(A.rowPtr)), cols, vals, dims) + elseif A isa GPUSparseMatrixCSC + GPUSparseMatrixCSC(compress(cols, dims[2], eltype(A.colPtr)), rows, vals, dims) + else + GPUSparseMatrixCOO(rows, cols, vals, dims) + end +end + +Base.getindex(A::GPUSparseMatrix, I::SparseRangeIndex, J::SparseRangeIndex) = + sparse_getindex(A, I, J) +Base.getindex(A::GPUSparseMatrix, i::Integer, J::SparseRangeIndex) = sparse_getindex(A, i, J) +Base.getindex(A::GPUSparseMatrix, I::SparseRangeIndex, j::Integer) = sparse_getindex(A, I, j) + +# Masks preserve coordinate order, so neither assembly nor sorting is needed. +function Base.getindex(A::GPUSparseMatrix,I::AbstractVector{Bool},J::AbstractVector{Bool}) + length(I)==size(A,1) && length(J)==size(A,2) || throw(BoundsError(A,(I,J))) + mask(X) = X isa AnyGPUVector ? X : copyto!(sparse_similar(A.nzVal,Bool,length(X)),Vector{Bool}(X)) + I,J = mask(I),mask(J) + check_sparse_storage(sparse_buffers(A)...,I,J) + rowrank = accumulate(+,Int.(I)) + colrank = accumulate(+,Int.(J)) + dims = (count(I),count(J)) + rows,cols = entry_coordinates(A) + kept = findall(I[rows] .& J[cols]) + Ti = indtype(A) + newrows = convert_index_buffer(Ti,rowrank[rows[kept]]) + newcols = convert_index_buffer(Ti,colrank[cols[kept]]) + values = A.nzVal[kept] + if A isa GPUSparseMatrixCSR + GPUSparseMatrixCSR(compress(newrows,dims[1],eltype(A.rowPtr)),newcols,values,dims) + elseif A isa GPUSparseMatrixCSC + GPUSparseMatrixCSC(compress(newcols,dims[2],eltype(A.colPtr)),newrows,values,dims) + else + GPUSparseMatrixCOO(newrows,newcols,values,dims) + end +end + +function sparse_getindex(x::GPUSparseVector, I::AbstractRange{<:Integer}) + @boundscheck checkbounds(x, I) + I = range_index(I, length(x)) + keep = similar(x.nzVal, Bool, nnz(x)) + new_inds = similar(x.nzInd, nnz(x)) + if nnz(x) > 0 + select_vector_entries_kernel(get_backend(x))(keep, new_inds, x.nzInd, + range_parameters(I); ndrange=nnz(x)) + end + kept = findall(keep) + return GPUSparseVector(new_inds[kept], x.nzVal[kept], length(I)) +end +# (separate methods to be more specific than SparseArrays' for any sparse vector) +Base.getindex(x::GPUSparseVector, I::AbstractUnitRange{<:Integer}) = sparse_getindex(x, I) +Base.getindex(x::GPUSparseVector, I::StepRange{<:Integer}) = sparse_getindex(x, I) diff --git a/src/host/sparse/linalg.jl b/src/host/sparse/linalg.jl new file mode 100644 index 000000000..5435a9347 --- /dev/null +++ b/src/host/sparse/linalg.jl @@ -0,0 +1,478 @@ +# products of sparse and dense arrays +# +# LinearAlgebra reduces products with transposed, adjoint, Symmetric and Hermitian operands +# to its storage-level `mul!(C, tA, [tB,] A, B, α, β)`, where a character describes each +# operand: 'N', 'T', 'C', and 'S'/'s' or 'H'/'h' for the upper or lower triangle of a +# Symmetric or Hermitian matrix. GPUArrays implements those methods for sparse operands +# (below), and back-ends override them for their vendor libraries, falling back to the +# generic implementations (`generic_spmv!`, `generic_spmm!`) where those do not apply. +# +# Every product is a gather over a compressed layout of the sparse operand, deterministic +# and valid for every element type. Transposed operands (Aᵀx for CSR, Ax for CSC) regroup +# the matrix first; an atomic scatter driven directly by the stored layout would avoid that. + +# the element type to accumulate products in: Float16 loses too much precision +product_accumulator(::Type{T}) where {T} = T +product_accumulator(::Type{Float16}) = Float32 +product_accumulator(::Type{ComplexF16}) = ComplexF32 +function product_accumulator(Ta::Type, Tb::Type) + T = typeof(zero(Ta) * zero(Tb) + zero(Ta) * zero(Tb)) + return product_accumulator(T) +end + +## the operands + +# The CSR buffers (pointers, indices, values) of `op(A)` for `t` in 'N', 'T' and 'C', with +# the conjugation of 'C' left to the kernel. Buffers of `A` itself are borrowed, others are +# temporary. +function csr_operand(A::GPUSparseMatrixCSR, t::AbstractChar) + B = t == 'N' ? A : transpose_view(generic_regroup(A)) + return B.rowPtr, B.colVal, B.nzVal +end +function csr_operand(A::GPUSparseMatrixCSC, t::AbstractChar) + B = t == 'N' ? generic_regroup(A) : transpose_view(A) + return B.rowPtr, B.colVal, B.nzVal +end +function csr_operand(A::GPUSparseMatrixCOO, t::AbstractChar) + t == 'N' || return csr_operand(generic_regroup(generic_compress(A)), t) + # a COO matrix is a CSR matrix with explicit row indices + return compress(A.rowInd, size(A, 1)), A.colInd, A.nzVal +end + +# The CSC buffers of `op(A)`, which are the CSR buffers of its transpose. The conjugation +# of 'C' is again left to the kernel. +csc_operand(A::GPUSparseMatrix, t::AbstractChar) = csr_operand(A, t == 'N' ? 'T' : 'N') + +# LinearAlgebra's characters for Symmetric and Hermitian operands combine two terms: the +# triangle of `A` that the wrapper uses, and the transpose (or adjoint) of that triangle +# without the diagonal. Each term is an operand character for the kernels, and a selection +# of the entries of `op(A)` by their position. +function operand_terms(t::AbstractChar) + t in ('N', 'T', 'C') && return ((t, all_entries),) + t == 'S' && return (('N', upper_entries), ('T', strict_lower_entries)) + t == 's' && return (('N', lower_entries), ('T', strict_upper_entries)) + t == 'H' && return (('N', upper_hermitian_entries), ('C', strict_lower_entries)) + t == 'h' && return (('N', lower_hermitian_entries), ('C', strict_upper_entries)) + throw(ArgumentError("unsupported operand character '$t'")) +end + +# the contribution of a stored entry `v` at `(i, j)`, as selected by a term +all_entries(i, j, v) = v +upper_entries(i, j, v) = j >= i ? v : zero(v) +lower_entries(i, j, v) = j <= i ? v : zero(v) +strict_upper_entries(i, j, v) = j > i ? v : zero(v) +strict_lower_entries(i, j, v) = j < i ? v : zero(v) +# the diagonal of a Hermitian matrix is real +upper_hermitian_entries(i, j, v) = j > i ? v : j == i ? oftype(v, real(v)) : zero(v) +lower_hermitian_entries(i, j, v) = j < i ? v : j == i ? oftype(v, real(v)) : zero(v) + +entry_selected(::typeof(all_entries), i, j) = true +entry_selected(::Union{typeof(upper_entries),typeof(upper_hermitian_entries)}, i, j) = j >= i +entry_selected(::Union{typeof(lower_entries),typeof(lower_hermitian_entries)}, i, j) = j <= i +entry_selected(::typeof(strict_upper_entries), i, j) = j > i +entry_selected(::typeof(strict_lower_entries), i, j) = j < i + +value_op(t::AbstractChar) = t == 'C' ? conj : identity + + +## kernels + +## COV_EXCL_START +# y = α op(A) x + β y, one thread per row of op(A), in CSR layout +@kernel function csr_matvec_kernel(y, ptr, ind, val, x, α, β, entry, valop, ::Type{T}) where {T} + i = @index(Global, Linear) + if i <= length(y) + acc = zero(T) + for k in @inbounds(ptr[i]):@inbounds(ptr[i+1] - one(eltype(ptr))) + j = @inbounds ind[k] + entry_selected(entry, i, j) || continue + v = entry(i, j, valop(@inbounds val[k])) + acc += T(v) * T(@inbounds x[j]) + end + # `y` is write-only when β is zero, so that it may hold NaN + @inbounds y[i] = iszero(β) ? α * acc : α * acc + β * y[i] + end +end + +# C = α op(A) op(B) + β C with a sparse op(A) in CSR layout, one thread per element of C +@kernel function csr_matmat_kernel(C, ptr, ind, val, B, α, β, entry, valop, tB::Char, + ::Type{T}) where {T} + i, j = @index(Global, NTuple) + if i <= size(C, 1) && j <= size(C, 2) + acc = zero(T) + for k in @inbounds(ptr[i]):@inbounds(ptr[i+1] - one(eltype(ptr))) + l = @inbounds ind[k] + entry_selected(entry, i, l) || continue + v = entry(i, l, valop(@inbounds val[k])) + b = tB == 'N' ? @inbounds(B[l, j]) : + tB == 'T' ? @inbounds(B[j, l]) : conj(@inbounds(B[j, l])) + acc += T(v) * T(b) + end + @inbounds C[i, j] = iszero(β) ? α * acc : α * acc + β * C[i, j] + end +end + +# C = α op(A) op(B) + β C with a sparse op(B) in CSC layout, one thread per element of C. +# Consecutive threads work down a column of C, reading a column of op(A) if it is not +# transposed, so that the dense accesses coalesce. +@kernel function csc_matmat_kernel(C, A, ptr, ind, val, α, β, entry, valop, tA::Char, + ::Type{T}) where {T} + i, j = @index(Global, NTuple) + if i <= size(C, 1) && j <= size(C, 2) + acc = zero(T) + for k in @inbounds(ptr[j]):@inbounds(ptr[j+1] - one(eltype(ptr))) + l = @inbounds ind[k] + entry_selected(entry, l, j) || continue + v = entry(l, j, valop(@inbounds val[k])) + a = tA == 'N' ? @inbounds(A[i, l]) : + tA == 'T' ? @inbounds(A[l, i]) : conj(@inbounds(A[l, i])) + acc += T(a) * T(v) + end + @inbounds C[i, j] = iszero(β) ? α * acc : α * acc + β * C[i, j] + end +end +## COV_EXCL_STOP + +## generic implementations + +# the dimensions of the result and of the operands of a product, as matrices +function check_product_dims(C::Dims, A::Dims, B::Dims) + mat(sz) = (sz[1], get(sz, 2, 1)) + C, A, B = mat(C), mat(A), mat(B) + A[2] == B[1] || + throw(DimensionMismatch("A has dimensions $A, B has dimensions $B")) + C == (A[1], B[2]) || + throw(DimensionMismatch("C has dimensions $C, needs $((A[1], B[2]))")) + return +end +op_size(A, t::AbstractChar) = t in ('T', 'C') ? reverse(size(A)) : size(A) + +# C = β C, treating C as write-only when β is zero +scale!(C, β) = iszero(β) ? fill!(C, zero(eltype(C))) : (C .*= β; C) + +## COV_EXCL_START +@kernel function materialize_symmetric_kernel(out, B, upper::Bool, hermitian::Bool) + i, j = @index(Global, NTuple) + if i <= size(out, 1) && j <= size(out, 2) + # the triangle of `B` that holds the values + stored = upper ? i <= j : i >= j + v = stored ? @inbounds(B[i, j]) : @inbounds(B[j, i]) + if hermitian + v = i == j ? oftype(v, real(v)) : stored ? v : conj(v) + end + @inbounds out[i, j] = v + end +end +## COV_EXCL_STOP + +# A dense operand wrapped in Symmetric or Hermitian is materialized; the kernels only index +# dense operands as they are, transposed, or adjoint. +function dense_operand(B, t::AbstractChar) + t in ('N', 'T', 'C') && return B, t + LinearAlgebra.checksquare(B) + out = sparse_similar(B, eltype(B), size(B)) + isempty(out) || + materialize_symmetric_kernel(get_backend(out))(out, B, t in ('S', 'H'), t in ('H', 'h'); + ndrange=size(out)) + return out, 'N' +end + +""" + GPUArrays.generic_spmv!(y, tA, A, x, α, β) + +Compute `y = α op(A) x + β y` for a sparse matrix `A` and dense vectors `x` and `y`, where +`op` is given by LinearAlgebra's character `tA` ('N', 'T', 'C', or 'S'/'s'/'H'/'h' for the +upper or lower triangle of a Symmetric or Hermitian `A`). This is the generic +implementation behind `mul!(y, tA, A, x, α, β)`, which back-ends can call when a vendor +routine does not apply. +""" +function generic_spmv!(y::AnyGPUVector, tA::AbstractChar, A::GPUSparseMatrix, + x::AnyGPUVector, α::Number, β::Number) + tA in ('S', 's', 'H', 'h') && LinearAlgebra.checksquare(A) + check_product_dims(size(y), op_size(A, tA), size(x)) + isempty(y) && return y + (isempty(x) || nnz(A) == 0) && return scale!(y, β) + T = product_accumulator(eltype(A), eltype(x)) + for (n, (t, entry)) in enumerate(operand_terms(tA)) + ptr, ind, val = csr_operand(A, t) + csr_matvec_kernel(get_backend(y))(y, ptr, ind, val, x, α, n == 1 ? β : true, + entry, value_op(t), T; ndrange=length(y)) + end + return y +end + +""" + GPUArrays.generic_spmm!(C, tA, tB, A, B, α, β) + +Compute `C = α op(A) op(B) + β C` where one of `A` and `B` is a sparse matrix and the other +one, like `C`, is dense; `tA` and `tB` are LinearAlgebra's characters as for +[`generic_spmv!`](@ref). This is the generic implementation behind +`mul!(C, tA, tB, A, B, α, β)`. +""" +function generic_spmm!(C::AnyGPUMatrix, tA::AbstractChar, tB::AbstractChar, + A::GPUSparseMatrix, B::AnyGPUMatrix, α::Number, β::Number) + tA in ('S', 's', 'H', 'h') && LinearAlgebra.checksquare(A) + B, tB = dense_operand(B, tB) + check_product_dims(size(C), op_size(A, tA), op_size(B, tB)) + isempty(C) && return C + (size(B, tB == 'N' ? 1 : 2) == 0 || nnz(A) == 0) && return scale!(C, β) + T = product_accumulator(eltype(A), eltype(B)) + for (n, (t, entry)) in enumerate(operand_terms(tA)) + ptr, ind, val = csr_operand(A, t) + csr_matmat_kernel(get_backend(C))(C, ptr, ind, val, B, α, n == 1 ? β : true, + entry, value_op(t), Char(tB), T; ndrange=size(C)) + end + return C +end +function generic_spmm!(C::AnyGPUMatrix, tA::AbstractChar, tB::AbstractChar, + A::AnyGPUMatrix, B::GPUSparseMatrix, α::Number, β::Number) + tB in ('S', 's', 'H', 'h') && LinearAlgebra.checksquare(B) + A, tA = dense_operand(A, tA) + check_product_dims(size(C), op_size(A, tA), op_size(B, tB)) + isempty(C) && return C + (size(A, tA == 'N' ? 2 : 1) == 0 || nnz(B) == 0) && return scale!(C, β) + T = product_accumulator(eltype(A), eltype(B)) + for (n, (t, entry)) in enumerate(operand_terms(tB)) + ptr, ind, val = csc_operand(B, t) + csc_matmat_kernel(get_backend(C))(C, A, ptr, ind, val, α, n == 1 ? β : true, + entry, value_op(t), Char(tA), T; ndrange=size(C)) + end + return C +end + + +## LinearAlgebra integration + +LinearAlgebra.mul!(C::AnyGPUVector, tA::AbstractChar, A::GPUSparseMatrix, B::AnyGPUVector, + α::Number, β::Number) = + spmv!(C, tA, A, B, α, β) +LinearAlgebra.mul!(C::AnyGPUMatrix, tA::AbstractChar, tB::AbstractChar, A::GPUSparseMatrix, + B::AnyGPUMatrix, α::Number, β::Number) = + spmm!(C, tA, tB, A, B, α, β) +LinearAlgebra.mul!(C::AnyGPUMatrix, tA::AbstractChar, tB::AbstractChar, A::AnyGPUMatrix, + B::GPUSparseMatrix, α::Number, β::Number) = + spmm!(C, tA, tB, A, B, α, β) + +# Julia < 1.13 dispatches on the non-public `generic_matvecmul!` and `generic_matmatmul!` +# instead, as for dense arrays (see `linalg.jl`) +@static if VERSION < v"1.13.0-rc4" + for (TA, TB) in ((:GPUSparseMatrix, :AnyGPUMatrix), (:AnyGPUMatrix, :GPUSparseMatrix)) + @eval begin + LinearAlgebra.generic_matmatmul!(C::AnyGPUMatrix, tA::AbstractChar, tB::AbstractChar, + A::$TA, B::$TB, a::Number, b::Number) = + LinearAlgebra.mul!(C, tA, tB, A, B, a, b) + LinearAlgebra.generic_matmatmul!(C::AnyGPUMatrix, tA::AbstractChar, tB::AbstractChar, + A::$TA, B::$TB, _add::MulAddMul=MulAddMul()) = + LinearAlgebra.mul!(C, tA, tB, A, B, _add.alpha, _add.beta) + end + end + LinearAlgebra.generic_matvecmul!(C::AnyGPUVector, tA::AbstractChar, A::GPUSparseMatrix, + B::AnyGPUVector, a::Number, b::Number) = + LinearAlgebra.mul!(C, tA, A, B, a, b) + LinearAlgebra.generic_matvecmul!(C::AnyGPUVector, tA::AbstractChar, A::GPUSparseMatrix, + B::AnyGPUVector, _add::MulAddMul=MulAddMul()) = + LinearAlgebra.mul!(C, tA, A, B, _add.alpha, _add.beta) +end + +# LinearAlgebra allocates the result of a product like its second operand, which gives a +# sparse destination for dense × sparse +const GPUSparseMatrixOperand = Union{GPUSparseMatrix, Transpose{<:Any,<:GPUSparseMatrix}, + Adjoint{<:Any,<:GPUSparseMatrix}, + Symmetric{<:Any,<:GPUSparseMatrix}, + Hermitian{<:Any,<:GPUSparseMatrix}} +function Base.:(*)(A::AnyGPUMatrix, B::GPUSparseMatrixOperand) + T = Base.promote_op(LinearAlgebra.matprod, eltype(A), eltype(B)) + return mul!(sparse_similar(A, T, (size(A, 1), size(B, 2))), A, B) +end + +# A sparse vector operand is densified, as CUDA.jl does. (A product with a sparse vector +# could skip its implicit zeros, but that only pays for very sparse vectors.) +densify(x::GPUSparseVector) = copyto!(sparse_similar(x.nzVal, eltype(x), length(x)), x) + +const GPUMatrixOperand = Union{GPUSparseMatrix, AnyGPUMatrix} +LinearAlgebra.mul!(C::AnyGPUVector, tA::AbstractChar, A::GPUMatrixOperand, B::GPUSparseVector, + α::Number, β::Number) = + LinearAlgebra.mul!(C, tA, A, densify(B), α, β) +@static if VERSION < v"1.13.0-rc4" + LinearAlgebra.generic_matvecmul!(C::AnyGPUVector, tA::AbstractChar, A::GPUMatrixOperand, + B::GPUSparseVector, a::Number, b::Number) = + LinearAlgebra.mul!(C, tA, A, B, a, b) + LinearAlgebra.generic_matvecmul!(C::AnyGPUVector, tA::AbstractChar, A::GPUMatrixOperand, + B::GPUSparseVector, _add::MulAddMul=MulAddMul()) = + LinearAlgebra.mul!(C, tA, A, B, _add.alpha, _add.beta) +end + +# like dense × sparse, LinearAlgebra would allocate a sparse destination +function Base.:(*)(A::Union{AnyGPUMatrix,GPUSparseMatrixOperand}, x::GPUSparseVector) + T = Base.promote_op(LinearAlgebra.matprod, eltype(A), eltype(x)) + return mul!(sparse_similar(x.nzVal, T, size(A, 1)), A, x) +end + + +## sparse × sparse + +# Nonnegative counts form an associative saturating monoid. Saturation is rejected +# before candidate allocation; neither a device scan nor its final count may wrap. +saturating_count_add(a, b) = b > typemax(Int64) - a ? typemax(Int64) : a + b + +## COV_EXCL_START +# the number of products every stored entry `A[i, l]` contributes: the length of row `l` +# of `B` +@kernel function spgemm_counts_kernel(counts, Aind, Bptr) + e = @index(Global, Linear) + if e <= length(counts) + l = @inbounds Aind[e] + @inbounds counts[e] = Int64(Bptr[l+1]) - Int64(Bptr[l]) + end +end + +# every thread computes one product `A[i, l] * B[l, j]`, finding its entry of `A` by binary +# search over the running total of the counts +@kernel function spgemm_expand_kernel(I, J, V, offsets, Arow, Aind, Aval, Bptr, Bind, Bval) + p = @index(Global, Linear) + if p <= length(I) + e = searchsortedfirst(offsets, Int64(p)) + q = p - (e == 1 ? 0 : @inbounds(offsets[e-1])) + l = @inbounds Aind[e] + k = @inbounds(Bptr[l]) + q - 1 + @inbounds I[p] = Arow[e] + @inbounds J[p] = Bind[k] + @inbounds V[p] = convert(eltype(V), Aval[e]) * convert(eltype(V), Bval[k]) + end +end +## COV_EXCL_STOP + +""" + GPUArrays.generic_spgemm(A, B) + +The product of two sparse matrices, in the format of `A`, computed on the device by +expansion, sorting and compression (ESC): every product of a stored entry of `A` with one +of `B` is materialized, then the products are summed per coordinate as in +`sparse(I, J, V, m, n, +)`. The memory needed grows with the number of products. The +generic implementation behind `A * B`; hash-based accumulation with atomic insertion would +avoid materializing all products. +""" +function generic_spgemm(A::GPUSparseMatrix, B::GPUSparseMatrix; max_temp_bytes::Integer=1 << 30) + size(A, 2) == size(B, 1) || + throw(DimensionMismatch("A has dimensions $(size(A)), B has dimensions $(size(B))")) + m, n = size(A, 1), size(B, 2) + T = Base.promote_op(LinearAlgebra.matprod, eltype(A), eltype(B)) + Ti = promote_type(indtype(A), indtype(B)) + Aptr, Aind, Aval = csr_operand(A, 'N') + Bptr, Bind, Bval = csr_operand(B, 'N') + Tp = promote_pointer_type(A, B) + Arow = expand_ptr(Aptr, length(Aval), indtype(A)) + + counts = sparse_similar(Aval, Int64, length(Aval)) + if !isempty(counts) + spgemm_counts_kernel(get_backend(counts))(counts, Aind, Bptr; ndrange=length(counts)) + end + offsets = accumulate(saturating_count_add, counts) + free_buffer!(counts) + total = isempty(offsets) ? 0 : Int(@allowscalar offsets[end]) + total < typemax(Int) || throw(ArgumentError("SpGEMM candidate count exceeds host indexing capacity")) + # Conservative bound for candidate coordinates/values, sort permutations and keys, + # assembly gathers, and offsets. Exact peak use depends on the sorting workspace. + bytes_per_candidate = Base.checked_add(8 * sizeof(Int), + Base.checked_add(4 * sizeof(Ti), 4 * sizeof(product_accumulator(eltype(A), eltype(B))))) + estimated = Base.checked_add(Base.checked_mul(total, bytes_per_candidate), + Base.checked_mul(length(Aval), 3 * sizeof(Int))) + estimated <= max_temp_bytes || throw(ArgumentError("portable SpGEMM temporary-memory estimate $estimated bytes exceeds max_temp_bytes=$max_temp_bytes; explicitly raise the limit or select a vendor implementation")) + + I = sparse_similar(Aind, Ti, total) + J = sparse_similar(Aind, Ti, total) + V = sparse_similar(Aval, product_accumulator(eltype(A), eltype(B)), total) + if total > 0 + spgemm_expand_kernel(get_backend(V))(I, J, V, offsets, Arow, Aind, Aval, Bptr, Bind, + Bval; ndrange=total) + end + free_buffer!(offsets) + free_buffer!(Arow) + fmt = A isa GPUSparseMatrixCSC ? :csc : A isa GPUSparseMatrixCSR ? :csr : :coo + C = assemble_matrix(I, J, V, m, n, +, fmt; pointer_type=Tp) + return eltype(C) == T ? C : with_values(C, T) +end + +# a sparse operand without a lazy wrapper +function materialize(S::Symmetric{<:Any,<:GPUSparseMatrix}) + A = parent(S) + T = S.uplo == 'U' ? triu(A) : tril(A) + strict = S.uplo == 'U' ? triu(A, 1) : tril(A, -1) + return T .+ copy(transpose(strict)) +end +function materialize(H::Hermitian{<:Any,<:GPUSparseMatrix}) + A = parent(H) + strict = H.uplo == 'U' ? triu(A, 1) : tril(A, -1) + return strict .+ copy(adjoint(strict)) .+ real.(band(A, 0, 0)) +end + +Base.:(*)(A::GPUSparseMatrixOperand, B::GPUSparseMatrixOperand) = + spgemm(materialize(A), materialize(B)) + +""" + GPUArrays.generic_spgemm!(C, tA, tB, A, B, α, β) + +Compute `C = α op(A) op(B) + β C` for sparse matrices, where `C` gets the union of the +structures of both terms (that of the product if `β` is zero). The buffers of `C` are +resized, so `C` must not share them with another array. The generic implementation behind +the sparse-output `mul!`. +""" +function generic_spgemm!(C::GPUSparseMatrix, tA::AbstractChar, tB::AbstractChar, + A::GPUSparseMatrix, B::GPUSparseMatrix, α::Number, β::Number) + P = generic_spgemm(materialize(LinearAlgebra.wrap(A, tA)), + materialize(LinearAlgebra.wrap(B, tB))) + size(P) == size(C) || + throw(DimensionMismatch("C has dimensions $(size(C)), needs $(size(P))")) + R = iszero(β) ? α .* P : α .* P .+ β .* C + return copyto!(C, convert(matrix_format(C), R)) +end + +LinearAlgebra.mul!(C::GPUSparseMatrix, tA::AbstractChar, tB::AbstractChar, + A::GPUSparseMatrix, B::GPUSparseMatrix, α::Number, β::Number) = + spgemm!(C, tA, tB, A, B, α, β) +@static if VERSION < v"1.13.0-rc4" + LinearAlgebra.generic_matmatmul!(C::GPUSparseMatrix, tA::AbstractChar, tB::AbstractChar, + A::GPUSparseMatrix, B::GPUSparseMatrix, a::Number, b::Number) = + LinearAlgebra.mul!(C, tA, tB, A, B, a, b) + LinearAlgebra.generic_matmatmul!(C::GPUSparseMatrix, tA::AbstractChar, tB::AbstractChar, + A::GPUSparseMatrix, B::GPUSparseMatrix, + _add::MulAddMul=MulAddMul()) = + LinearAlgebra.mul!(C, tA, tB, A, B, _add.alpha, _add.beta) +end + + +## matrix exponential + +""" + exp(A::GPUSparseMatrix; threshold=1e-7, nonzero_tol=1e-14) + +The matrix exponential of a square sparse matrix, by scaling and squaring a truncated +Taylor series, dropping entries below `nonzero_tol` as it goes. The series is truncated +when a term's largest magnitude falls below `threshold`. +""" +function LinearAlgebra.exp(A::GPUSparseMatrix; threshold=1e-7, nonzero_tol=1e-14) + n = LinearAlgebra.checksquare(A) + A = float(A) + T = eltype(A) + # scale A to norm at most 1 by a power of two + scaling = max(one(real(T)), nextpow(2, max(norm(A, Inf), floatmin(real(T))))) + A = A ./ T(scaling) + Id = spdiagm(0 => fill!(sparse_similar(nonzeros(A), n), one(T))) + P = convert(matrix_format(A), Id) + term = P + k = 1 + while true + term = (T(1 / k) * A) * term + droptol!(term, nonzero_tol) + P = P + term + norm(term, Inf) > threshold || break + k += 1 + end + for _ in 1:Int(log2(scaling)) + P = P * P + if nnz(P) / length(P) < 0.25 + droptol!(P, nonzero_tol) + end + end + return P +end diff --git a/src/host/sparse/primitives.jl b/src/host/sparse/primitives.jl new file mode 100644 index 000000000..108e30903 --- /dev/null +++ b/src/host/sparse/primitives.jl @@ -0,0 +1,161 @@ +# sparse-specific primitives, as internal kernels +# +# These build on the Base verbs a back-end provides for its dense arrays (`sortperm`, +# `accumulate!`, `findall`, `mapreduce`, and gathers through `getindex`). They are +# candidates for AcceleratedKernels once proven. +# +# The sparse code never launches a kernel over an empty range, because not every back-end +# supports that (JuliaGPU/Metal.jl#980). + +# release a temporary buffer early, without waiting for the GC +free_buffer!(x::AbstractGPUArray) = unsafe_free!(x) +free_buffer!(x) = nothing + +## COV_EXCL_START +@kernel function expand_ptr_kernel(major, ptr) + k = @index(Global, Linear) + if k <= length(major) + @inbounds major[k] = searchsortedlast(ptr, k % eltype(ptr)) % eltype(major) + end +end + +@kernel function compress_kernel(ptr, major) + j = @index(Global, Linear) + if j <= length(ptr) + @inbounds ptr[j] = searchsortedfirst(major, j) % eltype(ptr) + end +end + +@kernel function packed_key_kernel(keys, major, minor, nminor) + k = @index(Global, Linear) + if k <= length(keys) + @inbounds keys[k] = (Int64(major[k]) - 1) * nminor + Int64(minor[k]) + end +end +## COV_EXCL_STOP + +""" + expand_ptr(ptr, nnz) + +The slice index of each of the `nnz` stored entries of a compressed pointer buffer `ptr` +(e.g. the row of every entry of a CSR matrix), with the index type and storage of `ptr`. +Every stored entry is handled by its own thread, which finds its slice by binary search, +so that long slices do not serialize. +""" +function expand_ptr(ptr::AbstractVector, nnz::Integer, ::Type{Ti}=eltype(ptr)) where {Ti} + check_sparse_dims(Ti, (length(ptr) - 1,)) + major = sparse_similar(ptr, Ti, nnz) + if nnz > 0 + expand_ptr_kernel(get_backend(ptr))(major, ptr; ndrange=nnz) + end + return major +end + +""" + compress(major, n) + +The compressed pointer buffer, of length `n+1`, for stored entries whose sorted slice +indices are `major`: the inverse of [`expand_ptr`](@ref). Every pointer is found by binary +search; an atomic histogram followed by a scan would avoid the searches. +""" +function compress(major::AbstractVector, n::Integer, ::Type{Tp}=promote_type(Int, eltype(major))) where {Tp} + check_sparse_nnz(Tp, length(major)) + ptr = sparse_similar(major, Tp, Base.checked_add(n, 1)) + compress_kernel(get_backend(ptr))(ptr, major; ndrange=length(ptr)) + return ptr +end + +# Sort keys ordering entries by (major, minor) index. They are packed into a signed +# `Int64`, which fits for all practical sizes and which every back-end can sort on the +# device (Metal's `sortperm` does not support unsigned integers). +function packed_keys(major::AbstractVector, minor::AbstractVector, dims::Dims{2}) + nmajor, nminor = dims + if widemul(nmajor, nminor) > typemax(Int64) + # Tuple comparison is lexicographic and avoids multiplying the dimensions. + keys = sparse_similar(major, Tuple{eltype(major),eltype(minor)}, length(major)) + keys .= tuple.(major, minor) + return keys + end + keys = sparse_similar(major, Int64, length(major)) + if !isempty(keys) + packed_key_kernel(get_backend(keys))(keys, major, minor, Int64(nminor); + ndrange=length(keys)) + end + return keys +end + +# Regroup the stored entries of a compressed matrix along its other dimension: the CSC +# buffers of a CSR matrix, and vice versa. `dims` is (major, minor) of the input. +# +# The entries are sorted by (minor, major) and gathered. An atomic counting sort over the +# minor indices (histogram, scan, scatter) would be much faster than the general sort. +function regroup(ptr::AbstractVector{Tp}, ind::AbstractVector{Ti}, val::AbstractVector, + dims::Dims{2}) where {Tp,Ti} + nmajor, nminor = dims + n = length(val) + n == 0 && return empty_ptr(ptr, Tp, nminor), sparse_similar(ind, 0), sparse_similar(val, 0) + + major = expand_ptr(ptr, n, Ti) + keys = packed_keys(ind, major, (nminor, nmajor)) + perm = sortperm(keys) + free_buffer!(keys) + new_major = ind[perm] + new_ind = major[perm] + new_val = val[perm] + free_buffer!(major) + free_buffer!(perm) + new_ptr = compress(new_major, nminor, Tp) + free_buffer!(new_major) + return new_ptr, new_ind, new_val +end + +## COV_EXCL_START +@kernel function run_heads_kernel(heads, keys) + k = @index(Global, Linear) + if k <= length(keys) + @inbounds heads[k] = k == 1 || keys[k] != keys[k-1] + end +end + +# every thread folds one run of equal keys, in order +@kernel function reduce_runs_kernel(out_keys, out_vals, op, starts, keys, vals) + r = @index(Global, Linear) + if r <= length(starts) + first = @inbounds starts[r] + last = r == length(starts) ? length(vals) : @inbounds(starts[r+1]) - 1 + acc = @inbounds vals[first] + for k in first+1:last + acc = convert(eltype(out_vals), op(acc, @inbounds vals[k])) + end + @inbounds out_keys[r] = keys[first] + @inbounds out_vals[r] = acc + end +end +## COV_EXCL_STOP + +# whether each of the sorted `keys` starts a run of equal keys +function run_heads(keys::AbstractVector) + heads = sparse_similar(keys, Bool) + isempty(keys) || run_heads_kernel(get_backend(keys))(heads, keys; ndrange=length(keys)) + return heads +end + +""" + reduce_by_key(op, keys, vals) + +Combine the values of every run of equal, sorted `keys` with `op`, folding each run from +left to right. Returns the unique keys and the combined values. Every run is folded by +one thread, so many repetitions of a key serialize; a tiled segmented scan would not. +""" +function reduce_by_key(op, keys::AbstractVector, vals::AbstractVector) + starts = findall(run_heads(keys)) + n = length(starts) + out_keys = sparse_similar(keys, n) + out_vals = sparse_similar(vals, n) + if n > 0 + reduce_runs_kernel(get_backend(vals))(out_keys, out_vals, op, starts, keys, vals; + ndrange=n) + end + free_buffer!(starts) + return out_keys, out_vals +end diff --git a/src/host/sparse/reductions.jl b/src/host/sparse/reductions.jl new file mode 100644 index 000000000..cc906fa41 --- /dev/null +++ b/src/host/sparse/reductions.jl @@ -0,0 +1,175 @@ +# reductions + +# `op` folded over `n ≥ 1` copies of `x`, as the implicit zeros of a sparse array contribute +# to a reduction: by repeated squaring for an associative `op` (e.g. `n * x` for `+`), and a +# single `x` for idempotent ones. +@inline function _fold_repeated(op, x, n::Integer) + _isidempotent(op) && return x + result = x + power = x + n -= one(n) + while n > 0 + if isodd(n) + result = op(result, power) + end + power = op(power, power) + n >>= 1 + end + return result +end +_isidempotent(op) = false +_isidempotent(::Union{typeof(max), typeof(min), typeof(|), typeof(&)}) = true + +## COV_EXCL_START +# reduce the stored values `nzVal[first_ptr:last_ptr]` of a slice that also contains +# `nzeros` implicit zeros, and has at least one element. As in SparseArrays, the stored +# values are reduced first and the implicit zeros folded in afterwards, so `op` is assumed +# to be commutative as well as associative. +@inline function _reduce_slice(f, op, ::Type{T}, nzVal, first_ptr, last_ptr, nzeros) where {T} + if first_ptr <= last_ptr + val = convert(T, f(@inbounds nzVal[first_ptr])) + for ptr in first_ptr+one(first_ptr):last_ptr + val = convert(T, op(val, f(@inbounds nzVal[ptr]))) + end + if nzeros > 0 + val = convert(T, op(val, _fold_repeated(op, convert(T, f(zero(eltype(nzVal)))), nzeros))) + end + val + else + _fold_repeated(op, convert(T, f(zero(eltype(nzVal)))), nzeros) + end +end + +# every thread reduces one slice (a row of a CSR matrix, a column of a CSC one), which has +# `n` elements +@kernel function compressed_reduce_kernel(f::F, op::OP, output::AbstractArray{T}, ptr, nzVal, + n) where {F, OP, T} + j = @index(Global, Linear) + if j ≤ length(output) + first_ptr = @inbounds ptr[j] + last_ptr = @inbounds ptr[j+1] - one(first_ptr) + nzeros = n - (last_ptr - first_ptr + 1) + @inbounds output[j] = _reduce_slice(f, op, T, nzVal, first_ptr, last_ptr, nzeros) + end +end +## COV_EXCL_STOP + +# the element type of a reduction, relying on inference to reason through the map and +# reduce functions, or on the type of the initializer +function reduction_eltype(f, op, A, init) + ET = Broadcast.combine_eltypes(f, (A,)) + ET = Base.promote_op(op, ET, ET) + if init !== nothing + ET = Base.promote_op(op, typeof(init), ET) + end + (ET === Union{} || ET === Any) && + error("mapreduce cannot figure the output element type, please pass an explicit init value") + return ET +end + +# every slice reduction `r` becomes `op(init, r)`; empty slices give `init`, or the +# reduction of an empty collection as in Base +empty_reduction(f, op, ::Type{Tv}, init) where {Tv} = + init === nothing ? Base.mapreduce_empty(f, op, Tv) : init + +# reduce the stored values, then fold in the implicit zeros +function reduce_all(f, op, A::GPUSparseArray, init) + Tv = eltype(A) + length(A) == 0 && return empty_reduction(f, op, Tv, init) + ET = reduction_eltype(f, op, A, init) + nzeros = length(A) - nnz(A) + zeros = nzeros > 0 ? _fold_repeated(op, convert(ET, f(zero(Tv))), nzeros) : nothing + if nnz(A) > 0 + stored = init === nothing ? mapreduce(f, op, nonzeros(A)) : + mapreduce(f, op, nonzeros(A); init) + return zeros === nothing ? stored : op(stored, zeros) + else + return init === nothing ? zeros : op(init, zeros) + end +end + +function Base.mapreduce(f, op, x::GPUSparseVector; dims=:, init=nothing) + dims === Colon() && return reduce_all(f, op, x, init) + dims == 1 || error("only dims=: or dims=1 is supported for sparse vectors") + r = reduce_all(f, op, x, init) + return fill!(similar(x.nzVal, typeof(r), 1), r) +end + +function Base.mapreduce(f, op, A::GPUSparseMatrix; dims=:, init=nothing) + dims === Colon() && return reduce_all(f, op, A, init) + dims in (1, 2) || error("only dims=:, dims=1 or dims=2 is supported") + + # reduce along the compressed dimension, regrouping the entries if necessary. an atomic + # scatter could reduce along the other dimension directly, for `+` and friends. + B = dims == 1 ? GPUSparseMatrixCSC(A) : GPUSparseMatrixCSR(A) + ET = reduction_eltype(f, op, A, init) + output = similar(nonzeros(B), ET, dims == 1 ? (1, size(B, 2)) : (size(B, 1), 1)) + isempty(output) && return output + if size(B, dims) == 0 + return fill!(output, empty_reduction(f, op, eltype(A), init)) + end + compressed_reduce_kernel(get_backend(B))(f, op, output, compressed_ptr(B), nonzeros(B), + minor_dim(B); ndrange=length(output)) + if init !== nothing + output .= op.(init, output) + end + return output +end + +# `nnz(A) == 0` means nothing is stored, otherwise check the stored values +Base.iszero(A::GPUSparseArray) = nnz(A) == 0 || all(iszero, nonzeros(A)) + +# Implicit zeros participate in negative-order norms; min preserves a NaN magnitude. +function LinearAlgebra.norm(A::GPUSparseArray, p::Real=2) + r = norm(nonzeros(A), p) + return p < 0 && nnz(A) < length(A) ? min(r, zero(r)) : r +end + +function LinearAlgebra.opnorm(A::GPUSparseMatrix, p::Real=2) + if p == Inf + return maximum(sum(abs, A; dims=2)) + elseif p == 1 + return maximum(sum(abs, A; dims=1)) + else + throw(ArgumentError("p=$p is not supported")) + end +end + +# `f` of the stored values, and of the implicit zeros if there are any +function Base.count(f, A::GPUSparseArray) + n = nnz(A) == 0 ? 0 : count(f, nonzeros(A)) + return nnz(A) < length(A) && f(zero(eltype(A))) ? n + length(A) - nnz(A) : n +end +# (separate methods for functions, to be more specific than Base's) +for F in (:Function, :Any) + @eval begin + Base.any(f::$F, A::GPUSparseArray) = + (nnz(A) > 0 && any(f, nonzeros(A))) || (nnz(A) < length(A) && f(zero(eltype(A)))) + Base.all(f::$F, A::GPUSparseArray) = + (nnz(A) == 0 || all(f, nonzeros(A))) && (nnz(A) == length(A) || f(zero(eltype(A)))) + end +end +Base.any(A::GPUSparseArray) = any(identity, A) +Base.all(A::GPUSparseArray) = all(identity, A) + + +## dot products + +function LinearAlgebra.dot(x::GPUSparseVector, y::GPUSparseVector) + length(x) == length(y) || + throw(DimensionMismatch("x has length $(length(x)), y has length $(length(y))")) + # `+` rather than `sum`, which would widen small integers + return reduce(+, conj.(x) .* y) +end +function LinearAlgebra.dot(x::GPUSparseVector, y::AnyGPUVector) + length(x) == length(y) || + throw(DimensionMismatch("x has length $(length(x)), y has length $(length(y))")) + T = typeof(dot(zero(eltype(x)), zero(eltype(y)))) + nnz(x) == 0 && return zero(T) + return mapreduce((a, b) -> dot(a, b), +, x.nzVal, y[x.nzInd]) +end +LinearAlgebra.dot(y::AnyGPUVector, x::GPUSparseVector) = conj(dot(x, y)) +function LinearAlgebra.dot(x::AnyGPUVector,A::GPUSparseMatrix,y::AnyGPUVector) + length(x)==size(A,1) && size(A,2)==length(y) || throw(DimensionMismatch("dimensions must match")) + dot(x,A*y) +end diff --git a/src/host/sparse/selection.jl b/src/host/sparse/selection.jl new file mode 100644 index 000000000..317bba2a6 --- /dev/null +++ b/src/host/sparse/selection.jl @@ -0,0 +1,134 @@ +"Automatic sparse algorithm selection, including supported vendor implementations." +struct Automatic end +"Portable sparse execution, including all subordinate sparse conversions." +struct Portable end +"Require vendor execution; reject unsupported inputs before modifying the destination." +struct Vendor end + +struct UnsupportedSparseVendor + reason::String +end + +# Extensions return an already selected executor, with any vendor algorithm encoded in it. +# They must check types, both index widths, layout, placement and library capabilities. +vendor_spmv(y, tA, A, x, α, β) = UnsupportedSparseVendor("no supported sparse vendor extension for these operands") +vendor_spmm(C, tA, tB, A, B, α, β) = UnsupportedSparseVendor("no supported sparse vendor extension for these operands") +vendor_spgemm(A, B) = UnsupportedSparseVendor("no supported sparse vendor product for these operands") + +select_sparse(::Portable, candidate) = Portable() +select_sparse(::Automatic, candidate::UnsupportedSparseVendor) = Portable() +select_sparse(::Automatic, candidate) = candidate +select_sparse(::Vendor, candidate::UnsupportedSparseVendor) = throw(ArgumentError(candidate.reason)) +select_sparse(::Vendor, candidate) = candidate + +function check_sparse_product(C, tA, tB, A, B) + operand_terms(tA); operand_terms(tB) + tA in ('S', 's', 'H', 'h') && LinearAlgebra.checksquare(A) + tB in ('S', 's', 'H', 'h') && LinearAlgebra.checksquare(B) + check_product_dims(size(C), op_size(A, tA), op_size(B, tB)) + check_storage_compatibility(map(storage_parent, (C, product_buffers(A)..., product_buffers(B)...))...) + for input in (A, B) + any(buffer -> Base.mightalias(C,buffer),product_buffers(input)) && + throw(ArgumentError("sparse mul! requires a destination independent of its inputs")) + end + return nothing +end + +product_buffers(A::GPUSparseArray) = sparse_buffers(A) +product_buffers(A::AbstractGPUSparseArray) = (nonzeros(A),) +product_buffers(A::AbstractArray) = (A,) +normalize_sparse_product_operand(A,t,algorithm) = (A,t) + +const GPUSparseRootOperand = Union{AbstractGPUSparseMatrix, + Transpose{<:Any,<:AbstractGPUSparseMatrix},Adjoint{<:Any,<:AbstractGPUSparseMatrix}} +# Preserve the wrapper's chosen triangle. LinearAlgebra's two-argument fallback can +# wrap an already triangular diagonal operand again and miss the vendor executor. +LinearAlgebra.ldiv!(A::LinearAlgebra.UpperOrLowerTriangular{<:Any,<:GPUSparseRootOperand}, + B::Union{AnyGPUVector,AnyGPUMatrix}) = LinearAlgebra._ldiv!(B,A,B) + +""" + GPUArrays.spmv!(y, tA, A, x, α, β; algorithm=GPUArrays.Automatic()) + +Compute `y = α op(A) x + β y`. `tA` is a LinearAlgebra operand character. +Use `Portable()` to bypass vendor execution, or `Vendor()` to require it. +Validation and selection finish before output writes. An execution error is propagated; +it never triggers a retry against a potentially modified output. +""" +function spmv!(y, tA::AbstractChar, A::AbstractGPUSparseMatrix, x, α::Number, β::Number; + algorithm=Automatic()) + check_sparse_product(y, tA, 'N', A, x) + A,tA = normalize_sparse_product_operand(A,tA,algorithm) + executor = algorithm isa Portable ? Portable() : + select_sparse(algorithm, vendor_spmv(y, tA, A, x, α, β)) + return execute_spmv!(executor, y, tA, A, x, α, β) +end + +""" + GPUArrays.spmm!(C, tA, tB, A, B, α, β; algorithm=GPUArrays.Automatic()) + +Sparse × dense or dense × sparse matrix multiplication. Selection has the same contract +as [`spmv!`](@ref). Portable executors call portable conversions directly; they never +reenter selection for the same operation or a subordinate sparse conversion. +""" +function spmm!(C, tA::AbstractChar, tB::AbstractChar, A, B, α::Number, β::Number; + algorithm=Automatic()) + check_sparse_product(C, tA, tB, A, B) + A,tA = normalize_sparse_product_operand(A,tA,algorithm) + B,tB = normalize_sparse_product_operand(B,tB,algorithm) + executor = algorithm isa Portable ? Portable() : + select_sparse(algorithm, vendor_spmm(C, tA, tB, A, B, α, β)) + return execute_spmm!(executor, C, tA, tB, A, B, α, β) +end + +function execute_spmv!(::Portable, y,tA,A,x,α,β) + A isa GPUSparseMatrix || throw(ArgumentError("portable SpMV is not implemented for this sparse format")) + generic_spmv!(y,tA,A,x,α,β) +end +function execute_spmm!(::Portable,C,tA,tB,A,B,α,β) + A isa GPUSparseMatrix || B isa GPUSparseMatrix || + throw(ArgumentError("portable SpMM is not implemented for this sparse format")) + generic_spmm!(C,tA,tB,A,B,α,β) +end + +""" + GPUArrays.spgemm(A, B; algorithm=GPUArrays.Automatic(), max_temp_bytes=1<<30) + +Sparse matrix product with central selection. The portable ESC executor rejects its +temporary-memory estimate above `max_temp_bytes`; vendor executors manage their own +workspace. Both integer widths of the output follow the input-promotion contract. +""" +function spgemm(A::GPUSparseMatrix,B::GPUSparseMatrix;algorithm=Automatic(),max_temp_bytes::Integer=1<<30) + size(A,2) == size(B,1) || throw(DimensionMismatch("incompatible sparse product dimensions")) + check_storage_compatibility(map(storage_parent,(sparse_buffers(A)...,sparse_buffers(B)...))...) + executor = algorithm isa Portable ? Portable() : select_sparse(algorithm,vendor_spgemm(A,B)) + execute_spgemm(executor,A,B;max_temp_bytes) +end +execute_spgemm(::Portable,A,B;max_temp_bytes) = generic_spgemm(A,B;max_temp_bytes) + +function spgemm!(C::GPUSparseMatrix,tA::AbstractChar,tB::AbstractChar,A::GPUSparseMatrix,B::GPUSparseMatrix, + α::Number,β::Number;algorithm=Automatic(),max_temp_bytes::Integer=1<<30) + check_product_dims(size(C),op_size(A,tA),op_size(B,tB)) + check_sparse_mutation(C) + P = spgemm(materialize(LinearAlgebra.wrap(A,tA)),materialize(LinearAlgebra.wrap(B,tB)); + algorithm,max_temp_bytes) + R = iszero(β) ? α .* P : α .* P .+ β .* C + copyto!(C,convert(matrix_format(C),R)) +end + +# Open sparse roots can participate through a selected vendor executor without inheriting +# traversals that only understand GPUArrays' canonical layouts. +LinearAlgebra.mul!(y::AnyGPUVector,tA::AbstractChar,A::AbstractGPUSparseMatrix,x::AnyGPUVector,α::Number,β::Number) = + spmv!(y,tA,A,x,α,β) +LinearAlgebra.mul!(C::AnyGPUMatrix,tA::AbstractChar,tB::AbstractChar,A::AbstractGPUSparseMatrix,B::AnyGPUMatrix,α::Number,β::Number) = + spmm!(C,tA,tB,A,B,α,β) + +@static if VERSION < v"1.13.0-rc4" + LinearAlgebra.generic_matvecmul!(C::AnyGPUVector,tA::AbstractChar,A::AbstractGPUSparseMatrix,B::AnyGPUVector,a::Number,b::Number) = + LinearAlgebra.mul!(C,tA,A,B,a,b) + LinearAlgebra.generic_matvecmul!(C::AnyGPUVector,tA::AbstractChar,A::AbstractGPUSparseMatrix,B::AnyGPUVector,_add::MulAddMul=MulAddMul()) = + LinearAlgebra.mul!(C,tA,A,B,_add.alpha,_add.beta) + LinearAlgebra.generic_matmatmul!(C::AnyGPUMatrix,tA::AbstractChar,tB::AbstractChar,A::AbstractGPUSparseMatrix,B::AnyGPUMatrix,a::Number,b::Number) = + LinearAlgebra.mul!(C,tA,tB,A,B,a,b) + LinearAlgebra.generic_matmatmul!(C::AnyGPUMatrix,tA::AbstractChar,tB::AbstractChar,A::AbstractGPUSparseMatrix,B::AnyGPUMatrix,_add::MulAddMul=MulAddMul()) = + LinearAlgebra.mul!(C,tA,tB,A,B,_add.alpha,_add.beta) +end diff --git a/src/host/sparse/show.jl b/src/host/sparse/show.jl new file mode 100644 index 000000000..f4150715f --- /dev/null +++ b/src/host/sparse/show.jl @@ -0,0 +1,37 @@ +# printing, through a copy on the host + +# the name of the array type that stores the entries: "MtlArray", "CuArray", ... +storage_name(A) = nameof(typeof(A.nzVal)) + +# the concrete type is long and not what users write (they use the back-end aliases), so +# only show the format and the element and index types +function Base.summary(io::IO, A::GPUSparseArray) + print(io, Base.dims2string(size(A)), " ", nameof(typeof(A)), "{", eltype(A), ", ", + indtype(A), "} with ", nnz(A), " stored ", nnz(A) == 1 ? "entry" : "entries", + " in ", storage_name(A)) +end + +# the device-side representation, e.g. in an error message about a kernel argument, cannot +# be copied to the host +on_device(A) = A.nzVal isa AbstractDeviceArray + +Base.print_array(io::IO, A::GPUSparseMatrix) = + on_device(A) ? nothing : Base.print_array(io, SparseMatrixCSC(A)) + +Base.show(io::IO, A::GPUSparseMatrix) = + on_device(A) ? summary(io, A) : show(io, SparseMatrixCSC(A)) +Base.show(io::IO, x::GPUSparseVector) = + on_device(x) ? summary(io, x) : show(io, SparseVector(x)) +# (SparseArrays 1.10 has a method for this one) +Base.show(io::IOContext, x::GPUSparseVector) = + on_device(x) ? summary(io, x) : show(io, SparseVector(x)) + +# SparseArrays prints the stored entries of a vector as part of its `show` method, below +# a summary that we replace by our own +function Base.show(io::IO, mime::MIME"text/plain", x::GPUSparseVector) + summary(io, x) + (nnz(x) == 0 || on_device(x)) && return + println(io, ":") + entries = sprint(show, mime, SparseVector(x); context=io) + print(io, split(entries, '\n'; limit=2)[2]) +end diff --git a/src/host/sparse/structure.jl b/src/host/sparse/structure.jl new file mode 100644 index 000000000..f1c691a7f --- /dev/null +++ b/src/host/sparse/structure.jl @@ -0,0 +1,339 @@ +# structural operations + +# the row and column of every stored entry, borrowing the index buffers of `A` +entry_coordinates(A::GPUSparseMatrixCSR) = expand_ptr(A.rowPtr, nnz(A), indtype(A)), A.colVal +entry_coordinates(A::GPUSparseMatrixCSC) = A.rowVal, expand_ptr(A.colPtr, nnz(A), indtype(A)) +entry_coordinates(A::GPUSparseMatrixCOO) = A.rowInd, A.colInd + +## COV_EXCL_START +@kernel function keep_matrix_entries_kernel(keep, f, rows, cols, vals) + k = @index(Global, Linear) + if k <= length(keep) + @inbounds keep[k] = f(rows[k], cols[k], vals[k]) + end +end + +@kernel function keep_vector_entries_kernel(keep, f, inds, vals) + k = @index(Global, Linear) + if k <= length(keep) + @inbounds keep[k] = f(inds[k], vals[k]) + end +end +## COV_EXCL_STOP + +# replace the contents of `dst` by `src`, which may have a different length +function replace_buffer!(dst::AbstractVector, src::AbstractVector) + length(dst) == length(src) || resize!(dst, length(src)) + copyto!(dst, src) + return dst +end + +""" + fkeep!(f, A::GPUSparseMatrix) + fkeep!(f, x::GPUSparseVector) + +Keep the stored entries of `A` for which `f(i, j, v)` (or `f(i, v)` for a vector) is true, +on the device, as in SparseArrays. `f` runs in a kernel. The buffers of `A` are resized, so +`A` must not share them with another array. +""" +function SparseArrays.fkeep!(f::F, A::GPUSparseMatrix) where {F} + check_sparse_mutation(A) + nnz(A) == 0 && return A + rows, cols = entry_coordinates(A) + keep = sparse_similar(A.nzVal, Bool) + keep_matrix_entries_kernel(get_backend(A))(keep, f, rows, cols, A.nzVal; ndrange=nnz(A)) + kept = findall(keep) + free_buffer!(keep) + new_rows, new_cols, new_vals = rows[kept], cols[kept], A.nzVal[kept] + free_buffer!(kept) + if A isa GPUSparseMatrixCSR + replace_buffer!(A.rowPtr, compress(new_rows, size(A, 1), eltype(A.rowPtr))) + replace_buffer!(A.colVal, new_cols) + elseif A isa GPUSparseMatrixCSC + replace_buffer!(A.colPtr, compress(new_cols, size(A, 2), eltype(A.colPtr))) + replace_buffer!(A.rowVal, new_rows) + else + replace_buffer!(A.rowInd, new_rows) + replace_buffer!(A.colInd, new_cols) + end + replace_buffer!(A.nzVal, new_vals) + return A +end +function SparseArrays.fkeep!(f::F, x::GPUSparseVector) where {F} + check_sparse_mutation(x) + nnz(x) == 0 && return x + keep = sparse_similar(x.nzVal, Bool) + keep_vector_entries_kernel(get_backend(x))(keep, f, x.nzInd, x.nzVal; ndrange=nnz(x)) + kept = findall(keep) + free_buffer!(keep) + new_inds, new_vals = x.nzInd[kept], x.nzVal[kept] + free_buffer!(kept) + replace_buffer!(x.nzInd, new_inds) + replace_buffer!(x.nzVal, new_vals) + return x +end + +# the tolerance is captured by value, so that the predicate can run in a kernel +struct AboveTolerance{T} + tol::T +end +(p::AboveTolerance)(i, j, x) = abs(x) > p.tol +(p::AboveTolerance)(i, x) = abs(x) > p.tol + +SparseArrays.dropzeros!(A::GPUSparseMatrix) = fkeep!((i, j, x) -> !iszero(x), A) +SparseArrays.dropzeros!(x::GPUSparseVector) = fkeep!((i, x) -> !iszero(x), x) +SparseArrays.dropzeros(A::GPUSparseArray) = dropzeros!(copy(A)) +# the tolerance is compared in the precision of the values, so that a Float64 tolerance +# (like the default of `exp`) does not end up in kernels on back-ends without Float64 +droptol_tolerance(::Type{T}, tol) where {T<:Union{AbstractFloat,Complex{<:AbstractFloat}}} = + convert(real(T), tol) +droptol_tolerance(::Type, tol) = tol +SparseArrays.droptol!(A::GPUSparseArray, tol::Real) = + fkeep!(AboveTolerance(droptol_tolerance(eltype(A), tol)), A) + + +## triangular parts + +struct BandSelection + lo::Int + hi::Int +end +(b::BandSelection)(i, j, x) = b.lo <= j - i <= b.hi + +const GPUSparseMatrixOrTransposed = + Union{GPUSparseMatrix, Transpose{<:Any,<:GPUSparseMatrix}, Adjoint{<:Any,<:GPUSparseMatrix}} + +# a copy of the entries of `A` whose diagonal index `j - i` lies within `lo:hi` +band(A::GPUSparseMatrix, lo, hi) = fkeep!(BandSelection(lo, hi), copy(A)) +band(A::Union{Transpose,Adjoint}, lo, hi) = fkeep!(BandSelection(lo, hi), copy(A)) + +LinearAlgebra.triu(A::GPUSparseMatrixOrTransposed, k::Integer=0) = band(A, k, typemax(Int)) +LinearAlgebra.tril(A::GPUSparseMatrixOrTransposed, k::Integer=0) = band(A, typemin(Int), k) + +## COV_EXCL_START +@kernel function outside_band_kernel(outside, rows, cols, vals, lo, hi) + k = @index(Global, Linear) + if k <= length(outside) + d = @inbounds cols[k] - rows[k] + @inbounds outside[k] = !iszero(vals[k]) && !(lo <= d <= hi) + end +end +## COV_EXCL_STOP + +# whether all nonzero values lie within the diagonals `lo:hi` +function within_band(A::GPUSparseMatrix, lo, hi) + nnz(A) == 0 && return true + rows, cols = entry_coordinates(A) + outside = sparse_similar(A.nzVal, Bool) + outside_band_kernel(get_backend(A))(outside, rows, cols, A.nzVal, lo, hi; ndrange=nnz(A)) + return !any(outside) +end +LinearAlgebra.istriu(A::GPUSparseMatrix, k::Integer=0) = within_band(A, k, typemax(Int)) +LinearAlgebra.istril(A::GPUSparseMatrix, k::Integer=0) = within_band(A, typemin(Int), k) +LinearAlgebra.isdiag(A::GPUSparseMatrix) = within_band(A, 0, 0) + +# the wrapper determines where the nonzeros can be; LinearAlgebra's generic methods would +# index the parent (and are broken on some versions, JuliaLang/julia#55547) +const GPUSparseUpperOrUnitUpperTriangular = + LinearAlgebra.UpperOrUnitUpperTriangular{<:Any,<:GPUSparseMatrixOrTransposed} +const GPUSparseLowerOrUnitLowerTriangular = + LinearAlgebra.LowerOrUnitLowerTriangular{<:Any,<:GPUSparseMatrixOrTransposed} +LinearAlgebra.istriu(::GPUSparseUpperOrUnitUpperTriangular) = true +LinearAlgebra.istril(U::GPUSparseUpperOrUnitUpperTriangular) = + iszero(triu(materialize(parent(U)), 1)) +LinearAlgebra.istril(::GPUSparseLowerOrUnitLowerTriangular) = true +LinearAlgebra.istriu(L::GPUSparseLowerOrUnitLowerTriangular) = + iszero(tril(materialize(parent(L)), -1)) + +# `A` equals its transpose (or adjoint) when no element differs, which broadcasting over +# the union of both structures also checks for stored zeros on one side only +function LinearAlgebra.issymmetric(A::GPUSparseMatrix) + size(A, 1) == size(A, 2) || return false + return A == copy(transpose(A)) +end +function LinearAlgebra.ishermitian(A::GPUSparseMatrix) + size(A, 1) == size(A, 2) || return false + return A == copy(adjoint(A)) +end + + +## diagonals + +## COV_EXCL_START +# every thread looks up one element of the diagonal `k` in its slice +@kernel function diagonal_kernel(d, ptr, ind, val, k, csr::Bool) + t = @index(Global, Linear) + if t <= length(d) + # the diagonal element in row/column `major`, at column/row `minor` + i, j = k >= 0 ? (t, t + k) : (t - k, t) + major, minor = csr ? (i, j) : (j, i) + lo = @inbounds ptr[major] + hi = @inbounds ptr[major+1] - one(lo) + p = searchsortedfirst(ind, minor % eltype(ind), lo, hi, Base.Order.Forward) + @inbounds d[t] = (p <= hi && ind[p] == minor) ? val[p] : zero(eltype(d)) + end +end +## COV_EXCL_STOP + +function LinearAlgebra.diag(A::GPUSparseMatrix, k::Integer=0) + m, n = size(A) + k=Int(k) + (k<=-m || k>=n) && return sparse_similar(A.nzVal,0) + len = k >= 0 ? min(m, n - k) : min(m + k, n) + d = sparse_similar(A.nzVal, len) + len == 0 && return d + B = A isa GPUSparseMatrixCOO ? csr_view(A) : A + diagonal_kernel(get_backend(B))(d, compressed_ptr(B), compressed_ind(B), B.nzVal, k, + B isa GPUSparseMatrixCSR; ndrange=len) + return d +end + +function LinearAlgebra.diag(A::Transpose{<:Any,<:GPUSparseMatrix},k::Integer=0) + k=Int(k) + k==typemin(Int) ? sparse_similar(nonzeros(parent(A)),0) : diag(parent(A),-k) +end +function LinearAlgebra.diag(A::Adjoint{<:Any,<:GPUSparseMatrix},k::Integer=0) + k=Int(k) + k==typemin(Int) ? sparse_similar(nonzeros(parent(A)),0) : conj.(diag(parent(A),-k)) +end + +function LinearAlgebra.tr(A::GPUSparseMatrix) + LinearAlgebra.checksquare(A) + # `tr` folds with +, unlike sum's widening add_sum for narrow integers. + return mapreduce(identity,+,diag(A);init=zero(eltype(A))) +end + + +## reshaping + +## COV_EXCL_START +@kernel function reshape_kernel(rows, cols, m, m′) + k = @index(Global, Linear) + if k <= length(rows) + l = (Int64(cols[k]) - 1) * m + Int64(rows[k]) - 1 + @inbounds rows[k] = (l % m′ + 1) % eltype(rows) + @inbounds cols[k] = (l ÷ m′ + 1) % eltype(cols) + end +end + +@kernel function linear_index_kernel(inds, rows, cols, m) + k = @index(Global, Linear) + if k <= length(inds) + @inbounds inds[k] = ((Int64(cols[k]) - 1) * m + rows[k]) % eltype(inds) + end +end +## COV_EXCL_STOP + +# Column-major order of the entries (that of CSC) is the order of their linear indices, so +# it is preserved by any reshape: new coordinates, in the same order, form the CSC buffers +# of the result. +function Base.reshape(A::GPUSparseMatrix, dims::Dims{2}) + foldl(Base.checked_mul, dims; init=1) == length(A) || + throw(DimensionMismatch("new dimensions $dims must be consistent with array size $(length(A))")) + dims == size(A) && return copy(A) + C = A isa GPUSparseMatrixCSC ? copy(A) : GPUSparseMatrixCSC(A) + rows = C.rowVal + cols = expand_ptr(C.colPtr, nnz(C), indtype(C)) + check_sparse_dims(indtype(A), dims) + if nnz(C) > 0 + reshape_kernel(get_backend(C))(rows, cols, size(A, 1), dims[1]; ndrange=nnz(C)) + end + B = GPUSparseMatrixCSC(compress(cols, dims[2], eltype(C.colPtr)), rows, C.nzVal, dims) + free_buffer!(cols) + return convert(matrix_format(A), B) +end +Base.reshape(A::GPUSparseMatrix, dims::Tuple{Union{Int,Colon},Union{Int,Colon}}) = + reshape(A, Base._reshape_uncolon(A, dims)) + +function Base.vec(A::GPUSparseMatrix) + C = A isa GPUSparseMatrixCSC ? A : GPUSparseMatrixCSC(A) + check_sparse_dims(indtype(A), (length(A),)) + inds = sparse_similar(C.rowVal, nnz(C)) + if nnz(C) > 0 + cols = expand_ptr(C.colPtr, nnz(C), indtype(C)) + linear_index_kernel(get_backend(C))(inds, C.rowVal, cols, size(C, 1); ndrange=nnz(C)) + free_buffer!(cols) + end + return GPUSparseVector(inds, C === A ? copy(C.nzVal) : C.nzVal, length(A)) +end + + +## Kronecker products + +## COV_EXCL_START +# every thread fills a column of `kron(A, B)`: the products of column `j1` of `A` and +# column `j2` of `B`, in column-major order +@kernel function kron_kernel(colPtr, rowVal, nzVal, Aptr, Aind, Aval, Bptr, Bind, Bval, m2, n2) + c = @index(Global, Linear) + if c < length(colPtr) + j1 = (c - 1) ÷ n2 + 1 + j2 = (c - 1) % n2 + 1 + out = @inbounds colPtr[c] + for p in @inbounds(Aptr[j1]):@inbounds(Aptr[j1+1] - one(eltype(Aptr))) + for q in @inbounds(Bptr[j2]):@inbounds(Bptr[j2+1] - one(eltype(Bptr))) + @inbounds rowVal[out] = ((Aind[p] - 1) * m2 + Bind[q]) % eltype(rowVal) + @inbounds nzVal[out] = Aval[p] * Bval[q] + out += one(out) + end + end + end +end + +@kernel function kron_counts_kernel(counts, Aptr, Bptr, n2) + c = @index(Global, Linear) + if c <= length(counts) + j1 = (c - 1) ÷ n2 + 1 + j2 = (c - 1) % n2 + 1 + @inbounds counts[c] = (Aptr[j1+1] - Aptr[j1]) * (Bptr[j2+1] - Bptr[j2]) + end +end +## COV_EXCL_STOP + +function generic_kron(A::GPUSparseMatrixCSC, B::GPUSparseMatrixCSC) + m1, n1 = size(A) + m2, n2 = size(B) + Ti = promote_type(indtype(A), indtype(B)) + Tv = Base.promote_op(*, eltype(A), eltype(B)) + all(<=(typemax(Int)), (widemul(m1, m2), widemul(n1, n2))) || + throw(ArgumentError("the Kronecker product of $(size(A)) and $(size(B)) matrices is too large")) + dims = (m1 * m2, n1 * n2) + check_sparse_dims(Ti, dims) + Tp = promote_type(eltype(A.colPtr), eltype(B.colPtr)) + check_sparse_nnz(Tp, widemul(nnz(A), nnz(B))) + n = nnz(A) * nnz(B) + colPtr = sparse_similar(A.colPtr, Tp, Base.checked_add(dims[2], 1)) + rowVal = sparse_similar(A.rowVal, Ti, n) + nzVal = sparse_similar(A.nzVal, Tv, n) + fill!(view(colPtr, 1:1), one(Tp)) + if dims[2] > 0 + kron_counts_kernel(get_backend(A))(view(colPtr, 2:dims[2]+1), A.colPtr, B.colPtr, n2; + ndrange=dims[2]) + accumulate!(Base.add_sum, colPtr, colPtr) + if n > 0 + kron_kernel(get_backend(A))(colPtr, rowVal, nzVal, A.colPtr, A.rowVal, A.nzVal, + B.colPtr, B.rowVal, B.nzVal, m2, n2; ndrange=dims[2]) + end + end + return GPUSparseMatrixCSC(colPtr, rowVal, nzVal, dims) +end + +# the result has the format of `A` +function LinearAlgebra.kron(A::GPUSparseMatrixOrTransposed, B::GPUSparseMatrixOrTransposed) + C = generic_kron(convert(GPUSparseMatrixCSC, materialize(A)), + convert(GPUSparseMatrixCSC, materialize(B))) + return convert(matrix_format(materialize_format(A)), C) +end +LinearAlgebra.kron(A::GPUSparseMatrixOrTransposed, D::Diagonal{<:Any,<:AnyGPUVector}) = + kron(A, spdiagm(D.diag)) +LinearAlgebra.kron(D::Diagonal{<:Any,<:AnyGPUVector}, B::GPUSparseMatrixOrTransposed) = + convert(matrix_format(materialize_format(B)), kron(spdiagm(D.diag), B)) +LinearAlgebra.kron(A::GPUSparseMatrixOrTransposed,D::Diagonal) = + kron(A,Diagonal(move_buffer(nonzeros(materialize_format(A)),D.diag))) +LinearAlgebra.kron(D::Diagonal,B::GPUSparseMatrixOrTransposed) = + kron(Diagonal(move_buffer(nonzeros(materialize_format(B)),D.diag)),B) + +# a sparse matrix without a lazy wrapper, and the format that has +materialize(A::GPUSparseMatrix) = A +materialize(A::Union{Transpose,Adjoint}) = copy(A) +materialize_format(A::GPUSparseMatrix) = A +materialize_format(A::Union{Transpose,Adjoint}) = parent(A) diff --git a/src/host/sparse/types.jl b/src/host/sparse/types.jl new file mode 100644 index 000000000..02ce3cabc --- /dev/null +++ b/src/host/sparse/types.jl @@ -0,0 +1,512 @@ +# GPU sparse array types + +using SparseArrays: indtype, getcolptr, nonzeroinds + +""" + AbstractGPUSparseArray{Tv,Ti,N} + +Supertype of the sparse arrays whose storage lives on a GPU. GPUArrays provides the formats +[`GPUSparseVector`](@ref), [`GPUSparseMatrixCSR`](@ref), [`GPUSparseMatrixCSC`](@ref) and +[`GPUSparseMatrixCOO`](@ref); back-ends only subtype this directly for formats that +GPUArrays does not implement (e.g. block-sparse formats of a vendor library). +""" +abstract type AbstractGPUSparseArray{Tv,Ti,N} <: AbstractSparseArray{Tv,Ti,N} end +const AbstractGPUSparseVector{Tv,Ti} = AbstractGPUSparseArray{Tv,Ti,1} +const AbstractGPUSparseMatrix{Tv,Ti} = AbstractGPUSparseArray{Tv,Ti,2} +const AbstractGPUSparseVecOrMat = Union{AbstractGPUSparseVector,AbstractGPUSparseMatrix} + +# the checks below only look at host metadata; structural invariants (sorted, unique and +# in-bounds indices, consistent pointers) are the caller's responsibility, as for +# SparseArrays' raw constructors. `check_structure` verifies them on the device. +function check_sparse_dims(::Type{Ti}, dims::Dims) where {Ti} + all(>=(0), dims) || + throw(ArgumentError("dimensions must be non-negative, got $dims")) + all(d -> d <= typemax(Ti), dims) || + throw(ArgumentError("dimensions $dims do not fit index type $Ti")) + return +end +function check_sparse_nnz(::Type{Ti}, nnz::Integer) where {Ti} + # pointers go up to nnz+1 + 0 <= nnz < typemax(Ti) && nnz < typemax(Int) || + throw(ArgumentError("$nnz stored entries do not fit index type $Ti")) + return +end +function check_sparse_entries(::Type{Ti}, inds::AbstractVector, nzVal::AbstractVector) where {Ti} + length(inds) == length(nzVal) || + throw(ArgumentError("index and value buffers must have the same length, got $(length(inds)) and $(length(nzVal))")) +end +function check_sparse_ptr(ptr::AbstractVector, n::Integer) + length(ptr) == Base.checked_add(n, 1) || + throw(ArgumentError("the pointer buffer must have length $(n + 1), got $(length(ptr))")) + return +end + +# Query only host GPU wrappers. Device representations are plain DenseArrays in modern +# backends and must be reconstructible without invoking host runtime APIs. +function check_sparse_storage(buffers...) + if any(b -> b isa AnyGPUArray, buffers) + all(b -> b isa AnyGPUArray, buffers) || + throw(ArgumentError("sparse component buffers must use compatible storage")) + check_storage_compatibility(map(storage_parent, buffers)...) + end + return nothing +end + +""" + GPUSparseMatrixCSR(rowPtr, colVal, nzVal, dims) + +A sparse matrix in compressed sparse row format, stored in the dense vectors `rowPtr` +(length `m+1`), `colVal` and `nzVal` (length `nnz`). The stored entries of row `i` are at +positions `rowPtr[i]:rowPtr[i+1]-1`, with their columns in `colVal` and their values in +`nzVal`. The columns in a row must be sorted and unique; explicitly stored zeros are +allowed. Raw constructors borrow the supplied buffers, which must have compatible +storage and device placement. Coordinate storage `Vi` and compressed-pointer storage +`Vp` may use different integer types and wrappers. The stored count plus one must fit +the pointer type; both dimensions must fit the coordinate type. + +The index and value buffers are typically GPU vectors, whose type is part of the matrix +type (`GPUSparseMatrixCSR{Tv,Ti,Vp,Vi,Vv}`), so that generic code allocates new buffers with +`similar` on the existing ones. Back-ends provide aliases for their storage, e.g. +`const MtlSparseMatrixCSR{Tv,Ti} = GPUSparseMatrixCSR{Tv,Ti,<:MtlVector{<:Integer},<:MtlVector{Ti},<:MtlVector{Tv}}`. + +The same struct is used on the device: `adapt` converts the buffers, so kernels can read +the fields of a sparse matrix passed as an argument. +""" +struct GPUSparseMatrixCSR{Tv,Ti<:Integer,Vp<:AbstractVector{<:Integer},Vi<:AbstractVector{Ti},Vv<:AbstractVector{Tv}} <: + AbstractGPUSparseMatrix{Tv,Ti} + rowPtr::Vp + colVal::Vi + nzVal::Vv + dims::Dims{2} + + function GPUSparseMatrixCSR{Tv,Ti,Vp,Vi,Vv}(rowPtr::Vp, colVal::Vi, nzVal::Vv, + dims::NTuple{2,Integer}) where {Tv,Ti,Vp,Vi,Vv} + dims = Dims{2}(dims) + check_sparse_dims(Ti, dims) + check_sparse_ptr(rowPtr, dims[1]) + check_sparse_entries(Ti, colVal, nzVal) + check_sparse_nnz(eltype(rowPtr), length(nzVal)) + check_sparse_storage(rowPtr, colVal, nzVal) + new{Tv,Ti,Vp,Vi,Vv}(rowPtr, colVal, nzVal, dims) + end +end + +""" + GPUSparseMatrixCSC(colPtr, rowVal, nzVal, dims) + +A sparse matrix in compressed sparse column format, the layout of `SparseMatrixCSC`: the +stored entries of column `j` are at positions `colPtr[j]:colPtr[j+1]-1`, with their rows +(sorted and unique) in `rowVal` and their values in `nzVal`. See +[`GPUSparseMatrixCSR`](@ref) for the storage type parameters. +""" +struct GPUSparseMatrixCSC{Tv,Ti<:Integer,Vp<:AbstractVector{<:Integer},Vi<:AbstractVector{Ti},Vv<:AbstractVector{Tv}} <: + AbstractGPUSparseMatrix{Tv,Ti} + colPtr::Vp + rowVal::Vi + nzVal::Vv + dims::Dims{2} + + function GPUSparseMatrixCSC{Tv,Ti,Vp,Vi,Vv}(colPtr::Vp, rowVal::Vi, nzVal::Vv, + dims::NTuple{2,Integer}) where {Tv,Ti,Vp,Vi,Vv} + dims = Dims{2}(dims) + check_sparse_dims(Ti, dims) + check_sparse_ptr(colPtr, dims[2]) + check_sparse_entries(Ti, rowVal, nzVal) + check_sparse_nnz(eltype(colPtr), length(nzVal)) + check_sparse_storage(colPtr, rowVal, nzVal) + new{Tv,Ti,Vp,Vi,Vv}(colPtr, rowVal, nzVal, dims) + end +end + +""" + GPUSparseMatrixCOO(rowInd, colInd, nzVal, dims) + +A sparse matrix in coordinate format: stored entry `k` is at `(rowInd[k], colInd[k])` and +has value `nzVal[k]`. The entries must be sorted in row-major order without duplicate +coordinates, which makes the format a CSR matrix with explicit row indices. Unordered +coordinates with duplicates are the input of `sparse(I, J, V, m, n, combine)` instead. +See [`GPUSparseMatrixCSR`](@ref) for the storage type parameters. +""" +struct GPUSparseMatrixCOO{Tv,Ti<:Integer,Vi<:AbstractVector{Ti},Vv<:AbstractVector{Tv}} <: + AbstractGPUSparseMatrix{Tv,Ti} + rowInd::Vi + colInd::Vi + nzVal::Vv + dims::Dims{2} + + function GPUSparseMatrixCOO{Tv,Ti,Vi,Vv}(rowInd::Vi, colInd::Vi, nzVal::Vv, + dims::NTuple{2,Integer}) where {Tv,Ti,Vi,Vv} + dims = Dims{2}(dims) + check_sparse_dims(Ti, dims) + check_sparse_entries(Ti, rowInd, nzVal) + check_sparse_entries(Ti, colInd, nzVal) + check_sparse_storage(rowInd, colInd, nzVal) + new{Tv,Ti,Vi,Vv}(rowInd, colInd, nzVal, dims) + end +end + +""" + GPUSparseVector(nzInd, nzVal, len) + +A sparse vector of length `len` with stored entries at the sorted, unique indices `nzInd` +and values `nzVal`, the layout of `SparseVector`. See [`GPUSparseMatrixCSR`](@ref) for the +storage type parameters. +""" +struct GPUSparseVector{Tv,Ti<:Integer,Vi<:AbstractVector{Ti},Vv<:AbstractVector{Tv}} <: + AbstractGPUSparseVector{Tv,Ti} + nzInd::Vi + nzVal::Vv + len::Int + + function GPUSparseVector{Tv,Ti,Vi,Vv}(nzInd::Vi, nzVal::Vv, len::Integer) where {Tv,Ti,Vi,Vv} + check_sparse_dims(Ti, (Int(len),)) + check_sparse_entries(Ti, nzInd, nzVal) + check_sparse_storage(nzInd, nzVal) + new{Tv,Ti,Vi,Vv}(nzInd, nzVal, len) + end +end + +for (S, args) in ((:GPUSparseMatrixCSR, (:rowPtr, :colVal)), + (:GPUSparseMatrixCSC, (:colPtr, :rowVal))) + @eval $S($(args[1])::Vp, $(args[2])::Vi, nzVal::Vv, dims::NTuple{2,Integer}) where + {Vp<:AbstractVector{<:Integer},Ti<:Integer,Vi<:AbstractVector{Ti},Tv,Vv<:AbstractVector{Tv}} = + $S{Tv,Ti,Vp,Vi,Vv}($(args[1]), $(args[2]), nzVal, dims) +end +for (S, args) in ((:GPUSparseMatrixCOO, (:rowInd, :colInd)),) + @eval $S($(args[1])::Vi, $(args[2])::Vi, nzVal::Vv, dims::NTuple{2,Integer}) where + {Ti<:Integer,Vi<:AbstractVector{Ti},Tv,Vv<:AbstractVector{Tv}} = + $S{Tv,Ti,Vi,Vv}($(args[1]), $(args[2]), nzVal, dims) +end +GPUSparseVector(nzInd::Vi, nzVal::Vv, len::Integer) where + {Ti<:Integer,Vi<:AbstractVector{Ti},Tv,Vv<:AbstractVector{Tv}} = + GPUSparseVector{Tv,Ti,Vi,Vv}(nzInd, nzVal, len) + +const GPUSparseMatrixCompressed = Union{GPUSparseMatrixCSR,GPUSparseMatrixCSC} +const GPUSparseMatrix = Union{GPUSparseMatrixCSR,GPUSparseMatrixCSC,GPUSparseMatrixCOO} +const GPUSparseArray = Union{GPUSparseVector,GPUSparseMatrix} + +# Dispatch compatibility only: these are canonical containers, not abstract extension +# points. Independent formats must subtype the dimensional AbstractGPUSparseArray root. +Base.@deprecate_binding AbstractGPUSparseMatrixCSR GPUSparseMatrixCSR false +Base.@deprecate_binding AbstractGPUSparseMatrixCSC GPUSparseMatrixCSC false +Base.@deprecate_binding AbstractGPUSparseMatrixCOO GPUSparseMatrixCOO false + + +## accessors + +Base.size(A::GPUSparseMatrix) = A.dims +Base.length(A::GPUSparseMatrix) = Base.checked_mul(size(A, 1), size(A, 2)) +Base.size(x::GPUSparseVector) = (x.len,) +Base.size(A::GPUSparseArray,d::Integer) = d<1 ? + throw(ArgumentError("dimension must be positive, got $d")) : d<=ndims(A) ? size(A)[d] : 1 + +SparseArrays.nnz(A::GPUSparseArray) = length(A.nzVal) +SparseArrays.nonzeros(A::GPUSparseArray) = A.nzVal + +SparseArrays.getcolptr(A::GPUSparseMatrixCSC) = A.colPtr +SparseArrays.rowvals(A::GPUSparseMatrixCSC) = A.rowVal +SparseArrays.nzrange(A::GPUSparseMatrixCSC, col::Integer) = + @inbounds A.colPtr[col]:(A.colPtr[col+1] - one(eltype(A.colPtr))) + +# A full-column view borrows the corresponding stored slice. These accessors also +# work after adaptation to device storage, without a separate sparse device type. +const GPUSparseColumnView = SubArray{<:Any,1,<:GPUSparseMatrixCSC, + <:Tuple{Base.Slice,Integer}} +SparseArrays.nzrange(v::GPUSparseColumnView) = nzrange(parent(v),last(parentindices(v))) +SparseArrays.nnz(v::GPUSparseColumnView) = length(nzrange(v)) +SparseArrays.nonzeros(v::GPUSparseColumnView) = view(nonzeros(parent(v)),nzrange(v)) +SparseArrays.rowvals(v::GPUSparseColumnView) = view(rowvals(parent(v)),nzrange(v)) +SparseArrays.nonzeroinds(v::GPUSparseColumnView) = rowvals(v) + +SparseArrays.nonzeroinds(x::GPUSparseVector) = x.nzInd +SparseArrays.rowvals(x::GPUSparseVector) = x.nzInd + +KernelAbstractions.get_backend(A::GPUSparseArray) = get_backend(A.nzVal) + +# the pointer and index buffers of a compressed matrix, whatever its orientation +compressed_ptr(A::GPUSparseMatrixCSR) = A.rowPtr +compressed_ptr(A::GPUSparseMatrixCSC) = A.colPtr +compressed_ind(A::GPUSparseMatrixCSR) = A.colVal +compressed_ind(A::GPUSparseMatrixCSC) = A.rowVal + +function promote_pointer_type(A, B) + if A isa GPUSparseMatrixCompressed + return B isa GPUSparseMatrixCompressed ? + promote_type(eltype(compressed_ptr(A)), eltype(compressed_ptr(B))) : + eltype(compressed_ptr(A)) + elseif B isa GPUSparseMatrixCompressed + return eltype(compressed_ptr(B)) + end + return promote_type(Int, indtype(A), indtype(B)) +end + +# the size of the dimension the pointers compress, and of the other one +major_dim(A::GPUSparseMatrixCSR) = size(A, 1) +major_dim(A::GPUSparseMatrixCSC) = size(A, 2) +minor_dim(A::GPUSparseMatrixCSR) = size(A, 2) +minor_dim(A::GPUSparseMatrixCSC) = size(A, 1) + + +## allocation + +# Counts and byte sizes are host dimensions. Check arithmetic before asking a backend +# allocator, including allocations whose integer element type is narrower than Int. +function sparse_similar(proto, ::Type{T}, dims::Dims) where {T} + count = foldl(Base.checked_mul, dims; init=1) + count >= 0 || throw(ArgumentError("negative sparse allocation dimensions $dims")) + Base.checked_mul(count, sizeof(T)) + return similar(proto, T, dims) +end +sparse_similar(proto, ::Type{T}, n::Integer) where {T} = sparse_similar(proto, T, (Int(n),)) +sparse_similar(proto, ::Type{T}) where {T} = sparse_similar(proto, T, size(proto)) +sparse_similar(proto, dims::Dims) = sparse_similar(proto, eltype(proto), dims) +sparse_similar(proto, n::Integer) = sparse_similar(proto, eltype(proto), (Int(n),)) +sparse_similar(proto) = sparse_similar(proto, eltype(proto), size(proto)) + +# a copy of `src` converted to eltype `T`. Indices are converted without checks, as the +# callers have verified that the dimensions and number of entries fit. +function convert_buffer(::Type{T}, src::AbstractVector) where {T} + dst = similar(src, T) + dst .= src + return dst +end + +# Canonical dimensions/counts prove index representability before this conversion. +# Values use ordinary conversion semantics and must never take this unchecked path. +function convert_index_buffer(::Type{T}, src::AbstractVector) where {T<:Integer} + dst = sparse_similar(src, T) + dst .= src .% T + return dst +end + +value_representable(::Type{T}, value::Real) where {T<:Integer} = + isfinite(value) && isinteger(value) && typemin(T) <= value <= typemax(T) +value_representable(::Type{T}, value::Integer) where {T<:Integer} = + typemin(T) <= value <= typemax(T) +value_representable(::Type{T}, value::Complex) where {T<:Real} = + iszero(imag(value)) && value_representable(T, real(value)) +value_representable(::Type{Complex{T}}, value::Complex) where {T} = + value_representable(T, real(value)) && value_representable(T, imag(value)) +value_representable(::Type{Complex{T}}, value::Real) where {T} = value_representable(T, value) +value_representable(::Type, value) = true + +function check_sparse_values(::Type{T}, values::AbstractVector) where {T} + T === eltype(values) && return nothing + if !all(v -> value_representable(T, v), values) + throw(InexactError(:convert, T, "stored values cannot be represented by the destination type")) + end + return nothing +end + +# an empty compressed pointer buffer for `n` slices +empty_ptr(proto::AbstractVector, ::Type{Ti}, n::Integer) where {Ti} = + fill!(sparse_similar(proto, Ti, Base.checked_add(n, 1)), one(Ti)) + +# SparseArrays semantics: `similar(A[, Tv[, Ti]])` copies the structure and leaves the +# values uninitialized, while `similar(A, Tv, dims)` returns an array without stored +# entries (even when `dims == size(A)`): a sparse matrix of the same format for two +# dimensions, a sparse vector for one, and a dense array for more. +Base.similar(A::GPUSparseArray) = similar(A, eltype(A)) +Base.similar(A::GPUSparseArray, ::Type{Tv}) where {Tv} = similar(A, Tv, indtype(A)) +Base.similar(A::GPUSparseMatrixCSR, ::Type{Tv}) where {Tv} = + GPUSparseMatrixCSR(copy(A.rowPtr), copy(A.colVal), similar(A.nzVal, Tv), size(A)) +Base.similar(A::GPUSparseMatrixCSC, ::Type{Tv}) where {Tv} = + GPUSparseMatrixCSC(copy(A.colPtr), copy(A.rowVal), similar(A.nzVal, Tv), size(A)) +function Base.similar(A::GPUSparseMatrixCSR, ::Type{Tv}, ::Type{Ti}) where {Tv,Ti} + check_sparse_dims(Ti, size(A)) + check_sparse_nnz(Ti, nnz(A)) + GPUSparseMatrixCSR(convert_index_buffer(Ti, A.rowPtr), convert_index_buffer(Ti, A.colVal), + similar(A.nzVal, Tv), size(A)) +end +function Base.similar(A::GPUSparseMatrixCSC, ::Type{Tv}, ::Type{Ti}) where {Tv,Ti} + check_sparse_dims(Ti, size(A)) + check_sparse_nnz(Ti, nnz(A)) + GPUSparseMatrixCSC(convert_index_buffer(Ti, A.colPtr), convert_index_buffer(Ti, A.rowVal), + similar(A.nzVal, Tv), size(A)) +end +function Base.similar(A::GPUSparseMatrixCOO, ::Type{Tv}, ::Type{Ti}) where {Tv,Ti} + check_sparse_dims(Ti, size(A)) + GPUSparseMatrixCOO(convert_index_buffer(Ti, A.rowInd), convert_index_buffer(Ti, A.colInd), + similar(A.nzVal, Tv), size(A)) +end +function Base.similar(x::GPUSparseVector, ::Type{Tv}, ::Type{Ti}) where {Tv,Ti} + check_sparse_dims(Ti, size(x)) + GPUSparseVector(convert_index_buffer(Ti, x.nzInd), similar(x.nzVal, Tv), length(x)) +end + +Base.similar(A::GPUSparseArray, dims::Dims) = similar(A, eltype(A), dims) +Base.similar(A::GPUSparseArray, ::Type{Tv}, dims::Dims) where {Tv} = + length(dims)>2 ? similar(A.nzVal,Tv,dims) : similar(A, Tv, indtype(A), dims) +Base.similar(A::Union{GPUSparseVector,GPUSparseMatrixCOO}, ::Type{Tv}, dims::Dims{2}) where {Tv} = + empty_sparse(matrix_format(A), A.nzVal, Tv, indtype(A), dims, promote_type(Int, indtype(A))) +Base.similar(A::GPUSparseMatrixCompressed, ::Type{Tv}, dims::Dims{2}) where {Tv} = + empty_sparse(matrix_format(A), A.nzVal, Tv, indtype(A), dims, eltype(compressed_ptr(A))) +Base.similar(A::GPUSparseArray, ::Type{Tv}, ::Type{Ti}, dims::Dims) where {Tv,Ti} = + throw(MethodError(similar,(A,Tv,Ti,dims))) +Base.similar(A::GPUSparseArray,::Type{Tv},::Type{Ti},m::Integer) where {Tv,Ti} = + similar(A,Tv,Ti,(Int(m),)) +Base.similar(A::GPUSparseArray,::Type{Tv},::Type{Ti},m::Integer,n::Integer) where {Tv,Ti} = + similar(A,Tv,Ti,(Int(m),Int(n))) +function Base.similar(A::GPUSparseArray, ::Type{Tv}, ::Type{Ti}, dims::Dims{1}) where {Tv,Ti} + GPUSparseVector(similar(A.nzVal, Ti, 0), similar(A.nzVal, Tv, 0), dims[1]) +end +function Base.similar(A::GPUSparseArray, ::Type{Tv}, ::Type{Ti}, dims::Dims{2}) where {Tv,Ti} + empty_sparse(matrix_format(A), A.nzVal, Tv, Ti, dims) +end + +# the matrix format that allocations derived from `A` use; vectors become CSC, as +# `similar(::SparseVector, T, (m, n))` does in SparseArrays +matrix_format(::GPUSparseMatrixCSR) = GPUSparseMatrixCSR +matrix_format(::GPUSparseMatrixCSC) = GPUSparseMatrixCSC +matrix_format(::GPUSparseMatrixCOO) = GPUSparseMatrixCOO +matrix_format(::GPUSparseVector) = GPUSparseMatrixCSC + +# a matrix without stored entries, in format `S` and with the storage of `proto` +function empty_sparse(::Type{GPUSparseMatrixCSR}, proto, ::Type{Tv}, ::Type{Ti}, + dims::Dims{2}, ::Type{Tp}=Ti) where {Tv,Ti,Tp} + GPUSparseMatrixCSR(empty_ptr(proto, Tp, dims[1]), similar(proto, Ti, 0), + similar(proto, Tv, 0), dims) +end +function empty_sparse(::Type{GPUSparseMatrixCSC}, proto, ::Type{Tv}, ::Type{Ti}, + dims::Dims{2}, ::Type{Tp}=Ti) where {Tv,Ti,Tp} + GPUSparseMatrixCSC(empty_ptr(proto, Tp, dims[2]), similar(proto, Ti, 0), + similar(proto, Tv, 0), dims) +end +function empty_sparse(::Type{GPUSparseMatrixCOO}, proto, ::Type{Tv}, ::Type{Ti}, + dims::Dims{2}, ::Type{Tp}=Ti) where {Tv,Ti,Tp} + GPUSparseMatrixCOO(similar(proto, Ti, 0), similar(proto, Ti, 0), + similar(proto, Tv, 0), dims) +end + +Base.zero(A::GPUSparseArray) = similar(A, eltype(A), size(A)) + +# Allocating format constructors and `copy` return independent storage. Raw buffer +# constructors borrow their arguments; identity `convert` and `adapt` may share. +Base.copy(A::GPUSparseMatrixCSR) = + GPUSparseMatrixCSR(copy(A.rowPtr), copy(A.colVal), copy(A.nzVal), size(A)) +Base.copy(A::GPUSparseMatrixCSC) = + GPUSparseMatrixCSC(copy(A.colPtr), copy(A.rowVal), copy(A.nzVal), size(A)) +Base.copy(A::GPUSparseMatrixCOO) = + GPUSparseMatrixCOO(copy(A.rowInd), copy(A.colInd), copy(A.nzVal), size(A)) +Base.copy(x::GPUSparseVector) = + GPUSparseVector(copy(x.nzInd), copy(x.nzVal), length(x)) + +# Structural mutation requires resizable dense storage and exclusive access. Views are +# supported for reads and value updates, but cannot serve as structural destinations. +sparse_buffers(A::GPUSparseMatrixCSR) = (A.rowPtr, A.colVal, A.nzVal) +sparse_buffers(A::GPUSparseMatrixCSC) = (A.colPtr, A.rowVal, A.nzVal) +sparse_buffers(A::GPUSparseMatrixCOO) = (A.rowInd, A.colInd, A.nzVal) +sparse_buffers(A::GPUSparseVector) = (A.nzInd, A.nzVal) +Base.dataids(A::GPUSparseArray) = mapreduce(Base.dataids, (a, b) -> (a..., b...), sparse_buffers(A)) +Base.unaliascopy(A::GPUSparseArray) = copy(A) + +function check_sparse_mutation(A::GPUSparseArray) + all(supports_resize, sparse_buffers(A)) || + throw(ArgumentError("structural mutation requires resizable dense buffers; raw views support reads and values-only updates")) + return A +end + +function copy_buffer!(dst::AbstractVector, src::AbstractVector) + length(dst) == length(src) || resize!(dst, length(src)) + dst .= src + return dst +end + +function copy_sparse!(dst, src) + size(dst) == size(src) || + throw(DimensionMismatch("destination has size $(size(dst)), source has size $(size(src))")) + check_sparse_dims(indtype(dst), size(src)) + dst isa GPUSparseMatrixCompressed && check_sparse_nnz(eltype(compressed_ptr(dst)), nnz(src)) + check_sparse_mutation(dst) + check_sparse_values(eltype(dst), nonzeros(src)) + dst === src && return dst + # Copy before resizing/writing any component, including overlapping component views. + src = Base.mightalias(dst, src) ? copy(src) : src + foreach(copy_buffer!, sparse_buffers(dst), sparse_buffers(src)) + return dst +end +for S in (:GPUSparseMatrixCSR, :GPUSparseMatrixCSC, :GPUSparseMatrixCOO, :GPUSparseVector) + @eval Base.copyto!(dst::$S, src::$S) = copy_sparse!(dst, src) +end + + +## structural checks + +## COV_EXCL_START +# whether the indices in `ind[first:last]` are sorted, unique and within `1:n` +@inline function valid_slice(ind, first, last, n) + prev = zero(eltype(ind)) + for k in first:last + i = @inbounds ind[k] + (i > prev && i <= n) || return false + prev = i + end + return true +end + +@kernel function check_compressed_kernel(valid, ptr, ind, n) + j = @index(Global, Linear) + if j < length(ptr) + first = @inbounds ptr[j] + last = @inbounds ptr[j+1] - one(first) + @inbounds valid[j] = first >= 1 && last + 1 >= first && last <= length(ind) && + valid_slice(ind, first, last, n) + end +end + +@kernel function check_coo_kernel(valid, rowInd, colInd, dims) + k = @index(Global, Linear) + if k <= length(rowInd) + i = @inbounds rowInd[k] + j = @inbounds colInd[k] + ok = 1 <= i <= dims[1] && 1 <= j <= dims[2] + if k > 1 + # row-major order without duplicates + i′ = @inbounds rowInd[k-1] + j′ = @inbounds colInd[k-1] + ok &= i′ < i || (i′ == i && j′ < j) + end + @inbounds valid[k] = ok + end +end +## COV_EXCL_STOP + +""" + GPUArrays.check_structure(A) + +Verify on the device that the sparse array `A` satisfies the invariants of its format +(consistent pointers, sorted and unique indices within bounds), throwing an +`ArgumentError` otherwise. The constructors only check host metadata such as buffer +lengths, so this is useful for testing code that assembles sparse arrays from raw buffers. +""" +function check_structure(A::GPUSparseMatrixCompressed) + ptr, ind = compressed_ptr(A), compressed_ind(A) + n = major_dim(A) + valid = similar(A.nzVal, Bool, n) + if n > 0 + check_compressed_kernel(get_backend(A))(valid, ptr, ind, minor_dim(A); ndrange=n) + end + first, last = Array(ptr[[1, end]]) + (first == 1 && last == nnz(A) + 1 && all(valid)) || + throw(ArgumentError("invalid $(nameof(typeof(A))) structure")) + return A +end +function check_structure(A::GPUSparseMatrixCOO) + valid = similar(A.nzVal, Bool, nnz(A)) + if nnz(A) > 0 + check_coo_kernel(get_backend(A))(valid, A.rowInd, A.colInd, size(A); ndrange=nnz(A)) + end + all(valid) || throw(ArgumentError("invalid GPUSparseMatrixCOO structure")) + return A +end +function check_structure(x::GPUSparseVector) + valid = similar(x.nzVal, Bool, 1) + ptr = copyto!(similar(x.nzInd, Int, 2), Int[1, Base.checked_add(nnz(x), 1)]) + check_compressed_kernel(get_backend(x))(valid, ptr, x.nzInd, length(x); ndrange=1) + all(valid) || throw(ArgumentError("invalid GPUSparseVector structure")) + return x +end + +# Retained analysis can borrow a fixed pattern while binding immutable numerical data. +snapshot_sparse_values(A::GPUSparseMatrix) = + sparse_format_type(A)(Base.front(sparse_buffers(A))...,copy(nonzeros(A)),size(A)) diff --git a/test/runtests.jl b/test/runtests.jl index c2b9ea471..9bced09fb 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,6 +1,6 @@ using ParallelTestRunner: runtests, parse_args -using Test import GPUArrays +import Test include("testsuite.jl") @@ -8,9 +8,6 @@ const init_worker_code = quote using Test, JLArrays, SparseArrays include("testsuite.jl") - - TestSuite.sparse_types(::Type{<:JLArray}) = (JLSparseVector, JLSparseMatrixCSC, JLSparseMatrixCSR) - TestSuite.sparse_types(::Type{<:Array}) = (SparseVector, SparseMatrixCSC) end const init_code = quote @@ -26,7 +23,18 @@ for AT in (:JLArray, :Array), name in keys(TestSuite.tests) testsuite["$(AT)/$name"] = :(TestSuite.tests[$name]($AT)) end -# captured variables that get boxed (`Core.Box`) lose their type +# the sparse implementation should not add method ambiguities (the dense code has a few) +testsuite["sparse/v13-regressions"] = :(include("sparse-v13-regressions.jl")) +testsuite["sparse/ambiguities"] = quote + import GPUArrays + sparse_sources = (joinpath("src", "host", "sparse"), joinpath("JLArrays", "src", "sparse.jl"), + joinpath("JLArrays", "src", "algorithms.jl")) + ambiguities = filter(Test.detect_ambiguities(GPUArrays, JLArrays; recursive=true)) do (a, b) + any(m -> any(src -> occursin(src, string(m.file)), sparse_sources), (a, b)) + end + @test isempty(ambiguities) +end + if isdefined(Test, :detect_closure_boxes) testsuite["closure boxes"] = quote import GPUArrays @@ -34,10 +42,4 @@ if isdefined(Test, :detect_closure_boxes) end end -# a type without a COO counterpart must hit a MethodError, not recurse (#780) -testsuite["sparse/coo_type"] = quote - import GPUArrays - @test_throws MethodError GPUArrays.coo_type(sparse([1], [1], [1.0], 2, 2)) -end - runtests(GPUArrays, args; init_code, init_worker_code, testsuite) diff --git a/test/sparse-v13-regressions.jl b/test/sparse-v13-regressions.jl new file mode 100644 index 000000000..b7a0d4695 --- /dev/null +++ b/test/sparse-v13-regressions.jl @@ -0,0 +1,416 @@ +using Test, GPUArrays, JLArrays, SparseArrays, LinearAlgebra, Adapt +using GPUArrays: GPUSparseMatrixCSR, GPUSparseMatrixCSC, GPUSparseMatrixCOO, GPUSparseVector + +GPUArrays.allowscalar(false) + +struct FixedSparseStorage{T} <: GPUArrays.AbstractGPUArray{T,1} + data::JLVector{T} +end +Base.size(x::FixedSparseStorage) = size(x.data) +Base.Array(x::FixedSparseStorage) = Array(x.data) +GPUArrays.KernelAbstractions.get_backend(x::FixedSparseStorage) = GPUArrays.get_backend(x.data) +Base.resize!(::FixedSparseStorage, ::Integer) = throw(ArgumentError("fixed storage")) + +# Same CPU-transfer policy as MLDataDevices' GPUArraysSparseArraysExt, without +# resolving a downstream release whose compatibility does not yet include v13. +function downstream_cpu_transfer(::Type{T},x::Union{GPUSparseMatrixCSC,GPUSparseVector}) where T + target=x isa GPUSparseMatrixCSC ? SparseMatrixCSC : SparseVector + if T <: AbstractFloat + eltype(x) <: Complex && return target{Complex{T}}(x) + return target{T}(x) + end + return target(x) +end + +@testset "v13 sparse regressions" begin + @testset "host-accessible mixed sparse broadcast rejects" begin + x=GPUSparseVector(Int32[1,2],Float32[3,4],2) + for S in (GPUSparseMatrixCSR,GPUSparseMatrixCSC,GPUSparseMatrixCOO) + A=S===GPUSparseMatrixCOO ? S(Int32[1,2],Int32[1,2],Float32[1,2],(2,2)) : + S(Int32[1,2,3],Int32[1,2],Float32[1,2],(2,2)) + before=Array(A) + @test_throws ErrorException A .* x + @test_throws ErrorException x .+ A + @test_throws ErrorException 2 .* A .+ x + @test_throws ErrorException (A .= A .* x) + @test Array(A)==before + scalar=JLArray(fill(0f0)) + for make in ((A,x,z)->(A .* x) .+ z, + (A,x,z)->z .+ (A .* x), + (A,x,z)->A .* (z .+ x), + (A,x,z)->(x .+ z) .* A) + @test_throws r"both sparse vectors and sparse matrices" make(A,x,scalar) + end + @test_throws r"zero-dimensional GPU arrays" A .+ scalar + @test_throws r"zero-dimensional GPU arrays" scalar .+ A + for shape in ((2,2,1),(2,2,2),(2,2,1,1)), AT in (Array,JLArray) + dense=AT(zeros(Float32,shape)) + @test_throws r"both sparse vectors and sparse matrices" (A .* x) .+ dense + @test_throws r"both sparse vectors and sparse matrices" dense .+ (x .* A) + @test_throws r"both sparse vectors and sparse matrices" A .* (x .+ dense) + @test_throws r"both sparse vectors and sparse matrices" (dense .+ x) .* A + @test_throws r"expansion of sparse operands" A .+ dense + @test_throws r"expansion of sparse operands" dense .+ A + @test_throws DimensionMismatch (A .= A .+ dense) + @test Array(A)==before + end + end + @test_throws r"zero-dimensional GPU arrays" x .+ JLArray(fill(0f0)) + @test_throws r"zero-dimensional GPU arrays" JLArray(fill(0f0)) .+ x + end + @testset "downstream dispatch and storage migration" begin + for (name,S) in ((:AbstractGPUSparseMatrixCSR,GPUSparseMatrixCSR), + (:AbstractGPUSparseMatrixCSC,GPUSparseMatrixCSC), + (:AbstractGPUSparseMatrixCOO,GPUSparseMatrixCOO)) + legacy=getfield(GPUArrays,name) + @test legacy===S + @test legacy{Float32,Int32}===S{Float32,Int32} + @test !isabstracttype(legacy) + end + for host in (sparse(Float32[1 0;0 2]),sparse(ComplexF32[1 im;0 2]), + sparsevec([1,3],Float32[1,2],4),sparsevec([1,3],ComplexF32[1,im],4)) + A=adapt(JLArray,host) + for T in (Float32,Float64,Nothing) + B=downstream_cpu_transfer(T,A) + expected=T <: AbstractFloat ? (eltype(host) <: Complex ? Complex{T} : T) : eltype(host) + @test B isa typeof(host).name.wrapper + @test eltype(B)===expected + @test B==host + end + dense=similar(nonzeros(A),eltype(A),size(A)) + fill!(dense,convert(eltype(A),NaN)) + copyto!(dense,A) + @test dense isa JLArray + @test Array(dense)==Array(host) + end + end + @testset "generalized dot dimension preflight" begin + for S in (GPUSparseMatrixCSR,GPUSparseMatrixCSC,GPUSparseMatrixCOO) + A=S(adapt(JLArray,sparse(Int32[1 0 2;3 4 0]))) + x=JLArray(Int32[1,2]);y=JLArray(Int32[1,2,3]) + @test dot(x,A,y)==Int32(29) + @test_throws DimensionMismatch dot(JLArray(Int32[1,2,3]),A,y) + @test_throws DimensionMismatch dot(JLArray(Int32[1]),A,y) + @test_throws DimensionMismatch dot(x,A,JLArray(Int32[1,2])) + @test_throws DimensionMismatch dot(x,A,JLArray(Int32[1,2,3,4])) + empty=S(adapt(JLArray,spzeros(Int32,0,0))) + @test_throws DimensionMismatch dot(JLArray(Int32[1]),empty,JLArray(Int32[])) + @test_throws DimensionMismatch dot(JLArray(Int32[]),empty,JLArray(Int32[1])) + end + @test_throws DimensionMismatch dot(JLArray(Int32[1,2]),JLArray(Int32[3])) + end + @testset "diagonal wrapper and offset compatibility" begin + host=sparse(ComplexF32[1 im 0;2 3 4]) + for S in (GPUSparseMatrixCSR,GPUSparseMatrixCSC,GPUSparseMatrixCOO) + A=S(adapt(JLArray,host)) + for wrap in (identity,transpose,adjoint), k in (-20,-2,-1,0,1,3,20,typemin(Int),typemax(Int)) + if k in (typemin(Int),typemax(Int)) + @test isempty(diag(wrap(A),k)) + else + @test Array(diag(wrap(A),k))==diag(Matrix(wrap(host)),k) + end + end + @test Array(diag(A,UInt(1)))==diag(Matrix(host),1) + @test Array(diag(transpose(A),Int8(-1)))==diag(Matrix(transpose(host)),-1) + @test_throws InexactError diag(A,typemax(UInt)) + end + end + @testset "preflight narrowing before compression" begin + coo=GPUSparseMatrixCOO(JLArray(Int64[]),JLArray(Int64[]),JLArray(Float32[]),(2^32,1)) + @test_throws ArgumentError GPUSparseMatrixCSR{Float32,Int32}(coo) + @test_throws ArgumentError GPUSparseMatrixCSC{Float32,Int32}(coo) + wide=GPUSparseMatrixCOO(JLArray(Int64[]),JLArray(Int64[]),JLArray(Float32[]),(1,2^32)) + for S in (GPUSparseMatrixCSR,GPUSparseMatrixCSC), wrap in (transpose,adjoint) + @test_throws ArgumentError S{Float32,Int32}(wrap(wide)) + end + bad=GPUSparseMatrixCOO(JLArray(Int64[1]),JLArray(Int64[1]),JLArray(ComplexF32[1+im]),(2^32,1)) + @test_throws InexactError GPUSparseMatrixCSR{Float32}(bad) + @test_throws InexactError GPUSparseMatrixCSC{Float32}(transpose(bad)) + for S in (GPUSparseMatrixCSR,GPUSparseMatrixCSC) + @test_throws ArgumentError S{Float32,Int8}(Diagonal(JLArray(ones(Float32,127)))) + @test_throws ArgumentError S{Float32,Int8}(Diagonal(JLArray(zeros(Float32,127)))) + D=S{Float32,Int8}(Diagonal(JLArray(zeros(Float32,126)))) + @test nnz(D)==126 && size(D)==(126,126) && all(iszero,nonzeros(D)) + end + tiny=GPUSparseMatrixCOO(JLArray(Int8[1,1]),JLArray(Int8[1,2]),JLArray(Float32[1,2]),(1,2)) + @test_throws ArgumentError GPUSparseMatrixCSR{Float32,Bool}(tiny) + end + @testset "rectangular scaling and promoted wrapped products" begin + for S in (GPUSparseMatrixCSR,GPUSparseMatrixCSC,GPUSparseMatrixCOO) + host_cancel=sparse(reshape(Float32[16777216,1,-16777216],1,3)) + cancel=S(adapt(JLArray,host_cancel)) + rhs=JLArray(ones(Float32,3,1)) + @test eltype(1.0*cancel*rhs)===Float64 + @test Array(1.0*cancel*rhs)==ones(1,1) + @test Array(1.0*transpose(copy(transpose(cancel)))*rhs)==ones(1,1) + for dims in ((2,3),(3,2),(0,3),(3,0)) + host=sparse(reshape(Float32.(1:prod(dims)),dims)) + A=S(adapt(JLArray,host)) + # Preserve the existing GPU rectangular diagonal operation even + # when newer SparseArrays versions restrict UniformScaling to square. + for J in (I,2.5f0*I,-3f0*I), op in (+,-) + identity=zeros(Float32,dims) + for i in 1:min(dims...) + identity[i,i]=J.λ + end + @test Array(op(A,J)) == op(Matrix(host),identity) + @test Array(op(J,A)) == op(identity,Matrix(host)) + end + @test_throws DimensionMismatch one(A) + @test_throws DimensionMismatch oneunit(A) + end + for dims in ((0,0),(2,2)) + host=sparse(reshape(Float32.(1:prod(dims)),dims)) + A=S(adapt(JLArray,host)) + for J in (I,2.5f0*I,-3f0*I), op in (+,-) + @test Array(op(A,J)) == Matrix(op(host,J)) + @test Array(op(J,A)) == Matrix(op(J,host)) + end + end + host=sparse(Float32[1.1 2.3;0 3.7]) + A=S(adapt(JLArray,host)) + for wrap in (Symmetric,Hermitian) + @test Array(2*wrap(A))==Matrix(2*wrap(host)) + @test Array(wrap(A)*2)==Matrix(wrap(host)*2) + @test sparse(wrap(A)) isa S + @test Array(sparse(wrap(A)))==Matrix(wrap(host)) + @test SparseMatrixCSC(wrap(A))==sparse(wrap(host)) + for op in (+,-), outer in (identity,transpose,adjoint) + @test Array(op(outer(A),wrap(A)))≈Matrix(op(outer(host),wrap(host))) + @test Array(op(wrap(A),outer(A)))≈Matrix(op(wrap(host),outer(host))) + end + end + for wrap in (transpose,adjoint) + @test Array(1.1*wrap(A)) ≈ Matrix(1.1*wrap(host)) + @test Array(wrap(A)*1.1) ≈ Matrix(wrap(host)*1.1) + @test Array(1.1*wrap(A)*A) ≈ Matrix((1.1*wrap(host))*host) + @test Array(wrap(A)^2) ≈ Matrix(wrap(host)^2) + end + end + end + @testset "backend arithmetic parity" begin + host=sparse(ComplexF32[2 0;1+im 3]) + for S in (GPUSparseMatrixCSR,GPUSparseMatrixCSC,GPUSparseMatrixCOO) + A=S(adapt(JLArray,host)) + for B in (A,transpose(A),adjoint(A)), op in (+,-) + reference=B === A ? host : B isa Transpose ? transpose(host) : adjoint(host) + @test Array(op(A,B)) ≈ Array(op(host,reference)) + @test Array(op(B,A)) ≈ Array(op(reference,host)) + @test Array(op(B,2I)) ≈ Array(op(reference,2I)) + @test Array(op(2I,B)) ≈ Array(op(2I,reference)) + D=Diagonal(ComplexF32[2,4]) + @test Array(op(B,D)) ≈ Array(op(reference,D)) + @test Array(op(D,B)) ≈ Array(op(D,reference)) + end + @test Array(one(A))==Matrix{ComplexF32}(I,2,2) + @test Array(oneunit(A))==Matrix{ComplexF32}(I,2,2) + @test Array(A*2I)==Matrix(host)*2I + @test Array(2I*A)==2I*Matrix(host) + D=Diagonal(ComplexF32[2,4]) + @test Array(A*D)==Matrix(host*D) + @test Array(D*A)==Matrix(D*host) + @test Array(kron(A,D))≈Matrix(kron(host,D)) + @test Array(kron(D,A))≈Matrix(kron(D,host)) + @test iszero(zero(A)) + end + @test SparseMatrixCSC(JLArray(Matrix(host)))==host + host32=sparse(Int32[typemax(Int32) 0;0 typemax(Int32)]) + @test tr(adapt(JLArray,host32))===tr(host32) + @test Array(sparse(JLArray(Int32[1]),JLArray(Int32[1]),JLArray(Float32[2]),Int32(1),Int32(1)))==reshape(Float32[2],1,1) + end + @testset "borrowed CSC column accessors" begin + A=GPUSparseMatrixCSC(JLArray(Int64[1,3,3,4]),JLArray(Int32[1,3,2]), + JLArray(Float32[2,4,6]),(3,3)) + for j in 1:3 + column=view(A,:,j) + GPUArrays.@allowscalar begin + @test nnz(column)==(j==1 ? 2 : j==2 ? 0 : 1) + @test Array(nonzeros(column))==Array(nonzeros(A))[nzrange(A,j)] + @test Array(rowvals(column))==Array(rowvals(A))[nzrange(A,j)] + end + end + end + @testset "maximum coordinate" begin + x = GPUSparseVector(JLArray(Int8[127]), JLArray(Float32[2]), 127) + y = x .+ x + @test nnz(y) == 1 + @test Array(y)[end] == 4 + @test Array(GPUArrays.compress(JLArray(Int8[127]), 127))[end] == 2 + end + @testset "rejected mutation preserves every buffer" begin + p = JLArray([1, 2, 3]); i = JLArray([1, 2]) + v = JLArray([1., 7., 0., 9.]) + A = GPUSparseMatrixCSC(p, i, view(v, 1:2:4), (2, 2)) + before = (Array(p), Array(i), Array(v)) + @test_throws ArgumentError dropzeros!(A) + @test (Array(p), Array(i), Array(v)) == before + p = JLArray([1, 2, 3]); i = JLArray([1, 2]) + v = FixedSparseStorage(JLArray(Float32[1, 0])) + dest = GPUSparseMatrixCSC(p, i, v, (2, 2)) + src = GPUSparseMatrixCSC(JLArray([1, 2, 2]), JLArray([1]), JLArray(Float32[1]), (2, 2)) + before = (Array(p), Array(i), Array(v)) + @test_throws ArgumentError copyto!(dest, src) + @test (Array(p), Array(i), Array(v)) == before + end + @testset "symmetric nonfinite filtering" begin + S = sparse([1. 2.; 4. 3.]) + A = adapt(JLArray, S) + for wrapper in (Symmetric, Hermitian), uplo in (:U, :L) + @test Array(wrapper(A, uplo) * JLArray([Inf, 1.])) == wrapper(S, uplo) * [Inf, 1.] + end + end + @testset "scaled norms" begin + for v in (1f30, 1f-30) + S = sparse([1], [1], [v], 1, 1) + @test norm(adapt(JLArray, S)) ≈ norm(S) + end + end + @testset "logical negative-order norms" begin + for T in (Float32,ComplexF32) + fixtures=(sparsevec([1],T[2],2),sparsevec([1,2],T[2,3],2), + sparsevec([1,2],T[0,3],2),sparsevec([1],T[NaN],2), + sparsevec([1],T[Inf],2),spzeros(T,2),spzeros(T,0)) + if T===ComplexF32 + fixtures=(fixtures...,sparsevec([1],T[complex(Inf32,NaN32)],2)) + end + for host in fixtures + matrix=SparseMatrixCSC(length(host),1,Int[1,nnz(host)+1], + copy(SparseArrays.nonzeroinds(host)),copy(nonzeros(host))) + arrays=(adapt(JLArray,host), + (S(adapt(JLArray,matrix)) for S in + (GPUSparseMatrixCSR,GPUSparseMatrixCSC,GPUSparseMatrixCOO))...) + for A in arrays,p in (-Inf,-2,-1,-0.5,0,1,2,Inf) + # A complex BLAS nrm2 can report NaN for Inf+NaN*im even though + # its scalar magnitude is Inf. Test the scalar-magnitude contract. + reference=norm(norm.(Array(host)),p) + actual=norm(A,p) + @test typeof(actual)===typeof(reference) + @test isnan(reference) ? isnan(actual) : actual ≈ reference + end + # Julia 1.13's CPU norm with p=NaN can throw from an undefined local. + # Preserve delegation for this unordered exponent without that reference. + for A in arrays + @test isequal(norm(A,NaN),norm(nonzeros(A),NaN)) + end + end + end + x=GPUSparseVector(JLArray(Int32[1]),JLArray(Int32[2]),2) + @test norm(x,-Inf)===0.0 + end + @testset "component aliasing" begin + A = adapt(JLArray, sparse(Float32[1 0; 2 3])) + B = GPUSparseMatrixCSC(A.colPtr, A.rowVal, copy(A.nzVal), size(A)) + @test Base.mightalias(A, B) + @test !Base.mightalias(A, copy(A)) + end + @testset "independent compressed widths" begin + for S in (GPUSparseMatrixCSR, GPUSparseMatrixCSC) + A = S(JLArray(Int64[1, 3, 4]), JLArray(Int32[1, 2, 2]), + JLArray(Float32[1, 2, 3]), (2, 2)) + other = S === GPUSparseMatrixCSR ? GPUSparseMatrixCSC : GPUSparseMatrixCSR + for B in (copy(A), similar(A), similar(A, Float64), S{Float64}(A), other(A)) + @test eltype(GPUArrays.compressed_ptr(B)) == Int64 + @test SparseArrays.indtype(B) == Int32 + @test !Base.mightalias(A, B) + end + for B in (S{Float64,Int32}(A), similar(A, Float64, Int32)) + @test eltype(GPUArrays.compressed_ptr(B)) == Int32 + @test SparseArrays.indtype(B) == Int32 + end + @test SparseMatrixCSC(A) == SparseMatrixCSC(adapt(Array, A)) + @test eltype(SparseArrays.getcolptr(SparseMatrixCSC(A))) == Int64 + @test Array(GPUArrays.check_structure(A)) == Array(A) + @test Array(other(A)) == Array(A) + end + # Neither vector nor COO needs a terminal pointer in its coordinate type. + x = GPUSparseVector(JLArray(Int8.(1:127)), JLArray(ones(Float32, 127)), 127) + @test nnz(GPUArrays.check_structure(x)) == 127 + I = repeat(Int8.(1:20); inner=20) + J = repeat(Int8.(1:20); outer=20) + A = GPUSparseMatrixCOO(JLArray(I), JLArray(J), JLArray(ones(Float32, 400)), (20, 20)) + @test nnz(A) == 400 + B = GPUSparseMatrixCSR(A) + @test eltype(B.rowPtr) == Int + @test SparseArrays.indtype(B) == Int8 + @test nnz(GPUArrays.check_structure(B)) == 400 + @test_throws ArgumentError GPUSparseMatrixCSR{Float32,Int8}(B) + @test eltype(similar(x, Float32, (10, 10)).colPtr) == Int + @test eltype(similar(x, Float32, Int8, (10, 10)).colPtr) == Int8 + # The pointer-buffer length does not have to fit its element type. + ptr = ones(Int8, 201); ptr[end] = 2 + A = GPUSparseMatrixCSR(JLArray(ptr), JLArray(Int16[1]), JLArray(Float32[2]), (200, 1)) + B = GPUSparseMatrixCSR(JLArray(Int8[1, 2]), JLArray(Int16[1]), JLArray(Float32[3]), (1, 1)) + C = A * B + @test eltype(C.rowPtr) == Int8 + @test SparseArrays.indtype(C) == Int16 + @test Array(C)[end] == 6 + end + @testset "scalar equality can exceed input pointer widths" begin + a = sparse(ones(Int8, 72), Int8.(1:72), zeros(Float32, 72), 2, 72) + b = sparse(fill(Int8(2), 72), Int8.(1:72), zeros(Float32, 72), 2, 72) + A = GPUSparseMatrixCSR{Float32,Int8}(adapt(JLArray, a)) + B = GPUSparseMatrixCSR{Float32,Int8}(adapt(JLArray, b)) + @test A == B + @test_throws ArgumentError A .+ B + end + @testset "integer value conversion is checked" begin + x = GPUSparseVector(JLArray(Int32[1]), JLArray(Int64[300]), 2) + @test_throws InexactError GPUSparseVector{Int8,Int32}(x) + dest = GPUSparseVector(JLArray(Int32[1]), JLArray(Int8[1]), 2) + @test_throws InexactError copyto!(dest, x) + @test Array(dest) == Int8[1, 0] + @test_throws InexactError (dest .= 300 .* dest) + @test Array(dest) == Int8[1, 0] + for vals in (Float32[1.5], Float32[Inf], ComplexF32[1+im]) + src = GPUSparseVector(JLArray(Int32[2]), JLArray(vals), 2) + before = (Array(dest.nzInd), Array(dest.nzVal)) + @test_throws InexactError copyto!(dest, src) + @test (Array(dest.nzInd), Array(dest.nzVal)) == before + end + real_dest = GPUSparseVector(JLArray(Int32[1]), JLArray(Float32[4]), 2) + complex_src = GPUSparseVector(JLArray(Int32[2]), JLArray(ComplexF32[1+im]), 2) + @test_throws InexactError copyto!(real_dest, complex_src) + @test (Array(real_dest.nzInd), Array(real_dest.nzVal)) == (Int32[1], Float32[4]) + end + @testset "ordered boolean mask slices" begin + host = sparse(Float32[1 2 0; 3 0 4; 5 6 7]) + for S in (GPUSparseMatrixCSR,GPUSparseMatrixCSC,GPUSparseMatrixCOO) + A = S(adapt(JLArray,host)) + for mask in (Bool[true,false,true],falses(3)), device in (false,true) + I = device ? JLArray(mask) : mask + B = A[I,I] + @test Array(B) == Matrix(host[mask,mask]) + @test GPUArrays.check_structure(B) === B + @test !Base.mightalias(A,B) + end + @test_throws BoundsError A[JLArray(trues(2)),JLArray(trues(3))] + end + end + @testset "immutable numerical snapshots" begin + for values in (Float32[0],Float32[-0.0],reinterpret(Float32,UInt32[0x7fc00001])) + A=JLArray(values) + @test GPUArrays.same_value_bits(A,copy(A)) + end + @test !GPUArrays.same_value_bits(JLArray(Float32[0]),JLArray(Float32[-0.0])) + a=JLArray(reinterpret(Float32,UInt32[0x7fc00001])) + b=JLArray(reinterpret(Float32,UInt32[0x7fc00002])) + @test !GPUArrays.same_value_bits(a,b) + for S in (GPUSparseMatrixCSR,GPUSparseMatrixCSC,GPUSparseMatrixCOO) + A=S(adapt(JLArray,sparse(Float32[2 0;1 3]))) + B=GPUArrays.snapshot_sparse_values(A) + @test Array(A)==Array(B) + @test !Base.mightalias(nonzeros(A),nonzeros(B)) + @test all(pair -> pair[1] === pair[2],zip(Base.front(GPUArrays.sparse_buffers(A)),Base.front(GPUArrays.sparse_buffers(B)))) + end + end + @testset "concatenation rejects every operand position" begin + A = adapt(JLArray, sparse(Float32[1 0; 0 1])) + D = JLArray(ones(Float32, 2, 2)) + for f in (hcat, vcat), pos in 1:4 + args = ntuple(i -> i == pos ? transpose(A) : D, 4) + @test_throws ArgumentError f(args...) + mixed = ntuple(i -> i == pos ? A : ones(Float32, 2, 2), 4) + @test_throws ArgumentError f(mixed...) + end + end +end diff --git a/test/testsuite.jl b/test/testsuite.jl index 59dcbe05b..baa96056e 100644 --- a/test/testsuite.jl +++ b/test/testsuite.jl @@ -96,9 +96,6 @@ supported_eltypes() = (Int16, Int32, Int64, ComplexF16, ComplexF32, ComplexF64, Complex{Int16}, Complex{Int32}, Complex{Int64}) -# derived sparse container types that are supported by the array type -sparse_types(::Type{AT}) where {AT} = () - # some convenience predicates for filtering test eltypes isrealtype(T) = T <: Real iscomplextype(T) = T <: Complex diff --git a/test/testsuite/linalg.jl b/test/testsuite/linalg.jl index 8b8146a63..c6421a08c 100644 --- a/test/testsuite/linalg.jl +++ b/test/testsuite/linalg.jl @@ -422,6 +422,8 @@ @testset "dot" begin @testset "$T" for T in eltypes @test compare(dot, AT, rand(T,5), rand(T, 5)) + @test_throws DimensionMismatch dot(AT(rand(T,5)),AT(rand(T,4))) + @test_throws DimensionMismatch dot(AT(rand(T,4)),AT(rand(T,5))) end end @@ -494,6 +496,13 @@ end end @testsuite "linalg/diagonal" (AT, eltypes) -> begin + @testset "explicit host transfer" begin + host=Diagonal(Float32[1,2,3]) + D=Diagonal(AT(host.diag)) + @test Array(D)==Matrix(host) + @test Array{Float64}(D)==Matrix{Float64}(host) + @test Matrix{Float64}(D)==Matrix{Float64}(host) + end @testset "Array + Diagonal" begin n = 128 A = AT(rand(Float32, (n,n))) diff --git a/test/testsuite/sparse.jl b/test/testsuite/sparse.jl index e44733f3e..7174ff633 100644 --- a/test/testsuite/sparse.jl +++ b/test/testsuite/sparse.jl @@ -1,179 +1,495 @@ +using SparseArrays +using SparseArrays: nonzeroinds, nonzeros, rowvals, getcolptr +using GPUArrays: GPUSparseMatrixCSR, GPUSparseMatrixCSC, GPUSparseMatrixCOO, GPUSparseVector, + AbstractGPUSparseArray, check_structure + @testsuite "sparse" (AT, eltypes)->begin - sparse_ATs = sparse_types(AT) - @testset "sparse_AT = $sparse_AT" for sparse_AT in sparse_ATs - if sparse_AT <: AbstractSparseVector - vector(sparse_AT, eltypes) - vector_construction(sparse_AT, eltypes) - broadcasting_vector(sparse_AT, eltypes) - iszero_vector(sparse_AT, eltypes) - elseif sparse_AT <: AbstractSparseMatrix - matrix(sparse_AT, eltypes) - matrix_construction(sparse_AT, eltypes) - broadcasting_matrix(sparse_AT, eltypes) - mapreduce_matrix(sparse_AT, eltypes) - linalg(sparse_AT, eltypes) - transpose_matrix(sparse_AT, eltypes) - iszero_matrix(sparse_AT, eltypes) - end - end + # the generic sparse formats use `AT`'s vectors as storage; plain arrays have no device + AT <: AbstractGPUArray || return + # nothing may fall back to scalar indexing unless a test asks for it + GPUArrays.allowscalar(false) + + sparse_construction(AT, eltypes) + sparse_allocation(AT, eltypes) + sparse_transfer(AT, eltypes) + sparse_kernels(AT, eltypes) + sparse_display(AT, eltypes) + sparse_indexing(AT, eltypes) + sparse_conversions(AT, eltypes) + sparse_dense_conversions(AT, eltypes) + sparse_assembly(AT, eltypes) + sparse_products(AT, eltypes) + sparse_structure(AT, eltypes) + sparse_slicing(AT, eltypes) + sparse_misc(AT, eltypes) + sparse_spgemm(AT, eltypes) + broadcasting_vector(AT, eltypes) + broadcasting_matrix(AT, eltypes) + broadcasting_mixed(AT, eltypes) + mapreduce_matrix(AT, eltypes) + sparse_linalg(AT, eltypes) + iszero_sparse(AT, eltypes) + sparse_bool(AT) + sparse_v13(AT, eltypes) end -using SparseArrays -using SparseArrays: nonzeroinds, nonzeros, rowvals +include("sparse_v13.jl") + +const sparse_matrix_formats = (GPUSparseMatrixCSR, GPUSparseMatrixCSC, GPUSparseMatrixCOO) + +# A GPU sparse matrix of format `S` with the entries of `A`, built directly from buffers +# computed on the host, so that the tests do not depend on the conversions they test. +function gpu_sparse(AT, S::Type, A::SparseMatrixCSC) + if S <: GPUSparseMatrixCSC + GPUSparseMatrixCSC(AT(getcolptr(A)), AT(rowvals(A)), AT(nonzeros(A)), size(A)) + else + # the CSC buffers of the transpose are the CSR buffers of `A` + At = copy(transpose(A)) + if S <: GPUSparseMatrixCSR + GPUSparseMatrixCSR(AT(getcolptr(At)), AT(rowvals(At)), AT(nonzeros(At)), size(A)) + else + rows = similar(rowvals(At)) + for i in 1:size(A, 1), k in nzrange(At, i) + rows[k] = i + end + GPUSparseMatrixCOO(AT(rows), AT(rowvals(At)), AT(nonzeros(At)), size(A)) + end + end +end +gpu_sparse(AT, x::SparseVector) = GPUSparseVector(AT(nonzeroinds(x)), AT(nonzeros(x)), length(x)) # `sprand`/`sprandn` can produce *stored* zeros: with a coarse element type like Float16, # `rand` returns an exact zero about once every 2048 draws, so roughly one in 60 vectors # here contains one. That's a valid sparse array, but CPU and GPU implementations # legitimately disagree on what to do with it -- `x .* 1` drops stored zeros on the CPU # while our kernels preserve the structure -- so the structural comparisons below fail at -# random. Keep the inputs canonical instead. +# random. Keep the inputs canonical instead, and test stored zeros explicitly. sprand_nozeros(args...) = dropzeros!(sprand(args...)) sprandn_nozeros(args...) = dropzeros!(sprandn(args...)) -function vector(AT, eltypes) - dense_AT = GPUArrays.dense_array_type(AT) - @testset "Sparse vector properties" begin - @testset "$ET" for ET in eltypes - m = 25 - n = 35 - k = 10 - p = 5 - blockdim = 5 - x = sprand_nozeros(ET,m,0.2) - d_x = AT(x) - @test length(d_x) == m - @test size(d_x) == (m,) - @test size(d_x,1) == m - @test size(d_x,2) == 1 - @test ndims(d_x) == 1 - dense_d_x = dense_AT(x) - dense_d_x2 = dense_AT(d_x) - @allowscalar begin - @test Array(d_x[:]) == x[:] - @test d_x[firstindex(d_x)] == x[firstindex(x)] - @test d_x[div(end, 2)] == x[div(end, 2)] - @test d_x[end] == x[end] - @test Array(d_x[firstindex(d_x):end]) == x[firstindex(x):end] - @test Array(dense_d_x[firstindex(d_x):end]) == x[firstindex(x):end] - @test Array(dense_d_x2[firstindex(d_x):end]) == x[firstindex(x):end] - end - @test_throws BoundsError d_x[firstindex(d_x) - 1] - @test_throws BoundsError d_x[end + 1] - @test nnz(d_x) == nnz(x) - @test Array(nonzeros(d_x)) == nonzeros(x) - @test Array(nonzeroinds(d_x)) == nonzeroinds(x) - @test Array(rowvals(d_x)) == nonzeroinds(x) - @test nnz(d_x) == length(nonzeros(d_x)) +# a random matrix with an empty row and column, a dense row, and a stored zero +function sprand_awkward(::Type{T}, m, n; Ti=Int) where {T} + A = Matrix(sprand_nozeros(T, m, n, 0.3)) + A[1, :] .= rand(T, n) .+ one(T) + A[:, 2] .= zero(T) + A[3, :] .= zero(T) + S = SparseMatrixCSC{T,Ti}(sparse(A)) + nonzeros(S)[end] = zero(T) + return S +end + +# the same entries, and the same explicitly stored ones +same_sparse(A::SparseMatrixCSC, B::SparseMatrixCSC) = + size(A) == size(B) && getcolptr(A) == getcolptr(B) && rowvals(A) == rowvals(B) && + nonzeros(A) == nonzeros(B) +same_sparse(x::SparseVector, y::SparseVector) = + length(x) == length(y) && nonzeroinds(x) == nonzeroinds(y) && nonzeros(x) == nonzeros(y) +same_sparse(A::AbstractGPUSparseArray, B) = same_sparse(host(A), B) +function same_sparse_approx(A::AbstractGPUSparseArray,B::SparseMatrixCSC) + C=host(A) + size(C)==size(B) && getcolptr(C)==getcolptr(B) && rowvals(C)==rowvals(B) && + nonzeros(C) ≈ nonzeros(B) +end +host(A::AbstractGPUSparseArray) = ndims(A) == 1 ? SparseVector(A) : SparseMatrixCSC(A) + +# the result of a reduction on the host, whether it is a scalar or an array +reduced(x::Number) = x +reduced(x::AbstractArray) = Array(x) + +function sparse_construction(AT, eltypes) + @testset "construction and properties" begin + @testset "$S{$ET, $Ti}" for S in sparse_matrix_formats, ET in eltypes, Ti in (Int32, Int64) + A = sprand_awkward(ET, 10, 8; Ti) + dA = gpu_sparse(AT, S, A) + @test dA isa S{ET,Ti} + @test dA isa AbstractGPUSparseArray{ET,Ti,2} + @test size(dA) == (10, 8) + @test length(dA) == 80 + @test ndims(dA) == 2 + @test nnz(dA) == nnz(A) + @test issparse(dA) + @test SparseArrays.indtype(dA) == Ti + @test nonzeros(dA) isa AT{ET} + check_structure(dA) + @test same_sparse(SparseMatrixCSC(dA), A) + Thost = S === GPUSparseMatrixCOO ? promote_type(Int, Ti) : Ti + @test SparseMatrixCSC(dA) isa SparseMatrixCSC{ET,Thost} + @test Array(dA) == Array(A) + @test collect(dA) == Array(A) + @test get_backend(dA) == get_backend(AT(ET[])) + if S <: GPUSparseMatrixCSC + @test Array(getcolptr(dA)) == getcolptr(A) + @test Array(rowvals(dA)) == rowvals(A) + end + end + + @testset "vector $ET, $Ti" for ET in eltypes, Ti in (Int32, Int64) + x = SparseVector{ET,Ti}(sprand_nozeros(ET, 20, 0.3)) + dx = gpu_sparse(AT, x) + @test dx isa GPUSparseVector{ET,Ti} + @test size(dx) == (20,) + @test nnz(dx) == nnz(x) + @test Array(nonzeroinds(dx)) == nonzeroinds(x) + @test Array(rowvals(dx)) == nonzeroinds(x) + @test Array(nonzeros(dx)) == nonzeros(x) + check_structure(dx) + @test same_sparse(SparseVector(dx), x) + @test Array(dx) == Array(x) + end + + @testset "shapes" begin + ET = first(eltypes) + for (m, n) in ((0, 0), (0, 4), (4, 0), (5, 3)), S in sparse_matrix_formats + A = spzeros(ET, m, n) + dA = gpu_sparse(AT, S, A) + check_structure(dA) + @test same_sparse(SparseMatrixCSC(dA), A) + @test Array(dA) == zeros(ET, m, n) + end + for n in (0, 5) + x = spzeros(ET, n) + @test same_sparse(SparseVector(gpu_sparse(AT, x)), x) + end + end + + @testset "checks" begin + ET = first(eltypes) + ptr = AT(Int32[1, 2, 3]) + ind = AT(Int32[1, 2]) + val = AT(ET[1, 2]) + @test GPUSparseMatrixCSR(ptr, ind, val, (2, 2)) isa GPUSparseMatrixCSR + # pointer length + @test_throws ArgumentError GPUSparseMatrixCSR(ptr, ind, val, (3, 2)) + @test_throws ArgumentError GPUSparseMatrixCSC(ptr, ind, val, (2, 3)) + # buffer lengths + @test_throws ArgumentError GPUSparseMatrixCSR(ptr, ind, AT(ET[1]), (2, 2)) + @test_throws ArgumentError GPUSparseMatrixCOO(ind, AT(Int32[1]), val, (2, 2)) + @test_throws ArgumentError GPUSparseVector(ind, AT(ET[1]), 2) + # dimensions + @test_throws ArgumentError GPUSparseMatrixCOO(ind, ind, val, (-1, 2)) + @test_throws ArgumentError GPUSparseVector(AT(Int8[1]), AT(ET[1]), 200) + # the structure is only checked on request + @test check_structure(GPUSparseMatrixCSR(AT(Int32[1, 3, 3]), AT(Int32[1, 2]), val, (2, 2))) isa GPUSparseMatrixCSR + @test_throws ArgumentError check_structure(GPUSparseMatrixCSR(AT(Int32[1, 3, 3]), AT(Int32[2, 1]), val, (2, 2))) + @test_throws ArgumentError check_structure(GPUSparseMatrixCSC(AT(Int32[1, 2, 3]), AT(Int32[1, 3]), val, (2, 2))) + @test_throws ArgumentError check_structure(GPUSparseMatrixCOO(AT(Int32[2, 1]), AT(Int32[1, 1]), val, (2, 2))) + @test_throws ArgumentError check_structure(GPUSparseMatrixCOO(AT(Int32[1, 1]), AT(Int32[1, 1]), val, (2, 2))) + @test_throws ArgumentError check_structure(GPUSparseVector(AT(Int32[2, 2]), val, 3)) end end end -function vector_construction(AT, eltypes) - dense_AT = GPUArrays.dense_array_type(AT) - for ET in eltypes - m = 25 - n = 35 - k = 10 - p = 5 - blockdim = 5 - x = sprand_nozeros(ET, m, 0.2) - d_x = AT(x) - @test collect(d_x) == collect(x) - @test similar(d_x) isa AT{ET} - @test similar(d_x, Float32) isa AT{Float32} +function sparse_allocation(AT, eltypes) + @testset "allocation" begin + @testset "$S" for S in sparse_matrix_formats + ET = first(eltypes) + A = sprand_awkward(ET, 6, 7; Ti=Int32) + dA = gpu_sparse(AT, S, A) + + # structure-preserving + B = similar(dA) + @test B isa S{ET,Int32} + @test nnz(B) == nnz(dA) + copyto!(nonzeros(B), nonzeros(dA)) + @test same_sparse(B, A) + B = similar(dA, Float32) + @test B isa S{Float32,Int32} + @test nnz(B) == nnz(dA) + B = similar(dA, Float32, Int64) + @test B isa S{Float32,Int64} + @test nnz(B) == nnz(dA) + + # without stored entries + for dims in ((6, 7), (3, 4)) + B = similar(dA, Float32, dims) + @test B isa S{Float32,Int32} + @test size(B) == dims + @test nnz(B) == 0 + check_structure(B) + end + B = similar(dA, (3, 4)) + @test B isa S{ET,Int32} + B = similar(dA, Float32, (5,)) + @test B isa GPUSparseVector{Float32,Int32} + @test length(B) == 5 && nnz(B) == 0 + B = similar(dA, Float32, (2, 3, 4)) + @test B isa AT{Float32,3} + B = zero(dA) + @test B isa S{ET,Int32} + @test size(B) == size(dA) && nnz(B) == 0 + + # copies share nothing with the original + B = copy(dA) + @test B isa S{ET,Int32} + @test same_sparse(B, A) + fill!(nonzeros(B), zero(ET)) + @test same_sparse(dA, A) + B = S(dA) + fill!(nonzeros(B), zero(ET)) + @test same_sparse(dA, A) + + # copyto! replaces the structure + A2 = sprand_awkward(ET, 6, 7; Ti=Int32) + dA2 = gpu_sparse(AT, S, A2) + B = copy(dA) + @test copyto!(B, dA2) === B + @test same_sparse(B, A2) + @test_throws DimensionMismatch copyto!(B, gpu_sparse(AT, S, spzeros(ET, 2, 2))) + + # densify + D = AT(zeros(ET, 6, 7)) + @test copyto!(D, dA) === D + @test Array(D) == Array(A) + D = AT(ones(ET, 50)) + copyto!(D, dA) + @test Array(D) == [vec(Array(A)); ones(ET, 8)] + end + + @testset "vector" begin + ET = first(eltypes) + x = SparseVector{ET,Int32}(sprand_nozeros(ET, 20, 0.3)) + dx = gpu_sparse(AT, x) + @test similar(dx) isa GPUSparseVector{ET,Int32} + @test similar(dx, Float32) isa GPUSparseVector{Float32,Int32} + @test similar(dx, Float32, Int64) isa GPUSparseVector{Float32,Int64} + @test nnz(similar(dx, Float32, (20,))) == 0 + B = similar(dx, Float32, (4, 5)) + @test B isa GPUSparseMatrixCSC{Float32,Int32} + @test nnz(B) == 0 + y = copy(dx) + fill!(nonzeros(y), zero(ET)) + @test same_sparse(dx, x) + y = zero(dx) + @test nnz(y) == 0 + y = gpu_sparse(AT, SparseVector{ET,Int32}(sprand_nozeros(ET, 20, 0.5))) + copyto!(y, dx) + @test same_sparse(y, x) + D = AT(zeros(ET, 20)) + copyto!(D, dx) + @test Array(D) == Array(x) + end end end -function matrix(AT, eltypes) - dense_AT = GPUArrays.dense_array_type(AT) - @testset "Sparse matrix properties" begin - @testset "$ET" for ET in eltypes - m = 25 - n = 35 - k = 10 - p = 5 - blockdim = 5 - x = sprand_nozeros(ET, m, 0.2) - d_x = AT(x) - @test size(d_x) == (m, 1) - x = sprand_nozeros(ET, m, n, 0.2) - d_x = AT(x) - d_tx = AT(transpose(x)) - d_ax = AT(adjoint(x)) - @test size(d_tx) == (n, m) - @test size(d_ax) == (n, m) - @test length(d_x) == m*n - @test size(d_x) == (m, n) - @test size(d_x, 1) == m - @test size(d_x, 2) == n - @test size(d_x, 3) == 1 - @test ndims(d_x) == 2 +function sparse_transfer(AT, eltypes) + @testset "adapt" begin + ET = first(eltypes) + A = sprand_awkward(ET, 6, 7; Ti=Int32) + x = SparseVector{ET,Int32}(sprand_nozeros(ET, 20, 0.3)) + + # structural: keep format, element and index types + dA = adapt(AT, A) + @test dA isa GPUSparseMatrixCSC{ET,Int32} + @test nonzeros(dA) isa AT + @test same_sparse(dA, A) + dx = adapt(AT, x) + @test dx isa GPUSparseVector{ET,Int32} + @test same_sparse(dx, x) + @test adapt(AT, dA) === dA + + # to the host + B = adapt(Array, dA) + @test B isa SparseMatrixCSC{ET,Int32} + @test same_sparse(B, A) + @test adapt(Array, dx) isa SparseVector{ET,Int32} + for S in (GPUSparseMatrixCSR, GPUSparseMatrixCOO) + B = adapt(Array, gpu_sparse(AT, S, A)) + if S === GPUSparseMatrixCSR + @test B isa S{ET,Int32,Vector{Int32},Vector{Int32},Vector{ET}} + else + @test B isa S{ET,Int32,Vector{Int32},Vector{ET}} + end + @test same_sparse(SparseMatrixCSC(B), A) + end + + # a target with an element type only converts the values + T = ET <: Complex ? ComplexF32 : Float32 + if T in eltypes + dB = adapt(AT{T}, A) + @test dB isa GPUSparseMatrixCSC{T,Int32} + @test Array(dB) ≈ Array(A) + @test adapt(Array{T}, dA) isa SparseMatrixCSC{T,Int32} + end + + # to an explicit sparse type + dB = adapt(GPUSparseMatrixCSR, dA) + @test dB isa GPUSparseMatrixCSR{ET,Int32} + @test nonzeros(dB) isa AT + @test same_sparse(dB, A) + @test_throws ArgumentError adapt(GPUSparseMatrixCSR, A) + end +end + +@kernel function sparse_rowsum_kernel(out, A) + row = @index(Global, Linear) + acc = zero(eltype(out)) + for k in A.rowPtr[row]:(A.rowPtr[row+1] - 1) + acc += A.nzVal[k] + end + out[row] = acc +end + +@kernel function sparse_colsum_kernel(out, A) + col = @index(Global, Linear) + acc = zero(eltype(out)) + for k in nzrange(A, col) + acc += nonzeros(A)[k] + end + out[col] = acc +end + +function sparse_kernels(AT, eltypes) + @testset "user kernels" begin + ET = first(filter(isrealfloattype, eltypes)) + A = sprand_awkward(ET, 8, 6) + dA = gpu_sparse(AT, GPUSparseMatrixCSR, A) + out = AT(zeros(ET, 8)) + sparse_rowsum_kernel(get_backend(dA))(out, dA; ndrange=8) + @test Array(out) ≈ vec(sum(A; dims=2)) + dA = gpu_sparse(AT, GPUSparseMatrixCSC, A) + out = AT(zeros(ET, 6)) + sparse_colsum_kernel(get_backend(dA))(out, dA; ndrange=6) + @test Array(out) ≈ vec(sum(A; dims=1)) + end +end + +function sparse_display(AT, eltypes) + @testset "display" begin + ET = first(eltypes) + A = sprand_awkward(ET, 6, 7; Ti=Int32) + for S in sparse_matrix_formats + dA = gpu_sparse(AT, S, A) + str = summary(dA) + @test str == "6×7 $(nameof(S)){$ET, Int32} with $(nnz(A)) stored entries in $(nameof(AT))" + # GPUArrays delegates its entry grid to print_array, not SparseArrays' + # version-dependent MIME presentation (which can use braille patterns). + host_str = sprint(Base.print_array, A;context=(:compact=>true,:typeinfo=>ET)) + dev_str = sprint(show, MIME"text/plain"(), dA) + @test split(dev_str, '\n';limit=2)[2] == host_str + expected = S === GPUSparseMatrixCOO ? SparseMatrixCSC{ET,Int}(A) : A + @test sprint(show, dA) == sprint(show, expected) + end + x = SparseVector{ET,Int32}(sprand_nozeros(ET, 20, 0.3)) + dx = gpu_sparse(AT, x) + @test startswith(summary(dx), "20-element GPUSparseVector{$ET, Int32} with $(nnz(x)) stored") + host_str = sprint(show, MIME"text/plain"(), x) + dev_str = sprint(show, MIME"text/plain"(), dx) + @test split(dev_str, '\n')[2:end] == split(host_str, '\n')[2:end] + @test sprint(show, dx) == sprint(show, x) + @test sprint(show, MIME"text/plain"(), gpu_sparse(AT, spzeros(ET, 3))) == + summary(gpu_sparse(AT, spzeros(ET, 3))) + end +end + +function sparse_indexing(AT, eltypes) + @testset "indexing" begin + ET = first(eltypes) + A = sprand_awkward(ET, 6, 7) + probe = AT(ET[1]) + dense_scalar_rejected = try + probe[1] + false + catch err + err isa ErrorException || rethrow() + true + end + @testset "$S" for S in sparse_matrix_formats + dA = gpu_sparse(AT, S, A) + if dense_scalar_rejected + @test_throws ErrorException dA[1, 1] + else + @test dA[1, 1] == A[1, 1] + end @allowscalar begin - @test d_x[:, :] == x[:, :] - @test d_tx[:, :] == transpose(x)[:, :] - @test d_ax[:, :] == adjoint(x)[:, :] - @test d_x[(1, 1)] == x[1, 1] - @test d_x[firstindex(d_x)] == x[firstindex(x)] - @test d_x[div(end, 2)] == x[div(end, 2)] - @test d_x[end] == x[end] - @test d_x[firstindex(d_x), firstindex(d_x)] == x[firstindex(x), firstindex(x)] - @test d_x[div(end, 2), div(end, 2)] == x[div(end, 2), div(end, 2)] - @test d_x[end, end] == x[end, end] - for i in 1:size(x, 2) - @test Array(d_x[:, i]) == x[:, i] - end + @test all(dA[i, j] == A[i, j] for i in 1:6, j in 1:7) + @test dA[(2, 3)] == A[2, 3] + @test dA[end] == A[end] + @test dA[5] == A[5] end - @test_throws BoundsError d_x[firstindex(d_x) - 1] - @test_throws BoundsError d_x[end + 1] - @test_throws BoundsError d_x[firstindex(d_x) - 1, firstindex(d_x) - 1] - @test_throws BoundsError d_x[end + 1, end + 1] - @test_throws BoundsError d_x[firstindex(d_x) - 1:end + 1, :] - @test_throws BoundsError d_x[firstindex(d_x) - 1, :] - @test_throws BoundsError d_x[end + 1, :] - @test_throws BoundsError d_x[:, firstindex(d_x) - 1:end + 1] - @test_throws BoundsError d_x[:, firstindex(d_x) - 1] - @test_throws BoundsError d_x[:, end + 1] - @test nnz(d_x) == nnz(x) - @test nnz(d_x) == length(nonzeros(d_x)) - @test !issymmetric(d_x) - @test !ishermitian(d_x) + @test_throws BoundsError dA[0, 1] + @test_throws BoundsError dA[7, 1] + @test_throws BoundsError dA[1, 8] + @test same_sparse(dA[:, :], A) + @test_throws ArgumentError dA[1, 1] = one(ET) end + x = sprand_nozeros(ET, 20, 0.3) + dx = gpu_sparse(AT, x) + @allowscalar begin + @test all(dx[i] == x[i] for i in 1:20) + @test dx[end] == x[end] + end + @test_throws BoundsError dx[21] + @test same_sparse(dx[:], x) + @test_throws ArgumentError dx[1] = one(ET) end end -function matrix_construction(AT, eltypes) - dense_AT = GPUArrays.dense_array_type(AT) - for ET in eltypes - m = 25 - n = 35 - k = 10 - p = 5 - blockdim = 5 - x = sprand_nozeros(ET, m, n, 0.2) - d_x = AT(x) - @test collect(d_x) == collect(x) - @test similar(d_x) isa AT{ET} - @test similar(d_x, Float32) isa AT{Float32} - @test similar(d_x, Float32, n, m) isa AT{Float32} - @test similar(d_x, Float32, (n, m)) isa AT{Float32} - @test similar(d_x, (3, 4)) isa AT{ET} - @test size(similar(d_x, (3, 4))) == (3, 4) +function sparse_conversions(AT, eltypes) + @testset "conversions" begin + @testset "$ET" for ET in eltypes + A = sprand_awkward(ET, 9, 7; Ti=Int32) + dcsr = gpu_sparse(AT, GPUSparseMatrixCSR, A) + dcsc = gpu_sparse(AT, GPUSparseMatrixCSC, A) + + # between CSR and CSC + B = GPUSparseMatrixCSC(dcsr) + @test B isa GPUSparseMatrixCSC{ET,Int32} + check_structure(B) + @test same_sparse(B, A) + B = GPUSparseMatrixCSR(dcsc) + @test B isa GPUSparseMatrixCSR{ET,Int32} + check_structure(B) + @test same_sparse(B, A) + @test convert(GPUSparseMatrixCSR, dcsr) === dcsr + @test same_sparse(convert(GPUSparseMatrixCSR, dcsc), A) + + # element and index types + B = GPUSparseMatrixCSR{ET,Int64}(dcsc) + @test B isa GPUSparseMatrixCSR{ET,Int64} + @test same_sparse(B, SparseMatrixCSC{ET,Int64}(A)) + B = GPUSparseMatrixCSC{ET,Int64}(dcsc) + @test B isa GPUSparseMatrixCSC{ET,Int64} + @test same_sparse(B, A) + if ET <: Real && Float32 in eltypes + B = GPUSparseMatrixCSC{Float32}(dcsr) + @test B isa GPUSparseMatrixCSC{Float32,Int32} + @test Array(B) ≈ Array(A) + end + @test_throws ArgumentError GPUSparseMatrixCSR{ET,Int8}(gpu_sparse(AT, GPUSparseMatrixCSR, spzeros(ET, 200, 2))) + + # materializing transposes into the opposite format + B = GPUSparseMatrixCSC(transpose(dcsr)) + @test B isa GPUSparseMatrixCSC{ET,Int32} + @test same_sparse(B, copy(transpose(A))) + B = GPUSparseMatrixCSR(adjoint(dcsc)) + @test B isa GPUSparseMatrixCSR{ET,Int32} + @test same_sparse(B, copy(adjoint(A))) + + # shapes + for (m, n) in ((0, 0), (0, 4), (4, 0)) + Z = spzeros(ET, Int32, m, n) + @test same_sparse(GPUSparseMatrixCSC(gpu_sparse(AT, GPUSparseMatrixCSR, Z)), Z) + @test same_sparse(GPUSparseMatrixCSR(gpu_sparse(AT, GPUSparseMatrixCSC, Z)), Z) + end + + # the host arrays have no storage to convert to + @test_throws ArgumentError GPUSparseMatrixCSR(A) + end end end function broadcasting_vector(AT, eltypes) - dense_AT = GPUArrays.dense_array_type(AT) @testset "SparseVector broadcasting" begin @testset "$ET" for ET in eltypes m = 64 p = 0.5 x = sprand_nozeros(ET, m, p) - dx = AT(x) + dx = gpu_sparse(AT, x) # zero-preserving y = x .* ET(1) dy = dx .* ET(1) - @test dy isa AT{ET} + @test dy isa GPUSparseVector{ET} @test collect(SparseArrays.nonzeroinds(dy)) == collect(SparseArrays.nonzeroinds(dx)) @test collect(SparseArrays.nonzeroinds(dy)) == SparseArrays.nonzeroinds(y) @test collect(SparseArrays.nonzeros(dy)) == SparseArrays.nonzeros(y) @@ -182,293 +498,973 @@ function broadcasting_vector(AT, eltypes) # not zero-preserving y = x .+ ET(1) dy = dx .+ ET(1) - @test dy isa dense_AT{ET} + @test dy isa AT{ET} hy = Array(dy) @test Array(y) == hy # involving something dense y = x .+ ones(ET, m) - dy = dx .+ dense_AT(ones(ET, m)) - @test dy isa dense_AT{ET} + dy = dx .+ AT(ones(ET, m)) + @test dy isa AT{ET} @test Array(y) == Array(dy) # sparse to sparse - dx = AT(x) + dx = gpu_sparse(AT, x) y = sprand_nozeros(ET, m, p) - dy = AT(y) + dy = gpu_sparse(AT, y) z = x .* y dz = dx .* dy - @test dz isa AT{ET} + @test dz isa GPUSparseVector{ET} @test z ≈ SparseVector(dz) # multiple inputs y = sprand_nozeros(ET, m, p) w = sprand_nozeros(ET, m, p) - dy = AT(y) - dx = AT(x) - dw = AT(w) + dy = gpu_sparse(AT, y) + dx = gpu_sparse(AT, x) + dw = gpu_sparse(AT, w) z = @. x * y * w dz = @. dx * dy * dw - @test dz isa AT{ET} + @test dz isa GPUSparseVector{ET} @test z ≈ SparseVector(dz) y = sprand_nozeros(ET, m, p) w = sprand_nozeros(ET, m, p) dense_arr = rand(ET, m) - d_dense_arr = dense_AT(dense_arr) - dy = AT(y) - dw = AT(w) + d_dense_arr = AT(dense_arr) + dy = gpu_sparse(AT, y) + dw = gpu_sparse(AT, w) z = @. x * y * w * dense_arr dz = @. dx * dy * dw * d_dense_arr - @test dz isa dense_AT{ET} + @test dz isa AT{ET} @test Array(z) ≈ Array(dz) y = sprand_nozeros(ET, m, p) - dy = AT(y) - dx = AT(x) + dy = gpu_sparse(AT, y) + dx = gpu_sparse(AT, x) z = x .* y .* ET(2) dz = dx .* dy .* ET(2) - @test dz isa AT{ET} + @test dz isa GPUSparseVector{ET} @test z ≈ SparseVector(dz) # type-mismatching ## non-zero-preserving - dx = AT(x) + dx = gpu_sparse(AT, x) dy = dx .+ 1 y = x .+ 1 - @test dy isa dense_AT{promote_type(ET, Int)} + @test dy isa AT{promote_type(ET, Int)} @test Array(y) == Array(dy) ## zero-preserving dy = dx .* 1 y = x .* 1 - @test dy isa AT{promote_type(ET, Int)} + @test dy isa GPUSparseVector{promote_type(ET, Int)} @test collect(SparseArrays.nonzeroinds(dy)) == collect(SparseArrays.nonzeroinds(dx)) @test collect(SparseArrays.nonzeroinds(dy)) == SparseArrays.nonzeroinds(y) @test collect(SparseArrays.nonzeros(dy)) == SparseArrays.nonzeros(y) @test y == SparseVector(dy) + + # without stored entries + x = spzeros(ET, m) + dx = gpu_sparse(AT, x) + y = sprand_nozeros(ET, m, p) + dy = gpu_sparse(AT, y) + @test SparseVector(dx .* ET(2)) == x .* ET(2) + @test SparseVector(dx .* dy) == x .* y + @test Array(dx .+ dy) == Array(x .+ y) + @test length(gpu_sparse(AT, spzeros(ET, 0)) .* gpu_sparse(AT, spzeros(ET, 0))) == 0 end end + + # several sparse arguments, long enough that host-side processing of the combined + # structure (e.g. sorting it through scalar indexing) would be noticeable + @testset "long vectors" begin + ET = first(filter(isrealfloattype, eltypes)) + m = 10^4 + x = sprand_nozeros(ET, m, 0.1) + y = sprand_nozeros(ET, m, 0.1) + dx = gpu_sparse(AT, x) + dy = gpu_sparse(AT, y) + dz = dx .* dy + @test dz isa GPUSparseVector{ET} + @test SparseVector(dz) == x .* y + @test Array(dx .+ dy) == Array(x .+ y) + end end function broadcasting_matrix(AT, eltypes) - dense_AT = GPUArrays.dense_array_type(AT) @testset "SparseMatrix broadcasting" begin - @testset "$ET" for ET in eltypes + @testset "$S{$ET}" for S in (GPUSparseMatrixCSR, GPUSparseMatrixCSC), ET in eltypes m, n = 5, 6 p = 0.5 x = sprand_nozeros(ET, m, n, p) - dx = AT(x) + dx = gpu_sparse(AT, S, x) # zero-preserving y = x .* ET(1) dy = dx .* ET(1) - @test dy isa AT{ET} + @test dy isa S{ET} @test y == SparseMatrixCSC(dy) + # the output does not share buffers with the input + fill!(nonzeros(dy), zero(ET)) + @test same_sparse(dx, x) # not zero-preserving y = x .+ ET(1) dy = dx .+ ET(1) - @test dy isa dense_AT{ET} - hy = Array(dy) - dense_y = Array(y) + @test dy isa AT{ET} @test Array(y) == Array(dy) # involving something dense y = x .* ones(ET, m, n) - dy = dx .* dense_AT(ones(ET, m, n)) - @test dy isa dense_AT{ET} + dy = dx .* AT(ones(ET, m, n)) + @test dy isa AT{ET} @test Array(y) == Array(dy) # multiple inputs y = sprand_nozeros(ET, m, n, p) - dy = AT(y) + dy = gpu_sparse(AT, S, y) z = x .* y .* ET(2) dz = dx .* dy .* ET(2) - @test dz isa AT{ET} + @test dz isa S{ET} @test z ≈ SparseMatrixCSC(dz) # multiple inputs w = sprand_nozeros(ET, m, n, p) - dw = AT(w) + dw = gpu_sparse(AT, S, w) z = x .* y .* w dz = dx .* dy .* dw - @test dz isa AT{ET} + @test dz isa S{ET} @test z ≈ SparseMatrixCSC(dz) # create a matrix with nnz < leading_dim x = spdiagm(m, m, 2=>rand(ET, m - 2)) - dx = AT(x) + dx = gpu_sparse(AT, S, x) y = ET(3) * x dy = ET(3) * dx @test y == SparseMatrixCSC(dy) x = spdiagm(m, m, -2=>rand(ET, m - 2)) - dx = AT(x) + dx = gpu_sparse(AT, S, x) y = ET(3) * x dy = ET(3) * dx @test y == SparseMatrixCSC(dy) + + # without rows or columns + for dims in ((0, 4), (4, 0)) + x = spzeros(ET, dims...) + dx = gpu_sparse(AT, S, x) + @test same_sparse(SparseMatrixCSC(dx .* dx), x .* x) + @test same_sparse(SparseMatrixCSC(dx .* ET(2)), x .* ET(2)) + @test size(dx .+ dx) == dims + end + + # stored zeros are kept + x = sprand_awkward(ET, 6, 7) + dx = gpu_sparse(AT, S, x) + @test nnz(dx .* ET(2)) == nnz(x) + end + end +end + +function broadcasting_mixed(AT, eltypes) + @testset "mixed-format broadcasting" begin + @testset "$ET" for ET in eltypes + A = sprand_awkward(ET, 6, 7) + B = sprand_nozeros(ET, 6, 7, 0.4) + # the result has the format of the first sparse argument + for S in sparse_matrix_formats, S′ in sparse_matrix_formats + dA = gpu_sparse(AT, S, A) + dB = gpu_sparse(AT, S′, B) + dC = dA .* dB + @test dC isa S{ET} + check_structure(dC) + @test SparseMatrixCSC(dC) ≈ A .* B + dC = dA .+ dB + @test dC isa S{ET} + @test SparseMatrixCSC(dC) ≈ A .+ B + @test SparseMatrixCSC(dA + dB) ≈ A + B + @test SparseMatrixCSC(dA - dB) ≈ A - B + @test Array(dA .+ dB .+ ET(1)) ≈ Array(A .+ B .+ ET(1)) + end + for S in sparse_matrix_formats + dA = gpu_sparse(AT, S, A) + @test SparseMatrixCSC(ET(2) * dA) == ET(2) * A + @test SparseMatrixCSC(-dA) == -A + three = ET(3) + @test SparseMatrixCSC(map(v -> v * three, dA)) == map(v -> v * three, A) + @test Array(map(+, AT(Matrix(B)), dA)) ≈ Matrix(B) + A + @test SparseMatrixCSC(conj(dA)) == conj(A) + @test Array(dA .* AT(Matrix(B))) ≈ Array(A .* Matrix(B)) + end + @test_throws ErrorException gpu_sparse(AT, GPUSparseMatrixCSR, A) .* gpu_sparse(AT, sprand_nozeros(ET, 6, 0.5)) + + # SparseArrays' own methods for sparse vectors are replaced + x = sprand_nozeros(ET, 30, 0.3) + y = sprand_nozeros(ET, 30, 0.3) + dx, dy = gpu_sparse(AT, x), gpu_sparse(AT, y) + @test SparseVector(dx + dy) ≈ x + y + @test SparseVector(dx - dy) ≈ x - y + for f in (+, -, *) + @test SparseVector(map(f, dx, dy)) ≈ map(f, x, y) + @test SparseVector(broadcast(f, dx, dy)) ≈ broadcast(f, x, y) + end + if ET <: Real + @test SparseVector(map(max, dx, dy)) == map(max, x, y) + end + @test_throws DimensionMismatch dx + gpu_sparse(AT, sprand_nozeros(ET, 31, 0.3)) end end end function mapreduce_matrix(AT, eltypes) - dense_AT = GPUArrays.dense_array_type(AT) @testset "SparseMatrix mapreduce" begin - @testset "$ET" for ET in eltypes + @testset "$S{$ET}" for S in sparse_matrix_formats, ET in eltypes m,n = 5,6 p = 0.5 x = sprand_nozeros(ET, m, n, p) - dx = AT(x) + dx = gpu_sparse(AT, S, x) # dim=: y = sum(x) dy = sum(dx) @test y ≈ dy - y = mapreduce(abs, +, x) - dy = mapreduce(abs, +, dx) - @test y ≈ dy + # `abs` may give an element type the back-end does not support (Complex{Int16}) + if Base.promote_op(abs, ET) in eltypes + y = mapreduce(abs, +, x) + dy = mapreduce(abs, +, dx) + @test y ≈ dy + y = mapreduce(abs, +, x, dims=1) + dy = mapreduce(abs, +, dx, dims=1) + @test y ≈ Array(dy) + y = mapreduce(abs, +, x, dims=2) + dy = mapreduce(abs, +, dx, dims=2) + @test y ≈ Array(dy) + dy = mapreduce(abs, +, dx; init=zero(Base.promote_op(abs, ET))) + y = mapreduce(abs, +, x; init=zero(Base.promote_op(abs, ET))) + @test y ≈ dy + + # test with a matrix with fully empty rows + x = zeros(ET, m, n) + x[2, :] .= -one(ET) + x[2, end] = -ET(16) + dx = gpu_sparse(AT, S, sparse(x)) + y = mapreduce(abs, max, x) + dy = mapreduce(abs, max, dx) + @test y ≈ dy + end # dim=1 y = sum(x, dims=1) dy = sum(dx, dims=1) @test y ≈ Array(dy) - y = mapreduce(abs, +, x, dims=1) - dy = mapreduce(abs, +, dx, dims=1) - @test y ≈ Array(dy) - # dim=2 y = sum(x, dims=2) dy = sum(dx, dims=2) @test y ≈ Array(dy) - y = mapreduce(abs, +, x, dims=2) - dy = mapreduce(abs, +, dx, dims=2) - @test y ≈ Array(dy) - if ET in (Float32, Float64) - dy = mapreduce(abs, +, dx; init=zero(ET)) - y = mapreduce(abs, +, x; init=zero(ET)) - @test y ≈ dy + # result shapes follow Base + @test size(sum(dx; dims=1)) == (1, n) + @test size(sum(dx; dims=2)) == (m, 1) + + # functions that don't preserve zeros, with operators other than `+` + o = one(ET) + x = sprand_nozeros(ET, m, n, p) + dx = gpu_sparse(AT, S, x) + for dims in (:, 1, 2) + @test mapreduce(v -> v + o, *, x; dims) ≈ reduced(mapreduce(v -> v + o, *, dx; dims)) + if ET <: Real + @test mapreduce(v -> v + o, max, x; dims) ≈ reduced(mapreduce(v -> v + o, max, dx; dims)) + @test mapreduce(v -> v - o, min, x; dims) ≈ reduced(mapreduce(v -> v - o, min, dx; dims)) + end + @test mapreduce(v -> v + o, +, x; dims, init=zero(ET)) ≈ reduced(mapreduce(v -> v + o, +, dx; dims, init=zero(ET))) + @test mapreduce(v -> v + o, *, x; dims, init=o) ≈ reduced(mapreduce(v -> v + o, *, dx; dims, init=o)) + end + # an operator without a known neutral element, relying on `init` + @test mapreduce(identity, (a, b) -> a + b, dx; init=zero(ET)) ≈ mapreduce(identity, (a, b) -> a + b, x; init=zero(ET)) + y = sparse([1, 2], [1, 3], ET[1, 2], 3, 3) + dy = gpu_sparse(AT, S, y) + @test mapreduce(v -> v + o, +, dy) == mapreduce(v -> v + o, +, y) + @test mapreduce(v -> v + o, *, dy) == mapreduce(v -> v + o, *, y) + if ET <: Real + @test mapreduce(v -> v + o, max, dy) == mapreduce(v -> v + o, max, y) end - # test with a matrix with fully empty rows - x = zeros(ET, m, n) - x[2, :] .= -one(ET) - x[2, end] = -ET(16) - dx = AT(sparse(x)) - y = mapreduce(abs, max, x) - dy = mapreduce(abs, max, dx) - @test y ≈ dy + # the stored values of a row or column don't start from zero + if ET <: Real && !(ET <: Unsigned) + x = sparse(-rand(ET(1):ET(10), m, n)) + dx = gpu_sparse(AT, S, x) + @test nnz(dx) == m * n + for dims in (:, 1, 2) + @test maximum(x; dims) == reduced(maximum(dx; dims)) + @test minimum(-x; dims) == reduced(minimum(gpu_sparse(AT, S, -x); dims)) + end + # the same with some implicit zeros + x[1, :] .= 0 + dropzeros!(x) + dx = gpu_sparse(AT, S, x) + for dims in (:, 1, 2) + @test maximum(x; dims) == reduced(maximum(dx; dims)) + end + end + + # empty dimensions + for (k, l) in ((0, n), (m, 0), (0, 0)) + x = spzeros(ET, k, l) + dx = gpu_sparse(AT, S, x) + @test sum(dx) == sum(x) + @test reduced(sum(dx; dims=1)) == sum(x; dims=1) + @test reduced(sum(dx; dims=2)) == sum(x; dims=2) + if Base.promote_op(abs, ET) in eltypes + T = Base.promote_op(abs, ET) + @test reduced(mapreduce(abs, +, dx; dims=1, init=zero(T))) == mapreduce(abs, +, x; dims=1, init=zero(T)) + end + end + end + + @testset "vector $ET" for ET in eltypes + o = one(ET) + x = sprand_nozeros(ET, 20, 0.3) + dx = gpu_sparse(AT, x) + @test sum(dx) ≈ sum(x) + @test mapreduce(v -> v + o, *, dx) ≈ mapreduce(v -> v + o, *, x) + @test sum(gpu_sparse(AT, spzeros(ET, 0))) == zero(ET) end end end -function linalg(AT, eltypes) - dense_AT = GPUArrays.dense_array_type(AT) - @testset "Sparse matrix linear algebra" begin +function sparse_linalg(AT, eltypes) + @testset "linear algebra" begin # sprandn only works on real or complex float types - @testset "$ET" for ET in filter(isfloattype, eltypes) + @testset "$S{$ET}" for S in sparse_matrix_formats, ET in filter(isfloattype, eltypes) m = 10 A = sprandn_nozeros(ET, m, m, 0.2) - B = sprandn_nozeros(ET, m, m, 0.3) - ZA = spzeros(ET, m, m) - C = I(div(m, 2)) - dA = AT(A) - dB = AT(B) - dZA = AT(ZA) - @testset "opnorm and norm" begin - @test opnorm(A, Inf) ≈ opnorm(dA, Inf) - @test opnorm(A, 1) ≈ opnorm(dA, 1) - # SparseArrays.jl#859 implemented the 2-norm for SparseMatrixCSC - if dA isa SparseMatrixCSC && isdefined(SparseArrays, :opnorm2est) - @test opnorm(Matrix(A), 2) ≈ opnorm(dA, 2) - else - @test_throws ArgumentError opnorm(dA, 2) - end + dA = gpu_sparse(AT, S, A) + @test opnorm(A, Inf) ≈ opnorm(dA, Inf) + @test opnorm(A, 1) ≈ opnorm(dA, 1) + @test_throws ArgumentError opnorm(dA, 2) + for p in (1, 2, 3, Inf, -Inf, -2, -1, -0.5, 0) + # Older SparseArrays ignores implicit zeros for negative orders. + reference = p < 0 ? norm(Matrix(A), p) : norm(A, p) + @test reference ≈ norm(dA, p) end + @test norm(gpu_sparse(AT, S, spzeros(ET, 3, 3)), Inf) == 0 end end end -function transpose_matrix(AT, eltypes) - @testset "Sparse matrix transpose and permutedims" begin +function iszero_sparse(AT, eltypes) + @testset "iszero" begin @testset "$ET" for ET in eltypes m = 10 - n = 15 - x = sprand_nozeros(ET, m, n, 0.2) - d_x = AT(x) + x = sprand_nozeros(ET, m, 0.5) + while iszero(x) + x = sprand_nozeros(ET, m, 0.5) + end + @test !iszero(gpu_sparse(AT, x)) + @test iszero(gpu_sparse(AT, spzeros(ET, m))) + @test iszero(gpu_sparse(AT, x) .* zero(ET)) - @test collect(copy(transpose(d_x))) == collect(transpose(x)) - @test collect(copy(adjoint(d_x))) == collect(adjoint(x)) - @test collect(permutedims(d_x, (2, 1))) == collect(permutedims(x, (2, 1))) - @test collect(permutedims(d_x, (1, 2))) == collect(x) - @test collect(permutedims(d_x, [2, 1])) == collect(permutedims(x, [2, 1])) + A = sparse(Matrix(x * transpose(x))) + for S in sparse_matrix_formats + @test !iszero(gpu_sparse(AT, S, A)) + @test iszero(gpu_sparse(AT, S, spzeros(ET, m, m))) + # stored zeros + @test iszero(gpu_sparse(AT, S, A .* zero(ET))) + end + end + end +end - # the result has to stay on the device, in the format of the input - @test copy(transpose(d_x)) isa AT - @test copy(adjoint(d_x)) isa AT - @test permutedims(d_x, (2, 1)) isa AT +function sparse_bool(AT) + @testset "Bool" begin + A = sprand(Bool, 8, 9, 0.4) + B = sprand(Bool, 8, 9, 0.4) + for S in (GPUSparseMatrixCSR, GPUSparseMatrixCSC) + dA = gpu_sparse(AT, S, A) + dB = gpu_sparse(AT, S, B) + @test same_sparse(dA, A) + @test Array(dA .& dB) == Array(A .& B) + @test Array(dA .| dB) == Array(A .| B) + @test sum(dA) == sum(A) + @test reduced(sum(dA; dims=1)) == sum(A; dims=1) + @test reduced(mapreduce(identity, |, dA; dims=2)) == mapreduce(identity, |, A; dims=2) + @test same_sparse(GPUSparseMatrixCSC(gpu_sparse(AT, GPUSparseMatrixCSR, A)), A) + end + end +end - @test_throws ArgumentError permutedims(d_x, (2, 2)) +function sparse_dense_conversions(AT, eltypes) + @testset "format and dense conversions" begin + @testset "$ET" for ET in eltypes + A = sprand_awkward(ET, 9, 7; Ti=Int32) + # every pair of formats, exactly, including the stored zero + for S in sparse_matrix_formats, S′ in sparse_matrix_formats + dA = gpu_sparse(AT, S, A) + B = S′(dA) + @test B isa S′{ET,Int32} + check_structure(B) + @test same_sparse(B, A) + # no hidden sharing with the source + fill!(nonzeros(B), zero(ET)) + @test same_sparse(dA, A) + end + + # transposes, into the same and the other formats + for S in sparse_matrix_formats + dA = gpu_sparse(AT, S, A) + B = copy(transpose(dA)) + @test B isa S{ET,Int32} + check_structure(B) + @test same_sparse(B, copy(transpose(A))) + B = copy(adjoint(dA)) + @test same_sparse(B, copy(adjoint(A))) + @test same_sparse(permutedims(dA), permutedims(A)) + @test same_sparse(permutedims(dA, (1, 2)), A) + for S′ in sparse_matrix_formats + @test same_sparse(S′(transpose(dA)), copy(transpose(A))) + @test same_sparse(S′(adjoint(dA)), copy(adjoint(A))) + end + end + + # dense to sparse drops the zeros, like `sparse` + D = Array(A) + dD = AT(D) + for S in sparse_matrix_formats + B = S(dD) + @test B isa S{ET,Int} + check_structure(B) + @test same_sparse(B, sparse(D)) + B = S{ET,Int32}(dD) + @test B isa S{ET,Int32} + @test same_sparse(B, SparseMatrixCSC{ET,Int32}(sparse(D))) + end + default_Ti=GPUArrays.default_sparse_indtype(dD) + @test sparse(dD) isa GPUSparseMatrixCSC{ET,default_Ti} + @test same_sparse(sparse(dD), sparse(D)) + for (fmt,S) in ((:csc,GPUSparseMatrixCSC),(:csr,GPUSparseMatrixCSR),(:coo,GPUSparseMatrixCOO)) + result=sparse(dD;fmt) + @test result isa S{ET,default_Ti} + S===GPUSparseMatrixCOO || @test eltype(GPUArrays.compressed_ptr(result))===default_Ti + check_structure(result) + @test same_sparse(result,sparse(D)) + end + @test_throws ArgumentError sparse(dD; fmt=:unknown_format) + for (m, n) in ((0, 0), (0, 3), (3, 0), (3, 4)) + Z = zeros(ET, m, n) + for S in sparse_matrix_formats + @test same_sparse(S(AT(Z)), sparse(Z)) + end + end + + x = Vector(sprand_nozeros(ET, 30, 0.3)) + dx = sparse(AT(x)) + @test dx isa GPUSparseVector{ET,Int} + check_structure(dx) + @test same_sparse(dx, sparse(x)) + @test GPUSparseVector{ET,Int32}(AT(x)) isa GPUSparseVector{ET,Int32} + @test same_sparse(sparsevec(AT(x)), sparse(x)) + @test nnz(sparse(AT(zeros(ET, 0)))) == 0 end end + + @testset "findnz" begin + ET = first(eltypes) + A = sprand_awkward(ET, 9, 7; Ti=Int32) + I, J, V = findnz(A) + # in particular for COO, whose conversion once recursed without end (CUDA.jl#3189) + for S in sparse_matrix_formats + dI, dJ, dV = findnz(gpu_sparse(AT, S, A)) + @test dI isa AT{Int32} && dJ isa AT{Int32} && dV isa AT{ET} + @test Array(dI) == I && Array(dJ) == J && Array(dV) == V + end + x = SparseVector{ET,Int32}(sprand_nozeros(ET, 20, 0.3)) + dI, dV = findnz(gpu_sparse(AT, x)) + @test (Array(dI), Array(dV)) == findnz(x) + @test all(isempty, findnz(gpu_sparse(AT, GPUSparseMatrixCOO, spzeros(ET, 3, 3)))) + end end -function iszero_vector(AT, eltypes) - @testset "iszero SparseVector" begin +# a callable that is not a `Function` +struct SparseSubtract end +(::SparseSubtract)(a, b) = a - b + +function sparse_assembly(AT, eltypes) + @testset "assembly" begin @testset "$ET" for ET in eltypes - m = 25 + m, n = 7, 9 + N = 60 + # unordered, with many repeated coordinates + I = rand(1:m, N) + J = rand(1:n, N) + V = rand(ET, N) + dI, dJ, dV = AT(I), AT(J), AT(V) - # Test non-zero sparse vector - x = sprand_nozeros(ET, m, 0.5) - while iszero(x) - x = sprand_nozeros(ET, m, 0.5) + A = sparse(I, J, V, m, n) + for fmt in (:csc, :csr, :coo) + B = sparse(dI, dJ, dV, m, n; fmt) + @test B isa GPUArrays.sparse_format(fmt){ET,Int} + check_structure(B) + @test SparseMatrixCSC(B) ≈ A + end + if ET <: Real + @test same_sparse(sparse(dI, dJ, dV, m, n, max), sparse(I, J, V, m, n, max)) end - d_x = AT(x) - @test iszero(d_x) == iszero(x) - @test iszero(d_x) == false + # a combine function that depends on the order of the repeated entries + noncommutative(a, b) = a + a - b + B = sparse(dI, dJ, dV, m, n, noncommutative) + @test same_sparse(B, sparse(I, J, V, m, n, noncommutative)) + B = sparse(dI, dJ, dV, m, n, noncommutative; fmt=:csr) + @test same_sparse(B, sparse(I, J, V, m, n, noncommutative)) + + # repeated entries are folded in input order, also for `+`, which is not + # associative in floating point + if ET <: AbstractFloat + big = ET(2)^(precision(ET) + 1) + Is, Js, Vs = [2, 2, 2, 1], [3, 3, 3, 1], [big, -big, one(ET), one(ET)] + @test same_sparse(sparse(AT(Is), AT(Js), AT(Vs), 2, 3), sparse(Is, Js, Vs, 2, 3)) + @test same_sparse(sparsevec(AT(Is), AT(Vs), 2), sparsevec(Is, Vs, 2)) + end + # any callable + @test same_sparse(sparse(dI, dJ, dV, m, n, SparseSubtract()), sparse(I, J, V, m, n, -)) + @test same_sparse(sparsevec(dI, dV, m, SparseSubtract()), sparsevec(I, V, m, -)) + + # the dimensions follow from the coordinates + @test size(sparse(dI, dJ, dV)) == size(sparse(I, J, V)) + # a single value, and index types + @test same_sparse(sparse(dI, dJ, one(ET), m, n), sparse(I, J, one(ET), m, n)) + B = sparse(AT(Int32.(I)), AT(Int32.(J)), dV, m, n) + @test B isa GPUSparseMatrixCSC{ET,Int32} + @test SparseMatrixCSC(B) ≈ A + # explicit zeros are kept, as in SparseArrays + @test nnz(sparse(AT([1, 2]), AT([1, 2]), AT(zeros(ET, 2)), 2, 2)) == 2 + # nothing to assemble + @test nnz(sparse(AT(Int[]), AT(Int[]), AT(ET[]), 3, 4)) == 0 + @test_throws ArgumentError sparse(dI, dJ, dV, m - 1, n) + @test_throws ArgumentError sparse(dI, AT(J[1:end-1]), dV, m, n) - # Test zero sparse vector (no stored elements) - z = spzeros(ET, m) - d_z = AT(z) - @test iszero(d_z) == iszero(z) - @test iszero(d_z) == true + # vectors + x = sparsevec(I, V, m) + dx = sparsevec(dI, dV, m) + @test dx isa GPUSparseVector{ET,Int} + check_structure(dx) + @test SparseVector(dx) ≈ x + @test same_sparse(sparsevec(dI, dV, m, noncommutative), sparsevec(I, V, m, noncommutative)) + @test length(sparsevec(dI, dV)) == length(sparsevec(I, V)) + @test same_sparse(sparsevec(dI, dV, noncommutative), sparsevec(I, V, noncommutative)) - # Test sparse vector with stored zeros (e.g., after operations) - # Create a sparse vector then multiply by zero - x_zeros = x .* zero(ET) - d_x_zeros = d_x .* zero(ET) - @test iszero(d_x_zeros) == iszero(x_zeros) - @test iszero(d_x_zeros) == true + # diagonals + v = rand(ET, 5) + w = rand(ET, 4) + @test same_sparse(spdiagm(0 => AT(v), -1 => AT(w)), spdiagm(0 => v, -1 => w)) + @test same_sparse(spdiagm(7, 6, 1 => AT(w)), spdiagm(7, 6, 1 => w)) + @test same_sparse(spdiagm(AT(v)), spdiagm(v)) + @test_throws DimensionMismatch spdiagm(3, 3, 1 => AT(v)) + # without diagonals, SparseArrays applies + @test which(spdiagm, Tuple{Int,Int}).module === SparseArrays + end + + @testset "Bool" begin + I = rand(1:5, 40) + J = rand(1:6, 40) + V = rand(Bool, 40) + # repeated entries are combined with `|` by default + @test same_sparse(sparse(AT(I), AT(J), AT(V), 5, 6), sparse(I, J, V, 5, 6)) + @test same_sparse(sparse(AT(I), AT(J), true, 5, 6), sparse(I, J, true, 5, 6)) + @test same_sparse(sparsevec(AT(I), AT(V), 5), sparsevec(I, V, 5)) end end end -function iszero_matrix(AT, eltypes) - @testset "iszero SparseMatrix" begin +# the operand wrappers LinearAlgebra encodes as characters in the storage-level `mul!` +function sparse_operands(A::AbstractMatrix) + ops = Any[identity, transpose, adjoint] + if size(A, 1) == size(A, 2) + append!(ops, [x -> Symmetric(x, :U), x -> Symmetric(x, :L)]) + eltype(A) <: Complex && append!(ops, [x -> Hermitian(x, :U), x -> Hermitian(x, :L)]) + end + return ops +end + +function sparse_products(AT, eltypes) + @testset "products" begin + @testset "$S{$ET}" for S in sparse_matrix_formats, ET in eltypes + # every operand character for one real and one complex element type, a few for + # the others + all_ops = ET == first(eltypes) || ET == first(filter(iscomplextype, eltypes)) + A = sprand_awkward(ET, 7, 7; Ti=Int32) + R = sprand_awkward(ET, 7, 5; Ti=Int32) # rectangular + dA = gpu_sparse(AT, S, A) + dR = gpu_sparse(AT, S, R) + α, β = ET(2), ET(3) + + # sparse × dense vector (with dense references: SparseArrays lacks some of + # these products for some element types) + for (M, dM) in ((A, dA), (R, dR)), op in (all_ops ? sparse_operands(M) : [identity, adjoint]) + x = rand(ET, size(op(M), 2)) + y = rand(ET, size(op(M), 1)) + @test Array(op(dM) * AT(x)) ≈ op(Matrix(M)) * x + dy = AT(copy(y)) + @test mul!(dy, op(dM), AT(x), α, β) === dy + @test Array(dy) ≈ α * (op(Matrix(M)) * x) + β * y + end + + # sparse × dense matrix, and dense × sparse + dense_ops = all_ops ? [identity, transpose, adjoint] : [identity] + for (M, dM) in ((A, dA), (R, dR)), op in (all_ops ? sparse_operands(M) : [identity, transpose]), dop in dense_ops + B = rand(ET, 4, size(op(M), 2)) + B = dop === identity ? permutedims(B) : B # op(M) * dop(B) + ref = op(Matrix(M)) * dop(B) + @test Array(op(dM) * dop(AT(B))) ≈ ref + C = rand(ET, size(ref)) + dC = AT(copy(C)) + mul!(dC, op(dM), dop(AT(B)), α, β) + @test Array(dC) ≈ α * ref + β * C + + D = rand(ET, size(op(M), 1), 4) + D = dop === identity ? permutedims(D) : D # dop(D) * op(M) + ref = dop(D) * op(Matrix(M)) + @test Array(dop(AT(D)) * op(dM)) ≈ ref + @test dop(AT(D)) * op(dM) isa AT + C = rand(ET, size(ref)) + dC = AT(copy(C)) + mul!(dC, dop(AT(D)), op(dM), α, β) + @test Array(dC) ≈ α * ref + β * C + end + if all_ops + # dense Symmetric and Hermitian operands + B = rand(ET, 7, 7) + for wrap in (Symmetric, Hermitian), uplo in (:U, :L) + @test Array(dA * wrap(AT(B), uplo)) ≈ A * wrap(B, uplo) + @test Array(wrap(AT(B), uplo) * dA) ≈ wrap(B, uplo) * A + end + end + + # with β = 0 the destination is write-only + if ET <: AbstractFloat + dy = AT(fill(ET(NaN), 7)) + mul!(dy, dA, AT(ones(ET, 7)), true, false) + @test Array(dy) ≈ A * ones(ET, 7) + dC = AT(fill(ET(NaN), 7, 3)) + mul!(dC, dA, AT(ones(ET, 7, 3)), true, false) + @test Array(dC) ≈ A * ones(ET, 7, 3) + dC = AT(fill(ET(NaN), 3, 7)) + mul!(dC, AT(ones(ET, 3, 7)), dA, true, false) + @test Array(dC) ≈ ones(ET, 3, 7) * A + end + + # sparse vector operands are densified + x = sprand_nozeros(ET, 7, 0.5) + dx = gpu_sparse(AT, x) + @test dA * dx isa AT{ET,1} + @test Array(dA * dx) ≈ A * Vector(x) + @test Array(AT(Matrix(A)) * dx) ≈ Matrix(A) * Vector(x) + dy = AT(ones(ET, 7)) + mul!(dy, dA, dx, α, β) + @test Array(dy) ≈ α * (A * Vector(x)) + β * ones(ET, 7) + + # empty operands + Z = spzeros(ET, 3, 0) + @test Array(gpu_sparse(AT, S, Z) * AT(zeros(ET, 0))) == zeros(ET, 3) + @test Array(gpu_sparse(AT, S, spzeros(ET, 0, 3)) * AT(ones(ET, 3))) == zeros(ET, 0) + @test Array(gpu_sparse(AT, S, spzeros(ET, 3, 4)) * AT(ones(ET, 4, 2))) == zeros(ET, 3, 2) + @test Array(AT(ones(ET, 2, 3)) * gpu_sparse(AT, S, spzeros(ET, 3, 4))) == zeros(ET, 2, 4) + @test_throws DimensionMismatch dR * AT(ones(ET, 7)) + end + + @testset "$T" for T in (Int, Bool) + A = sprand(T, 6, 5, 0.5) + x = rand(T, 5) + B = rand(T, 5, 3) + D = rand(T, 3, 6) + for S in sparse_matrix_formats + dA = gpu_sparse(AT, S, A) + @test Array(dA * AT(x)) == A * x + @test Array(transpose(dA) * AT(rand(T, 6))) isa Vector + @test Array(dA * AT(B)) == A * B + @test Array(AT(D) * dA) == D * A + end + end + + # Float16 products accumulate in Float32 + if Float16 in eltypes + A = sprand(Float16, 50, 200, 0.5) + x = rand(Float16, 200) + ref = Float16.(Float32.(A) * Float32.(x)) + for S in sparse_matrix_formats + @test Array(gpu_sparse(AT, S, A) * AT(x)) == ref + end + end + end +end + +function sparse_structure(AT, eltypes) + @testset "structure" begin + @testset "$S{$ET}" for S in sparse_matrix_formats, ET in eltypes + A = sprand_awkward(ET, 7, 6; Ti=Int32) + dA = gpu_sparse(AT, S, A) + + # removing entries, in place + B = dropzeros!(copy(dA)) + @test B isa S{ET,Int32} + check_structure(B) + @test same_sparse(B, dropzeros(A)) + @test same_sparse(dropzeros(dA), dropzeros(A)) + @test nnz(dA) == nnz(A) + if ET <: Real + B = copy(dA) + z = zero(ET) + @test fkeep!((i, j, v) -> i != j && v > z, B) === B + check_structure(B) + @test same_sparse(B, fkeep!((i, j, v) -> i != j && v > z, copy(A))) + end + if !(ET <: Integer) && !(ET <: Complex{<:Integer}) + tol = real(ET)(0.5) + @test same_sparse(droptol!(copy(dA), tol), droptol!(copy(A), tol)) + end + + # triangular parts + for k in (-2, 0, 1) + @test same_sparse(triu(dA, k), triu(A, k)) + @test same_sparse(tril(dA, k), tril(A, k)) + @test triu(dA, k) isa S{ET,Int32} + @test same_sparse(triu(transpose(dA), k), triu(copy(transpose(A)), k)) + @test same_sparse(tril(adjoint(dA), k), tril(copy(adjoint(A)), k)) + end + @test same_sparse(triu(dA), triu(A)) + @test istriu(triu(dA)) && !istriu(dA) + @test istril(tril(dA, -1), -1) && !istril(triu(dA, 1)) + @test isdiag(gpu_sparse(AT, S, spdiagm(0 => rand(ET, 4)))) + @test !isdiag(dA) + Q = copy(A[1:6, 1:6]) + Q[1, 2] = Q[2, 1] = one(ET) + dQ = gpu_sparse(AT, S, Q) + @test istriu(UpperTriangular(dQ)) && !istril(UpperTriangular(dQ)) + @test istril(LowerTriangular(dQ)) && !istriu(LowerTriangular(dQ)) + dD = gpu_sparse(AT, S, spdiagm(0 => rand(ET, 4))) + @test istril(UpperTriangular(dD)) && istriu(LowerTriangular(dD)) + + # symmetry, also with a stored zero on one side only + @test !issymmetric(dA) && !ishermitian(dA) + H = sparse(Matrix(A[1:6, 1:6]) + Matrix(A[1:6, 1:6])') + @test ishermitian(gpu_sparse(AT, S, H)) + @test issymmetric(gpu_sparse(AT, S, H)) == issymmetric(H) + Z = sparse([1, 2], [2, 1], ET[1, 0], 2, 2) + @test issymmetric(gpu_sparse(AT, S, Z)) == issymmetric(Z) + if ET <: AbstractFloat + Z = sparse([1, 2], [2, 1], ET[Inf, Inf], 2, 2) + @test issymmetric(gpu_sparse(AT, S, Z)) + end + + # diagonals + for k in (-3, 0, 2, 6, -7) + d = diag(dA, k) + @test d isa AT{ET} + @test Array(d) == diag(A, k) + end + @test isempty(diag(dA, 7)) + @test tr(gpu_sparse(AT, S, A[1:6, 1:6])) ≈ tr(A[1:6, 1:6]) + + # reshaping (SparseArrays gives a lazy wrapper), keeping the stored zero + B = reshape(dA, 14, 3) + @test B isa S{ET,Int32} + check_structure(B) + @test SparseMatrixCSC(B) == reshape(A, 14, 3) + @test nnz(B) == nnz(A) + @test SparseMatrixCSC(reshape(dA, (3, :))) == reshape(A, (3, :)) + @test_throws DimensionMismatch reshape(dA, 5, 5) + x = vec(dA) + @test x isa GPUSparseVector{ET,Int32} + check_structure(x) + @test SparseVector(x) == vec(A) + @test nnz(x) == nnz(A) + + # Kronecker products + B = SparseMatrixCSC{ET,Int32}(sprand_nozeros(ET, 3, 4, 0.5)) + dB = gpu_sparse(AT, S, B) + @test same_sparse_approx(kron(dA, dB), kron(A, B)) + @test kron(dA, dB) isa S{ET,Int32} + @test same_sparse_approx(kron(transpose(dA), dB), kron(copy(transpose(A)), B)) + v = rand(ET, 3) + @test same_sparse_approx(kron(dA, Diagonal(AT(v))), kron(A, Diagonal(v))) + @test same_sparse_approx(kron(Diagonal(AT(v)), dB), kron(Diagonal(v), B)) + huge = gpu_sparse(AT, GPUSparseMatrixCSC, spzeros(ET, 2^32, 0)) + @test_throws ArgumentError kron(huge, huge) + end + + @testset "vector $ET" for ET in eltypes + x = SparseVector{ET,Int32}(sprand_nozeros(ET, 30, 0.4)) + nonzeros(x)[1] = zero(ET) + dx = gpu_sparse(AT, x) + @test same_sparse(dropzeros(dx), dropzeros(x)) + @test same_sparse(dropzeros!(copy(dx)), dropzeros(x)) + B = fkeep!((i, v) -> isodd(i), copy(dx)) + @test same_sparse(B, fkeep!((i, v) -> isodd(i), copy(x))) + end + end +end + +function sparse_slicing(AT, eltypes) + @testset "slicing" begin + @testset "$S{$ET}" for S in sparse_matrix_formats, ET in eltypes + A = sprand_awkward(ET, 9, 8; Ti=Int32) + dA = gpu_sparse(AT, S, A) + for (I, J) in ((2:5, 1:3:8), (:, 2:4), (3:3, :), (1:0, :), (9:-1:9, 2:2), (2:-1:2, 3:-5:3)) + B = dA[I, J] + @test B isa S{ET,Int32} + check_structure(B) + @test same_sparse(B, A[I, J]) + end + for (I, J) in ((:, 3), (4, :), (2:6, 2), (1, 2:2:8), (3, :)) + x = dA[I, J] + @test x isa GPUSparseVector{ET,Int32} + check_structure(x) + @test same_sparse(x, A[I, J]) + end + @test_throws BoundsError dA[1:10, :] + @test_throws BoundsError dA[:, 9] + @test_throws ArgumentError dA[5:-1:1, :] + end + @testset "vector $ET" for ET in eltypes + x = SparseVector{ET,Int32}(sprand_nozeros(ET, 30, 0.4)) + dx = gpu_sparse(AT, x) + for I in (3:17, 1:4:30, 31:30, 3:-1:3) + y = dx[I] + @test y isa GPUSparseVector{ET,Int32} + @test same_sparse(y, x[I]) + end + @test_throws BoundsError dx[0:3] + end + end +end + +function sparse_misc(AT, eltypes) + @testset "dot, count and conversions" begin @testset "$ET" for ET in eltypes - m, n = 10, 10 - - # Test non-zero sparse matrix - A = sprand_nozeros(ET, m, n, 0.5) - while iszero(A) - A = sprand_nozeros(ET, m, n, 0.5) - end - dA = AT(A) - @test iszero(dA) == iszero(A) - @test iszero(dA) == false - - # Test zero sparse matrix (no stored elements) - ZA = spzeros(ET, m, n) - dZA = AT(ZA) - @test iszero(dZA) == iszero(ZA) - @test iszero(dZA) == true - - # Test sparse matrix with stored zeros - A_zeros = A .* zero(ET) - dA_zeros = dA .* zero(ET) - @test iszero(dA_zeros) == iszero(A_zeros) - @test iszero(dA_zeros) == true + x = sprand_nozeros(ET, 30, 0.4) + y = sprand_nozeros(ET, 30, 0.4) + dx, dy = gpu_sparse(AT, x), gpu_sparse(AT, y) + v = rand(ET, 30) + @test dot(dx, dy) ≈ dot(x, y) + @test dot(dx, AT(v)) ≈ dot(x, v) + @test dot(AT(v), dx) ≈ dot(v, x) + @test dot(gpu_sparse(AT, spzeros(ET, 30)), AT(v)) == zero(ET) + @test_throws DimensionMismatch dot(dx, AT(rand(ET, 29))) + A = sprand_awkward(ET, 30, 30) + for S in sparse_matrix_formats + dA = gpu_sparse(AT, S, A) + w = rand(ET, 30) + @test dot(AT(w), dA, AT(v)) ≈ dot(w, A, v) + @test count(!iszero, dA) == count(!iszero, A) + @test count(iszero, dA) == count(iszero, A) + @test any(iszero, dA) == any(iszero, A) + @test all(iszero, dA) == all(iszero, A) + @test sparse(dA) isa GPUSparseMatrixCSC{ET} + @test same_sparse(sparse(dA), A) + @test sparse(dA; fmt=:coo) isa GPUSparseMatrixCOO{ET} + @test SparseVector(sparsevec(dA)) == vec(A) + end + @test Array(sum(dx; dims=1)) ≈ sum(x; dims=1) + @test Vector(dx) == Vector(x) + @test same_sparse(sparse(dx), x) + # (which may give an element type the back-end does not support) + if float(ET) in eltypes + @test SparseVector(float(dx)) == float(x) + end + if complex(ET) in eltypes + @test SparseVector(complex(dx)) == complex(x) + end + @test SparseVector(imag(dx)) == imag(x) + @test SparseVector(real(dx)) == real(x) + end + @testset "Bool" begin + A = sprand(Bool, 8, 9, 0.3) + for S in sparse_matrix_formats + dA = gpu_sparse(AT, S, A) + @test count(dA) == count(A) + @test any(dA) == any(A) + @test all(dA) == all(A) + @test !all(gpu_sparse(AT, S, sparse(trues(3, 3))) .& false) + end + end + end +end + +function sparse_spgemm(AT, eltypes) + @testset "sparse × sparse" begin + @testset "$S{$ET}" for S in sparse_matrix_formats, ET in eltypes + A = sprand_awkward(ET, 8, 6; Ti=Int32) + B = SparseMatrixCSC{ET,Int32}(sprand_nozeros(ET, 6, 7, 0.4)) + dA = gpu_sparse(AT, S, A) + for S′ in sparse_matrix_formats + dB = gpu_sparse(AT, S′, B) + C = dA * dB + @test C isa S{Base.promote_op(LinearAlgebra.matprod, ET, ET),Int32} + check_structure(C) + @test SparseMatrixCSC(C) ≈ A * B + end + dB = gpu_sparse(AT, S, B) + @test SparseMatrixCSC(transpose(dB) * transpose(dA)) ≈ transpose(B) * transpose(A) + @test SparseMatrixCSC(dA' * dA) ≈ A' * A + Q = sprand_nozeros(ET, 6, 6, 0.4) + dQ = gpu_sparse(AT, S, Q) + @test SparseMatrixCSC(Symmetric(dQ) * dB) ≈ Symmetric(Matrix(Q)) * B + if ET <: Complex + @test SparseMatrixCSC(Hermitian(dQ, :L) * dB) ≈ Hermitian(Matrix(Q), :L) * B + end + @test nnz(dA * gpu_sparse(AT, S, spzeros(ET, Int32, 6, 3))) == 0 + @test size(gpu_sparse(AT, S, spzeros(ET, Int32, 0, 4)) * gpu_sparse(AT, S, spzeros(ET, Int32, 4, 2))) == (0, 2) + @test_throws DimensionMismatch dA * dA + + # the sparse-output mul! takes the union of both structures + if ET <: Union{AbstractFloat, Complex{<:AbstractFloat}} + α, β = ET(2), ET(3) + C = SparseMatrixCSC{ET,Int32}(sprand_nozeros(ET, 8, 7, 0.3)) + dC = gpu_sparse(AT, S, C) + @test mul!(dC, dA, dB, α, β) === dC + check_structure(dC) + @test SparseMatrixCSC(dC) ≈ α * (A * B) + β * C + # a destination with as many entries as the product, but elsewhere + P = A * B + D = copy(P) + D.rowval .= mod1.(D.rowval .+ 1, 8) + D = sparse(findnz(D)..., 8, 7) + dD = gpu_sparse(AT, S, D) + mul!(dD, dA, dB, true, true) + @test SparseMatrixCSC(dD) ≈ P + D + # with β = 0, the structure of the product + dD = gpu_sparse(AT, S, SparseMatrixCSC{ET,Int32}(D)) + mul!(dD, dA, dB) + @test SparseMatrixCSC(dD) ≈ P + @test nnz(dD) == nnz(dA * dB) + end + end + + @testset "$T" for T in (Int, Bool) + A = sprand(T, 6, 5, 0.5) + B = sprand(T, 5, 4, 0.5) + for S in sparse_matrix_formats + @test SparseMatrixCSC(gpu_sparse(AT, S, A) * gpu_sparse(AT, S, B)) == A * B + end + end + + @testset "exp" begin + # of an integer matrix (which gives Float64 values) + if Float64 in eltypes + A = sparse([1, 2], [1, 2], [1, 2], 2, 2) + @test Array(exp(gpu_sparse(AT, GPUSparseMatrixCSR, A))) ≈ exp(Matrix{Float64}(A)) + end + for ET in filter(T -> T <: Union{AbstractFloat, Complex{<:AbstractFloat}} && + real(T) != Float16, eltypes) + for scale in (0.25, 4) + A = sprand(ET, 7, 7, 0.4) * real(ET)(scale) + for S in sparse_matrix_formats + E = exp(gpu_sparse(AT, S, A)) + @test E isa S{ET} + @test Array(E) ≈ exp(Matrix(A)) rtol=1e-4 + end + end + end end end end diff --git a/test/testsuite/sparse_v13.jl b/test/testsuite/sparse_v13.jl new file mode 100644 index 000000000..5979aad86 --- /dev/null +++ b/test/testsuite/sparse_v13.jl @@ -0,0 +1,144 @@ +# Shared semantic gates for reference and physical backends. + +@kernel function sparse_lookup_v13_kernel(out, A) + i = @index(Global, Linear) + out[i] = A[i, i] +end + +# Affine-map composition modulo a prime: exactly associative, noncommutative, +# represented by an isbits integer that every backend can transfer. +function compose_affine_v13(x::UInt32, y::UInt32) + a, b = x & 0xffff, x >> 16 + c, d = y & 0xffff, y >> 16 + ((a * c) % 251) | (((a * d + b) % 251) << 16) +end + +function sparse_v13(AT, eltypes) + @testset "v13 contracts" begin + @testset "$S" for S in sparse_matrix_formats + host = sparse(Float32[1 0 0; 2 0 0; 0 0 3]) + A = gpu_sparse(AT, S, host) + v = gpu_sparse(AT, sparsevec([1,3],Float32[2,4],3)) + dense_v = AT(Float32[2,3,4]) + before = copy(A) + @test_throws ErrorException A .* v + @test_throws ErrorException v .* A + @test_throws ErrorException 2f0 .* A .+ v + @test_throws ErrorException A .* v .+ dense_v + @test_throws ErrorException dense_v .+ v .* A + scalar = AT(fill(0f0)) + @test_throws r"both sparse vectors and sparse matrices" (A .* v) .+ scalar + @test_throws r"both sparse vectors and sparse matrices" scalar .+ (A .* v) + @test_throws r"both sparse vectors and sparse matrices" A .* (scalar .+ v) + @test_throws r"both sparse vectors and sparse matrices" (v .+ scalar) .* A + @test_throws r"zero-dimensional GPU arrays" A .+ scalar + @test_throws r"zero-dimensional GPU arrays" scalar .+ A + @test_throws r"zero-dimensional GPU arrays" A .+ view(dense_v,1) + @test_throws r"zero-dimensional GPU arrays" (A .= A .+ scalar) + @test_throws r"zero-dimensional GPU arrays" v .+ scalar + @test_throws r"zero-dimensional GPU arrays" scalar .+ v + for shape in ((3,3,1),(3,3,2),(3,3,1,1)) + dense = AT(zeros(Float32,shape)) + @test_throws r"both sparse vectors and sparse matrices" (A .* v) .+ dense + @test_throws r"both sparse vectors and sparse matrices" dense .+ (v .* A) + @test_throws r"both sparse vectors and sparse matrices" A .* (v .+ dense) + @test_throws r"both sparse vectors and sparse matrices" (dense .+ v) .* A + @test_throws r"expansion of sparse operands" A .+ dense + @test_throws r"expansion of sparse operands" dense .+ A + @test_throws DimensionMismatch (A .= A .+ dense) + end + @test_throws ErrorException (A .= A .* v) + @test A == before + @test Array(A .* dense_v) == Array(host) .* Float32[2,3,4] + @test Array(dense_v .+ A) == Float32[2,3,4] .+ Array(host) + @test Array(A .+ AT(Float32[2])) == Array(host) .+ 2f0 + row = gpu_sparse(AT,S,sparse(reshape(Float32[1,0,2],1,3))) + @test_throws ArgumentError row .+ AT(Float32[2,3]) + @test_throws ArgumentError AT(Float32[2,3]) .* row + column = gpu_sparse(AT,S,sparse(reshape(Float32[1,0,2],3,1))) + @test_throws ArgumentError column .+ AT(ones(Float32,3,2)) + singleton = gpu_sparse(AT,sparsevec([1],Float32[2],1)) + @test_throws ArgumentError singleton .+ v + out = AT(zeros(Float32, 3)) + sparse_lookup_v13_kernel(get_backend(out))(out, A; ndrange=3) + @test Array(out) == diag(host) + @test A == copy(A) + B = copy(A) + B .*= 2 + @test Array(B) == 2 .* Array(host) + B .= B .+ A + @test Array(B) == 3 .* Array(host) + left = sparse(Float32[1 2 3; 0 0 0; 0 0 0]) + right = sparse(Float32[0 0 0; 4 0 5; 0 6 0]) + fused = 2f0 .* gpu_sparse(AT,S,left) .+ 3f0 .* gpu_sparse(AT,S,right) + @test Array(GPUArrays.check_structure(fused)) == 2f0 .* Array(left) .+ 3f0 .* Array(right) + @test Array(A[:]) == vec(Array(host)) + @test nnz(transpose(A)) == nnz(A) + @test sum(transpose(A)) == sum(host) + @test Array(sum(adjoint(A); dims=1)) == sum(adjoint(host); dims=1) + @test_throws ArgumentError hcat(A, A) + @test_throws ArgumentError vcat(A, A) + before = copy(A) + @test_throws ArgumentError (A .= A .+ 1) + @test A == before + y = AT(fill(Float32(NaN), 3)); x = AT(ones(Float32, 3)) + GPUArrays.spmv!(y, 'N', A, x, 1f0, 0f0; algorithm=GPUArrays.Portable()) + @test Array(y) == host * ones(Float32, 3) + candidate = GPUArrays.vendor_spmv(y, 'N', A, x, 1f0, 0f0) + if candidate isa GPUArrays.UnsupportedSparseVendor + @test_throws ArgumentError GPUArrays.spmv!(y, 'N', A, x, 1f0, 0f0; + algorithm=GPUArrays.Vendor()) + else + GPUArrays.spmv!(y, 'N', A, x, 1f0, 0f0; algorithm=GPUArrays.Vendor()) + end + @test Array(y) == host * ones(Float32, 3) + for wrapper in (Symmetric, Hermitian), uplo in (:U, :L) + host = sparse(Float32[1 2; 4 3]) + A = gpu_sparse(AT, S, host) + @test Array(wrapper(A, uplo) * AT(Float32[Inf, 1])) == + wrapper(host, uplo) * Float32[Inf, 1] + end + end + @testset "mixed pointer and coordinate storage" begin + for S in (GPUSparseMatrixCSR, GPUSparseMatrixCSC) + A = S(AT(Int64[1, 3, 4]), AT(Int32[1, 2, 2]), AT(Float32[1, 2, 3]), (2, 2)) + for B in (copy(A), similar(A), similar(A, Float32), S{Float32}(A), zero(A)) + @test eltype(GPUArrays.compressed_ptr(B)) == Int64 + @test SparseArrays.indtype(B) == Int32 + @test !Base.mightalias(A, B) + end + B = A .+ A + @test eltype(GPUArrays.compressed_ptr(B)) == Int64 + @test SparseArrays.indtype(B) == Int32 + @test Array(B) == 2 .* Array(A) + @test Array(GPUArrays.check_structure(B)) == Array(B) + backing = AT(Float32[1, 9, 2, 9, 3, 9]) + bad = S(copy(GPUArrays.compressed_ptr(A)), copy(GPUArrays.compressed_ind(A)), + view(backing, 1:2:6), size(A)) + before = map(Array, GPUArrays.sparse_buffers(bad)) + @test_throws ArgumentError copyto!(bad, zero(A)) + @test map(Array, GPUArrays.sparse_buffers(bad)) == before + for Tp in (Int32,Int64) + shape = S === GPUSparseMatrixCSR ? (1,4) : (4,1) + four = S(AT(Tp[1,5]),AT(Int32[1,2,3,4]),AT(Float32[1,2,3,4]),shape) + empty = zero(four) + merged = 2f0 .* four .+ 3f0 .* empty + @test nnz(merged) == 4 + @test Array(GPUArrays.check_structure(merged)) == 2f0 .* Array(four) + end + end + end + @testset "ordered duplicate assembly" begin + I = AT(Int32[2, 1, 2, 1, 2, 1]) + V = AT(UInt32[2 | (3 << 16), 4 | (5 << 16), 6 | (7 << 16), + 8 | (9 << 16), 10 | (11 << 16), 12 | (13 << 16)]) + x = sparsevec(I, V, 3, compose_affine_v13) + expected = sparsevec(Array(I), Array(V), 3, compose_affine_v13) + @test Array(x) == Array(expected) + # Packed-key overflow has a tiny lexicographic test independent of allocation. + keys = GPUArrays.packed_keys(AT(Int64[2, 1, 2]), AT(Int64[1, 2, 1]), + (typemax(Int), 2)) + @test Array(sortperm(keys; alg=Base.Sort.MergeSort)) == [2, 1, 3] + end + end +end diff --git a/validation/HISTORY.sparse-v13.md b/validation/HISTORY.sparse-v13.md new file mode 100644 index 000000000..b393a389c --- /dev/null +++ b/validation/HISTORY.sparse-v13.md @@ -0,0 +1,787 @@ +# Historical sparse-v13 implementation checkpoints + +Superseded by HANDOFF.sparse-v13.md. Retained as chronological evidence, not current status. + +# GPUArrays v13 implementation handoff + +## Current state (Oct21, final source validation) + +Final peer follow-up: zero-dimensional GPU fusion also needed sparse style joins. +Twelve focused cases fail red with incidental scalar indexing, then pass the precise +mixed-rank rejection after the fix. Positive GPU0 operands explicitly reject before +host scalar capture; device-scalar broadcast is not implemented. The new runtime +patch passes1914regressions,184contracts,display52,linalg195,mixedbroadcast275 and +ambiguity1 locally on Julia1.13.2; Sol79 checked ordering and zero-dimensional views. +GPUArrays64edd6d is pushed to both topic/PR refs with one TEMP CI commitca680c3; +GHA39641730432 and Buildkite2085 target that exact head, but the GPU0 follow-up still +needs pushing and its own final-head validation. Earlier exact GA64edd6d +Julia1.10.12 focused run passes1894regressions,display52,linalg195,broadcast275 and +ambiguity1. Actual final CI helper at GA64edd6d/OpenCLe0a passes seven native +array/memory/context/queue/event/synchronization groups155 on Pegasus; log +ga13-opencl-svm-env/final-native64.log, exact pins/clean source logged. +CUDAa3/GA64ea focused run completed151contracts+195linalg+275broadcast+ambiguity1. + +This section supersedes every historical checkpoint below. GPUArrays local4083b78 +contains four reviewed fixes after pushed ef7659a: mixed sparse broadcast styles and +preallocation rejection, logical negative-order norms, CPU-MIME-independent display +tests, and scalar-magnitude nonfinite complex norm references. Runtime source at +4083b78 equals frozen64ea2d8; only the final regression reference differs. Final +central pins are CUDAa3e0c60, AMDf5b6976, oneAPId1cb45d, Metal1aaa653 and OpenCLe0a09f4, +plus AKa915b41/Atomix1ffd4ae. Pin/helper/timeout changes will be folded into the single +TEMP CI commit, not pushed as another series of debug commits. Preserve tested source +SHAs and the old remote tip before that rewrite; do not relabel historical tests as +literal tests of a new commit hash. + +Exact clean-source results: + +- GA64ea2d8/OpenCLe0a09f4: actual root Pkg.test sparse/SVM suites pass11,759 on + EACH host, Pegasus RTX5080 and Taurus RTX6000Ada. Shared sparse11,744 across all + ten supported types, SVM15, normal GC/synchronization, one worker. An empty + OCL_ICD_VENDORS directory plus native OCL_ICD_FILENAMES constrained enumeration + to NVIDIA only. Both logs: ga13-opencl-svm-env/actual-final-entry-full.log. + This is actual migrated GPU coverage, not PoCL evidence or a whole-backend pass. +- GA64ea2d8/oneAPId1cb45d: actual checked Pkg.test, allow_reresolve=false, passes + 13,138 plus24 pre-existing broken checks: sparse ABI96, shared11,641, native + oneMKL1279+24broken, resources122. Taurus ga13-oneapi-env/final-marked-stack.log. + The loaded rebuilt support SHA256 is + c19915c2965bc4f7153e4c16d240deb107ecfe0a902f07936445d5f9969932c9 at + ~/Julia/tmp/gpuarrays-v13-stack-lyFPFO/support-scratch/8f75cd03-7ff8-4ecb-9b8f-daf728133b1b/deps/lib/liboneapi_support.so. + This is a source-built stack, not the registered artifact. Actual published0.11.0 + is rejected for its absent version marker; the new library reports2. Dense oneAPI + still executes with sparse eligibility disabled; real-artifact probes6/6 pass. +- GA64ea2d8 full shared JL sparse on Julia1.15-DEV61 passes14,957; ambiguity1. + Expanded regressions initially1890pass/4fail exposed CPU BLAS versus scalar complex + Inf/NaN reference differences, not a runtime sparse failure. Corrected GA4083b78 + actual nightly Pkg.test passes1894regressions+ambiguity1. Pegasus + ga13-nightly-final-env/{nightly-logical-final,nightly-norm-final-regressions}.log. +- Local Julia1.13.2, runtime4083b78: regressions1894 and contracts154 pass; + focused display52, linalg195, mixed broadcast275, ambiguity1 pass. The broadcast + regression was independently red9/9 then green9/9. Negative orders now include + implicit zeros on every Julia version, preserving NaN magnitudes. Dense operands + may broadcast-expand; sparse expansion/mixed sparse ranks reject before allocation. +- GA64ea2d8/AMDf5b6976: GPUArrays-first contracts151+ambiguity1 pass; earlier backend- + first ef7659a/f5b6976 contracts109+ambiguity1 pass. Pegasus + ga13-amdgpu-loadorder-final-env/core64-gpuarrays-first.log. +- Metal1aaa653/GAef7659a: fresh actual Pkg.test spot53/53, --jobs=1 and explicit + --threads=1. Local ga13-metal-final-spot-prefix.log. No complete local Metal suite + and no physical Metal retest of the final central broadcast/norm changes claimed. + The user's limit remains single-threaded spot testing, not full local testing. +- CUDAa3e0c60/GA64ea2d8 refreshed focused contracts/linalg/broadcast run is pending; + Pegasus ga13-cuda-env/final-core-focused.log. Retain older full automatic/portable + and native solver/factorization/BSR/batch evidence below at its exact revisions. + +The inherited NVIDIA dense-allocation churn remains independently reproducible on +OpenCL main9c27048/registered GA12.1.1, including strict NVIDIA-only discovery. A +native C shim observes NULL(size44,alignment4), alloc/fail/free=(15,1,3) at failure +and(15,1,11) after unwind, after ten checks; baseline-strict-nvidia.log. The root +sparse suites above do not reproduce it. Cause unresolved; neither a v13 regression +nor a driver defect is established. The independent NULL guard and vendor-driver +test initialization fix are production changes, not alleged causal fixes. See +validation/REPORT.opencl-svm.md and its dense-only reproducer. + +Actual CI run39625043230 tested synthetic merge32c670f409c65fd095df6f0cdff9a541cc3ac788 +(main6fa6490 + ef7659a), NOT the local fixes above. Ubuntu/Windows1.10–1.13 and docs +pass; Windows1.12 actual35,416 assertions. Both nightly jobs fail21 (18 implicit-zero +norm assertions,3 CPU display-format references), now addressed by the frozen-source +tests above but awaiting new CI. OpenCL hit its exact60-minute timeout while both +IL/C sparse suites and dense jobs were still running; no completed aggregate pass. +Logs were inspected. The final isolated CI layer gives this unshortened two-route +suite180minutes, rather than skipping tests or allowing failure. macOS jobs remain +queued/running. Buildkite2082 canceled and2083 all-broken are NOT backend validation. + +oneAPI's native artifact gate probes wrapper version2 plus wide CSC using one loaded +handle. All87 raw sparse wrappers gate before argument conversion/native entry; +96 ABI checks verify this behavior. Regeneration retains unrelated generated files. +The official0.11.0+0 library lacks this marker despite exporting wide CSC, so must +not be accepted as safe. Sparse support fails closed; dense behavior remains usable. +Yggdrasil topic maleadt/Yggdrasil:tb/oneapi-sparse-v13-support is pushed at31c79455b, +on latest master0defdb4d8. Separate e0dc775 fixes an existing extra-parenthesis parse +error. Version0.11.1 retains upstream source5093529 and Intel2026.1, carrying only +the focused native cpp/header patch; patched files cmp equal tested oneAPId1cb45d. +Runtime dependency floors are the validated Loader1.32/OpenCL2024.10.24/Hwloc2.15, +not claimed minimal ABI requirements. Julia1.13.2 root-project BinaryBuilder metadata +generation passes for x86_64-linux-gnu-cxx11. Actual crossbuild/generated-JLL test is +pending on Pegasus (ga13-ygg-build.log), with task-owned JULIA_PKG_DEVDIR. Metadata +and the source-built Arc result are not a generated artifact validation. + +Release/maintainer checklist (temporary pins are not release-ready resolution): + +1. Finish refreshed exact-stack CI and inspect checkout + STACK_REVISION lines, + including CUDA's root/all13 split packages and their test environments. Use the + explicit develop/setup path on Julia1.10; [sources] alone is ineffective there. +2. Build/test/publish and register oneAPI_Support0.11.1 with wrapper contract2; + test the generated artifact WITHOUT the source-build preference. Raise oneAPI's + support minimum to the actual registered version, retaining the runtime gate. +3. Release compatibility-only AK0.5.9/Atomix1.6.1, GPUArrays13/JLArrays0.5 (Core stays + 0.2.1), then coordinated CUDA7 (all14 packages), AMDGPU3, oneAPI3, OpenCL0.11, + Metal1.12 as their weak-dependency ordering permits. Verify every test project. +4. Land downstream compatibility/API updates recorded in + validation/REPORT.sparse-v13-downstream.md; whole consumers were not tested. +5. Remove the isolated TEMP CI/bootstrap pins when registered resolution works; + rerun normal Pkg.test/docs on Julia1.10 and supported modern/nightly versions. + Before then describe results as exact pinned-stack validation only. +6. Keep the inherited OpenCL dense NULL reproducer open; old cuSPARSE algorithm + branch performance, additional GPU generations and full local Metal remain + untested. CUDA-only measured performance/memory is in perf/REPORT.sparse-v13.md. + +Astra architectural decisions through23 and Sol implementation review through76 are +in peer conversations sparse-v13-architecture and sparse-v13-review. Findings were +checked and fixes rerun. No GitHub discussions or PR descriptions changed yet. The +user authorizes succinct existing PR745/Metal982 description updates once work is +finished; new backend/dependency PR descriptions remain local drafts, not posted PRs. + +## Historical checkpoints + +## Latest checkpoint (Oct21, CI history cleanup and final native gates) + +This section supersedes the chronological checkpoints below. GPUArrays ef7659a pins +CUDAa3e0c60ee, AMDf5b69764c, oneAPI4858db9, Metal89143a6ed and OpenCL0760462. +All topic and existing PR branches are pushed. Per the user's concern,14 incremental +TEMP CI commits were consolidated into ONE isolated CI commit. Production commits +and contributor authorship remain; the full tree before repinning was byte-identical +to fc9bda1. Old remote65ff6ec and full fc9bda1 are preserved in local/remote backup refs +backup/sparse-v13-{ci-debug,before-ci-cleanup}-20261021. Backend bootstraps pin cleaned +dd0ee48, retained under backup/sparse-v13-bootstrap-20261021; new central pins are the +only tree difference from that cleanup. Backend previous tips have their own local/ +remote backup/sparse-v13-before-bootstrap-repin-20261021. All rewrites checked remote +heads and used explicit leases/refspecs. This remains stack validation, not normal +registered dependency resolution. + +Every backend's CI now explicitly resolves the pinned stack rather than letting an +early test-plugin instantiate registered incompatible versions. Existing toolkit, +allocator/memory, sanitizer, multi-GPU, minimum-dependency, Enzyme, debug/validation, +OpenCL driver/memory/translation and docs/performance variants are retained. Nightly +and dependency installation failures are hard failures. CUDA's memory matrix now +writes CUDACore's preference (not obsolete CUDA's), and the helper asserts the loaded +memory mode. Custom dependency jobs change the disposable root environment and keep +their selections through the checked runner. Preferences retain recursive __clear__ +semantics and transitive UUID/name bindings via extras. Binding, actual Pkg.test +sandbox propagation and preference-clearing probes pass on Julia1.10.12 and1.13.2. +Sol turns28-30 reviewed these changes; two verified wiring issues were corrected. +YAML parsing and diff checks pass. Actual checked Pkg.test runner ecbe532 with CUDA +42d9ec7 passes228 in confirmed unified memory, allow_reresolve=false/check-bounds=yes; +AMD47a61c2 passes3189 native sparse+dense solver; oneAPI80a88b3 passes1401 native/ +resource checks plus24 pre-existing broken CSC checks. Their runtime files are unchanged +by the repin; do not label the new bootstrap SHAs hardware-tested yet. Logs Pegasus +ga13-cuda-ci-entry/self-unified-final-green.log, ga13-amdgpu-env/self-final-green.log; +Taurus ga13-oneapi-env/self-final-green.log. oneAPI rebuilt support SHA256 +6d0ee79ec8b9a91034b0442e8e9d7e54e493019aaf36db76376fa4241be13e55 at +/tmp/gpuarrays-v13-stack-sttpHy/support-scratch/8f75cd03-7ff8-4ecb-9b8f-daf728133b1b/deps/lib/liboneapi_support.so. +Initial invocations accidentally appended daemon startup ARGS; reruns cleared ARGS +before the checked runner. That was a harness error, not a production CI failure. +No further full local Metal runs: single-worker/thread spot tests only. + +Clean frozen GA0f2d80f / CUDAe37f85d19 full shared six-type suites now pass automatic +7712 and forced-portable7713 (2004 portable selections), with an empty sparse ambiguity +check each. Clean GA0f2d80f / AMDe3a3a604c shared four-type suites pass automatic5540 +and forced-portable5541 (1557 selections), ambiguity1 each. Pegasus logs: +ga13-{cuda,amdgpu}-env/shared-0f2-{auto,portable}.log. Clean GA c8dd580 / +oneAPI26fb776 forced-portable shared four-type run passes5517 (1560 selections), +ambiguity1; Taurus ga13-oneapi-env/portable-final.log. These supplement the native +backend suites and oneAPI12904+24 pre-existing broken checks recorded below. + +New deprecated CSR/CSC/COO dispatch names (Astra16/Sol31) alias canonical containers, +not abstract extension points. BSR/device aliases and dense_array_type remain removed. +CPU precision-policy and initialized dense-copy fixtures pass:858 on Julia1.10/1.13; +clean frozen781a0b2 actual Julia1.15 Pkg.test passes859 including ambiguity. Log Pegasus +ga13-nightly-final-env/nightly-final.log. History-cleaned equivalent is3ceeb6e. +Julia1.10 docs makedocs also passes locally (ga13-docs-build-only110.log); merely +including docs/make.jl in interactive jld does NOT call main. No deployment performed. +See validation/REPORT.sparse-v13-downstream.md for exact consumer audit revisions; +MLDataDevices, QuantumToolboxCore and SparseMatrixColorings still need coordinated +compatibility/API releases. No whole downstream-package pass claimed. +CUDAcb5389b13 / AMD8c0ff4687 remove four no-longer-included legacy integration files; +their history remains recoverable and the active centralized methods are unchanged. + +OpenCL Pegasus clean781a0b2/88d88e8: PoCL CPU forced-portable four-type full shared +suite5544 passes (1554 selections), ambiguity1; forced OpenCL-C contracts112+ambiguity1 +pass. NVIDIA RTX5080 fresh contracts112+ambiguity1 pass, but broader sparse run fails +first at native SVM upload(-30); separately the requested Float16 is unsupported there. +The driver now records resolved IL/C route, capability-tests Float16 rejection when +absent and omits it only from default types; explicit type selections remain honored. +No complete NVIDIA sparse suite pass. Logs Pegasus ga13-opencl-final-env. + +Minimizer ga13-opencl-dense-triplet-churn.jl uses ONLY dense CLArray buffers and CPU +SparseArrays reconstruction: current stack fails872 or4 checks in fresh contexts; +matched minimal v13 environment also fails952. Same-host OpenCL main9c27048 with +registered GA12.1.1 passes960 twice. Other simple dense/GC probes pass. This narrows +the failing stack but does NOT establish GPUArrays or driver causality. GPUToolbox +3.4.1 matches both Pegasus envs; AK/Atomix src/ext match their registered baseline +trees exactly. Default preferences match. Guarded noinline uploader passes once; +Native-blocking and ordinary-Ptr-return diagnostics also fail. Do not enable GC-off or +blocking synchronization as a production fix without causal evidence. + +Subsequent controls supersede that baseline correlation: fresh same-host OpenCL +main9c27048 / registered GA12.1.1 also fails10+1 on this minimizer, first native +SVM memcpy CL_INVALID_VALUE(-30), log ga13-opencl-baseline-env/baseline-long.log. +Classify the intermittent allocation/transfer failure as reproduced on baseline +and v13, cause unresolved; this does not establish a driver or v13 regression. +The ordinary-Ptr native-return diagnostic also fails824+1, so changing the custom +CLPtr return type is not a sufficient fix. A same-call native C shim confirms actual +NULL on the baseline, with the native out-parameter matching Julia's return. With +OCL_ICD_FILENAMES=libnvidia-opencl.so.1,344 checks pass before size36/alignment4 +returns NULL: alloc/fail/free=(516,1,237) at failure and (516,1,514) after unwind. +That environment variable adds ICDs, not exclusive discovery. The allocating task's +finalizer-inhibition counter was0 on the default pool. No fixed resource cap is +established. See validation/REPORT.opencl-svm.md and its dense-only reproducer. + +Independent OpenCL029274e is pushed: correctly detects CL_NULL (not cross-address- +space C_NULL), retries through existing bounded reclamation, preserves the Context +across GC and throws before retaining its native context or publishing storage. +Mocked retry/error/argument checks and actual native context-reference checks pass15. +The new error reports native NULL, not an unproven OOM cause. This does NOT fix the +underlying intermittent native failure. Astra20/Sol43 reviewed the focused patch; +post-initialization cleanup was deliberately not added because failed completion +cannot justify freeing pending memory. Existing Intel initialization cleanup is a +separate audit. Actual clean root Pkg.test with GAef7659a/OpenCL029274e crashed +before tests in versioninfo, mixing JLL/system PoCL libraries: the runner imported +pocl_jll even for --platform=cuda. Independent runner fix6ba2cf4 conditionally +imports CPU drivers in all three runner contexts, leaving target/test selection +unchanged. The matching GA setup fix remains uncommitted in the single CI layer. +Actual root GAef7659a/OpenCL6ba2cf4 on both hosts now passes the15 SVM checks and is +running the entire shared sparse group, one worker. Logs on both hosts: +ga13-opencl-svm-env/actual-driver-entry-full.log. Complete result pending. + +Upstream Metal/OpenCL main advanced during validation. Rebased/pushed Metal +1aaa6534c9770c28403a46d8c4006cfbaac5edb4 onto e0dd041 (cooperative execution test +hardening) and OpenCL e0a09f4994faedfaa1606cb9e4f5ff4913e4da41 onto c0ee73d +(versioninfo subgroup-ballot display). Metal's four CI iterations are consolidated +into one isolated CI commit. Previous tips89143a6 and6ba2cf4 remain local/remote +backup/sparse-v13-before-main-refresh-20261021; pushes used explicit leases/refspecs. +Runtime sparse implementations are unchanged by these upstream deltas. The current +root runs remain tied to frozen6ba2cf4, not the rebased OpenCL SHA. A fresh Metal +53-check spot run is pending, --jobs=1 and explicit --threads=1; no full local suite. +All other upstream main refs were rechecked and unchanged. + +Astra17/Sol33 independently confirmed an existing OpenCL callback lifetime defect: +both Context constructors drop runtime Base.CFunction handles. Wrapper-only rooting +would not cover native context lifetime, and foreign-thread callbacks need an audit. +Keep this separate: not shown to cause the upload failure, not patched or posted. + +GitHub reports a PR/Actions/webhook delay incident starting14:52UTC Oct21; PR heads +lagged verified git refs. GHA39625043230 documentation passes, actual checkout +32c670f409c65fd095df6f0cdff9a541cc3ac788 (synthetic merge of main6fa6490 and +topic ef7659a), not the topic SHA alone. Logs verify exact AK/Atomix pins, actual +doctests and CheckDocument; preview deployment declined. Windows Julia1.12 also +passes35416 actual assertions on that same merge checkout, with exact AK/Atomix +pins verified in ga13-ci-windows112-ef7659a.log. Other jobs are queued or running, +no whole-stack CI pass. Buildkite2082 on ef7659a was auto-canceled with +build_skipping after delayed older webhooks;2083 on old781a0b2 says passed but ALL +backend jobs are broken/never executed. This is NOT backend validation. +Inspect exact head SHAs/logs after recovery; earlier Ubuntu1.13 35017 and Metal53 spot +passes retain their recorded older revisions. Do not cancel unrelated busy GPU jobs. +Official oneAPI support JLL still lacks the native exception-boundary change: a new +artifact plus coordinated releases remain mandatory. User now authorizes updating +existing PR descriptions ONCE EVERYTHING IS DONE, clearly and extremely succinctly; +draft descriptions for new migration PRs too. Do not post premature completion claims. +No PR descriptions/discussion text have been changed yet. Succinct drafts for GA, +all five backends, AK and Atomix are /Users/tim/Julia/tmp/sparse-v13-*-description.md; +Sol57 identified and corrected an overclaim about cuSPARSE direct publication: +it avoids redundant canonical reassembly, not vendor candidate expansion. + +## Latest checkpoint (Oct21, policy ownership and final-stack checks) + +This section supersedes older chronological checkpoints below. Latest pushed GPUArrays +is1234787 on both topic/PR branches; production0f23249 is committed locally and awaits the +coordinated dependency-pin push. CUDAe37f85d19 and AMDe3a3a604c are pushed to tb/sparse-v13; +Metalb8dc6bf1f is pushed to both branches and bootstraps GA1234787. oneAPI26fb776, +OpenCL9526a11, AKa915b41 and Atomix1ffd4ae unchanged. PR745 targets main and is detached +from its old stack; the other three PRs' bases and heads were verified unchanged. + +Production0f23249 moves dense sparse(A) policy fully into GPUArrays. Two scoped helpers +preserve CUDA/AMD uniform Cint defaults and existing BSR constructor/block-size behavior; +explicit canonical constructors and coordinate assembly retain their width rules. Removed +backend public overrides in CUDAbcf4a222a/AMDc5ae5ef37. Astra turn14/Sol turn24 approve. +Shared tests now check the declared default including pointer width; independent backend +tests require Cint and BSR parity, invalid-format tests use unknown_format. No tests skipped. +Candidate physical conversions pass600 shared+8 findnz each on CUDA/AMD, plus each native +conversion suite and12 dedicated policy checks. JL reference208 checks pass. These are +candidate-worktree results, not yet clean final-commit full shared runs. + +Independent explicit SparseMatricesCSR upload fix in CUDAe37f85d19/AMDe3a3a604c: +typed CSC/COO now construct the requested uniform-width CSR directly instead of first +narrowing through the backend's default Cint adaptor. Old CUDA8a9379519 red repro rejects +explicit Int64 COO with dimensions(1,2147483648) despite one stored entry. New physical +12 regressions each pass, plus CUDA48/AMD53 existing extension checks. No giant allocation. + +Clean frozen GA c8dd580 / oneAPI26fb776 Pkg.test on Taurus ArcA770 passes12904, +with24 pre-existing broken CSC triangular checks: shared sparse11503, resources122, +oneMKL1279. Custom support path verified and native exception containment included. +AKa915b41 and Atomix1ffd4ae source trees verified clean. Official support artifact still +lacks the C++ catch fix, so this remains stack validation, not normal release resolution. +Log Taurus: ga13-oneapi-env/frozen-root.log. + +Clean frozen GA c8dd580 / AMD08277ef7c actual Pkg.test on Pegasus passes all selected +native sparse and dense solver groups: sparse_v13 208, solver1089, interfaces625, +generic596, preconditioners312, broadcast65, linalg168, SparseMatricesCSR53, +conversions34, device15. Log ga13-amdgpu-env/frozen-native.log. This precedes the latest +policy/upload commits; final rerun remains required. CUDA clean c8dd580/8a9379519 shared +six-type runs expose ONLY24 dense policy expectation failures: automatic7652 passing, +forced-portable7653 passing. The policy correction above addresses those exact failures; +do not call those whole runs passing. + +TEMP CI1234787 (reviewed by Sol turn23) resolves each actual CUDA library test project +with the same pins; Julia<1.12's generated root-test Manifest is moved into scratch before +Pkg.test because old Pkg unconditionally rejects duplicate developed UUIDs even at the +same path. Library manifests stay for runner project switches. Actual Julia1.10.12 CUDA +entrypoint SparseMatricesCSR test passes48, log ga13-cuda-ci-entry/entry110-libs2.log. +Self-backend checkouts are cloned clean into disposable scratch (no stale manifests or +source worktree mutations). oneAPI source build is unconditional build_local, with supplied +preferences retained through the build and propagated to tests. Metal own spot tests53 +pass one-worker, log /Users/tim/Julia/tmp/ga13-metal-spot-clean.log; daemon stopped. +Local Metal testing must remain focused and single-worker per user's request. + +GHA39608465677 / Buildkite2067 test exact GA1234787, not local0f23249. At last check all17 +GHA jobs queued; Buildkite uploader passed, backend jobs scheduled. Earlier GHA7d2123d +Ubuntu1.13 passed35017. New pins/commits will need revision verification and real logs. +GPU agents are online but occupied; do not cancel unrelated jobs. Other backend self-CI +still needs coordinated bootstrap wiring; Metal is wired. No CI stack pass claimed. + +Performance checkpoint0cc9461 commits the harness,115 native-case comparisons, raw data, +and observed Nsight device memory. Exact GA c8dd580 / CUDA8a9379519 native SpGEMM is +within1% of old CUDA44ac4ebb3, same203605548-byte incremental peak. See perf/REPORT.sparse-v13.md. + +Fresh NVIDIA OpenCL Taurus reproduction: frozen GA c8dd580/OpenCL9526a11, RTX6000Ada, +GPUToolbox3.4.2. Normal GC/cooperative synchronization gives82 passing/3 errors; first +-9999 occurs after tiny fused CSR broadcast, then context is poisoned. Same full contracts +with GC disabled pass112 + ambiguity1; blocking preference (normal GC) also passes112+1. +Isolated tiny broadcast with explicit synchronization+GC passes3 repetitions. These variants +are diagnostics, not acceptable test skips/fixes. Sol turn25 found no confirmed missing +kernel root; cooperative finalizer marker path and timing need dense-only minimization +against OpenCL main9c27048. Taurus logs ga13-opencl-frozen-env/{contracts-nvidia4.log, +contracts-nvidia-nogc.log,min-csr32.log}; blocking variant ga13-opencl-blocking-env. +Dense pending-input-GC minimizer ga13-opencl-dense-gc.jl is running; baseline environment +ga13-opencl-baseline-env and clean main source opencl-v13-main-baseline are prepared. +Do not attribute the failure to sparse code or treat PoCL CPU as GPU coverage. + +Validation driver validation/sparse-v13-backend.jl is committed in5c477a3. +It supports full automatic/portable shared runs and fixed contract fixtures, two public +import scenarios (not impossible dependency-order reversals), extension assertions, +source revision/dirty metadata, and ambiguity filtering including split extension files. +No permanent global algorithm switch; portable instrumentation is process-local only. +Finish clean exact-revision suites, OpenCL minimization/both-host coverage, oneAPI prepared +resource tests with forced-portable shared products, backend self-CI wiring and CI logs. +Keep the release/artifact checklist precise. No PR discussion text has been posted. + +## Current checkpoint (Oct21, after PR retarget) + +The sections below are chronological checkpoints; this section supersedes stale state +claims in them. PR745 now targets main and is no longer stacked, with the user's explicit +approval. GitHub only supports unstacking the whole stack: saved stack797 to +`/Users/tim/Julia/tmp/ga13-stack797-before.json`, dissolved it, recreated unchanged +PRs795/796/798 as stack832, and retargeted745. Verified their base relationships and +head SHAs unchanged. No other branches, PR descriptions or discussion comments changed. +PR745 is mergeable. Its previous tip and Metal982's previous tip are preserved in both +local and remote backup refs documented below. Both PR branches have been safely rewritten. + +Latest pushed GPUArrays31bd874 is on tb/sparse-v13 and PR branch tb/sparse. Metal f8b9125 +is on both corresponding branches. CUDA8a9379519 adds direct publication of completed +canonical SpGEMM output; 168 physical RTX5080 assertions pass across CSR/CSC and +default/ALG1/ALG2/ALG3, including empty products, cancellation, retained-storage GC and +check_structure. Older cuSPARSE library versions retain assembly (not hardware-tested). +CUDA213e11fff and oneAPI26fb776 also remove obsolete TestSuite.sparse_types registration +hooks that prevented the actual root runners from starting. The real focused CUDA Pkg.test +entrypoint passes221 at CUDA213e11fff / GPUArrays31bd874, clean detached checkouts. +Other pushed tips: AMD08277ef7c, OpenCL9526a11, AKa915b41, Atomix1ffd4ae. + +GitHub Actions39597816964 tests exact GPUArrays7d2123d, not the newest tips. Ubuntu1.13 +passed35017 assertions including dense and sparse. Documentation failed because setup-only +mktempdir dependencies vanished at process exit;31bd874 fixes retention but needs a new +run. Other jobs were running/queued at the last check. Buildkite2065 at31bd874 had only +the uploader passed; all8 backend jobs scheduled. Authenticated `bk api` / `bk build view` +works with pipeline julialang/gpuarrays-dot-jl; public web JSON omits jobs. Obsolete builds +2060..2064 and2058 were automatically canceled. Do not report scheduled jobs as coverage. + +Local full Metal Pkg.test (--all,96 groups,5 workers) was interrupted at the user's request +to reduce local load. Source stack: GA c0c2482 / Metal a763c7fb6 / KA0.10.1. Many dense, +native and own sparse tests passed, kernelabstractions failed (details not emitted before +interruption), and the shared sparse group had not finished. Log: +`/Users/tim/Julia/tmp/ga13-metal-stack-full.log`. This is NOT a passing full suite. +Use only focused spot tests locally, one worker (--jobs=1, Julia threads1); leave full +Metal suites in CI. The full daemon and workers are stopped. Prior focused Metal results +remain valid only for their recorded revisions. + +Performance harness is currently untracked at perf/sparse-v13-cuda.jl; old/new warm timings +and Nsight peak-memory traces exist in Pegasus ga13-cuda-{baseline,benchmark}-env. +Pre-optimization heavy SpGEMM: old vendor0.589ms, migrated vendor0.853ms, portable7.391ms; +incremental peak device allocation203605548 bytes for both native paths,553648304 portable. +These are exploratory changing-stack results, not the final report. Rebenchmark8a9379519 +serially against frozen baseline before recording conclusions. Record cumulative GPU +allocation separately from observed live peak. New dependency pins must be pushed and +their exact tested revisions verified. Official oneAPI support still needs a new artifact. + +## Worktrees and branches + +All implementation uses fresh isolated worktrees under `/Users/tim/Julia/tmp`: +`gpuarrays-v13`, `metal-v13`, `cuda-v13`, `amdgpu-v13`, `oneapi-v13`, `opencl-v13`. +Each branch is `tb/sparse-v13`. The investigation worktree and unrelated backend +working copies are untouched. + +Fetched upstream on 2026-10-21: GPUArrays main `6fa6490`, PR #745 `2a50d28`; +Metal main `e4412098c`, PR #982 `ce1ba502f`; CUDA `44ac4ebb3`; +AMDGPU `284e54858`; oneAPI `705aeb0`; OpenCL `9c27048`. +Local backup refs `backup/sparse-v13-pr745-20261021` and +`backup/sparse-v13-pr982-20261021` preserve the previous PR tips. +The existing remote PR branches have not yet been rewritten. + +GPUArrays retains all ten original contributor commits rebased onto main. Metal +retains its two integration commits, omitting the zero-resize fix already upstream. +GPUArrays is 13.0.0; JLArrays is 0.5.0. Existing AK dependency/compat and main's dense +alias, wrapper and reference-algorithm fixes are retained. + +## Implemented and pushed + +GPUArrays `2c797a8` is pushed to `JuliaGPU/GPUArrays.jl:tb/sparse-v13`: +- `9b116ed`: robust sparse norms through the scaled dense implementation. +- `52b2e7e`: explicit dense `supports_resize` capability, including JLArrays. +- `9b6b82b`: separate compressed-pointer storage, mixed-width conversions, + borrowing/mutation/alias contracts, device lookup, checked counts and allocation + sizes, stable assembly with lexicographic fallback, nonfinite filtering, and + bounded ESC SpGEMM estimates. +- `d5e5050`: initial GPUArrays-owned SpMV/SpMM selection, value equality, + in-place broadcast, common transpose metadata and explicit concat rejection. +- `2c797a8`: shared v13 contracts. + +OpenCL `9526a11` is pushed to `JuliaGPU/OpenCL.jl:tb/sparse-v13`: v0.11, +GPUArrays 13 compat, resizing capability rejecting system-memory wrappers. +Physical execution and package release resolution are still pending. + +Metal `f52178f83` is local: five-parameter aliases, resizing capability, checked +transfer counts, removal of the old dependency pin, and default-storage-aware tests. +No vendor code is needed for Metal. Do not claim complete migration until the shared +suite and load-order/ambiguity tests pass. + +CUDA has an uncommitted direct-alias migration in `/Users/tim/Julia/tmp/cuda-v13`, +with matching GPUArrays `ext/CUSPARSEExt` selection/solves. Its split packages are +provisionally v7.0.0. Syntax parsing passes, but package loading and hardware parity +are still pending; do NOT call this a completed migration. AMDGPU and oneAPI still +only have uncommitted dense resize-capability declarations. Their sparse migrations +and oneAPI resource ownership remain unimplemented. + +## Validation so far + +Julia 1.13.2 / JLArrays / KA 0.9.45 / AK 0.5.8: +- Rebased baseline shared Float32 sparse suite passes. +- Focused red run: 1 pass, 12 failures, reproducing the report. +- Corrected focused suite: 72/72 passes, including mixed-width and mutation cases. +- New shared contract suite: 101/101 passes, with scalar indexing disabled. +- Existing shared Float32 sparse suite passes after the type-parameter updates. +- Relevant dense broadcasting/scan/predicate checks pass; full dense gates remain. +- Sparse ambiguity detection is empty on Julia 1.13.2. + +Julia nightly 1.14.0-DEV.3743: sparse ambiguity detection with GPUArrays, JLArrays, +SparseArrays and LinearAlgebra passes (empty ambiguity list). Nightly full suite is +not yet run. The exact binary is +`~/.julia/juliaup/julia-nightly/Julia-1.14.app/Contents/Resources/julia/bin/julia`. + +Metal M3 Pro / Julia 1.13.2 / KA 0.10.1 / AK 0.5.8: +50/50 backend constructor/transfer/densification checks pass. The shared contract +suite is being exercised with private storage. A constrained UnionAll array type +has no constructor; use a callable `x -> MtlArray{eltype(x),ndims(x),Metal.PrivateStorage}(x)` for +the focused helper, or configure the ordinary MtlArray default for the whole suite. +Main now defaults to SharedStorage; do not replay the old private-default assertion. + +Validation used changing worktree snapshots. Rerun release-relevant tests after +final commits and report exact revisions. No CI stack validation has passed yet. + +## Reviews and architectural reasoning + +`peer log sparse-v13-architecture` (Astra) confirms central selection and recommends: +- Qualified cuSPARSE/rocSPARSE routines as executors, with GPUArrays-owned selection + and normalization. Preserve CUDA #3416 integer/Bool input normalization. +- A synchronous explicit oneAPI owner retaining the actual SYCL queue, originating + stream/context/device and arrays; integrate current managed-memory ownership and + batch flushing, check Cint statuses, and dispose deterministically. +- Prepared reuse must be explicit, borrowing fixed storage with a prohibition on + structural edits, resize, rebinding and free. Values may update between completed + operations; cross-domain execution initially rejects before writes. +- oneAPI's installed support artifact must match the checked-in synchronous ABI. + Existing complex-CSC wrappers mutate dense inputs; replace this with scratch. +- Canonical arrays may not represent unfinished/noncanonical vendor intermediates. + CUDA column-sorted COO and sorted=false APIs require explicit migration notes. + +`peer log sparse-v13-review` (Sol) has identified nine issues in two passes. They +were reproduced and corrected: instance resize capability; independent SpGEMM +pointer promotion; coordinate-aware expansion; default matrix pointers without a +compressed prototype; checked empty-pointer bytes; broadcast dispatch intersections; +mixed COO/compressed pointer promotion; direct portable subordinate conversions; +and wrapper/mixed concat guards. A refreshed third review is running. Verify all +findings independently and continue the conversation for later migrations. + +## Running validation and remote scratch + +Daemons use unique names and 2h idle timeouts: +`ga13-baseline`, `ga13-green`, `ga13-nightly` on GPUArrays test project; +`metal13` on Metal root and test project. Test daemons use --no-revise. +jld root/test separation matters; KernelAbstractions is a Metal weakdep and is +directly importable in its test project. Pkg.develop adds local sources/direct deps: +keep those changes out of production commits. Manifest files remain local. + +Pegasus scratch checkout `~/Julia/tmp/gpuarrays-v13` is at pushed `2c797a8`. +Its origin is GitHub (a --shared clone initially pointed at the local source). +No existing remote ~/Julia/pkg working copy was modified. Both hosts became unreachable during implementation; see the latest checkpoint below. + +## Next work and release gates + +1. Finish refreshed core review, broader types, direct GPU lookup/host guards, + overflow/resource-limit and dense regression coverage; run Julia 1.10. +2. Complete Metal shared tests and ambiguity/load-order checks; push migration. +3. Implement actual CUDA/cuSPARSE vendor selector and aliases, preserve BSR/batched + representations and cuSOLVER consumers, canonicalize producing adapters, and run + physical portable/vendor/solver suites. Legacy Cint ABI boundaries must check + pointer width as well as coordinate width. +4. Migrate oneAPI explicit resource owner and prepared operations before removing + old containers. Then AMDGPU and its solves/BSR/factorizations. +5. Test OpenCL NVIDIA failures from fresh contexts against main, PoCL CPU separately, + and taurus routes/devices; never label CPU portability as GPU validation. +6. Hard-fail downstream install errors and nightly tests. Add exact coordinated + temporary pins separately, including CUDA split-package test environments and + explicit Julia 1.10 Pkg.develop setup. Inspect logs and exact tested revisions. +7. Performance/memory measurements and release-resolution checklist, local PR drafts, + lease-checked updates of existing PR branches. Do not post GitHub discussions. + +The selector is provisional until real CUDA and oneAPI integrations validate it. +Nothing here is a release-ready coordinated stack yet. + +## Latest checkpoint (supersedes earlier validation counts) + +GPUArrays pushed through `44b9e3b`; local central SpGEMM/open-format selection is +`3b72005`. New fixes include `7bba4e2` (CI installation/nightly hard failures), +`bcf3422` (wide equality and all-position concat guards), `7e23537` (checked values +and single-column conversions), `c09c1af` (real/complex value preflight and storage +placement), `1b6757e` (Metal count-loop workaround), `44b9e3b` (scalar policy tests). +Focused JLArrays regressions pass 103/103 on Julia 1.13.2. Full shared sparse runs +are ongoing against these changes; final exact-commit reruns remain required. + +Metal's full private-storage suite exposed a genuine count-kernel miscompilation: +CSR/CSC with one slice [1,2,3,4] and an empty slice, Int32 pointers and Int64 counts, +can count 1 instead of 4 with an unchecked final store. Sol independently reproduced +it, including fixed-arity kernels. Removing that store's @inbounds restores the +counts; optimized LLVM loop bodies are identical except SSA names. The failing +stage beyond optimized LLVM is not established. Keep wide counts. The production +workaround retains a bounds-checked store and explicit layout tag; shared tests +cover both formats and Int32/Int64 pointers. The prior layout-tag-only snapshot +passed 200 repeated product/update checks and 69 focused SpGEMM assertions. Current +full private suite must pass before claiming this fixed across the stack. +Scratch repros: `/Users/tim/Julia/tmp/ga13-metal-{copy,iterator}-repro.jl`. + +Pegasus fresh NVIDIA OpenCL context (RTX 5080, CUDA driver 13.4.96) passed the +101 shared v13 contracts using GPUArrays `2c797a8`, OpenCL `9526a11`, AK `b13f97e`, +Atomix `3486963`. The full suite stopped at scalar-rejection expectations because +CLArray's SVM storage deliberately permits host scalar access. `44b9e3b` makes the +check follow the component policy, not skip sparse results. Rerun full NVIDIA and +separate PoCL CPU suites after synchronizing final revisions; neither is a full +backend pass yet. No hardware equivalence is inferred from PoCL CPU. + +AK `a915b41` and Atomix `1ffd4ae` are pushed to their `tb/sparse-v13` branches: +OpenCL 0.11 and provisional CUDA7/AMDGPU3/oneAPI3 compatibility. These are justified +by actual package-resolution failures against the new OpenCL major/minor. A split +CUDA scratch stack now resolves with explicit Pkg.develop paths in +`pegasus:~/Julia/tmp/ga13-cuda-env`; installation is not migration validation. +All remote work is in ~/Julia/tmp, with uniquely named, two-hour-idle jld daemons. + +## Checkpoint after physical CUDA and refreshed Metal runs + +GPUArrays latest committed extension/selection snapshot is `22df6fb`; central mask +slicing is `70fdfca`, triangular-wrapper preservation/component alias preflight is +`2f996a1`. JLArrays focused regressions now pass **142/142**, sparse ambiguities +empty on Julia 1.13.2. The entire shared sparse suite passed all default JLArray +value types before the mask/triangular additions. Julia **1.10.12** explicit +Pkg.develop resolution and strict precompilation pass (AK0.5.8, KA0.9.45, +Atomix1.6.0). Focused regressions142 pass there; full suite initially stopped because +the scratch harness lacked a direct KA dependency, being corrected. + +Metal migration `a763c7fb6` is pushed to `tb/sparse-v13`. Full shared sparse tests +pass Float32/ComplexF32/Int32/Float16 against GPUArrays `3b72005` plus the extension +worktree snapshot (including checked count-store). Dense broadcast specialization +previously covered arbitrary AbstractArray destinations, creating a sparse copyto! +ambiguity; narrowing it to AnyGPUArray resolves this and preserves sparse errors. +Fresh current Metal tests: backend53 assertions, shared112 contracts, supported +dense broadcast groups, and sparse ambiguity check all pass. The ignored test +LocalPreferences.toml still selects private storage. + +Pegasus CUDA stack (Julia1.13.2, runtime13.4, cuSPARSE12.8.6, RTX5080) physically +passed: +- Shared v13 contracts109 and forced portable/vendor product comparison60. +- Migrated cuSPARSE construction217 plus Float16 construction4. +- Existing BSR solve groups80 + 80 + 432; IC/ILU preconditioners20 + 20 + 16 + 24. +- Qualified SpMV/SpMM, dense conversions and ordinary triangular vendor groups + passed in the generic suite. SpGEMM updates passed after replacing an obsolete + different-pattern error expectation with the now-supported numerical result. + The suite then exposed sparse-vector -> compressed default pointer widening in + qualified gemv; fixed locally with explicit uniform output construction, not yet + rerun. Batched/device suites have NOT run because generic.jl stopped first. +These used changing uncommitted CUDA/extension snapshots, not final commit hashes. + +CUDA local history includes resizing `6b15bb1cf`, family-v7 declaration +`e530fab77`, descriptor storage retention `7435b2a08`, legacy both-width guards +`dcb0568af`. The large alias/output-builder migration remains uncommitted, as do +the latest physical-test follow-ups. Sol source review found and prompted fixes: +legacy Cint IC/ILU/coloring guards; descriptor-owned operand lifetimes; accumulator +precision eligibility; BSR component alias checks; shared Boolean masks; missing +SpMM version guards; destination-format conversion in qualified gemm!. New CUDA +sparse_v13.jl rejection/cancellation/alias regressions are not hardware validated. +The latest BSR metadata/copy preflight and scalar indexing corrections are also +not rerun. Do not describe all CUDA functionality as migrated. + +oneAPI now has an uncommitted direct-alias/resource/qualified-operation rewrite, +plus GPUArrays ONEMKLExt (`22df6fb`). sparse_resources.jl owns captured actual SYCL +queue, stream/context/device/task and buffers; prepared objects borrow fixed +storage and constrain optimized-operation parameters. Scoped calls close +deterministically. All support statuses are checked. Complex CSC uses scratch; +effective RHS widths are used. Astra source-reviewed and required removal of the +blocking reclaim callback (could invert allocation locks), preservation of handles +on pre-release failure, full-resource quarantine after uncertain native release, +skipping already-closed finalizers, foreign-task drain without mkl_dirty changes, +and solve/scalar preflight. Those fixes are local. Exact RHS/output alias support +still needs vendor confirmation; stage transposed in-place TRSM until proven. +No oneAPI code has loaded or executed on hardware yet; installed ABI unverified. +AMDGPU migration is still unimplemented beyond resizing capability. + +At approximately 12:25 local time, **both pegasus and taurus began returning +"No route to host" on SSH**, reproducible across several attempts. Taurus setup +disconnected during Pkg.resolve after jld bootstrap. Its new scratch trees: +~/Julia/tmp/{oneapi-v13,gpuarrays-v13,ak-v13,atomix-v13,ga13-oneapi-env}. +jld was absent there; installed JuliaDaemon from its actual app-manifest repository +https://github.com/KristofferC/JuliaDaemon.jl (master), then used unique two-hour +daemons. No ~/Julia/pkg edits. Ask user about LAN/VPN restoration; continue local +work. Pegasus latest source sync predates the descriptor-retention/ABI/precision/ +mask follow-ups. Explicitly resynchronize and rerun after access returns. + +Fresh NVIDIA-only ICD run reproduced full-suite failure: construction144 and +allocation97 pass, transfer16 then packed-key kernel compilation fails, followed +by CL_UNKNOWN_ERROR_CODE/finalizer errors. Fresh isolated packed_keys on three +entries passes; a separate fresh isolated sparse_transfer run passes20/20. +Therefore the failure is sequence/context dependent, not yet a proven bad +packed-key kernel. Full logs are in jld transcripts of ga13-nvidia-isolated, +ga13-nvidia-min and ga13-nvidia-adapt in pegasus ga13-opencl-env. The earlier +nonisolated run had CL_OUT_OF_HOST_MEMORY creating a context; explicitly creating +a new NVIDIA context subsequently worked. Keep Intel ICD enumeration warnings +separate from NVIDIA-only reproduction. + +## Current checkpoint (supersedes incomplete status above) + +LAN access works: Pegasus directly; Taurus with `ssh -J pegasus.maleadt.lan +taurus.maleadt.lan`. Work remains under `~/Julia/tmp`, never remote pkg trees. + +GPUArrays HEAD is 7889d6c (pushed through f8e31c4); current uncommitted changes +include ROCSPARSEExt, constructor/shape compatibility, diagonal arithmetic, +trace overflow semantics and Kronecker test rounding. Fresh Julia 1.13.2 strict +precompilation and focused JLArrays regressions pass 303/303; no sparse +ambiguities. This does not establish a globally ambiguity-free package: eight +unrelated dense ambiguities remain. Julia 1.10.12 full shared sparse tests passed +Float32/ComplexF32/Int32/Float16 before the newest compatibility additions. + +CUDA RTX5080: qualified generic cuSPARSE suite, batched53, canonical device14, +CSC column views3, ABI16, cancellation36, BSR alias6, cuSOLVER reorder12/numeric100 +and factorization36 pass. Remaining actual suite stops in conversion.jl on old +constructor-identity expectations and missing typed BSR conversions. Array490, +duplicate9, utility32, batch7, conversion52, half12, Bool29 passed in that run. +The full shared suite exposed integer trace widening and exact complex Kronecker +rounding expectations; fixes are local, full rerun pending. Large CUDA migration +is uncommitted; pushed branch only contains earlier preparatory commits. + +oneAPI Arc A770: full actual oneMKL suite passes746 sparse assertions with24 +pre-existing broken CSC triangular cases; supported dense BLAS/LAPACK also pass. +Full shared sparse suite passes Float32/ComplexF32/Int32/Float16. Resource and +prepared-value-refresh tests pass119. Latest source HEAD1f0cb7c is not pushed +(remote f93f70b). Optimized bindings use independent bit-exact value snapshots, +re-preparing when source values change; PLAN records Astra's contract review. +Transposed dense SpMM is rejected before writes and automatic selection uses +portable execution. An unhandled native oneMKL exception previously aborted the +process; C++ generator/epilogue/generated catches now contain all std::exception +and unknown exceptions. A task-owned support build was actually loaded and +native error containment passes2. Regenerated final C++ source still needs +synchronizing locally/rebuilding/full resource rerun. Official support artifact +does NOT contain this fix: coordinated artifact release remains mandatory. + +AMDGPU gfx1036: migration loads and strict-precompiles; shared contracts109, +forced portable/vendor/solve43 pass. Existing generic/vendor solve groups, +preconditioners24, triangular solves288 and conversion groups pass. Rectangular +SpMM54 layouts across three policies passes165, ABI preflight9. COO transpose +descriptor normalization fixes unsorted vendor interpretation and retains its +central canonical temporary. Remaining interfaces suite failures concern BSR +arithmetic normalization, rectangular UniformScaling, scalar wrapped arithmetic +and scaled-product precision. Entire AMD migration remains uncommitted. + +Sol reviews through turn13 (`peer log sparse-v13-review`) verified descriptor, +ABI, placement and geometry fixes. Latest remaining finding: qualified AMD +gemm! must reject zero-based index before central delegation and mutation. +Astra reviews through turn11 are in sparse-v13-architecture. No GitHub discussion +was posted. Metal remains a763c7fb6, OpenCL9526a11, AKa915b41, Atomix1ffd4ae. + +No coordinated CI stack has run yet; neither existing PR branch has been +rewritten. Need exact dependency pins (including Julia1.10 explicit develop and +CUDA split workspace), remote-tip backups/concurrency checks, commit all logical +changes, push, then inspect real job logs and revisions. Full dense regressions, +final extension/load-order checks, representative performance/memory measurements, +OpenCL sequence minimization and hardware coverage remain incomplete. These are +not release-ready results or completed backend migrations. + +## CI and parity checkpoint (Oct21, after first coordinated push) + +GPUArrays PR745 is now safely rewritten onto current main history: old tip2a50d28 +is preserved remotely as backup/sparse-v13-pr745-20261021. Both tb/sparse and +tb/sparse-v13 include temporary exact-stack CI commit7d2123d. The PR itself still +targets tb/permutedims-empty and is conflicted: `gh pr edit --base main` fails +with "Cannot change the base branch because the pull request is part of a stack." +An asynchronous question asks the user to detach it or retain stacking. Do not +modify the old base branch. No PR description/comment was posted or edited. + +Direct workflow dispatch avoids the conflicted merge ref: GitHub run39597816964 +tests exact GPUArrays7d2123d. All17 jobs were queued at the last check. Buildkite +2058 also tests7d2123d, with five backend steps scheduled (nine jobs including +matrix expansion). Public build JSON statistics work, but its jobs/steps arrays +are empty; retrieve actual job logs before accepting results. These are NOT +passing CI results. Backend pins in .buildkite/sparse-v13-stack.toml are exact; +follow-ups must update pins and verify tested revisions. The helper performs +named Pkg.test from a separate environment, explicitly develops CUDA's complete +split workspace (including test-only libraries), runs --all library tests and +asserts runtime/vendor functionality. oneAPI builds support in an isolated module +and task-owned scratch with cleaned ARGS, propagates the artifact preference and +tests the native exception boundary only after verifying the expected library. + +Latest pushed production backend tips at this checkpoint: CUDAcdca56af1, +AMDGPU08277ef7c, oneAPI01b71ad (Metal/OpenCL/AK/Atomix unchanged). AMD actual suite8 +and suite9 completed successfully, including43 preflight/empty checks,165 forced +rectangular products, all native generic/solver groups, preconditioners24, +triangular288, interfaces620, SparseArrays5, device15 and SparseMatricesCSR53. +This is changing-worktree evidence, not final fresh-checkout commit validation. + +CUDA suites through conversions, broadcast, operations, reductions and linalg +passed. interfaces/mul then found missing canonical +/- Symmetric/Hermitian +materialization; centrally fixed, focused601 assertions + sparse ambiguity check +pass. cuda-rest17.log is the current fresh physical rerun. Earlier tests also +found missing generalized-dot dimensions (silently accepted trailing x entries), +now fixed both in sparse dot(x,A,y) and generic dense dot. Typed conversion now +checks requested widths/values before regrouping or wrapper materialization; +the huge-COO narrowing regression previously attempted32GiB allocation. +Diagonal extraction handles transposes, narrow offsets and extreme empty results; +explicit host Array(Diagonal(GPUVector)) transfers only its diagonal first. +495 focused + all JLArray linalg/diagonal groups passed that independent fix. + +Full JLArrays shared sparse default-type suite completed in +ga13-parity-shared-final.log before the latest compatibility additions. Full +Julia1.10 CPU+dense+sparse Pkg.test is currently running under ga13-ci-stack110b +from the new explicit pinned setup; log /Users/tim/Julia/tmp/ga13-ci-stack110b.log. +Its first run exposed a real Test import omission at the runner entrypoint, +fixed separately in ebda95b. The initial setup experiment rewrote an ignored +root manifest with Julia1.10 stdlibs; it was moved recoverably to +/Users/tim/Julia/tmp/ga13-root-manifest-ci110.saved.toml and fresh1.13 strict +precompilation passes. Production Project.toml was restored; the final helper no +longer develops into it. Do not confuse dependency setup with completed tests. + +Sol review through turn20: dense diagonal transfer20 local checks pass; wrapper +preflight, empty-BSR guards, CI functionality checks and build namespace fixes +verified. Source claims about a missing AMD logical-dimension guard were checked +and withdrawn: require_cint_storage already checks it. Astra turn12 approves the +explicit scale-and-promote numerical policy with documented scope/limitations. + +Still required: finish CUDA interface/solver parity, final exact-revision hardware +runs (including portable-only shared products), fresh Metal, OpenCL GPU sequence +minimization and both-host coverage, load-order/ambiguities, performance/peak-temp +benchmarks, actual CI logs, backend self-CI wiring/release coordination, and Metal +PR982 update. Official oneAPI support artifact still lacks the C++ exception fix; +new artifact release and normal registered dependency resolution are release gates. +Local draft PR text: /Users/tim/Julia/tmp/sparse-v13-pr-description.md. + diff --git a/validation/REPORT.oneapi-support.md b/validation/REPORT.oneapi-support.md new file mode 100644 index 000000000..858c5f11c --- /dev/null +++ b/validation/REPORT.oneapi-support.md @@ -0,0 +1,80 @@ +# oneAPI sparse support artifact + +The sparse migration requires native wrapper contract version2, not merely the +presence of wide CSC symbols. Published oneAPI_Support0.11.0+0 has those symbols +but lacks the catch-all exception boundary. oneAPI rejects it for sparse execution +without disabling dense operations. The runtime marker guard remains necessary +for preferences and custom overrides after a new minimum version is registered. + +## Recipe and generated artifact + +Recipe: maleadt/Yggdrasil branch tb/oneapi-sparse-v13-support, commit +31c79455bc9f0b415a75ceb9faac59a26148e8f4, based on master0defdb4d8. Independent +e0dc7751df3b832bdc67f0061b01c0a3a727ee2e fixes an existing parse error. The recipe +keeps upstream oneAPI source5093529311e36f530c5990e4b0b3669452c45d8c and Intel2026.1, +applying only the native cpp/header patch. Patched files compare equal to tested +oneAPId1cb45d (runtime-equivalent successor c7e27cb). Dependency floors are the +validated Loader1.32/OpenCL2024.10.24/Hwloc2.15, not asserted minimum ABI versions. + +Pegasus crossbuild, Julia1.13.2 with the Yggdrasil root project: + +- x86_64-linux-gnu-cxx11, version0.11.1+0; compiler/link/install completed, exit0. +- Artifact tree876c51345ae21089df99bd35dfc29cf292c2a4ce. +- Tarball SHA25622eadbfd15512fa8aa9867970f96c1907ce5314e767144a33f44410a1b8963eb. +- Loaded support SHA256d5e977efdfe577477c74827cd9b827b61d14f49fa2c8466fb3387a3e3c933fd7. +- Recipe products live under ~/Julia/tmp/yggdrasil-oneapi-v13/O/oneAPI_Support/products. +- Generated wrapper is ~/Julia/tmp/ga13-ygg-jll-dev/oneAPI_Support_jll on both hosts; + local file downloads are not a published/registered artifact. +- Full build log: ~/Julia/tmp/ga13-ygg-build-no-update.log. Metadata-only generation + passed separately, but is not the crossbuild result. + +Run the recipe with --verbose --deploy=local x86_64-linux-gnu-cxx11 through a unique +jld daemon, --no-revise --idle-timeout=2h. Clear ARGS first. Set +BinaryBuilder.devdir[] to a task-owned directory after importing BinaryBuilder: +the precompiled Ref can retain ~/.julia/dev despite JULIA_PKG_DEVDIR. The newly +generated wrapper was moved recoverably into task scratch; existing pkg checkouts +were not modified. No artifact was uploaded or registered. + +## Runtime validation without a source override + +Taurus Arc A770, GPUArrays4e20de2 and oneAPId1cb45d, exact AK/Atomix migration pins. +Fresh environments explicitly develop the generated JLL and all coordinated packages; +there is no liboneapi_support_path preference. Actual loaded path is +~/.julia/artifacts/876c51345ae21089df99bd35dfc29cf292c2a4ce/lib/liboneapi_support.so. +Artifact identity, marker eligibility and device functionality pass three checks. + +Julia1.10.12 actual Pkg.test, allow_reresolve=false: native oneMKL1279, +resources121, ABI96 pass, plus24 pre-existing broken CSC checks. Total1496pass/24broken; +no shared-suite pass is claimed for this run. Log: +~/Julia/tmp/ga13-oneapi-generated-jll110-env/test.log. + +Julia1.13.2 actual Pkg.test passes13167, plus24 pre-existing broken: shared sparse11671, +native oneMKL1279, resources121, ABI96. Log: +~/Julia/tmp/ga13-oneapi-generated-jll-env/test.log. Do not confuse the earlier separately +source-built support result13138pass/24broken with this generated artifact. The final +all-dimensional broadcast guard postdates this frozen GA source: its2058 local +regressions,247contracts and focused groups pass separately at34d125e. It does not +change the native support ABI; final physical guard checks remain separate evidence. + +Final physical guard checks at34d125e/oneAPI06f2f77 pass244+ambiguity1 using the same +generated artifact. Separately, the actual final pinned helper9a9126f/06f2f77 builds +support from source and passes13231+24broken: shared11734,native1279,resources122, +ABI96. Loaded support SHA256b3f319f2972b03890f14b97c0616c67992ccaa1662075136db55fd684c89cd9a; +log ~/Julia/tmp/ga13-oneapi-generated-jll-env/final9a-root.log. This verifies actual +checked setup/Pkg.test plumbing; it remains distinct from the generated JLL and CI. + +## Inherited audit warnings and release gates + +BinaryBuilder's binutils2.25.1 objdump aborts while auditing some Intel libraries. +Standalone vendor MKL dlopen probes also report undefined symbols, and the linker +script libmkl_sycl.so is not itself an ELF library. The baseline0.11.0 recipe at +e0dc775 (only the parse fix, no native patch/version/compat changes) reproduces these +warnings and produces its tarball. Log: ~/Julia/tmp/ga13-ygg-baseline-build.log. + +This establishes that the warnings are inherited, not that the interrupted audits +passed. No audit bypass or soft installation failure was added. Normal generated-JLL +loading/native operations are tested separately above. Resolve or explicitly review +the incomplete audit checks before artifact publication, then test the published +artifact and raise oneAPI's compatibility minimum to the actual registered0.11.1. +Until then the source-built CI override and exact stack pins are temporary validation +plumbing, not normal registered release resolution. diff --git a/validation/REPORT.opencl-svm.md b/validation/REPORT.opencl-svm.md new file mode 100644 index 000000000..d6f9ff63d --- /dev/null +++ b/validation/REPORT.opencl-svm.md @@ -0,0 +1,81 @@ +# NVIDIA OpenCL allocation/transfer failure + +Status: reproduced on baseline and v13; cause unresolved. This is not established +as a sparse redesign regression or a driver defect. PoCL CPU results do not validate +this GPU path. + +`opencl-svm-transfer-churn.jl` uploads three ordinary dense buffers and reconstructs +a CPU SparseMatrixCSC after downloading them. It uses no GPU sparse containers or +kernels. Run through a fresh uniquely named jld daemon with normal GC and the default +synchronization policy. Forty repetitions contain 960 checks; request more with +`OPENCL_TRIPLET_REPEATS`. A passing short run is not sufficient: fresh baseline runs +initially passed 960 twice, then failed after ten checks. + +## Same-host baseline control + +Pegasus, RTX 5080, NVIDIA 615.78.08, Julia 1.13.2: + +- OpenCL main `9c27048a45501f0022dbeff47a24305db33efeb6`. +- Registered GPUArrays 12.1.1, tree `f72b034a94bf1578dcc6e5410990f257b2091ada`. +- GPUToolbox 3.4.1; AK 0.5.8 and Atomix 1.6.0 implementations match the migration + branches' source/extension trees. Those branches change compatibility metadata. +- Baseline direct run: ten checks then native SVM memcpy `CL_INVALID_VALUE` (-30). +- Baseline C-shim run with `OCL_ICD_FILENAMES=libnvidia-opencl.so.1`: 344 checks, + then native `clSVMAlloc` returned NULL for size 36, alignment 4. A native out-parameter + matched the pointer returned to Julia, excluding return-value loss in that call. + Completed allocation/failure/free counts were (516,1,237) at failure and + (516,1,514) after unwinding. No fixed live-allocation threshold is established. +- Strict NVIDIA-only discovery (empty OCL_ICD_VENDORS directory plus the native + NVIDIA ICD filename) also fails after ten checks: native NULL for size44, + alignment4, counts(15,1,3) at failure and(15,1,11) after unwind. The actual + enumeration contains only NVIDIA CUDA; baseline-strict-nvidia.log. + +Logs and the process-local C instrumentation are under Pegasus +`~/Julia/tmp/ga13-opencl-baseline-env/{baseline-long,baseline-shim,baseline-nvidia-only}.log` +and `~/Julia/tmp/ga13-opencl-alloc-probe.{c,jl,so}`. The ICD filename setting adds an +ICD; it does not promise exclusive system-ICD discovery. + +## Independent correction + +OpenCL `9e55fc3` (rebased successor of `029274e6b517464084f00248e3df2631191633a5`) +rejects native NULL before +publishing nonempty storage or retaining a native context. The old cross-address- +space comparison with host C_NULL missed CLPtr NULL. The fix uses existing bounded +reclamation, preserves the context across GC and reports NULL without inventing a +precise native error cause. Fifteen focused mocked/native-reference checks pass. +This corrects error handling; it does not resolve the allocation failure. + +An ordinary-Ptr return declaration and native blocking-copy diagnostics also fail; +neither is a sufficient fix. The allocating task's finalizer-inhibition counter was +zero on the default pool. GC-off and blocking synchronization passes remain +diagnostics, not production fixes or completed migration results. + +## Separate runner initialization failure + +Actual Pkg.test on GA `ef7659a` / OpenCL `029274e` crashed before tests in +`versioninfo`: JLL libpocl, system libpocl and the system pthread device module were +mixed while enumerating platforms. The runner imported pocl_jll even for an explicit +vendor request. OpenCL `6ba2cf4` imports CPU drivers only for default/CPU requests; +the target generation and test bodies are unchanged. The corresponding temporary +GPUArrays setup follows the same rule. This removes injected CPU drivers, not all +installed system ICDs. + +## Actual migrated GPU suite + +Fresh actual root Pkg.test at GPUArrays64ea2d8/OpenCLe0a09f4 passes11,759 checks on +EACH host: Pegasus RTX5080 and Taurus RTX6000Ada. Shared sparse11,744 across all ten +supported types plus15 SVM checks, one worker, normal GC/synchronization, default +program route. Strict ICD discovery enumerates only NVIDIA. Both logs are +~/Julia/tmp/ga13-opencl-svm-env/actual-final-entry-full.log. These are actual migrated +GPU sparse/SVM results, not PoCL CPU or whole-backend CI results. They do not resolve +or conceal the independently reproduced baseline dense-allocation failure above. + +The final exact pinned helper at GPUArrays9a9126f/OpenCL94fae12 also passes11,852 +on Pegasus: shared11,837 plus15 SVM, including the final all-dimensional broadcast +guards. Clean sources and exact pins are logged in +~/Julia/tmp/ga13-opencl-svm-env/final9a-root.log. This is actual checked Pkg.test +stack validation on a test host, not a hosted-CI or normal registered-resolution pass. + +Existing Context callback-handle lifetime and Intel SVM initialization-failure +cleanup need separate audits. Neither has been shown to cause this failure, and +neither was silently changed as part of the NULL guard. diff --git a/validation/REPORT.sparse-v13-downstream.md b/validation/REPORT.sparse-v13-downstream.md new file mode 100644 index 000000000..8c38ec65c --- /dev/null +++ b/validation/REPORT.sparse-v13-downstream.md @@ -0,0 +1,22 @@ +# Downstream source audit + +This is a scoped source audit, not a passing downstream package test suite. Existing +compatibility bounds deliberately exclude the unreleased stack; they were not loosened +to manufacture a passing installation. + +| Source revision | Finding | Required migration | +| --- | --- | --- | +| [Lux 393dc2175f](https://github.com/LuxDL/Lux.jl/tree/393dc2175fa242a13c40673ba0cb6098b63db6a8/lib/MLDataDevices), MLDataDevices1.17.10 | GPUArraysSparseArraysExt dispatches on AbstractGPUSparseMatrixCSC / AbstractGPUSparseVector. GPUArrays compat11.3.2, older backend majors. | Deprecated CSC binding preserves dispatch; the CPU-transfer body is covered by the focused regression fixture. Review other extensions and release compatibility for the coordinated stack. No whole-package pass claimed. | +| [QuantumToolbox 6cf548f2f0](https://github.com/qutip/QuantumToolbox.jl/tree/6cf548f2f0931ca6de04a0bf29a81d7d58af5425/lib/QuantumToolboxCore), QuantumToolboxCore0.3.1 | GPUArraysExt imports dense_array_type for sparse-to-dense conversion. GPUArrays compat11.4.1/12.1; CUDACore6. | Allocate from nonzeros(A), then copyto! from A. Keep element-type conversion and initialize structural zeros. Release updated compatibility after backend validation. | +| [SparseMatrixColorings 3ec3aef981](https://github.com/JuliaDiff/SparseMatrixColorings.jl/tree/3ec3aef981e48520904655ef82dd7def66466467),0.4.29 | GPUArraysExt uses dense_array_type after CPU compression. GPUArrays compat11.5/12. | Allocate on A's buffer storage, then copy the compressed CPU result into that destination. Audit its CUDA extension and update compatibility. | + +The v13 regression suite checks all three deprecated format bindings with partial +parameters, the MLDataDevices-style real/complex matrix/vector CPU precision policy, +and buffer-based sparse-to-dense allocation/copy with NaN-seeded destinations. These +tests validate the relevant idioms, not extension loading or complete consumer behavior. + +The old format names are canonical aliases, not abstract extension points. Independent +formats must subtype AbstractGPUSparseArray directly. Separate device sparse types, +BSR abstract aliases, removed fields and old complete type-parameter layouts are not +restored. CUDA BSR/batched and AMD BSR remain actual backend representations and are +tested by their native suites, separately from this audit. diff --git a/validation/opencl-svm-transfer-churn.jl b/validation/opencl-svm-transfer-churn.jl new file mode 100644 index 000000000..52bd6807d --- /dev/null +++ b/validation/opencl-svm-transfer-churn.jl @@ -0,0 +1,38 @@ +# Dense-only minimizer: run with jld --no-revise in an isolated OpenCL environment. +# No GPU sparse arrays, kernels, dependency setup or persistent preferences are used. +using OpenCL, SparseArrays, Random, Test + +OpenCL.cl.platform!(get(ENV,"SPARSE_OPENCL_PLATFORM","NVIDIA")) +println("OPENCL_SOURCE ",pkgdir(OpenCL)," version=",pkgversion(OpenCL)) +println("OPENCL_DEVICE ",OpenCL.cl.device().name, + " driver=",OpenCL.cl.device().driver_version) +println("OPENCL_MEMORY ",OpenCL.memory_type()) +OpenCL.memory_type() === OpenCL.cl.SharedVirtualMemory || + error("This SVM reproducer requires SharedVirtualMemory, not another memory backend") +println("JULIA_VERSION ",VERSION) +Random.seed!(0x745) + +function host_fixture(T,Ti) + dense=Matrix(dropzeros!(sprand(T,10,8,0.3))) + dense[1,:].=rand(T,8).+one(T) + dense[:,2].=zero(T) + dense[3,:].=zero(T) + host=SparseMatrixCSC{T,Ti}(sparse(dense)) + nonzeros(host)[end]=zero(T) + return host +end + +repeats=parse(Int,get(ENV,"OPENCL_TRIPLET_REPEATS","40")) +@testset "dense triplet transfer churn" begin + for repeat in 1:repeats,T in (Float32,ComplexF32,Int32),Ti in (Int32,Int64) + host=host_fixture(T,Ti) + for input in (host,copy(transpose(host))) + ptr=CLArray(SparseArrays.getcolptr(input)) + ind=CLArray(rowvals(input)) + val=CLArray(nonzeros(input)) + @test Array(val)==nonzeros(input) + back=SparseMatrixCSC(size(input)...,Array(ptr),Array(ind),Array(val)) + @test back==input + end + end +end diff --git a/validation/sparse-v13-backend.jl b/validation/sparse-v13-backend.jl new file mode 100644 index 000000000..aa4a400d4 --- /dev/null +++ b/validation/sparse-v13-backend.jl @@ -0,0 +1,110 @@ +# Run in a task-owned backend environment through jld --no-revise. +# Arguments: backend [automatic|portable] [contracts|full] [gpuarrays-first|backend-first] +# No package setup or persistent configuration changes are performed here. +using Test, SparseArrays, LinearAlgebra + +backend = get(ARGS,1,"CUDA") +policy = get(ARGS,2,"automatic") +suite = get(ARGS,3,"contracts") +order = get(ARGS,4,"gpuarrays-first") +backend in ("CUDA","AMDGPU","oneAPI","Metal","OpenCL") || error("unknown backend") +policy in ("automatic","portable") || error("unknown policy") +suite in ("contracts","full") || error("unknown suite") +order in ("gpuarrays-first","backend-first") || error("unknown import order") +load_package(name) = Core.eval(Main,Expr(:using,Expr(:.,Symbol(name)))) +order=="gpuarrays-first" && load_package("GPUArrays") +if backend=="CUDA" + load_package("CUDACore") + # Public import scenarios, not arbitrary dependency orders: CUDACore requires + # GPUArrays, and cuSOLVER itself requires cuSPARSE in this package stack. + for name in (order=="gpuarrays-first" ? ("cuSPARSE","cuSOLVER") : ("cuSOLVER","cuSPARSE")) + load_package(name) + end + array_type=CUDACore.CuArray + modules=(CUDACore,cuSPARSE,cuSOLVER) +else + load_package(backend) + backend_module=getfield(Main,Symbol(backend)) + array_type=getfield(backend_module,Dict("AMDGPU"=>:ROCArray,"oneAPI"=>:oneArray, + "Metal"=>:MtlArray,"OpenCL"=>:CLArray)[backend]) + modules=(backend_module,) +end +order=="backend-first" && load_package("GPUArrays") +if backend in ("CUDA","AMDGPU","oneAPI") + extension=Dict("CUDA"=>:CUSPARSEExt,"AMDGPU"=>:ROCSPARSEExt,"oneAPI"=>:ONEMKLExt)[backend] + @test Base.get_extension(GPUArrays,extension)!==nothing +end +if backend=="OpenCL" + platform=get(ENV,"SPARSE_OPENCL_PLATFORM","NVIDIA") + if platform=="Portable Computing Language" + load_package("pocl_jll") + end + OpenCL.cl.platform!(platform) + OpenCL.program_backend!(Symbol(get(ENV,"SPARSE_OPENCL_PROGRAM_BACKEND","auto"))) + println("OPENCL_DEVICE ",OpenCL.cl.device().name) + println("OPENCL_PROGRAM_BACKEND ",OpenCL.program_backend()," -> ", + OpenCL.resolve_program_backend(OpenCL.cl.device())) +end +GPUArrays.allowscalar(false) +for mod in (GPUArrays,modules...) + path=dirname(dirname(pathof(mod))) + println("VALIDATION_PACKAGE ",nameof(mod),' ',pkgversion(mod),' ',path) + if success(pipeline(`git -C $path rev-parse HEAD`;stdout=devnull,stderr=devnull)) + revision=readchomp(`git -C $path rev-parse HEAD`) + dirty=!isempty(readchomp(`git -C $path status --porcelain`)) + println("VALIDATION_REVISION ",nameof(mod),' ',revision," dirty=",dirty) + end +end +println("VALIDATION_MODE ",join((backend,policy,suite,order),' ')) + +const portable_selections=Ref(0) +if policy=="portable" + # Process-local test instrumentation. Public operations must use the portable + # executor even for vendor-supported operands; production methods are unchanged. + @eval GPUArrays begin + select_sparse(::Automatic,candidate) = + ($(portable_selections)[]+=1; Portable()) + select_sparse(::Automatic,candidate::UnsupportedSparseVendor) = + ($(portable_selections)[]+=1; Portable()) + end +end + +include(joinpath(pkgdir(GPUArrays),"test","testsuite.jl")) +default_types="Float32,ComplexF32,Int32,Float16" +if backend=="OpenCL" && !("cl_khr_fp16" in OpenCL.cl.device().extensions) + # Match the backend's device-capability contract, not a sparse-test exemption. + # Explicit caller-selected types remain unchanged and can deliberately test errors. + default_types="Float32,ComplexF32,Int32" + @testset "OpenCL unsupported Float16 rejects before allocation" begin + @test_throws ErrorException array_type{Float16}(undef,1) + end +end +types=[getfield(Base,Symbol(name)) for name in + split(get(ENV,"SPARSE_VALIDATION_TYPES",default_types),',')] +println("VALIDATION_FIXTURES ",suite=="full" ? string(types) : + "fixed v13 fixtures: primarily Float32, with explicit integer/index edge cases") +@testset "$backend $policy $suite" begin + if suite=="full" + TestSuite.tests["sparse"](array_type;eltypes=types) + else + TestSuite.sparse_v13(array_type,types) + end + if policy=="portable" && suite=="full" + @test portable_selections[]>0 + end +end +println("PORTABLE_SELECTIONS ",portable_selections[]) + +@testset "sparse extension ambiguities" begin + ambiguities=filter(Test.detect_ambiguities(GPUArrays,SparseArrays,LinearAlgebra,modules...; + recursive=true)) do methods + any(methods) do method + file=replace(string(method.file),'\\'=>'/') + occursin("host/sparse/",file) || any(name->occursin(name*".jl",file) || + occursin("ext/"*name*"/",file), + ("CUSPARSEExt","CUSOLVERExt","ROCSPARSEExt","ONEMKLExt")) + end + end + isempty(ambiguities) || show(stdout,"text/plain",ambiguities) + @test isempty(ambiguities) +end