Random updates

.JuliaFormatter.toml

@@ -0,0 +1 @@
+style = "default"

.github/workflows/CI.yml

@@ -46,14 +46,16 @@ jobs:
           - libmpich-dev
           - libopenmpi-dev
         julia_version:
-          - "1"
+          - "1.10"
       fail-fast: false
     env:
       P4ESTTYPES_TEST: P4ESTTYPES_CUSTOM_MPI_CUSTOM
       P4ESTTYPES_TEST_BINARY: system
       OMPI_MCA_btl_base_warn_component_unused: 0
       OMPI_ALLOW_RUN_AS_ROOT: 1
       OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
+      OMPI_MCA_rmaps_base_oversubscribe: true
+      PRTE_MCA_rmaps_default_mapping_policy: :oversubscribe
     steps:
       - uses: actions/checkout@v4
       - name: Install MPI via apt

Project.toml

@@ -26,7 +26,7 @@ RavenCUDAExt = "CUDA"
 RavenWriteVTKExt = "WriteVTK"
 
 [compat]
-Adapt = "3"
+Adapt = "3, 4"
 CUDA = "4, 5"
 Compat = "3.42, 4"
 GPUArraysCore = "0.1"

docs/.JuliaFormatter.toml

@@ -0,0 +1 @@
+style = "default"

examples/advection/.JuliaFormatter.toml

@@ -0,0 +1 @@
+style = "default"

examples/grids/.JuliaFormatter.toml

@@ -0,0 +1 @@
+style = "default"

ext/RavenCUDAExt.jl

@@ -3,6 +3,7 @@ module RavenCUDAExt
 import Raven
 import Adapt
 import MPI
+import StaticArrays
 
 isdefined(Base, :get_extension) ? (using CUDA) : (using ..CUDA)
 isdefined(Base, :get_extension) ? (using CUDA.CUDAKernels) : (using ..CUDA.CUDAKernels)
@@ -58,4 +59,11 @@ else
     Adapt.adapt_storage(::CUDA.KernelAdaptor, ::MPI.Comm) = nothing
 end
 
+CUDA.@device_override function Base.checkbounds(A::StaticArrays.MArray, I...)
+    @inline
+    checkbounds(Bool, A, I...) ||
+        CUDA.@print_and_throw("BoundsError while indexing an MArray.")
+    nothing
+end
+
 end # module RavenCUDAExt

src/Raven.jl

@@ -65,15 +65,21 @@ if !isdefined(Base, :get_extension)
     using Requires
 end
 
-@static if !isdefined(Base, :get_extension)
-    function __init__()
+function __init__()
+    @static if !isdefined(Base, :get_extension)
         @require CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" include(
             "../ext/RavenCUDAExt.jl",
         )
         @require WriteVTK = "64499a7a-5c06-52f2-abe2-ccb03c286192" include(
             "../ext/RavenWriteVTKExt.jl",
         )
     end
+
+    MPI.add_finalize_hook!() do
+        for cm in COMM_MANAGERS
+            finalize(cm.value)
+        end
+    end
 end
 
 end # module Raven

src/coarsegrids.jl

@@ -65,7 +65,8 @@ end
 
 function cubeshellgrid(R::Real, r::Real)
     @assert R > r "R (outer radius) must be greater that r (inner radius)"
-    vertices = zeros(SVector{3,Float64}, 16)
+    T = promote_type(typeof(R), typeof(r))
+    vertices = zeros(SVector{3,T}, 16)
 
     vertices[1] = SVector(+R, +R, -R)
     vertices[2] = SVector(+R, -R, -R)
@@ -109,7 +110,7 @@ function cubeshellgrid(R::Real, r::Real)
         z_x = tan(η)
 
         # Compute the new points
-        x = point[3] / hypot(1, y_x, z_x)
+        x = point[3] / hypot(one(y_x), y_x, z_x)
         y = x * y_x
         z = x * z_x
 
@@ -147,7 +148,7 @@ end
     A cube shell is a 2D connectivity.
 """
 function cubeshell2dgrid(R::Real)
-    vertices = zeros(SVector{3,Float64}, 8)
+    vertices = zeros(SVector{3,typeof(R)}, 8)
 
     vertices[1] = SVector(+R, +R, -R)
     vertices[2] = SVector(+R, -R, -R)
@@ -175,7 +176,7 @@ function cubeshell2dgrid(R::Real)
         z_x = tan(η)
 
         # Compute the new points
-        x = point[3] / hypot(1, y_x, z_x)
+        x = point[3] / hypot(one(y_x), y_x, z_x)
         y = x * y_x
         z = x * z_x
 

src/communication.jl

@@ -1,3 +1,5 @@
+const COMM_MANAGERS = WeakRef[]
+
 struct CommPattern{AT,RI,RR,RRI,SI,SR,SRI}
     recvindices::RI
     recvranks::RR
@@ -74,7 +76,7 @@ end
 
 abstract type AbstractCommManager end
 
-struct CommManagerBuffered{CP,RBD,RB,SBD,SB} <: AbstractCommManager
+mutable struct CommManagerBuffered{CP,RBD,RB,SBD,SB} <: AbstractCommManager
     comm::MPI.Comm
     pattern::CP
     tag::Cint
@@ -86,8 +88,8 @@ struct CommManagerBuffered{CP,RBD,RB,SBD,SB} <: AbstractCommManager
     sendrequests::MPI.UnsafeMultiRequest
 end
 
-struct CommManagerTripleBuffered{CP,RBC,RBH,RBD,RB,RS,SBC,SBH,SBD,SB,SS} <:
-       AbstractCommManager
+mutable struct CommManagerTripleBuffered{CP,RBC,RBH,RBD,RB,RS,SBC,SBH,SBD,SB,SS} <:
+               AbstractCommManager
     comm::MPI.Comm
     pattern::CP
     tag::Cint
@@ -184,7 +186,7 @@ function commmanager(
         )
     end
 
-    return if triplebuffer
+    cm = if triplebuffer
         backend = get_backend(arraytype(pattern))
         recvstream = Stream(backend)
         sendstream = Stream(backend)
@@ -218,6 +220,18 @@ function commmanager(
             sendrequests,
         )
     end
+
+    finalizer(cm) do cm
+        for reqs in (cm.recvrequests, cm.sendrequests)
+            for req in reqs
+                MPI.free(req)
+            end
+        end
+    end
+
+    push!(COMM_MANAGERS, WeakRef(cm))
+
+    return cm
 end
 
 get_backend(cm::AbstractCommManager) = get_backend(arraytype(cm.pattern))

test/.JuliaFormatter.toml

@@ -0,0 +1 @@
+style = "default"

test/configure_packages.jl

@@ -7,6 +7,7 @@ Pkg.add("UUIDs")
 @static if VERSION >= v"1.8"
     Pkg.compat("MPIPreferences", "0.1")
     Pkg.compat("Preferences", "1")
+    Pkg.compat("UUIDs", "1")
 end
 
 const RAVEN_TEST = get(ENV, "RAVEN_TEST", "RAVEN_JLL_MPI_DEFAULT")

test/grids.jl

@@ -59,14 +59,14 @@ function grids_testsuite(AT, FT)
 
     let
         N = (3, 3, 3)
-        R = 2
-        r = 1
+        R = FT(2)
+        r = FT(1)
 
         coarse_grid = Raven.cubeshellgrid(R, r)
 
         gm = GridManager(LobattoCell{Tuple{N...},FT,AT}(), coarse_grid, min_level = 2)
 
-        indicator = rand((Raven.AdaptNone, Raven.AdaptRefine), length(gm))
+        indicator = rand(rng, (Raven.AdaptNone, Raven.AdaptRefine), length(gm))
         adapt!(gm, indicator)
 
         grid = generate(gm)
@@ -89,13 +89,13 @@ function grids_testsuite(AT, FT)
 
     let
         N = (3, 3)
-        R = 1
+        R = FT(1)
 
         coarse_grid = Raven.cubeshell2dgrid(R)
 
         gm = GridManager(LobattoCell{Tuple{N...},FT,AT}(), coarse_grid, min_level = 2)
 
-        indicator = rand((Raven.AdaptNone, Raven.AdaptRefine), length(gm))
+        indicator = rand(rng, (Raven.AdaptNone, Raven.AdaptRefine), length(gm))
         adapt!(gm, indicator)
 
         grid = generate(gm)