From 7ce0eda489f97a42ed908f1c2a973ffbc748e7c5 Mon Sep 17 00:00:00 2001 From: Lucas C Wilcox Date: Mon, 1 Apr 2024 11:03:51 -0700 Subject: [PATCH 01/10] Add default formatting options for JuliaFormatter --- .JuliaFormatter.toml | 1 + docs/.JuliaFormatter.toml | 1 + examples/advection/.JuliaFormatter.toml | 1 + examples/grids/.JuliaFormatter.toml | 1 + test/.JuliaFormatter.toml | 1 + 5 files changed, 5 insertions(+) create mode 100644 .JuliaFormatter.toml create mode 100644 docs/.JuliaFormatter.toml create mode 100644 examples/advection/.JuliaFormatter.toml create mode 100644 examples/grids/.JuliaFormatter.toml create mode 100644 test/.JuliaFormatter.toml diff --git a/.JuliaFormatter.toml b/.JuliaFormatter.toml new file mode 100644 index 0000000..021527d --- /dev/null +++ b/.JuliaFormatter.toml @@ -0,0 +1 @@ +style = "default" diff --git a/docs/.JuliaFormatter.toml b/docs/.JuliaFormatter.toml new file mode 100644 index 0000000..021527d --- /dev/null +++ b/docs/.JuliaFormatter.toml @@ -0,0 +1 @@ +style = "default" diff --git a/examples/advection/.JuliaFormatter.toml b/examples/advection/.JuliaFormatter.toml new file mode 100644 index 0000000..021527d --- /dev/null +++ b/examples/advection/.JuliaFormatter.toml @@ -0,0 +1 @@ +style = "default" diff --git a/examples/grids/.JuliaFormatter.toml b/examples/grids/.JuliaFormatter.toml new file mode 100644 index 0000000..021527d --- /dev/null +++ b/examples/grids/.JuliaFormatter.toml @@ -0,0 +1 @@ +style = "default" diff --git a/test/.JuliaFormatter.toml b/test/.JuliaFormatter.toml new file mode 100644 index 0000000..021527d --- /dev/null +++ b/test/.JuliaFormatter.toml @@ -0,0 +1 @@ +style = "default" From e35cbf8abebcf091baea12dd383d3a24398ef990 Mon Sep 17 00:00:00 2001 From: Lucas C Wilcox Date: Mon, 1 Apr 2024 11:04:21 -0700 Subject: [PATCH 02/10] Upgrade Adapt to version 4 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 9051e05..eebdf71 100644 --- a/Project.toml +++ b/Project.toml @@ -26,7 +26,7 @@ RavenCUDAExt = "CUDA" RavenWriteVTKExt = "WriteVTK" [compat] -Adapt = "3" +Adapt = "3, 4" CUDA = "4, 5" Compat = "3.42, 4" GPUArraysCore = "0.1" From cc43269cc65db2251ce397948f07ff9ea88218c2 Mon Sep 17 00:00:00 2001 From: Lucas C Wilcox Date: Mon, 1 Apr 2024 11:10:30 -0700 Subject: [PATCH 03/10] Make grid tests reproducible Use StableRNGs to ensure that our random adaptive tests are reproducible. --- test/grids.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/grids.jl b/test/grids.jl index 7558f0c..59c114a 100644 --- a/test/grids.jl +++ b/test/grids.jl @@ -66,7 +66,7 @@ function grids_testsuite(AT, FT) gm = GridManager(LobattoCell{Tuple{N...},FT,AT}(), coarse_grid, min_level = 2) - indicator = rand((Raven.AdaptNone, Raven.AdaptRefine), length(gm)) + indicator = rand(rng, (Raven.AdaptNone, Raven.AdaptRefine), length(gm)) adapt!(gm, indicator) grid = generate(gm) @@ -95,7 +95,7 @@ function grids_testsuite(AT, FT) gm = GridManager(LobattoCell{Tuple{N...},FT,AT}(), coarse_grid, min_level = 2) - indicator = rand((Raven.AdaptNone, Raven.AdaptRefine), length(gm)) + indicator = rand(rng, (Raven.AdaptNone, Raven.AdaptRefine), length(gm)) adapt!(gm, indicator) grid = generate(gm) From 8f1d95b9ff226625cadca37c7d9fc54ee16c6f57 Mon Sep 17 00:00:00 2001 From: Lucas C Wilcox Date: Mon, 1 Apr 2024 13:36:12 -0700 Subject: [PATCH 04/10] Cube shell coarse grids get `eltype` from radii --- src/coarsegrids.jl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/coarsegrids.jl b/src/coarsegrids.jl index 80d1aed..40b8046 100644 --- a/src/coarsegrids.jl +++ b/src/coarsegrids.jl @@ -65,7 +65,8 @@ end function cubeshellgrid(R::Real, r::Real) @assert R > r "R (outer radius) must be greater that r (inner radius)" - vertices = zeros(SVector{3,Float64}, 16) + T = promote_type(typeof(R), typeof(r)) + vertices = zeros(SVector{3,T}, 16) vertices[1] = SVector(+R, +R, -R) vertices[2] = SVector(+R, -R, -R) @@ -109,7 +110,7 @@ function cubeshellgrid(R::Real, r::Real) z_x = tan(η) # Compute the new points - x = point[3] / hypot(1, y_x, z_x) + x = point[3] / hypot(one(y_x), y_x, z_x) y = x * y_x z = x * z_x @@ -147,7 +148,7 @@ end A cube shell is a 2D connectivity. """ function cubeshell2dgrid(R::Real) - vertices = zeros(SVector{3,Float64}, 8) + vertices = zeros(SVector{3,typeof(R)}, 8) vertices[1] = SVector(+R, +R, -R) vertices[2] = SVector(+R, -R, -R) @@ -175,7 +176,7 @@ function cubeshell2dgrid(R::Real) z_x = tan(η) # Compute the new points - x = point[3] / hypot(1, y_x, z_x) + x = point[3] / hypot(one(y_x), y_x, z_x) y = x * y_x z = x * z_x From 6debc906f1a233881d475e3fedf797a1c5464c5b Mon Sep 17 00:00:00 2001 From: Lucas C Wilcox Date: Mon, 1 Apr 2024 13:38:03 -0700 Subject: [PATCH 05/10] Use `FT` for cube shell coarse grid tests --- test/grids.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/grids.jl b/test/grids.jl index 59c114a..d70fcf6 100644 --- a/test/grids.jl +++ b/test/grids.jl @@ -59,8 +59,8 @@ function grids_testsuite(AT, FT) let N = (3, 3, 3) - R = 2 - r = 1 + R = FT(2) + r = FT(1) coarse_grid = Raven.cubeshellgrid(R, r) @@ -89,7 +89,7 @@ function grids_testsuite(AT, FT) let N = (3, 3) - R = 1 + R = FT(1) coarse_grid = Raven.cubeshell2dgrid(R) From 8c16db2c42d5c6da3ec204193fe9923db55f592a Mon Sep 17 00:00:00 2001 From: Lucas C Wilcox Date: Wed, 3 Apr 2024 16:20:51 -0700 Subject: [PATCH 06/10] Avoid capturing `MArray`s in `BoundsError` This quirk allows bounds checking of `MArray`s on device in more cases. This is a workaround to address . This change was proposed upstream here and is no longer needed if accepted upstream. --- ext/RavenCUDAExt.jl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ext/RavenCUDAExt.jl b/ext/RavenCUDAExt.jl index 83c1efc..833b739 100644 --- a/ext/RavenCUDAExt.jl +++ b/ext/RavenCUDAExt.jl @@ -3,6 +3,7 @@ module RavenCUDAExt import Raven import Adapt import MPI +import StaticArrays isdefined(Base, :get_extension) ? (using CUDA) : (using ..CUDA) isdefined(Base, :get_extension) ? (using CUDA.CUDAKernels) : (using ..CUDA.CUDAKernels) @@ -58,4 +59,11 @@ else Adapt.adapt_storage(::CUDA.KernelAdaptor, ::MPI.Comm) = nothing end +CUDA.@device_override function Base.checkbounds(A::StaticArrays.MArray, I...) + @inline + checkbounds(Bool, A, I...) || + CUDA.@print_and_throw("BoundsError while indexing an MArray.") + nothing +end + end # module RavenCUDAExt From dcc3295418c22fe85383b4591662d30fc8f6b56a Mon Sep 17 00:00:00 2001 From: Lucas C Wilcox Date: Wed, 17 Apr 2024 12:56:30 -0700 Subject: [PATCH 07/10] Add compat entry for UUIDs when testing system MPI --- test/configure_packages.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/configure_packages.jl b/test/configure_packages.jl index ad20185..fbc0713 100644 --- a/test/configure_packages.jl +++ b/test/configure_packages.jl @@ -7,6 +7,7 @@ Pkg.add("UUIDs") @static if VERSION >= v"1.8" Pkg.compat("MPIPreferences", "0.1") Pkg.compat("Preferences", "1") + Pkg.compat("UUIDs", "1") end const RAVEN_TEST = get(ENV, "RAVEN_TEST", "RAVEN_JLL_MPI_DEFAULT") From 2efccbcd79a63fe62e743eda60b179dd43ce5850 Mon Sep 17 00:00:00 2001 From: Lucas C Wilcox Date: Wed, 17 Apr 2024 12:57:39 -0700 Subject: [PATCH 08/10] Use julia v1.10 in github system MPI CI --- .github/workflows/CI.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 7a8f258..8189d11 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -46,7 +46,7 @@ jobs: - libmpich-dev - libopenmpi-dev julia_version: - - "1" + - "1.10" fail-fast: false env: P4ESTTYPES_TEST: P4ESTTYPES_CUSTOM_MPI_CUSTOM From 288b5a62209ca0c68892955e99c4f0b10d7b823f Mon Sep 17 00:00:00 2001 From: Lucas C Wilcox Date: Wed, 17 Apr 2024 13:00:06 -0700 Subject: [PATCH 09/10] Allow openmpi to over subscribe a node The environment variables were obtained from . --- .github/workflows/CI.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 8189d11..ba0380b 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -54,6 +54,8 @@ jobs: OMPI_MCA_btl_base_warn_component_unused: 0 OMPI_ALLOW_RUN_AS_ROOT: 1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1 + OMPI_MCA_rmaps_base_oversubscribe: true + PRTE_MCA_rmaps_default_mapping_policy: :oversubscribe steps: - uses: actions/checkout@v4 - name: Install MPI via apt From dc14f707287ce85061b52d8e4d199b3e8c035d49 Mon Sep 17 00:00:00 2001 From: Lucas C Wilcox Date: Wed, 17 Apr 2024 13:51:36 -0700 Subject: [PATCH 10/10] Cleanup MPI Requests --- src/Raven.jl | 10 ++++++++-- src/communication.jl | 22 ++++++++++++++++++---- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/src/Raven.jl b/src/Raven.jl index d2321b8..2dbb57c 100644 --- a/src/Raven.jl +++ b/src/Raven.jl @@ -65,8 +65,8 @@ if !isdefined(Base, :get_extension) using Requires end -@static if !isdefined(Base, :get_extension) - function __init__() +function __init__() + @static if !isdefined(Base, :get_extension) @require CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" include( "../ext/RavenCUDAExt.jl", ) @@ -74,6 +74,12 @@ end "../ext/RavenWriteVTKExt.jl", ) end + + MPI.add_finalize_hook!() do + for cm in COMM_MANAGERS + finalize(cm.value) + end + end end end # module Raven diff --git a/src/communication.jl b/src/communication.jl index 20be420..a865d59 100644 --- a/src/communication.jl +++ b/src/communication.jl @@ -1,3 +1,5 @@ +const COMM_MANAGERS = WeakRef[] + struct CommPattern{AT,RI,RR,RRI,SI,SR,SRI} recvindices::RI recvranks::RR @@ -74,7 +76,7 @@ end abstract type AbstractCommManager end -struct CommManagerBuffered{CP,RBD,RB,SBD,SB} <: AbstractCommManager +mutable struct CommManagerBuffered{CP,RBD,RB,SBD,SB} <: AbstractCommManager comm::MPI.Comm pattern::CP tag::Cint @@ -86,8 +88,8 @@ struct CommManagerBuffered{CP,RBD,RB,SBD,SB} <: AbstractCommManager sendrequests::MPI.UnsafeMultiRequest end -struct CommManagerTripleBuffered{CP,RBC,RBH,RBD,RB,RS,SBC,SBH,SBD,SB,SS} <: - AbstractCommManager +mutable struct CommManagerTripleBuffered{CP,RBC,RBH,RBD,RB,RS,SBC,SBH,SBD,SB,SS} <: + AbstractCommManager comm::MPI.Comm pattern::CP tag::Cint @@ -184,7 +186,7 @@ function commmanager( ) end - return if triplebuffer + cm = if triplebuffer backend = get_backend(arraytype(pattern)) recvstream = Stream(backend) sendstream = Stream(backend) @@ -218,6 +220,18 @@ function commmanager( sendrequests, ) end + + finalizer(cm) do cm + for reqs in (cm.recvrequests, cm.sendrequests) + for req in reqs + MPI.free(req) + end + end + end + + push!(COMM_MANAGERS, WeakRef(cm)) + + return cm end get_backend(cm::AbstractCommManager) = get_backend(arraytype(cm.pattern))