Globus Rework (#129)

* try perlmutter build

* try frontier test

* frontier typo

* simultanious

* try just frontier

* remove env file

* add perlmutter test

* increment version

.github/workflows/frontier/install.sh

@@ -0,0 +1,92 @@
+# #!/bin/bash
+
+branch=$1
+
+cd /lustre/orion/phy122/scratch/castia5/globus-compute/pumi-pic-test
+
+module load rocm
+module load craype-accel-amd-gfx90a
+module load cray-mpich
+export CRAYPE_LINK_TYPE=dynamic
+export MPICH_GPU_SUPPORT_ENABLED=1
+
+# # Kokkos
+# git clone -b 4.1.00 [email protected]:Kokkos/kokkos.git
+# bdir=$PWD/build-kokkos
+# rm -rf $bdir
+# cmake -S kokkos -B $bdir \
+#  -DCMAKE_BUILD_TYPE=RelWithDebInfo\
+#  -DCMAKE_CXX_COMPILER=CC\
+#  -DCMAKE_CXX_EXTENSIONS=OFF\
+#  -DKokkos_ENABLE_TESTS=OFF\
+#  -DKokkos_ENABLE_EXAMPLES=OFF\
+#  -DKokkos_ENABLE_SERIAL=ON\
+#  -DKokkos_ENABLE_OPENMP=OFF\
+#  -DKokkos_ENABLE_HIP=ON\
+#  -DKokkos_ARCH_VEGA90A=ON\
+#  -DKokkos_ENABLE_DEBUG=OFF\
+#  -DCMAKE_INSTALL_PREFIX=$bdir/install
+# cmake --build $bdir -j8 --target install
+
+# #Omega_h
+# git clone -b scorec-v10.8.4 [email protected]:SCOREC/omega_h.git
+# bdir=$PWD/build-omega_h
+# rm $bdir -rf
+# cmake -S omega_h -B $bdir \
+#   -DCMAKE_INSTALL_PREFIX=$bdir/install \
+#   -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+#   -DBUILD_SHARED_LIBS=OFF \
+#   -DOmega_h_USE_CUDA=OFF \
+#   -DOmega_h_USE_MPI=ON \
+#   -DOmega_h_USE_OpenMP=OFF \
+#   -DCMAKE_CXX_COMPILER=CC \
+#   -DOmega_h_USE_Kokkos=ON \
+#   -DOmega_h_USE_CUDA_AWARE_MPI=ON \
+#   -DKokkos_PREFIX=$PWD/build-kokkos/install \
+#   -DBUILD_TESTING=ON
+# cmake --build $bdir -j8 --target install
+
+# #Engpar
+# git clone [email protected]:SCOREC/EnGPar.git
+# bdir=$PWD/build-EnGPar
+# cmake -S EnGPar -B $bdir \
+#   -DCMAKE_INSTALL_PREFIX=$bdir/install \
+#   -DCMAKE_C_COMPILER="mpicc" \
+#   -DCMAKE_CXX_COMPILER="mpicxx" \
+#   -DCMAKE_CXX_FLAGS="-std=c++11" \
+#   -DENABLE_PARMETIS=OFF \
+#   -DENABLE_PUMI=OFF \
+#   -DIS_TESTING=OFF
+# cmake --build $bdir -j8 --target install
+
+# #Cabana
+# git clone https://github.com/ECP-copa/Cabana.git cabana
+# bdir=$PWD/build-cabana
+# rm $bdir -rf
+# cmake -S cabana -B $bdir \
+#   -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+#   -DCMAKE_CXX_COMPILER=CC \
+#   -DCMAKE_PREFIX_PATH=$PWD/build-kokkos/install \
+#   -DCMAKE_INSTALL_PREFIX=$bdir/install
+# cmake --build $bdir -j8 --target install
+
+#Pumi-Pic
+bdir=$PWD/build-pumi-pic
+rm pumi-pic -rf
+rm $bdir -rf
+git clone --recursive [email protected]:SCOREC/pumi-pic.git
+cd pumi-pic && git checkout $branch && cd -
+cmake -S pumi-pic -B $bdir \
+  -DCMAKE_CXX_COMPILER=CC \
+  -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+  -DIS_TESTING=ON \
+  -DPS_IS_TESTING=ON \
+  -DPS_USE_GPU_AWARE_MPI=ON \
+  -DTEST_DATA_DIR=$PWD/pumi-pic/pumipic-data \
+  -DOmega_h_PREFIX=$PWD/build-omega_h/install \
+  -DKokkos_PREFIX=$PWD/build-kokkos/install \
+  -DEnGPar_PREFIX=$PWD/build-EnGPar/install \
+  -DCabana_PREFIX=$PWD/build-cabana/install  \
+  -DENABLE_CABANA=on \
+  -DCMAKE_INSTALL_PREFIX=$bdir/install
+cmake --build $bdir -j8 --target install

.github/workflows/frontier/run.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+name=pumi-pic
+
+cd /lustre/orion/phy122/scratch/castia5/globus-compute/$name-test
+
+module load rocm
+module load craype-accel-amd-gfx90a
+module load cray-mpich
+export CRAYPE_LINK_TYPE=dynamic
+export MPICH_GPU_SUPPORT_ENABLED=1
+
+cd build-$name
+salloc --account=PHY122 --time=00:20:00 -q debug --nodes=1 --ntasks=1 --cpus-per-task=1 --gpus-per-task=1 --gpus=1 ctest
+cat $PWD/Testing/Temporary/LastTest.log

.github/workflows/globus-test.yml

@@ -0,0 +1,19 @@
+name: Globus-Systems
+on:
+  schedule:
+    # Monday 5:35 UTC or 01:35 EDT
+    - cron: '35 5 * * 1'
+
+jobs:
+
+  perlmutter-test:
+    uses: SCOREC/github-actions/.github/workflows/globus-test.yml@main
+    secrets: inherit
+    with:
+      machine: "perlmutter"
+
+  frontier-test:
+    uses: SCOREC/github-actions/.github/workflows/globus-test.yml@main
+    secrets: inherit
+    with:
+      machine: "frontier" 

.github/workflows/perlmutter/install.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+
+branch=$1
+
+cd $SCRATCH/globus-compute/pumi-pic-test
+
+export root=$PWD
+module load cmake
+
+function getname() {
+  name=$1
+  machine=perlmutter
+  buildSuffix=${machine}-cuda
+  echo "build-${name}"
+}
+export engpar=$root/`getname engpar`/install # This is where engpar will be (or is) installed
+export kk=$root/`getname kokkos`/install   # This is where kokkos will be (or is) installed
+export oh=$root/`getname omegah`/install  # This is where omega_h will be (or is) installed
+export cab=$root/`getname cabana`/install # This is where cabana will be (or is) installed
+export pumipic=$root/`getname pumi-pic`/install # This is where PumiPIC will be (or is) installed
+export CMAKE_PREFIX_PATH=$engpar:$kk:$kk/lib64/cmake:$oh:$cab:$pumipic:$CMAKE_PREFIX_PATH
+export MPICH_CXX=$root/kokkos/bin/nvcc_wrapper
+
+# #kokkos
+# git clone -b 4.1.00 https://github.com/kokkos/kokkos.git
+# cmake -S kokkos -B ${kk%%install} \
+#   -DCMAKE_INSTALL_PREFIX=$kk \
+#   -DCMAKE_CXX_COMPILER=$root/kokkos/bin/nvcc_wrapper \
+#   -DKokkos_ARCH_AMPERE80=ON \
+#   -DKokkos_ENABLE_SERIAL=ON \
+#   -DKokkos_ENABLE_OPENMP=off \
+#   -DKokkos_ENABLE_CUDA=on \
+#   -DKokkos_ENABLE_CUDA_LAMBDA=on \
+#   -DKokkos_ENABLE_DEBUG=on
+# cmake --build ${kk%%install} -j 24 --target install
+
+# #engpar
+# unset MPICH_CXX #don't want nvcc_wrapper for engpar
+# git clone https://github.com/SCOREC/EnGPar.git
+# cmake -S EnGPar -B ${engpar%%install} \
+#   -DCMAKE_INSTALL_PREFIX=$engpar \
+#   -DCMAKE_BUILD_TYPE="Release" \
+#   -DCMAKE_C_COMPILER=cc \
+#   -DCMAKE_CXX_COMPILER=CC \
+#   -DCMAKE_CXX_FLAGS="-std=c++11" \
+#   -DENABLE_PARMETIS=OFF \
+#   -DENABLE_PUMI=OFF \
+#   -DIS_TESTING=OFF
+# cmake --build ${engpar%%install} -j 24 --target install
+# export MPICH_CXX=$root/kokkos/bin/nvcc_wrapper #restore use of nvcc_wrapper
+
+# #omegah
+# git clone -b scorec-v10.8.4 https://github.com/SCOREC/omega_h.git
+# cmake -S omega_h -B ${oh%%install} \
+#   -DCMAKE_INSTALL_PREFIX=$oh \
+#   -DCMAKE_BUILD_TYPE="Release" \
+#   -DBUILD_SHARED_LIBS=OFF \
+#   -DOmega_h_USE_Kokkos=ON \
+#   -DOmega_h_USE_CUDA=on \
+#   -DOmega_h_CUDA_ARCH=80 \
+#   -DOmega_h_USE_MPI=on  \
+#   -DBUILD_TESTING=off  \
+#   -DCMAKE_C_COMPILER=cc \
+#   -DCMAKE_CXX_COMPILER=CC \
+#   -DKokkos_PREFIX=$kk/lib64/cmake
+# cmake --build ${oh%%install} -j 24 --target install
+
+# #cabana
+# git clone -b 0.6.1 https://github.com/ECP-copa/Cabana.git cabana
+# cmake -S cabana -B ${cab%%install} \
+#   -DCMAKE_INSTALL_PREFIX=$cab \
+#   -DCMAKE_BUILD_TYPE="Release" \
+#   -DCMAKE_CXX_COMPILER=$root/kokkos/bin/nvcc_wrapper \
+#   -DCabana_ENABLE_TESTING=OFF \
+#   -DCabana_ENABLE_EXAMPLES=OFF
+# cmake --build ${cab%%install} -j 24 --target install
+
+#pumipic
+rm $pumipic -rf
+rm pumi-pic -rf
+git clone --recursive https://github.com/SCOREC/pumi-pic.git
+cd pumi-pic && git checkout $branch && cd -
+cmake -S pumi-pic -B ${pumipic%%install} \
+  -DCMAKE_INSTALL_PREFIX=$pumipic \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_CXX_COMPILER=CC \
+  -DENABLE_CABANA=ON \
+  -DTEST_DATA_DIR=$root/pumi-pic/pumipic-data \
+  -DOmega_h_PREFIX=$oh \
+  -DEnGPar_PREFIX=$engpar \
+  -DIS_TESTING=ON \
+  -DBUILD_TESTING=ON \
+  -DPS_IS_TESTING=ON
+cmake --build ${pumipic%%install} -j 24 --target install

.github/workflows/perlmutter/run.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+name=pumi-pic
+
+cd $SCRATCH/globus-compute/$name-test
+
+export root=$PWD
+module load cmake
+export MPICH_CXX=$root/kokkos/bin/nvcc_wrapper
+
+cd build-$name
+salloc --time 00:20:00 --constrain=gpu --qos=interactive --nodes=1 --ntasks-per-node=40 --cpus-per-task=1 --gpus=1 --account=m4564 ctest
+cat $PWD/Testing/Temporary/LastTest.log

CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.0.0)
 
-project(pumipic VERSION 2.1.2 LANGUAGES CXX)
+project(pumipic VERSION 2.1.3 LANGUAGES CXX)
 
 include(cmake/bob.cmake)
 

SUPPORT.md

@@ -0,0 +1 @@
+If you have any issues you can contact us at [email protected]