Merge pull request #740 from ExtremeFLOW/release/0.5

NEKO v0.5.0
ExtremeFLOW · Dec 14, 2022 · 79c37fd · 79c37fd
2 parents 9f97285 + dd80e48
commit 79c37fd
Show file tree

Hide file tree

Showing 295 changed files with 13,242 additions and 3,399 deletions.
diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml
@@ -3,34 +3,63 @@ name: develop
 # Controls when the action will run. 
 on:
   pull_request:
-     branches: [develop]
+     branches: [develop,release/*]
   workflow_dispatch:
 
 jobs:
+  linting:
+      name: "Flint"
+      runs-on: ubuntu-20.04
+      steps:
+        - name: Setup env.
+          run: |
+            sudo apt-get update && sudo apt-get install python-dev python3-tk
+            pip install nobvisual==0.2.0 flinter==0.4.0
+        - name: Checkout
+          uses: actions/checkout@v3
+          with:
+            fetch-depth: 1
+        - name: Lint
+          run: |
+            flint score src/ -d 10 -r flinter_rc.yml | tee flint.txt
+            score=$(awk '$1==0{print $3}' flint.txt)
+            if (( $(echo "$score < 8.13" |bc -l) )) ; then
+              exit 1
+            fi
+        - name: Archive linter report
+          uses: actions/upload-artifact@v3
+          with:
+            name: flint-report
+            path: flint.txt
+            retention-days: 5
   GNU:
-
+    needs: linting
     runs-on: ${{matrix.os}}
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
-        os: [ubuntu-20.04, macos-latest]
-        compiler: [gfortran-10, gfortran-11]
-        backend: [cpu, cuda, opencl]
+        os: [ubuntu-20.04, macos-12]
+        compiler: [gfortran-10, gfortran-11, gfortran-12]
+        backend: [cpu, cuda, hip, opencl]
         precision: [sp, dp]
         exclude:
         - os: ubuntu-20.04
           compiler: gfortran-11
+        - os: ubuntu-20.04
+          compiler: gfortran-12
         - os: ubuntu-20.04
           backend: opencl
-        - os: macos-latest
+        - os: macos-12
           compiler: gfortran-10
-        - os: macos-latest
+        - os: macos-12
           backend: cuda
+        - os: macos-12
+          backend: hip
         include:
         - os: ubuntu-20.04
-          setup-env: sudo apt-get update && sudo apt-get install -y openmpi-bin libopenmpi-dev autoconf automake autotools-dev libopenblas-dev make git m4 python3  cmake-curses-gui nvidia-cuda-toolkit
-        - os: macos-latest
-          setup-env: brew update && brew install openmpi && brew install automake && brew install gcc@11
+          setup-env: sudo apt-get update && sudo apt-get install -y openmpi-bin libopenmpi-dev autoconf automake autotools-dev libopenblas-dev make git m4 python3  cmake-curses-gui 
+        - os: macos-12
+          setup-env: brew update && brew install openmpi && brew install automake && brew install gcc@11 && brew install gcc@12
     env:
       FC: ${{ matrix.compiler }}
       OMPI_FC: ${{ matrix.compiler }}
@@ -44,33 +73,59 @@ jobs:
         run: ${{ matrix.setup-env }}
       - name: Cache pFUnit
         id: cache-pfunit
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/pkg/pfunit
           key: pfunit-${{ runner.os }}-${{ matrix.os }}-${{ matrix.compiler }}
       - name: Build pFUnit
         if: ${{ (steps.cache-pfunit.outputs.cache-hit != 'true') && (matrix.backend == 'cpu') }}
         run: |
           git clone https://github.com/Goddard-Fortran-Ecosystem/pFUnit.git
-          cd pFUnit && mkdir b && cd b
+          cd pFUnit
+          cat >> pfunit_error_stop.patch << _ACEOF
+          diff --git a/src/funit/FUnit.F90 b/src/funit/FUnit.F90
+          index 7df7b65..4f7dbf5 100644
+          --- a/src/funit/FUnit.F90
+          +++ b/src/funit/FUnit.F90
+          @@ -168,7 +168,7 @@ contains
+           #if defined(PGI)
+                    call exit(-1)
+           #else
+          -         stop '*** Encountered 1 or more failures/errors during testing. ***'
+          +         error stop '*** Encountered 1 or more failures/errors during testing. ***'
+           #endif
+                 end if
+           
+          _ACEOF
+          git apply pfunit_error_stop.patch && mkdir b && cd b
           cmake -DCMAKE_INSTALL_PREFIX=${HOME}/pkg/pfunit ..
           make -j$(nproc) && make install && cd ../../
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
       - name: Build (CPU backend)
         if: matrix.backend == 'cpu'
         run: |
           ./regen.sh 
-          ./configure FC=${FC} FCFLAGS="-O2 -pedantic -std=f2008"  --with-pfunit=${HOME}/pkg/pfunit/PFUNIT-4.2 --enable-real=${RP}
+          ./configure FC=${FC} FCFLAGS="-O2 -pedantic -std=f2008"  --with-pfunit=${HOME}/pkg/pfunit/PFUNIT-4.4 --enable-real=${RP}
           make -j$(nproc)
       - name: Build (CUDA backend)        
         if: matrix.backend == 'cuda'
         run: |
+          sudo apt-get install -y nvidia-cuda-toolkit
           ./regen.sh 
           ./configure FC=${FC} FCFLAGS="-O2 -pedantic -std=f2008"  --enable-real=${RP} --with-cuda=/usr
           make -j$(nproc)
+      - name: Build (HIP backend)        
+        if: matrix.backend == 'HIP'
+        run: |
+          wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key | sudo apt-key add -
+          echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/debian/ ubuntu main' | sudo tee /etc/apt/sources.list.d/rocm.list
+          sudo apt-get update && sudo apt-get install -y rocm-dev
+          ./regen.sh 
+          ./configure FC=${FC} FCFLAGS="-O2 -pedantic -std=f2008" HIP_HIPCC_FLAGS="-O2 -fPIE" --enable-real=${RP} --with-hip=/opt/rocm/hip
+          make -j$(nproc)
       - name: Build (OpenCL backend)
         if: matrix.backend == 'opencl'
         run: |
@@ -99,6 +154,15 @@ jobs:
           cd releng/neko-*
           ./configure FC=${FC} --enable-real=${RP} --with-cuda=/usr
           make -j $(nproc)
+      - name: Dist (HIP backend)
+        if: matrix.backend == 'hip'
+        run: |
+          make dist
+          mkdir releng
+          tar xf neko-*.tar.gz -C releng
+          cd releng/neko-*
+          ./configure FC=${FC} FCFLAGS="-fPIE" --enable-real=${RP} HIP_HIPCC_FLAGS="-O2 -fPIE" --with-hip=/opt/rocm/hip
+          make -j $(nproc)
       - name: Dist (OpenCL backend)
         if: matrix.backend == 'opencl'
         run: |
@@ -110,6 +174,7 @@ jobs:
           make -j $(nproc)
     
   Intel:
+    needs: linting
     runs-on: ${{matrix.os}}
     strategy:
       fail-fast: false
@@ -133,14 +198,94 @@ jobs:
       - name: Setup env.
         run: ${{ matrix.setup-env }}
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 1
       - name: Build
         run: |
           ./regen.sh
           ./configure FC=${FC} CC=${CC} MPIFC"=mpiifort -fc=${FC}" --enable-real=${RP} 
           make FCFLAGS="-O2 -stand f08 -warn errors" -j$(nproc)
-            
-            
- 
+  
+  NVIDIA:
+    needs: linting
+    runs-on: ${{matrix.os}}
+    strategy:
+      fail-fast: true
+      matrix:
+        os: [ubuntu-20.04]
+        compiler: [nvfortran]
+        backend: [cpu, cuda]
+        precision: [dp]
+        include:
+        - os: ubuntu-20.04
+          setup-env: sudo apt-get update && sudo apt-get install -y autoconf automake autotools-dev make git m4 libopenblas-dev && curl https://developer.download.nvidia.com/hpc-sdk/ubuntu/DEB-GPG-KEY-NVIDIA-HPC-SDK | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg && echo 'deb [signed-by=/usr/share/keyrings/nvidia-hpcsdk-archive-keyring.gpg] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /' | sudo tee /etc/apt/sources.list.d/nvhpc.list && sudo apt-get update -y && sudo apt-get install -y nvhpc-22-11-cuda-multi &&  NVARCH=`uname -s`_`uname -m`; export NVARCH && NVCOMPILERS=/opt/nvidia/hpc_sdk; export NVCOMPILERS && PATH=$NVCOMPILERS/$NVARCH/22.11/compilers/bin:$PATH; export PATH &&  export PATH=$NVCOMPILERS/$NVARCH/22.11/comm_libs/mpi/bin:$PATH && printenv >> $GITHUB_ENV 
+    env:
+      CC: gcc
+      FC: ${{ matrix.compiler }}
+      OMPI_FC: ${{ matrix.compiler }}
+      OMPI_CC: gcc
+      OMPI_ALLOW_RUN_AS_ROOT: 1
+      OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
+      OMPI_MCA_btl_vader_single_copy_mechanism: none
+      RP: ${{ matrix.precision }}
+    name: ${{ matrix.os }} / ${{ matrix.compiler }} / ${{ matrix.backend }} / ${{ matrix.precision }}
+    steps:
+      - name: Setup env.
+        run: ${{ matrix.setup-env }}
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+      - name: Build (CPU backend)
+        if: matrix.backend == 'cpu'
+        run: |
+          git apply patches/nvhpc_bge.patch
+          nvfortran --version
+          ./regen.sh
+          ./configure FC=${FC} FCFLAGS="-O3" --enable-real=${RP}
+          make -j$(nproc)
+      - name: Build (CUDA backend)
+        if: matrix.backend == 'cuda'
+        run: |
+          sudo apt-get install -y nvidia-cuda-toolkit
+          git apply patches/nvhpc_bge.patch
+          nvfortran --version
+          ./regen.sh
+          ./configure FC=${FC} FCFLAGS="-O3" --enable-real=${RP} --with-cuda=/usr
+          make -j$(nproc)          
+  ReFrame:
+      needs: GNU
+      runs-on: ubuntu-20.04
+      strategy:
+        fail-fast: true
+        matrix:
+          precision: [sp, dp]
+          compiler: [gfortran-10]
+      env:
+        FC: ${{ matrix.compiler }}
+        OMPI_FC: ${{ matrix.compiler}}
+        OMPI_ALLOW_RUN_AS_ROOT: 1
+        OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
+        OMPI_MCA_btl_vader_single_copy_mechanism: none
+        RP: ${{ matrix.precision }}
+      name: ReFrame / ${{ matrix.compiler}} / ${{ matrix.precision }}
+      steps:
+        - name: Setup env.
+          run: |
+            sudo apt-get update && sudo apt-get install -y openmpi-bin libopenmpi-dev autoconf automake autotools-dev libopenblas-dev make git m4 python3
+            pip install reframe-hpc 
+        - name: Topology
+          run: |
+            reframe --detect-host-topology
+        - name: Checkout
+          uses: actions/checkout@v3
+          with:
+            fetch-depth: 1
+        - name: Regression tests
+          run: |
+            ./regen.sh
+            cd reframe
+            env NEKO_REAL=${RP} NEKO_SCHEME=pnpn reframe -v -C settings.py -c checks.py -n MiniHemi -n MiniTgv8 -n MiniRB -r --performance-report --system github-actions:cpu
+
+
diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml
@@ -1,8 +1,8 @@
 name: Documentation
 
 on:
-  push:
-    branches: [ master ]
+ # push:
+ #   branches: [ master ]
   workflow_dispatch:
 
 jobs:

diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -2,8 +2,8 @@ name: Linting
 
 # Controls when the action will run. 
 on:
-  pull_request:
-     branches: [develop,release/*,master]
+#  pull_request:
+#     branches: [develop,release/*,master]
   workflow_dispatch:
 
 jobs: