Build on ALCF Theta

Setup

Using Intel Compilers and MPICH (currently the only MPI choice available through modules) and Ben's cmake build:

/gpfs/theta-fs1/projects/UnsAdaptCFD_tesp/kjansen/cmake/v3.6.1/bin

Currently Loaded Modulefiles:
  1) modules/3.2.10.4                  5) craype/2.5.6                      9) pmi/5.0.10-1.0000.11050.0.0.ari  13) job/1.5.5-3.58                   17) atp/2.0.2
  2) eswrap/2.0.9-2.1                  6) cray-libsci/16.07.1              10) dmapp/7.1.0-12.37                14) dvs/2.7_0.9.0-2.148              18) PrgEnv-intel/6.0.3
  3) intel/16.0.3.210                  7) udreg/2.3.2-4.6                  11) gni-headers/5.0.7-3.1            15) alps/6.1.6-20.1                  19) craype-mic-knl
  4) craype-network-aries              8) ugni/6.0.12-2.1                  12) xpmem/0.1-4.5                    16) rca/1.0.0-6.21                   20) cray-mpich/7.4.2

ParMETIS

#!/bin/bash -e
metis=/gpfs/theta-fs1/projects/UnsAdaptCFD_tesp/cwsmith/develop/deps/parmetis/parmetis-4.0.3/metis/
cmake \
-DCMAKE_INSTALL_PREFIX=$1 \
-DMETIS_PATH=$metis \
-DGKLIB_PATH=$metis/GKlib \
-DCMAKE_C_COMPILER=cc \
-DCMAKE_CXX_COMPILER=CC \
-DCMAKE_BUILD_TYPE=None \
-DCMAKE_C_FLAGS="-O2 -static " \
-DCMAKE_CXX_FLAGS="-O2 -static " \
..
```

## Zoltan

```
#!/bin/bash -ex
unset CXXCPP
../configure \
--prefix=$1 \
--with-parmetis \
--with-parmetis-libdir=$PARMETIS_INSTALL_DIR/lib \
--with-parmetis-incdir=$PARMETIS_INSTALL_DIR/include \
--enable-mpi \
--disable-examples \
--host=x86_64-unknown-linux-gnu \
--with-gnumake \
--with-id-type=ulong \
CXXCPP='CC -E ' \
FC=ftn \
CC=cc \
CXX=CC
```

## BZip2

```
flags="-Wall -Winline -O2 -g -D_FILE_OFFSET_BITS=64 "
make CC=cc clean
make bzip2 CC=cc CFLAGS="$flags"
make install PREFIX=$prefix
```

## Core

cmake configuration script using Intel compilers `doConfigureIntel.sh`
```
#!/bin/bash -ex
export CMAKE_PREFIX_PATH=/lus/theta-fs0/home/cwsmith/develop/deps/install-intel:$CMAKE_PREFIX_PATH
flags="-O3 -Wall"
cmake \
-DCMAKE_C_FLAGS="${flags} " \
-DCMAKE_CXX_FLAGS="${flags} " \
-DCMAKE_CXX_LINK_FLAGS="${flags} " \
-DCMAKE_CXX_COMPILER=CC \
-DCMAKE_C_COMPILER=cc \
-DENABLE_ZOLTAN=ON \
-DPCU_COMPRESS=ON \
-DMDS_SET_MAX=1024 \
-DMDS_ID_TYPE=long \
-DMPIRUN=/lus/theta-fs0/home/cwsmith/develop/core/runTheta.sh \
-DMPIRUN_PROCFLAG="" \
-DIS_TESTING=True \
-DMESHES=/lus/theta-fs0/home/cwsmith/develop/meshes \
../
```

cobalt job submission script for ctest `submitCtest.sh`
```
#!/bin/bash -ex
rundir=/lus/theta-fs0/home/cwsmith/develop/core/buildintel
qsub --cwd $rundir -q cache-quad -n 1 -t 10 -A UnsAdaptCFD_tesp /lus/theta-fs0/home/cwsmith/tools/cmake/v3.6.1/bin/ctest -VV
```

cray job execution script for each test
```
#!/bin/bash -x
processes=$1
exe=$2
myparam=${@:3}
aprun -n $processes -N $processes -j 1 $exe $myparam
```