Static Build on Cori Phase II

Here's what my config.sh script for HPX looks like:

#! /bin/bash
#
# Copyright (c) 2011-2012 Bryce Adelstein-Lelbach
#
# Distributed under the Boost Software License, Version 1.0. (See accompanying
# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)

source ${HOME}/.bashrc_real

module unload PrgEnv-intel
module load PrgEnv-gnu/6.0.3

if [ -f CMakeCache.txt ]; then
  rm -rf CMakeCache.txt
fi

SOURCE=${PWD}/..
INSTALL=${PWD}/../INSTALL_CORI_GCC_CRAY_AVX512_RELEASE_STATIC

LOG=config_$(date "+%Y.%m.%d_%H.%M.%S").log
LOGLEVEL="WARN"

CMAKE=/usr/common/software/cmake/3.3.2/bin/cmake

${CMAKE} -DCMAKE_TOOLCHAIN_FILE="${SOURCE}/cmake/toolchains/CrayKNLStatic.cmake"       \
         -DCMAKE_BUILD_TYPE=Release                                                    \
         -DHPX_WITH_THREAD_STACK_MMAP=ON                                               \
         -DHPX_WITH_THREAD_MANAGER_IDLE_BACKOFF=OFF                                    \
         -DHPX_WITH_THREAD_BACKTRACE_ON_SUSPENSION=OFF                                 \
         -DHPX_WITH_THREAD_TARGET_ADDRESS=OFF                                          \
         -DHPX_WITH_THREAD_QUEUE_WAITTIME=OFF                                          \
         -DHPX_WITH_THREAD_IDLE_RATES=OFF                                              \
         -DHPX_WITH_THREAD_CUMULATIVE_COUNTS=OFF                                       \
         -DHPX_WITH_THREAD_STEALING_COUNTS=OFF                                         \
         -DHPX_WITH_THREAD_LOCAL_STORAGE=OFF                                           \
         -DHPX_WITH_SCHEDULER_LOCAL_STORAGE=OFF                                        \
         -DHPX_WITH_THREAD_GUARD_PAGE=OFF                                              \
         -DHPX_WITH_PARCELPORT_MPI=ON                                                  \
         -DHPX_WITH_PARCELPORT_MPI_MULTITHREADED=ON                                    \
         -DHPX_WITH_PARCELPORT_LIBFABRIC=OFF                                           \
         -DHPX_WITH_DATAPAR_VC=ON                                                      \
         -DHPX_WITH_DATAPAR_VC_NO_LIBRARY=ON                                           \
         -DCMAKE_SKIP_INSTALL_RPATH=ON                                                 \
         -DBoost_COMPILER="-gcc"                                                       \
         -DVc_INCLUDE_DIR="${HOME}/development/Vc/INSTALL_ALL/include"                 \
         -DBOOST_ROOT="${HOME}/development/boost-1.61.0/INSTALL_CORI_GCC_CRAY_AVX512"  \
         -DTCMALLOC_ROOT="${HOME}/development/gperftools-2.5/INSTALL_CORI_RELEASE"     \
         -DJEMALLOC_ROOT="${HOME}/development/jemalloc/INSTALL_CORI_RELEASE"           \
         -DLIBFABRIC_ROOT="${HOME}/development/libfabric-cray/INSTALL_CORI_RELEASE"    \
         -DHWLOC_ROOT="${HOME}/install"                                                \
         -DHPX_WITH_MALLOC="jemalloc"                                                  \
         -DCMAKE_INSTALL_PREFIX="${INSTALL}"                                           \
         -DHPX_CMAKE_LOGLEVEL="${LOGLEVEL}"                                            \
         -Wdev ${SOURCE} 2>&1 | tee ${LOG}

And here's what it looks like for Octotiger:

#! /bin/bash
#
# Copyright (c) 2011-2012 Bryce Adelstein-Lelbach
#
# Distributed under the Boost Software License, Version 1.0. (See accompanying
# file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)

if [ -f CMakeCache.txt ]; then
  rm -rf CMakeCache.txt
fi

HPX=${HOME}/development/hpx/INSTALL_CORI_GCC_CRAY_AVX512_RELEASE_STATIC
SOURCE=${PWD}/..
INSTALL=${PWD}/../INSTALL_CORI_GCC_CRAY_AVX512_RELWITHDEBINFO_STATIC

LOG=config_$(date "+%Y.%m.%d_%H.%M.%S").log
LOGLEVEL="WARN"

CMAKE=/usr/common/software/cmake/3.3.2/bin/cmake

${CMAKE} -DCMAKE_TOOLCHAIN_FILE="${HPX}/lib/cmake/HPX/toolchains/CrayKNLStatic.cmake"           \
         -DCMAKE_BUILD_TYPE=RelWithDebInfo                                                      \
         -DCMAKE_PREFIX_PATH="${HPX}"                                                           \
         -DHPX_WITH_MALLOC="jemalloc"                                                           \
         -DSilo_LIBRARY="${HOME}/development/silo-4.10.2/INSTALL_CORI_RELEASE/lib/libsilo.a"    \
         -DSilo_INCLUDE_DIR="${HOME}/development/silo-4.10.2/INSTALL_CORI_RELEASE/include"      \
         -DCMAKE_INSTALL_PREFIX="${INSTALL}"                                                    \
         -DHPX_CMAKE_LOGLEVEL="${LOGLEVEL}"                                                     \
         -Wdev ${SOURCE} 2>&1 | tee ${LOG}

My .modules looks like this:

if [ "$NERSC_HOST" = "edison" ] || [ "$NERSC_HOST" = "cori" ]; then
    module unload darshan
    module unload atp
    module unload altd

    module unload memkind
    module unload hwloc
    module unload numactl

    module load vtune/2016.up4

    module load scons
    module load mercurial
    module load subversion

    module load swig

    module unload PrgEnv-intel
 
    if [ "$NERSC_HOST" = "cori" ]; then
        if [ "$CORI_TARGET" = "knl" ]; then
            module unload craype-haswell
            module load craype-mic-knl
        fi

        module unload PrgEnv-intel
        module load PrgEnv-gnu/6.0.3

        module load intel/17.0.2.174

        module load gcc/6.3.0 

        module load boost/1.61

        module load python/3.5-anaconda
    fi

    if [ "$NERSC_HOST" = "edison" ]; then
        module unload PrgEnv-intel
        module load PrgEnv-gnu/5.2.56

        module load intel/17.0.1.132

        module load gcc/6.1.0 

        module load boost/1.59

        module load python/3.4
    fi
fi

if [ "$NERSC_HOST" = "carl" ] && [ -f /opt/Modules/default/init/bash ]; then
    module load intel/2017.beta.up2
    module load impi impi/5.1.3.181
    module load vtune vtune/2016.up2
fi

Important Notes:

• You need to build all of the dependencies (Boost, jemalloc, hwloc, silo, libfabric-cray) as static libraries.
• You must module unload hwloc and module unload numactl, just to be safe. The hwloc module is detected by the cray compiler wrappers and does not have a static library; it depends on libnuma (and the numactl module also doesn't have a static library). Build your own hwloc. You'll need to turn off a bunch of things when building hwloc: --disable-cairo --without-x --disable-libxml2 --disable-pci --disable-libnuma --enable-shared --enable-static.
• TCMalloc segfaulted in pre-main initialization code for me, so I have been using jemalloc.