ELM->ATS coupling

.ELMenv

@@ -0,0 +1,25 @@
+
+export CC_ROOT=/usr/local
+export FC_ROOT=/usr/local
+export MPI_ROOT=${OPENMPI_DIR}
+
+
+export HDF5_PATH=${AMANZI_TPLS_DIR}
+export NETCDF_PATH=${AMANZI_TPLS_DIR}
+#export PNETCDF_PATH=${AMANZI_TPLS_DIR}
+export PNETCDF_PATH=/Users/80x/Software/kernel_test_E3SM/pnetcdf/pnetcdf-install
+export BLASLAPACK_DIR=/Library/Developer/CommandLineTools/SDKs/MacOSX13.3.sdk/usr/lib
+
+
+export PROJECT_E3SM=${ATS_BASE}/E3SM_jb_branch
+export E3SM_ROOT=${PROJECT_E3SM}/E3SM
+export E3SM_INPUTDATA=${E3SM_ROOT}/pt-e3sm-inputdata
+
+
+export MACH_NAME=mymac
+
+export OMPI_CC=gcc
+export OMPI_FC=gfortran
+export OMPI_CXX=g++
+
+export PYTHONPATH=${E3SM_ROOT}/python:$PYTHONPATH

.gitignore

@@ -26,3 +26,4 @@ buildlib_cmakec
 
 # Ignore emacs backup files
 *~
+buildlib_cmakec

bld_elm_ats.sh

@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+
+# set NCOLS=1 to run single-column example
+# set NCOLS=5 to run 5-column hillslope example
+export NCOLS=5
+
+# set compset name
+export COMPSET=ICB1850CNPRDCTCBC
+export COMPDIR="${PROJECT_E3SM}/cases/${COMPSET}"
+export RUNDIR="${PROJECT_E3SM}/scratch/${COMPSET}/run"
+
+cd ${E3SM_ROOT}/cime/scripts && 
+./create_newcase --case ${COMPDIR} --res ELM_USRDAT --mach ${MACH_NAME} --compiler gnu --compset ${COMPSET} --walltime 06:00:00
+
+cd ${COMPDIR} && 
+./xmlchange --id ELM_USRDAT_NAME --val "${NCOLS}x1pt_Oakharbor-GRID"
+./xmlchange NTASKS=1
+./xmlchange NTASKS_PER_INST=1
+./xmlchange PIO_TYPENAME=netcdf
+./xmlchange RUN_STARTDATE=2000-07-15
+./xmlchange STOP_N=100
+./xmlchange HIST_N=1
+
+# expect to error here
+# part of pt-mode ELM hack
+cd ${COMPDIR}; ./case.setup
+
+# exit on error beyond this point
+set -e
+
+# write to user_nl_elm
+if [[ $NCOLS -eq 1 ]]
+then
+## single-column
+  cd ${COMPDIR} &&
+  echo "metdata_type = 'gswp3'
+ metdata_bypass = '/Users/80x/Software/ats_newstate/repos/E3SM/pt-e3sm-inputdata/atm/datm7/atm_forcing.datm7.GSWP3.0.5d.v2.c180716_Oakharbor-Grid/cpl_bypass_full'
+ fsurdat = '/Users/80x/Software/ats_newstate/repos/E3SM/pt-e3sm-inputdata/lnd/clm2/surfdata_map/surfdata_1x1pt_Oakharbor-GRID_simyr1850_c360x720_c20230522.nc'
+ aero_file = '/Users/80x/Software/ats_newstate/repos/E3SM/pt-e3sm-inputdata/atm/cam/chem/trop_mozart_aero/aero/aerosoldep_monthly_1850_mean_1.9x2.5_c090803.nc'
+ CO2_file = '/Users/80x/Software/ats_newstate/repos/E3SM/pt-e3sm-inputdata/atm/datm7/CO2/fco2_datm_1765-2007_c100614.nc'
+ nyears_ad_carbon_only = 0
+ spinup_mortality_factor = 10
+ hist_empty_htapes = .true.
+ hist_nhtfrq = -24
+ hist_fincl1 = 'TBOT', 'PBOT','RH','RAIN','SNOW', 'TLAI', 'ZWT', 'SMP', 'SOILLIQ', 'SOILICE','SOIL_PRESSURE'
+ hist_mfilt = 1
+ use_ats = .true.
+ ats_inputdir = '/Users/80x/Software/ats_newstate/repos/E3SM/pt-e3sm-inputdata/lnd/clm2/ats'
+ ats_inputfile = 'column_jb.xml'
+&dynamic_subgrid
+ do_harvest = .false.
+ do_transient_pfts = .false.
+ flanduse_timeseries = ''
+/
+&finidat_consistency_checks
+ check_finidat_fsurdat_consistency = .false.
+ check_finidat_year_consistency = .false.
+/
+ " >> user_nl_elm;
+fi
+
+if [[ $NCOLS -eq 5 ]]
+then
+## 5-column 2D hillslope
+  cd ${COMPDIR} &&
+  echo "metdata_type = 'gswp3'
+ fsurdat = '/Users/80x/Software/ats_newstate/repos/E3SM/pt-e3sm-inputdata/lnd/clm2/surfdata_map/surfdata_5x1pt_Oakharbor-GRID_simyr1850_c360x720_c20230522.nc'
+ flanduse_timeseries = '/Users/80x/Software/ats_newstate/repos/E3SM/pt-e3sm-inputdata/lnd/clm2/surfdata_map/surfdata_5x1pt_Oakharbor-GRID_simyr1850_c360x720_c20230522.nc'
+ nyears_ad_carbon_only = 25
+ spinup_mortality_factor = 10
+ hist_empty_htapes = .true.
+ hist_nhtfrq = -24
+ hist_fincl1 = 'TBOT', 'PBOT','RH','RAIN','SNOW', 'TLAI', 'ZWT', 'SMP', 'SOILLIQ', 'SOILICE','SOIL_PRESSURE'
+ hist_mfilt = 365
+ metdata_bypass = '/Users/80x/Software/ats_newstate/repos/E3SM/pt-e3sm-inputdata/atm/datm7/atm_forcing.datm7.GSWP3.0.5d.v2.c180716_Oakharbor-Grid/cpl_bypass_full'
+ aero_file = '/Users/80x/Software/ats_newstate/repos/E3SM/pt-e3sm-inputdata/atm/cam/chem/trop_mozart_aero/aero/aerosoldep_monthly_1850_mean_1.9x2.5_c090421.nc'
+ CO2_file = '/Users/80x/Software/ats_newstate/repos/E3SM/pt-e3sm-inputdata/atm/datm7/CO2/fco2_datm_1765-2007_c100614.nc'
+ use_ats = .true.
+ ats_inputdir = '/Users/80x/Software/ats_newstate/repos/E3SM/pt-e3sm-inputdata/lnd/clm2/ats'
+ ats_inputfile = 'hillslope_jb.xml'
+&dynamic_subgrid
+ do_harvest = .false.
+ do_transient_pfts = .false.
+ flanduse_timeseries = ''
+/
+&finidat_consistency_checks
+ check_finidat_fsurdat_consistency = .false.
+ check_finidat_year_consistency = .false.
+/
+ " >> user_nl_elm;
+fi
+
+# setup again - this time with correct nl variables in place
+cd ${COMPDIR}; ./case.setup
+
+# build
+cd ${COMPDIR}; ./case.build

cime_config/machines/bashrc_gnu

@@ -0,0 +1,86 @@
+export PATH=/usr/sbin:/sbin:/usr/bin:/bin:/opt/X11/bin
+export PATH=/usr/local/CMake.app/Contents/bin:$PATH
+export LD_LIBRARY_PATH=/usr/lib:/opt/X11/lib
+
+export PATH=/Library/Frameworks/Python.framework/Versions/3.6/bin:$PATH
+
+# blas-lapack library dir
+export BLASLAPACK_DIR=/usr/local/blas-lapack-dir
+export LD_LIBRARY_PATH=$BLASLAPACK_DIR:$LD_LIBRARY_PATH
+
+##--------------------------------------------------------------------------------------------------##
+
+export PATH=/usr/local/bin:$PATH
+export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
+
+# GNU tools, built with clang/clang++
+export PATH=/usr/local/gnutools/bin:$PATH
+export LD_LIBRARY_PATH=/usr/local/autotools/lib:$LD_LIBRARY_PATH
+
+# whole package of GCC, built by clang/clang++
+export PACKAGE_ROOT=/usr/local/gcc-x
+export FC_ROOT=$PACKAGE_ROOT/gcc-x-clang
+export CC_ROOT=$PACKAGE_ROOT/gcc-x-clang
+if [ -d $CC_ROOT ] ; then
+  export PATH=$CC_ROOT/bin:$PATH
+  export LD_LIBRARY_PATH=$CC_ROOT/lib:$LD_LIBRARY_PATH
+fi;
+
+##--------------------------------------------------------------------------------------------------##
+#############  NOTE: the following are built using clang OR gcc, and gfortran #########
+
+# MPICH-3.3 or openmpi-4
+export MPINAME=mpich
+export MPI_CURRENT=mpich-3.3
+if [ -d $PACKAGE_ROOT/$MPI_CURRENT ] ; then
+  export MPI_ROOT=$PACKAGE_ROOT/$MPI_CURRENT
+  export PATH=$MPI_ROOT/bin:$PATH
+  export LD_LIBRARY_PATH=$MPI_ROOT/lib:$LD_LIBRARY_PATH
+fi;
+
+  # HDF5-1.10 built with mpich/gcc or mpich/clang-gfortran
+  # a note here - HDF5-1.10 seems have unknown issues with pflotran
+  export HDF5_CURRENT=hdf5-1.10
+  if [ -d $PACKAGE_ROOT/$HDF5_CURRENT ] ; then
+    export HDF5_PATH=$PACKAGE_ROOT/$HDF5_CURRENT
+    export PATH=$HDF5_PATH/bin:$PATH
+    export LD_LIBRARY_PATH=$HDF5_PATH/lib:$LD_LIBRARY_PATH
+  fi;
+
+  # NETCDF-4.x built with HDF5 and mpich
+  export NC_CURRENT=netcdf-4.x-hdf5
+  if [ -d $PACKAGE_ROOT/$NC_CURRENT ] ; then
+    export NETCDF_PATH=$PACKAGE_ROOT/$NC_CURRENT
+    export PATH=$NETCDF_PATH/bin:$PATH
+    export LD_LIBRARY_PATH=$NETCDF_PATH/lib:$LD_LIBRARY_PATH 
+
+    export NETCDF_C_PATH=$NETCDF_PATH
+    export NETCDF_FORTRAN_PATH=$NETCDF_PATH
+
+  fi;
+
+  export PNETCDF_CURRENT=pnetcdf-1.11.2
+  if [ -d $PACKAGE_ROOT/$PNETCDF_CURRENT ] ; then
+    export PNETCDF_PATH=$PACKAGE_ROOT/$PNETCDF_CURRENT
+    export PATH=$PNETCDF_PATH/bin:$PATH
+    export LD_LIBRARY_PATH=$PNETCDF_PATH/lib:$LD_LIBRARY_PATH
+  else
+    export PNETCDF_PATH=
+  fi;
+
+#-------------------------------------------------------------------
+# Jupyter Notebook installation
+export PATH=/Users/f9y/Library/Python/3.6/bin:$PATH
+
+#-----------------------------------------------------------------
+# Mac doesn't automatically set all ENV for GUI applications
+# have to by 'launchd'
+launchctl setenv PATH $PATH
+launchctl setenv CC_ROOT $CC_ROOT
+launchctl setenv FC_ROOT $FC_ROOT
+launchctl setenv HDF5_PATH $HDF5_PATH
+launchctl setenv NETCDF_PATH $NETCDF_PATH
+launchctl setenv MPI_ROOT $MPI_ROOT
+launchctl setenv LD_LIBRARY_PATH $LD_LIBRARY_PATH
+launchctl setenv PYTHONPATH $PYTHONPATH
+

cime_config/machines/bashrc_wsl

@@ -0,0 +1,144 @@
+# ~/.bashrc: executed by bash(1) for non-login shells.
+# see /usr/share/doc/bash/examples/startup-files (in the package bash-doc)
+# for examples
+
+# If not running interactively, don't do anything
+case $- in
+    *i*) ;;
+      *) return;;
+esac
+
+# don't put duplicate lines or lines starting with space in the history.
+# See bash(1) for more options
+HISTCONTROL=ignoreboth
+
+# append to the history file, don't overwrite it
+shopt -s histappend
+
+# for setting history length see HISTSIZE and HISTFILESIZE in bash(1)
+HISTSIZE=1000
+HISTFILESIZE=2000
+
+# check the window size after each command and, if necessary,
+# update the values of LINES and COLUMNS.
+shopt -s checkwinsize
+
+# If set, the pattern "**" used in a pathname expansion context will
+# match all files and zero or more directories and subdirectories.
+#shopt -s globstar
+
+# make less more friendly for non-text input files, see lesspipe(1)
+[ -x /usr/bin/lesspipe ] && eval "$(SHELL=/bin/sh lesspipe)"
+
+# set variable identifying the chroot you work in (used in the prompt below)
+if [ -z "${debian_chroot:-}" ] && [ -r /etc/debian_chroot ]; then
+    debian_chroot=$(cat /etc/debian_chroot)
+fi
+
+# set a fancy prompt (non-color, unless we know we "want" color)
+case "$TERM" in
+    xterm-color|*-256color) color_prompt=yes;;
+esac
+
+# uncomment for a colored prompt, if the terminal has the capability; turned
+# off by default to not distract the user: the focus in a terminal window
+# should be on the output of commands, not on the prompt
+#force_color_prompt=yes
+
+if [ -n "$force_color_prompt" ]; then
+    if [ -x /usr/bin/tput ] && tput setaf 1 >&/dev/null; then
+	# We have color support; assume it's compliant with Ecma-48
+	# (ISO/IEC-6429). (Lack of such support is extremely rare, and such
+	# a case would tend to support setf rather than setaf.)
+	color_prompt=yes
+    else
+	color_prompt=
+    fi
+fi
+
+if [ "$color_prompt" = yes ]; then
+    PS1='${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '
+else
+    PS1='${debian_chroot:+($debian_chroot)}\u@\h:\w\$ '
+fi
+unset color_prompt force_color_prompt
+
+# If this is an xterm set the title to user@host:dir
+case "$TERM" in
+xterm*|rxvt*)
+    PS1="\[\e]0;${debian_chroot:+($debian_chroot)}\u@\h: \w\a\]$PS1"
+    ;;
+*)
+    ;;
+esac
+
+# enable color support of ls and also add handy aliases
+if [ -x /usr/bin/dircolors ]; then
+    test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)"
+    alias ls='ls --color=auto'
+    #alias dir='dir --color=auto'
+    #alias vdir='vdir --color=auto'
+
+    alias grep='grep --color=auto'
+    alias fgrep='fgrep --color=auto'
+    alias egrep='egrep --color=auto'
+fi
+
+# colored GCC warnings and errors
+#export GCC_COLORS='error=01;31:warning=01;35:note=01;36:caret=01;32:locus=01:quote=01'
+
+# some more ls aliases
+alias ll='ls -alF'
+alias la='ls -A'
+alias l='ls -CF'
+
+# Add an "alert" alias for long running commands.  Use like so:
+#   sleep 10; alert
+alias alert='notify-send --urgency=low -i "$([ $? = 0 ] && echo terminal || echo error)" "$(history|tail -n1|sed -e '\''s/^\s*[0-9]\+\s*//;s/[;&|]\s*alert$//'\'')"'
+
+# Alias definitions.
+# You may want to put all your additions into a separate file like
+# ~/.bash_aliases, instead of adding them here directly.
+# See /usr/share/doc/bash-doc/examples in the bash-doc package.
+
+if [ -f ~/.bash_aliases ]; then
+    . ~/.bash_aliases
+fi
+
+# enable programmable completion features (you don't need to enable
+# this, if it's already enabled in /etc/bash.bashrc and /etc/profile
+# sources /etc/bash.bashrc).
+if ! shopt -oq posix; then
+  if [ -f /usr/share/bash-completion/bash_completion ]; then
+    . /usr/share/bash-completion/bash_completion
+  elif [ -f /etc/bash_completion ]; then
+    . /etc/bash_completion
+  fi
+fi
+
+#--------------------------------------------------------------------
+# the following are likely required for E3SM Land Model (ELM) settings
+# 
+export PATH=~/.local/bin:$PATH 
+
+# user-defined
+export FC_ROOT=/usr
+export CC_ROOT=/usr
+export MPI_ROOT=/usr
+
+export LD_LIBRARY_PATH=/usr/local/lib:/lib
+export BLASLAPACK_LIBDIR=/usr/local/lib
+
+export ZLIB_DIR=/usr/lib/x86_64-linux-gnu
+export LD_LIBRARY_PATH=${ZLIB_DIR}:${LD_LIBRARY_PATH}
+
+export HDF5_PATH=/usr/local/hdf5-1.10-parallel
+export PATH=${HDF5_PATH}/bin:${PATH}
+export LD_LIBRARY_PATH=${HDF5_PATH}/lib:${LD_LIBRARY_PATH}
+
+export NETCDF_PATH=/usr/local/netcdf-4.x-hdf5-parallel
+export PATH=${NETCDF_PATH}/bin:${PATH}
+export LD_LIBRARY_PATH=${NETCDF_PATH}/lib:${LD_LIBRARY_PATH}
+export NETCDF_C_PATH=${NETCDF_PATH}
+export NETCDF_FORTRAN_PATH=${NETCDF_PATH}
+

cime_config/machines/cmake_macros/Darwin.cmake

@@ -1,4 +1,4 @@
 string(APPEND CPPDEFS " -DSYSDARWIN")
 if (COMP_CLASS STREQUAL cpl)
-  string(APPEND LDFLAGS " -all_load")
+  string(APPEND LDFLAGS " ")
 endif()

cime_config/machines/cmake_macros/Macros.cmake

@@ -14,13 +14,14 @@ set(OS_MACRO ${MACROS_DIR}/${OS}.cmake)
 set(MACHINE_MACRO ${MACROS_DIR}/${MACH}.cmake)
 set(COMPILER_OS_MACRO ${MACROS_DIR}/${COMPILER}_${OS}.cmake)
 set(COMPILER_MACHINE_MACRO ${MACROS_DIR}/${COMPILER}_${MACH}.cmake)
+set(USERDEFINED_MACRO ${MACROS_DIR}/userdefined.cmake)
 
 if (CONVERT_TO_MAKE)
   get_cmake_property(VARS_BEFORE_BUILD_INTERNAL_IGNORE VARIABLES)
 endif()
 
 # Include order defines precedence
-foreach (MACRO_FILE ${UNIVERSAL_MACRO} ${COMPILER_MACRO} ${OS_MACRO} ${MACHINE_MACRO} ${COMPILER_OS_MACRO} ${COMPILER_MACHINE_MACRO})
+foreach (MACRO_FILE ${UNIVERSAL_MACRO} ${COMPILER_MACRO} ${OS_MACRO} ${MACHINE_MACRO} ${COMPILER_OS_MACRO} ${COMPILER_MACHINE_MACRO} ${USERDEFINED_MACRO})
   if (EXISTS ${MACRO_FILE})
     include(${MACRO_FILE})
   else()

cime_config/machines/cmake_macros/gnu.cmake

@@ -31,6 +31,9 @@ endif()
 set(CXX_LINKER "FORTRAN")
 string(APPEND FC_AUTO_R8 " -fdefault-real-8")
 string(APPEND FFLAGS " -mcmodel=medium -fconvert=big-endian -ffree-line-length-none -ffixed-line-length-none")
+if (CMAKE_Fortran_COMPILER_VERSION VERSION_GREATER_EQUAL 10.0.0)
+  string(APPEND FFLAGS " -fallow-argument-mismatch")
+endif()
 if (compile_threaded)
   string(APPEND FFLAGS " -fopenmp")
 endif()