From cd22178d7f3b3a991c0fe59788c355aa46490cf8 Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Mon, 17 Oct 2022 19:34:09 +0100 Subject: [PATCH 01/11] Add CosmoSIS and standard library build tool to packlist --- conda/packlist.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda/packlist.txt b/conda/packlist.txt index 4adb851..c486c5c 100644 --- a/conda/packlist.txt +++ b/conda/packlist.txt @@ -3,6 +3,8 @@ celerite cmdstan cmdstanpy coloredlogs +cosmosis +cosmosis-build-standard-library emcee extinction f90nml From de0a1b8ac3841ec4a0a4d14000bbe429a2961e3f Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Tue, 18 Oct 2022 10:24:16 +0100 Subject: [PATCH 02/11] Testing adding an additional setup script --- Dockerfile | 9 +++++---- conda/build-nersc.sh | 5 +++++ conda/packlist.txt | 4 ++-- conda/post-build.sh | 9 +++++++++ docker/update-docker.sh | 3 +++ 5 files changed, 24 insertions(+), 6 deletions(-) create mode 100644 conda/post-build.sh diff --git a/Dockerfile b/Dockerfile index e604efd..3450409 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,7 @@ ARG LSST_GROUP=lsst USER lsst WORKDIR $LSST_STACK_DIR - + RUN echo "Environment: \n" && env | sort && \ curl -LO https://ls.st/lsstinstall && \ bash ./lsstinstall ${LSST_TAG:+"-X"} $LSST_TAG && \ @@ -41,7 +41,7 @@ RUN echo "Environment: \n" && env | sort && \ mkdir -p /tmp/gh && \ cd /tmp/gh && \ git clone https://github.com/LSSTDESC/td_env && \ - cd td_env && \ + cd td_env && \ git checkout $GH_SHA USER root @@ -52,11 +52,12 @@ RUN cd /tmp/gh/td_env && \ USER lsst RUN cd /tmp/gh/td_env/conda && \ bash /tmp/gh/td_env/docker/update-docker.sh w_2022_10 && \ + bash post-build.sh && \ echo "source $LSST_STACK_DIR/loadLSST.bash" >> ~/.bashrc && \ echo "setup lsst_distrib" >> ~/.bashrc && \ rm -Rf /tmp/gh - - + + ENV HDF5_USE_FILE_LOCKING FALSE ENV PYTHONSTARTUP '' diff --git a/conda/build-nersc.sh b/conda/build-nersc.sh index 6cc66a3..f3afcbf 100755 --- a/conda/build-nersc.sh +++ b/conda/build-nersc.sh @@ -40,6 +40,7 @@ fi mkdir -p $curBuildDir cp conda/packlist.txt $curBuildDir +cp conda/post-build.sh $curBuildDir cp conda/piplist.txt $curBuildDir cp nersc/setup_td_env.sh $curBuildDir cp nersc/sitecustomize.py $curBuildDir @@ -73,6 +74,10 @@ cd resspect python setup.py install cd .. +# Additional build steps +bash ./post-build.sh + + python -m compileall $curBuildDir # Hard-coding this for now for building at NERSC diff --git a/conda/packlist.txt b/conda/packlist.txt index c486c5c..41d757e 100644 --- a/conda/packlist.txt +++ b/conda/packlist.txt @@ -3,8 +3,8 @@ celerite cmdstan cmdstanpy coloredlogs -cosmosis -cosmosis-build-standard-library +cosmosis>=2.2.2 +cosmosis-build-standard-library>=1.2 emcee extinction f90nml diff --git a/conda/post-build.sh b/conda/post-build.sh new file mode 100644 index 0000000..e2f99f0 --- /dev/null +++ b/conda/post-build.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# Project- or dependency-specific setup commands to be +# run after a conda installation. + + +# Download and build the CosmoSIS standard library +# in a directory under the CosmoSIS python dir +cosmosis-build-standard-library -i diff --git a/docker/update-docker.sh b/docker/update-docker.sh index a8b6006..d753a25 100644 --- a/docker/update-docker.sh +++ b/docker/update-docker.sh @@ -18,6 +18,9 @@ conda clean -y -a # Import of ligo.em_bright to cause associated data files to be downloaded into the image # python -c "import ligo.em_bright" +# Additional build steps +bash ./post-build.sh + conda env export --no-builds > /opt/lsst/software/stack/td_env-image-nobuildinfo.yml conda env export > /opt/lsst/software/stack/td_env-image.yml From be6733c77a3850f0bc671a83bb28b44f3d785643 Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Tue, 18 Oct 2022 10:26:03 +0100 Subject: [PATCH 03/11] require newer cosmosis versiong --- conda/packlist.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/packlist.txt b/conda/packlist.txt index c486c5c..41d757e 100644 --- a/conda/packlist.txt +++ b/conda/packlist.txt @@ -3,8 +3,8 @@ celerite cmdstan cmdstanpy coloredlogs -cosmosis -cosmosis-build-standard-library +cosmosis>=2.2.2 +cosmosis-build-standard-library>=1.2 emcee extinction f90nml From c3ebc8d6112db85c8d2c7d9f0619c35142c2063a Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Tue, 18 Oct 2022 10:30:35 +0100 Subject: [PATCH 04/11] rename file to something more useful --- Dockerfile | 2 +- conda/build-nersc.sh | 4 ++-- conda/{post-build.sh => post-conda-build.sh} | 5 +++-- docker/update-docker.sh | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) rename conda/{post-build.sh => post-conda-build.sh} (53%) diff --git a/Dockerfile b/Dockerfile index 3450409..940b35a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -52,7 +52,7 @@ RUN cd /tmp/gh/td_env && \ USER lsst RUN cd /tmp/gh/td_env/conda && \ bash /tmp/gh/td_env/docker/update-docker.sh w_2022_10 && \ - bash post-build.sh && \ + bash post-conda-build.sh && \ echo "source $LSST_STACK_DIR/loadLSST.bash" >> ~/.bashrc && \ echo "setup lsst_distrib" >> ~/.bashrc && \ rm -Rf /tmp/gh diff --git a/conda/build-nersc.sh b/conda/build-nersc.sh index f3afcbf..7b57ca4 100755 --- a/conda/build-nersc.sh +++ b/conda/build-nersc.sh @@ -40,7 +40,7 @@ fi mkdir -p $curBuildDir cp conda/packlist.txt $curBuildDir -cp conda/post-build.sh $curBuildDir +cp conda/post-conda-build.sh $curBuildDir cp conda/piplist.txt $curBuildDir cp nersc/setup_td_env.sh $curBuildDir cp nersc/sitecustomize.py $curBuildDir @@ -75,7 +75,7 @@ python setup.py install cd .. # Additional build steps -bash ./post-build.sh +bash ./post-conda-build.sh python -m compileall $curBuildDir diff --git a/conda/post-build.sh b/conda/post-conda-build.sh similarity index 53% rename from conda/post-build.sh rename to conda/post-conda-build.sh index e2f99f0..c726413 100644 --- a/conda/post-build.sh +++ b/conda/post-conda-build.sh @@ -1,9 +1,10 @@ #!/bin/bash # Project- or dependency-specific setup commands to be -# run after a conda installation. +# run after a conda installation, for example to download +# additional data files # Download and build the CosmoSIS standard library -# in a directory under the CosmoSIS python dir +# in a directory under the CosmoSIS python directory cosmosis-build-standard-library -i diff --git a/docker/update-docker.sh b/docker/update-docker.sh index d753a25..c1c0934 100644 --- a/docker/update-docker.sh +++ b/docker/update-docker.sh @@ -19,7 +19,7 @@ conda clean -y -a # python -c "import ligo.em_bright" # Additional build steps -bash ./post-build.sh +bash ./post-conda-build.sh conda env export --no-builds > /opt/lsst/software/stack/td_env-image-nobuildinfo.yml conda env export > /opt/lsst/software/stack/td_env-image.yml From e4face689db1ec7e176e2f9cf0e6694fa43d16b4 Mon Sep 17 00:00:00 2001 From: Joe Zuntz Date: Wed, 26 Oct 2022 15:37:13 +0100 Subject: [PATCH 05/11] no-op commit to try and re-run integration now that there is a cosmosis built for mpich 3 --- conda/packlist.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/conda/packlist.txt b/conda/packlist.txt index 41d757e..5cf26dc 100644 --- a/conda/packlist.txt +++ b/conda/packlist.txt @@ -29,4 +29,3 @@ multiprocess>=0.70.8 nestcheck pymultinest pyxdg - From 016a81c192a0ee936fa6702d3c43c98b174929f8 Mon Sep 17 00:00:00 2001 From: Heather Kelly Date: Thu, 1 Dec 2022 19:09:36 -0500 Subject: [PATCH 06/11] define CSL_DIR and FIRECROWN_SITE_PACKAGES --- nersc/setup_td_env.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nersc/setup_td_env.sh b/nersc/setup_td_env.sh index 1587736..3e57a68 100755 --- a/nersc/setup_td_env.sh +++ b/nersc/setup_td_env.sh @@ -54,3 +54,7 @@ else source $DESC_LSST_INSTALL_DIR/loadLSST.zsh fi setup lsst_distrib + +# For cosmosis and firecrown. Should try to find a better way to set these +export CSL_DIR=$CONDA_PREFIX/lib/python3.8/site-packages/cosmosis/cosmosis-standard-library +export FIRECROWN_SITE_PACKAGES=$CONDA_PREFIX/lib/python3.8/site-packages From 8f6b45bcfd8a11fdd0f46315c49227f10d7cbc8b Mon Sep 17 00:00:00 2001 From: Heather Kelly Date: Thu, 1 Dec 2022 19:53:43 -0500 Subject: [PATCH 07/11] download a copy of firecrown so we have the examples subdirectory --- conda/build-nersc.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/conda/build-nersc.sh b/conda/build-nersc.sh index c069670..2aae6a6 100755 --- a/conda/build-nersc.sh +++ b/conda/build-nersc.sh @@ -80,6 +80,12 @@ cd resspect python setup.py install cd .. +# Grab firecrown source so we have the examples subdirectory +firecrown_ver=$(conda list firecrown | grep firecrown|tr -s " " | cut -d " " -f 2) +echo $firecrown_ver +curl -LO https://github.com/LSSTDESC/firecrown/archive/refs/tags/v$firecrown_ver.tar.gz +tar xvzf v$firecrown_ver.tar.gz + # Additional build steps bash ./post-conda-build.sh From 16bef2a143a4ea4e9788e0572d3d941047c0a664 Mon Sep 17 00:00:00 2001 From: Heather Kelly Date: Fri, 9 Dec 2022 13:52:57 -0500 Subject: [PATCH 08/11] symlink firecrown directory --- conda/build-nersc.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda/build-nersc.sh b/conda/build-nersc.sh index 2aae6a6..bf94213 100755 --- a/conda/build-nersc.sh +++ b/conda/build-nersc.sh @@ -85,6 +85,8 @@ firecrown_ver=$(conda list firecrown | grep firecrown|tr -s " " | cut -d " " -f echo $firecrown_ver curl -LO https://github.com/LSSTDESC/firecrown/archive/refs/tags/v$firecrown_ver.tar.gz tar xvzf v$firecrown_ver.tar.gz +# Set up a common directory name without version info to set FIRECROWN_DIR more easily +ln -s firecrown-$firecrown_ver firecrown # Additional build steps bash ./post-conda-build.sh From 481a0e78ab1663b4fc4e7d97f26c55d593bcba15 Mon Sep 17 00:00:00 2001 From: Heather Kelly Date: Fri, 9 Dec 2022 13:54:33 -0500 Subject: [PATCH 09/11] define FIRECROWN_DIR --- nersc/setup_td_env.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/nersc/setup_td_env.sh b/nersc/setup_td_env.sh index 3e57a68..7203e35 100755 --- a/nersc/setup_td_env.sh +++ b/nersc/setup_td_env.sh @@ -58,3 +58,4 @@ setup lsst_distrib # For cosmosis and firecrown. Should try to find a better way to set these export CSL_DIR=$CONDA_PREFIX/lib/python3.8/site-packages/cosmosis/cosmosis-standard-library export FIRECROWN_SITE_PACKAGES=$CONDA_PREFIX/lib/python3.8/site-packages +export FIRECROWN_DIR=$DESC_LSST_INSTALL_DIR/firecrown From 98efbb1dbc1208971453e10454309835fa70e2cb Mon Sep 17 00:00:00 2001 From: Heather Kelly Date: Fri, 9 Dec 2022 14:02:27 -0500 Subject: [PATCH 10/11] add option to set up cosmosis --- nersc/setup_td_int.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/nersc/setup_td_int.sh b/nersc/setup_td_int.sh index 4bea50c..8f851cd 100644 --- a/nersc/setup_td_int.sh +++ b/nersc/setup_td_int.sh @@ -17,6 +17,7 @@ SCRIPT=${BASH_SOURCE[0]} usage() { # Function: Print a help message. echo -e \\n"Help documentation for ${BOLD}${SCRIPT}"\\n echo "Command line switches are optional. The following switches are recognized." + echo "-c --Setup cosmosis." echo "-k --Setup the env without doing module purge." echo "-n --Setup the env without the LSST Sci Pipelines." echo "-s --Setup the env for shifter." @@ -28,9 +29,10 @@ usage() { # Function: Print a help message. # -h help # -n Do not setup the LSST Sci Pipelines #while getopts e:n: flag -while getopts "hkns" flag +while getopts "chkns" flag do case "${flag}" in + c) cosmosis=1;; h) usage;; k) keepenv=1;; n) nolsst=1;; @@ -120,6 +122,13 @@ case $NERSC_HOST in export SNANA_SCRATCH="/global/cscratch1/sd/kessler" ;; esac + + +if [[ "$cosmosis" ]]; +then + source cosmosis-configure +fi + export SNANA_LSST_SIM="$SNANA_SCRATCH/SNANA_LSST_SIM" export SCRATCH_SIMDIR="$SNANA_LSST_SIM" From ea0309979c1fd12ebe307e1bec9dbd414b41b51e Mon Sep 17 00:00:00 2001 From: Heather Kelly Date: Fri, 9 Dec 2022 14:16:45 -0500 Subject: [PATCH 11/11] add some words about cosmosis and firecrown --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index bf9bac5..005b9cb 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,13 @@ Defines, builds and installs a conda environment including the LSST Science Pipe ## Current Stable Release https://github.com/LSSTDESC/td_env/releases/latest +## Running cosmosis + firecrown +Cosmosis is available in the td_env integration builds. Users are encouraged to try this out. + +The default `setup_td_int.sh` will prepare the environment by settting $CSL_DIR, $FIRECROWN_DIR, $FIRECROWN_SITE_PACKAGES variables which points to the current integration build directory. Currently the default setup does NOT entirely complete the cosmosis set up and does not run `source cosmosis-configure`. However, this can be enabled by doing: `source setup_td_int.sh -c`. + +To run a test of cosmosis + firecrown after doing `source setup_td_int.sh` please try: `cosmosis $FIRECROWN_DIR/examples/des_y1_3x2pt/des_y1_3x2pt.ini` + ## To Request Additional Packages, Report Problems, or Submit Questions Please [open an issue](https://github.com/LSSTDESC/td_env/issues) on this repository.