Skip to content

Commit cb80e79

Browse files
julianmorillojulianmorillo
authored
Merge branch 'main' into main
2 parents aed30ea + 6084589 commit cb80e79

File tree

11 files changed

+69
-29
lines changed

11 files changed

+69
-29
lines changed

‎EESSI-extend-easybuild.eb‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -228,6 +228,7 @@ if mode() == "unload" or mode() == "dependencyCk" or convertToCanonical(easybuil
228228
setenv ("EASYBUILD_CUDA_SANITY_CHECK_ERROR_ON_FAILED_CHECKS", "1")
229229
setenv ("EASYBUILD_FAIL_ON_MOD_FILES_GCCCORE", "1")
230230
setenv ("EASYBUILD_LOCAL_VAR_NAMING_CHECK", "error")
231+
setenv ("EASYBUILD_PARALLEL_EXTENSIONS_INSTALL", "1")
231232
-- Set environment variables that are EESSI version specific
232233
if convertToCanonical(eessi_version) > convertToCanonical("2023.06") then
233234
setenv ("EASYBUILD_PREFER_PYTHON_SEARCH_PATH", "EBPYTHONPREFIXES")

‎EESSI-install-software.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,7 @@ fi
207207

208208
# the install_scripts.sh script relies on knowing the location of the PR diff
209209
# assume there's only one diff file that corresponds to the PR patch file
210-
pr_diff=$(ls [0-9]*.diff | head -1)
210+
pr_diff=$(ls [0-9]*.diff | head -n 1)
211211
export PR_DIFF="$PWD/$pr_diff"
212212

213213
# Only run install_scripts.sh if not in dev.eessi.io for security

‎bot/build.sh‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,22 @@ else
171171
fi
172172
echo "bot/build.sh: EESSI_ACCELERATOR_TARGET_OVERRIDE='${EESSI_ACCELERATOR_TARGET_OVERRIDE}'"
173173

174+
# Log the full lscpu and os-release info:
175+
lscpu > _bot_job${SLURM_JOB_ID}.lscpu
176+
cat /etc/os-release > _bot_job${SLURM_JOB_ID}.os
177+
178+
# Also: fetch CPU flags into an array, so that we can implement a hard check against a reference
179+
lscpu_flags_line=$(lscpu | grep "Flags:")
180+
# strip leading "Flags:" and spaces, and put result in a bash array
181+
if [[ $lscpu_flags =~ Flags:\ (.*) ]]; then lscpu_flags=(${BASH_REMATCH[1]}); fi
182+
# for now, just print
183+
echo "bot/build.sh: CPU flags=${lscpu_flags[@]}"
184+
# TODO: an actual comparison with a reference bash array, e.g. through
185+
# diff_result=$(diff <(printf "%s\n" "${lscpu_flags[@]}" | sort) <(printf "%s\n" "${lscpu_flags_ref[@]}" | sort))
186+
# if [ ! -z "$diff_result" ]; then
187+
# echo "bot/build.sh: ERROR: difference between reported lscpu flags and reference for this ($EESSI_SOFTWARE_SUBDIR_OVERRIDE) CPU architecture. This could mean an incorrect build host was used to build for this target.
188+
# fi
189+
174190
# get EESSI_OS_TYPE from .architecture.os_type in ${JOB_CFG_FILE} (default: linux)
175191
EESSI_OS_TYPE=$(cfg_get_value "architecture" "os_type")
176192
export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux}
@@ -265,16 +281,24 @@ BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2)
265281
TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}")
266282

267283
timestamp=$(date +%s)
284+
# determine compression/extension for tarball, check in order of preference
285+
if [[ -x "$(command -v zstd)" ]]; then
286+
tarball_extension="tar.zst"
287+
elif [[ -x "$(command -v gzip)" ]]; then
288+
tarball_extension="tar.gz"
289+
else
290+
tarball_extension="tar"
291+
fi
268292
# to set EESSI_VERSION we need to source init/eessi_defaults now
269293
source $software_layer_dir/init/eessi_defaults
270294
# Note: if ${EESSI_DEV_PROJECT} is defined (building for dev.eessi.io), then we
271295
# append the project (subdirectory) name to the end tarball name. This is information
272296
# then used at the ingestion stage. If ${EESSI_DEV_PROJECT} is not defined, nothing is
273297
# appended
274298
if [[ -z ${EESSI_ACCELERATOR_TARGET_OVERRIDE} ]]; then
275-
export TGZ=$(printf "eessi-%s-software-%s-%s-%b%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp})
299+
export TARBALL=$(printf "eessi-%s-software-%s-%s-%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp})
276300
else
277-
export TGZ=$(printf "eessi-%s-software-%s-%s-%s-%b%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_ACCELERATOR_TARGET_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp})
301+
export TARBALL=$(printf "eessi-%s-software-%s-%s-%s-%b%d.${tarball_extension}" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_ACCELERATOR_TARGET_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp})
278302
fi
279303

280304
# Export EESSI_DEV_PROJECT to use it (if needed) when making tarball
@@ -288,8 +312,8 @@ export EESSI_DEV_PROJECT=${EESSI_DEV_PROJECT}
288312
TMP_IN_CONTAINER=/tmp
289313
echo "Executing command to create tarball:"
290314
echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}"
291-
echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET_OVERRIDE}\" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}"
315+
echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET_OVERRIDE}\" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr}"
292316
$software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \
293-
-- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET_OVERRIDE}" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
317+
-- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET_OVERRIDE}" /eessi_bot_job/${TARBALL} 2>&1 | tee -a ${tar_outerr}
294318

295319
exit 0

‎bot/check-build.sh‎

Lines changed: 22 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
# - SUCCESS (all of)
1919
# - working directory contains slurm-JOBID.out file
20-
# - working directory contains eessi*tar.gz
20+
# - working directory contains eessi*tar*
2121
# - no message FATAL
2222
# - no message ERROR
2323
# - no message FAILED
@@ -165,19 +165,19 @@ if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
165165
fi
166166

167167
if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
168-
TGZ=-1
168+
TARBALL_CREATED=-1
169169
TARBALL=
170170
if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
171-
GP_tgz_created="\.tar\.gz created!"
172-
grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tgz_created}" | sort -u)
171+
GP_tarball_created="\.tar.* created!"
172+
grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tarball_created}" | sort -u)
173173
if [[ $? -eq 0 ]]; then
174-
TGZ=1
174+
TARBALL_CREATED=1
175175
TARBALL=$(echo ${grep_out} | sed -e 's@^.*/\(eessi[^/ ]*\) .*$@\1@')
176176
else
177-
TGZ=0
177+
TARBALL_CREATED=0
178178
fi
179179
# have to be careful to not add searched for pattern into slurm out file
180-
[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'"
180+
[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tarball_created}"'"
181181
[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
182182
fi
183183
fi
@@ -190,7 +190,7 @@ fi
190190
[[ ${VERBOSE} -ne 0 ]] && echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)"
191191
[[ ${VERBOSE} -ne 0 ]] && echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
192192
if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
193-
[[ ${VERBOSE} -ne 0 ]] && echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
193+
[[ ${VERBOSE} -ne 0 ]] && echo " TARBALL_CREATED: $([[ $TARBALL -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
194194
fi
195195

196196
# Here, we try to do some additional analysis on the output file
@@ -219,7 +219,7 @@ if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]] && \
219219
[[ ${FAILED} -eq 0 ]] && \
220220
[[ ${MISSING} -eq 0 ]] && \
221221
[[ ${NO_MISSING} -eq 1 ]] && \
222-
[[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -ne 0 || ${TGZ} -eq 1 ]] && \
222+
[[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -ne 0 || ${TARBALL_CREATED} -eq 1 ]] && \
223223
[[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -ne 0 || -n ${TARBALL} ]]; then
224224
# SUCCESS
225225
status="SUCCESS"
@@ -429,9 +429,9 @@ failure_msg="no message matching <code>${GP_no_missing}</code>"
429429
comment_details_list=${comment_details_list}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
430430

431431
if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
432-
success_msg="found message matching <code>${GP_tgz_created}</code>"
433-
failure_msg="no message matching <code>${GP_tgz_created}</code>"
434-
comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
432+
success_msg="found message matching <code>${GP_tarball_created}</code>"
433+
failure_msg="no message matching <code>${GP_tarball_created}</code>"
434+
comment_details_list=${comment_details_list}$(add_detail ${TARBALL_CREATED} 1 "${success_msg}" "${failure_msg}")
435435
fi
436436

437437
# Now, do the actual replacement of __DETAILS_FMT__
@@ -478,7 +478,16 @@ if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then
478478
size="$(stat --dereference --printf=%s ${TARBALL})"
479479
size_mib=$((${size} >> 20))
480480
tmpfile=$(mktemp --tmpdir=. tarfiles.XXXX)
481-
tar tf ${TARBALL} > ${tmpfile}
481+
if [[ "${TARBALL}" == *.tar.zst ]]; then
482+
tar --use-compress-program=zstd -tf ${TARBALL} > ${tmpfile}
483+
elif [[ "${TARBALL}" == *.tar.gz ]]; then
484+
tar --use-compress-program=gzip -tf ${TARBALL} > ${tmpfile}
485+
elif [[ "${TARBALL}" == *.tar ]]; then
486+
tar -tf ${TARBALL} > ${tmpfile}
487+
else
488+
echo "ERROR: Unsupported tarball extension!" >&2
489+
exit 1
490+
fi
482491
entries=$(cat ${tmpfile} | wc -l)
483492
# determine prefix from job config: VERSION/software/OS_TYPE/CPU_FAMILY/ARCHITECTURE
484493
# e.g., 2023.06/software/linux/x86_64/intel/skylake_avx512

‎bot/test.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then
100100
fi
101101

102102
# try to determine tmp directory from build job
103-
RESUME_DIR=$(grep 'Using .* as tmp directory' slurm-${SLURM_JOBID}.out | head -1 | awk '{print $2}')
103+
RESUME_DIR=$(grep 'Using .* as tmp directory' slurm-${SLURM_JOBID}.out | head -n 1 | awk '{print $2}')
104104

105105
if [[ -z ${RESUME_DIR} ]]; then
106106
RESUME_TGZ=${PWD}/previous_tmp/build_step/$(ls previous_tmp/build_step)

‎create_tarball.sh‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ eessi_tmpdir=$1
1212
eessi_version=$2
1313
cpu_arch_subdir=$3
1414
accel_subdir=$4
15-
target_tgz=$5
15+
target_tarball=$5
1616

1717
tmpdir=`mktemp -d`
1818
echo ">> tmpdir: $tmpdir"
@@ -114,10 +114,9 @@ fi
114114

115115
topdir=${cvmfs_repo}/versions/
116116

117-
echo ">> Creating tarball ${target_tgz} from ${topdir}..."
118-
tar cfvz ${target_tgz} -C ${topdir} --files-from=${files_list}
119-
120-
echo ${target_tgz} created!
117+
echo ">> Creating tarball ${target_tarball} from ${topdir}..."
118+
tar cavf ${target_tarball} -C ${topdir} --files-from=${files_list}
119+
echo ${target_tarball} created!
121120

122121
echo ">> Cleaning up tmpdir ${tmpdir}..."
123122
rm -r ${tmpdir}

‎init/eessi_archdetect.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -187,7 +187,7 @@ accelpath() {
187187
nvidia_smi_out=$(mktemp -p /tmp nvidia_smi_out.XXXXX)
188188
nvidia-smi --query-gpu=gpu_name,count,driver_version,compute_cap --format=csv,noheader 2>&1 > $nvidia_smi_out
189189
if [[ $? -eq 0 ]]; then
190-
nvidia_smi_info=$(head -1 $nvidia_smi_out)
190+
nvidia_smi_info=$(head -n 1 $nvidia_smi_out)
191191
cuda_cc=$(echo $nvidia_smi_info | sed 's/, /,/g' | cut -f4 -d, | sed 's/\.//g')
192192
log "DEBUG" "accelpath: CUDA compute capability '${cuda_cc}' derived from nvidia-smi output '${nvidia_smi_info}'"
193193
res="accel/nvidia/cc${cuda_cc}"
@@ -225,3 +225,4 @@ case "$ARGUMENT" in
225225
"accelpath") accelpath; exit;;
226226
*) echo "$USAGE"; log "ERROR" "Missing <action> argument (possible actions: 'cpupath', 'accelpath')";;
227227
esac
228+

‎init/eessi_environment_variables‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ if [ -d $EESSI_PREFIX ]; then
5959
fi
6060

6161
if [[ $accelpath_exit_code -eq 0 ]]; then
62-
export EESSI_ACCEL_SUBDIR=$(tail -1 $tmpout && rm -f $tmpout)
62+
export EESSI_ACCEL_SUBDIR=$(tail -n 1 $tmpout && rm -f $tmpout)
6363
if [ -z ${EESSI_ACCEL_SUBDIR} ]; then
6464
error "accelerator detection with archdetect worked, but no result was returned?!"
6565
else
@@ -177,3 +177,4 @@ if [ -d $EESSI_PREFIX ]; then
177177
else
178178
error "EESSI repository at $EESSI_PREFIX not found!"
179179
fi
180+

‎run_tests.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ source ${base_dir}/init/eessi_defaults
2020
# Make sure we clone the latest version. This assumes versions are of the format "v1.2.3", then picks the latest
2121
# then checks it out
2222
TEST_CLONE="git clone https://github.com/EESSI/test-suite EESSI-test-suite && cd EESSI-test-suite"
23-
LATEST_VERSION="VERSION=\$(git tag | grep '^v[0-9]\+\.[0-9]\+\.[0-9]\+$' | sort -t. -k 1,1n -k 2,2n -k 3,3n | tail -1)"
23+
LATEST_VERSION="VERSION=\$(git tag | grep '^v[0-9]\+\.[0-9]\+\.[0-9]\+$' | sort -t. -k 1,1n -k 2,2n -k 3,3n | tail -n 1)"
2424
CHECKOUT_LATEST="git checkout \${VERSION}"
2525

2626
# Git clone has to be run in compat layer, to make the git command available

‎scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -223,7 +223,7 @@ check_nvidia_smi_info() {
223223

224224
if nvidia-smi --query-gpu=gpu_name,count,driver_version,compute_cap --format=csv,noheader > "$nvidia_smi_out" 2>&1
225225
then
226-
nvidia_smi_info=$(head -1 "${nvidia_smi_out}")
226+
nvidia_smi_info=$(head -n 1 "${nvidia_smi_out}")
227227
HOST_GPU_CUDA_VERSION=$(echo "${nvidia_smi_info}" | sed 's/, /,/g' | cut -f4 -d,)
228228
HOST_GPU_DRIVER_VERSION=$(echo "${nvidia_smi_info}" | sed 's/, /,/g' | cut -f3 -d,)
229229
echo_green "Found host CUDA version ${HOST_GPU_CUDA_VERSION}"
@@ -689,3 +689,4 @@ symlink_mode
689689

690690
# If everything went OK, show success message
691691
echo_green "Host NVIDIA GPU drivers linked successfully for EESSI"
692+

0 commit comments

Comments
 (0)