Skip to content

Commit 519a654

Browse files
MartinsNadiaMartinsNadia
committed
deleting licenses
adding PR handling, build and comparison of modules and envs
1 parent 062b624 commit 519a654

File tree

1 file changed

+203
-0
lines changed

1 file changed

+203
-0
lines changed

‎modules/module_check.sh‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
#!/bin/bash
2+
3+
# This script checks the consistency of EB-generated modules and identifies broken or missing modules.
4+
# Usage: ./module_check.sh <path to easystack file> [<optional path to PR diff>]
5+
6+
# It uses an adapted approach from check_missing_installations.sh to handling PRs/unmerged PRs
7+
TOPDIR=$(dirname $(realpath $0))
8+
9+
if [ "$#" -eq 1 ]; then
10+
echo "No PR diff provided. Processing all modules in the easystack file."
11+
pr_exceptions=""
12+
elif [ "$#" -eq 2 ]; then
13+
echo "Using $2 to create exceptions for PR filtering of easystack"
14+
pr_diff="$2"
15+
pr_exceptions=$(grep '^+' "$pr_diff" | grep 'from-pr' | uniq | awk '{print $3}' | xargs -I {} echo " || /'{}'/")
16+
else
17+
echo "ERROR: Usage: $0 <path to easystack file> [<optional path to PR diff>]" >&2
18+
exit 1
19+
fi
20+
21+
easystack="$1"
22+
23+
LOCAL_TMPDIR=$(mktemp -d)
24+
mkdir -p "$LOCAL_TMPDIR"
25+
26+
# Clone the develop branch of EasyBuild and use that to search for easyconfigs
27+
git clone -b develop https://github.com/easybuilders/easybuild-easyconfigs.git $LOCAL_TMPDIR/easyconfigs
28+
export EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs
29+
30+
# All PRs used in EESSI are supposed to be merged, so we can strip ou all cases of from-pr
31+
tmp_easystack="${LOCAL_TMPDIR}/$(basename "${easystack}")"
32+
grep -v 'from-pr' "${easystack}" > "${tmp_easystack}"
33+
34+
# If PR exceptions exist, modify the easystack file to include exceptions
35+
if [ -n "$pr_exceptions" ]; then
36+
# Use awk to exclude lines containing PR numbers specified in pr_exceptions
37+
awk_command="awk '!/from-pr/ EXCEPTIONS' ${easystack}"
38+
awk_command=${awk_command/\\/}
39+
eval "${awk_command/EXCEPTIONS/$pr_exceptions}" > "${tmp_easystack}"
40+
fi
41+
42+
# Set up temporary directories for module installation and lock files
43+
TMPDIR=${TMPDIR:-/tmp}/$USER
44+
module_install_dir="$TMPDIR/EESSI/module-only"
45+
locks_dir="$TMPDIR/EESSI/locks"
46+
mkdir -p "$module_install_dir" "$locks_dir"
47+
48+
# Log file to record broken modules
49+
broken_modules_log="broken_modules.log"
50+
> "$broken_modules_log"
51+
52+
# To keep track of already-checked modules and avoid re-checking
53+
declare -A checked_modules
54+
55+
# Identify missing easyconfigs based on the temporary easystack file
56+
echo "Identifying missing easyconfigs using the temporary easystack file..."
57+
missing_easyconfigs=$(eb --easystack "${tmp_easystack}" --missing --robot 2>&1)
58+
59+
if [ -z "$missing_easyconfigs" ]; then
60+
echo "No missing easyconfigs to install."
61+
rm -rf "$LOCAL_TMPDIR"
62+
exit 0
63+
fi
64+
65+
# Process each missing easyconfig file
66+
for easyconfig_file in $missing_easyconfigs; do
67+
package_name=$(basename "$easyconfig_file" .eb)
68+
69+
# Building of the easyconfig
70+
echo "Building $package_name using EasyBuild..."
71+
eb "$easyconfig_file" --robot
72+
if [ $? -ne 0 ]; then
73+
echo "EasyBuild build failed for $package_name. Skipping..."
74+
echo "$package_name: EasyBuild build failed" >> "$broken_modules_log"
75+
continue
76+
fi
77+
78+
# Generate the module using --module-only
79+
echo "Generating module for $package_name using --module-only..."
80+
eb "$easyconfig_file" --module-only --installpath-modules "$module_install_dir" --locks-dir "$locks_dir" --force --robot
81+
if [ $? -ne 0 ]; then
82+
echo "EasyBuild --module-only command failed for $package_name. Skipping..."
83+
echo "$package_name: EasyBuild --module-only command failed" >> "$broken_modules_log"
84+
continue
85+
fi
86+
87+
# Find the module file generated from the build
88+
module_relpath=$(eb "$easyconfig_file" --show-module --robot 2>/dev/null)
89+
if [ -z "$module_relpath" ]; then
90+
echo "Failed to get module relative path for $package_name"
91+
echo "$package_name: Failed to get module relative path" >> "$broken_modules_log"
92+
continue
93+
fi
94+
95+
# Modules names and version
96+
module_software=$(echo "$module_relpath" | sed 's/\.lua$//')
97+
98+
# Check if the module has already been validated to avoid redundant checks
99+
if [ -n "${checked_modules[$module_software]}" ]; then
100+
echo "Module $module_software already checked. Skipping."
101+
continue
102+
fi
103+
104+
# Paths to the module files generated from build and the --module-only
105+
module_file_build="${EASYBUILD_INSTALLPATH}/modules/all/${module_relpath}"
106+
module_file_module_only="${module_install_dir}/all/${module_relpath}"
107+
108+
# Check if both module files exist
109+
if [ ! -f "$module_file_build" ]; then
110+
echo "Module file from full build not found: $module_file_build"
111+
echo "$package_name: Module file from full build not found" >> "$broken_modules_log"
112+
continue
113+
fi
114+
115+
if [ ! -f "$module_file_module_only" ]; then
116+
echo "Module file from --module-only build not found: $module_file_module_only"
117+
echo "$package_name: Module file from --module-only build not found" >> "$broken_modules_log"
118+
continue
119+
fi
120+
121+
# Compare the module files
122+
if diff -q "$module_file_build" "$module_file_module_only" >/dev/null; then
123+
echo "Module files for $package_name match"
124+
else
125+
echo "Module files for $package_name differ"
126+
echo "$package_name: Module files differ" >> "$broken_modules_log"
127+
# Save differences
128+
diff_file="${module_software//\//_}_module_diff.txt"
129+
diff "$module_file_build" "$module_file_module_only" > "$diff_file"
130+
echo "Module file differences saved to $diff_file"
131+
fi
132+
133+
# Proceed to compare the environments
134+
echo "Testing module: $module_software"
135+
136+
# Function to get filtered environment variables, excluding lmod-related vars
137+
get_filtered_env() {
138+
env | grep -v -E '^(LMOD_|MODULEPATH|MODULESHOME|LOADEDMODULES|BASH_FUNC_module|_ModuleTable_|PWD=|SHLVL=|OLDPWD=|PS1=|PS2=|_LMFILES_)=.*$' | sort
139+
}
140+
141+
# Compare the environments of the modules
142+
module purge
143+
module unuse "$module_install_dir"
144+
module load EasyBuild
145+
146+
# Load the module from the full build
147+
if module --ignore_cache load "$module_software" 2>/dev/null; then
148+
original_env=$(get_filtered_env)
149+
module unload "$module_software"
150+
else
151+
echo "Failed to load module from full build: $module_software."
152+
original_env=""
153+
fi
154+
155+
# Load the module from the --module-only
156+
module purge
157+
module use "$module_install_dir"
158+
159+
if module --ignore_cache load "$module_software" 2>/dev/null; then
160+
new_env=$(get_filtered_env)
161+
module unload "$module_software"
162+
else
163+
echo "Failed to load module from --module-only build: $module_software."
164+
echo "$package_name: Failed to load module from --module-only build" >> "$broken_modules_log"
165+
module unuse "$module_install_dir"
166+
continue
167+
fi
168+
169+
# Compare the environments
170+
if [ -n "$original_env" ]; then
171+
if diff <(echo "$original_env") <(echo "$new_env") >/dev/null; then
172+
echo "$module_software loaded with identical environment."
173+
else
174+
echo "$module_software environment mismatch."
175+
echo "$package_name: $module_software (environment mismatch)" >> "$broken_modules_log"
176+
diff_file="${module_software//\//_}_env_diff.txt"
177+
diff <(echo "$original_env") <(echo "$new_env") > "$diff_file"
178+
echo "Environment differences saved to $diff_file"
179+
fi
180+
else
181+
echo "Original environment not available for comparison for $module_software."
182+
echo "$package_name: $module_software (failed to load module from full build)" >> "$broken_modules_log"
183+
fi
184+
185+
186+
module unuse "$module_install_dir"
187+
188+
# Mark module as checked
189+
checked_modules[$module_software]=1
190+
191+
done
192+
193+
# Report
194+
if [ -f "$broken_modules_log" ] && [ -s "$broken_modules_log" ]; then
195+
echo "Some modules did not match. See $broken_modules_log for details."
196+
exit 1
197+
else
198+
echo "All modules match between build and --module-only build."
199+
fi
200+
201+
# Clean up temporary directories
202+
rm -rf "$LOCAL_TMPDIR"
203+

0 commit comments

Comments
 (0)