Skip to content

Make matching component sources in Bundle easyblock more reliable #3778

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

Make matching component sources in Bundle easyblock more reliable #3778

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 39 additions & 50 deletions easybuild/easyblocks/generic/bundle.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,10 @@ def __init__(self, *args, **kwargs):
# (like adding component sources to top-level sources easyconfig parameter)
self.cfg = self.cfg.copy()

# Keep track of sources for each component to restore them:
# (Component instance, start idx [in self.src], end idx)
self.comp_to_src_idxs = []

# disable templating to avoid premature resolving of template values
# Note that self.cfg.update also resolves templates!
with self.cfg.disable_templating():
Expand Down Expand Up @@ -197,6 +201,10 @@ def __init__(self, *args, **kwargs):

comp_cfg.generate_template_values()

# Combine all component sources into the top-level sources parameter
# This allows reusing top-level source_urls and unpacking them all in the extract_step

old_num_srcs = len(self.cfg.get('sources', resolve=False))
# Don't require that all template values can be resolved at this point but still resolve them.
# This is important to ensure that template values like %(name)s and %(version)s
# are correctly resolved with the component name/version before values are copied over to self.cfg
Expand Down Expand Up @@ -226,22 +234,15 @@ def __init__(self, *args, **kwargs):
comp_checksums = comp_cfg['checksums']
if comp_checksums:
src_cnt = len(comp_sources)

# add per-component checksums for sources to list of checksums
self.cfg.update('checksums', comp_checksums[:src_cnt])

# add per-component checksums for patches to list of checksums for patches
checksums_patches.extend(comp_checksums[src_cnt:])

with comp_cfg.allow_unresolved_templates():
comp_patches = comp_cfg['patches']
comp_postinstall_patches = comp_cfg['postinstallpatches']
if comp_patches:
self.cfg.update('patches', comp_patches)
# Patch step is skipped so adding postinstall patches of components here is harmless
self.cfg.update('patches', comp_postinstall_patches)

self.comp_instances.append((comp_cfg, comp_cfg.easyblock(comp_cfg, logfile=self.logfile)))
new_num_srcs = len(self.cfg.get('sources', resolve=False))
self.comp_to_src_idxs.append((self.comp_instances[-1], old_num_srcs, new_num_srcs))
# check if sanity checks are enabled for the component
if self.cfg['sanity_check_all_components'] or comp_cfg['name'] in self.cfg['sanity_check_components']:
self.comp_cfgs_sanity_check.append(self.comp_instances[-1])

self.cfg.update('checksums', checksums_patches + orig_checksums)

Expand All @@ -263,6 +264,26 @@ def check_checksums(self):

return checksum_issues

def fetch_step(self):
"""Fetch sources of all extensions"""
super().fetch_step()
# Init src attribute as usually done by fetch_step
for (_, comp), start_idx, end_idx in self.comp_to_src_idxs:
comp.src = self.src[start_idx:end_idx]
# need to run fetch_patches to ensure per-component patches are gathered
comp.fetch_patches()

def prepare_step(self, *args, **kwargs):
"""Init required properties for components"""
super().prepare_step(*args, **kwargs)
for _, comp in self.comp_instances:
# correct build/install dirs
comp.builddir = self.builddir
comp.install_subdir, comp.installdir = self.install_subdir, self.installdir

# make sure we can build in parallel
comp.set_parallel()

def patch_step(self):
"""Patch step must be a no-op for bundle, since there are no top-level sources/patches."""
pass
Expand Down Expand Up @@ -319,47 +340,15 @@ def install_step(self):
(comp.name, comp.version, idx + 1, comp_cnt))
self.log.info("Installing component %s v%s using easyblock %s", comp.name, comp.version, cfg.easyblock)

# correct build/install dirs
comp.builddir = self.builddir
comp.install_subdir, comp.installdir = self.install_subdir, self.installdir

# make sure we can build in parallel
comp.set_parallel()

# figure out correct start directory
comp.guess_start_dir()

# need to run fetch_patches to ensure per-component patches are applied
comp.fetch_patches()

# Compatibility with ECs expecting the previous behavior where src wasn't populated at this point
tmp_src = comp.src
comp.src = []
comp.guess_start_dir()
comp.src = tmp_src

# find matching entries in self.src for this component
with comp.cfg.allow_unresolved_templates():
comp_sources = comp.cfg['sources']
for source in comp_sources:
if isinstance(source, str):
comp_src_fn = source
elif isinstance(source, dict):
if 'filename' in source:
comp_src_fn = source['filename']
else:
raise EasyBuildError("Encountered source file specified as dict without 'filename': %s", source)
else:
raise EasyBuildError("Specification of unknown type for source file: %s", source)

found = False
for src in self.src:
if src['name'] == comp_src_fn:
self.log.info("Found spec for source %s for component %s: %s", comp_src_fn, comp.name, src)
comp.src.append(src)
found = True
break
if not found:
raise EasyBuildError("Failed to find spec for source %s for component %s", comp_src_fn, comp.name)

# location of first unpacked source is used to determine where to apply patch(es)
comp.src[-1]['finalpath'] = comp.cfg['start_dir']
# location of first unpacked source is used to determine where to apply patch(es)
comp.src[0]['finalpath'] = comp.cfg['start_dir']

# check if sanity checks are enabled for the component
if self.cfg['sanity_check_all_components'] or comp.name in self.cfg['sanity_check_components']:
Expand Down