From a29bca1cd1f0a0ae3a1a6c3037920ff29d80a75e Mon Sep 17 00:00:00 2001 From: crivella Date: Wed, 28 Oct 2026 17:29:54 +0100 Subject: [PATCH 1/3] Add hook to force `pip` to retry multiple times when installing a `CargoPythonPackage` on 2025.06 --- eb_hooks.py | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index bfe34762..ff005679 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -2086,11 +2086,92 @@ def pre_test_hook_Siesta_ignore_failure_with_crosscompilation(self, *args, **kwa self.cfg['testopts'] = re.sub(r"-E '(.*)'", rf"-E '\1|{extra_testopts}'", testopts) + +def pre_cargo_python_extension_2025(self, *args, **kwargs): + """Ensure that Cargo builds in python packages use the correct cpu-target for the current EESSI CPU family. + Also mitigate the issue described in https://gitlab.com/eessi/support/-/work_items/270 where random + SIGSEGV 11 errors are seen in cargo builds.""" + eessi_version = get_eessi_envvar('EESSI_VERSION') + if eessi_version != '2025.06': + return + print_msg(f"Running pre_python_cargo_2025 for {self.name} {self.version}") + from easybuild.easyblocks.generic.cargopythonpackage import CargoPythonPackage + CPU_TARGET_MAP = { + # CPU_TARGET_A64FX: 'a64fx', + CPU_TARGET_HASWELL: 'haswell', + } + # print_msg(f"Checking {self.name} for Cargo instances to set cpu-target") + if isinstance(self, CargoPythonPackage): + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + env_vars = {} + def set_env_var(var_name, var_value): + env.setvar(var_name, var_value) + env_vars[var_name] = var_value + if cpu_target in CPU_TARGET_MAP: + print_msg( + f"Setting RUSTFLAGS for Cargo build of {self.name} {self.version} to use cpu-target {cpu_target}" + ) + rust_target = CPU_TARGET_MAP[cpu_target] + old_rustflags = os.getenv('RUSTFLAGS', '') + rustflags = old_rustflags + if '-C target-cpu=' not in rustflags: + rustflags += f' -C target-cpu={rust_target}' + else: + rustflags = re.sub(r'-C target-cpu=[^ "]+', f'-C target-cpu={rust_target}', rustflags) + set_env_var('RUSTFLAGS', rustflags) + + self.cfg.update('installopts', '-Cbuild-args=--verbose') + + + # Avoid SIGSEGV 11 errors on Haswell + EESSI/2025.06 seen randomly with Cargo builds ran trhough maturin + # for python packages: + # - https://github.com/EESSI/software-layer/pull/1543#issuecomment-5328338503 + # - https://github.com/EESSI/software-layer/pull/1871#issuecomment-6441762518 + # Mitigate this by: + # - increase the available stack size for Rust to 160MB (RUST_MIN_STACK=167772160) + # - disable LTO (linker-plugin-lto=no) to reduce memory usage + # - Force optimization level to 2 (-C opt-level=2) usually set to 3 by maturin/pip + # if cpu_target in [ + # CPU_TARGET_HASWELL + # ]: + # print_msg(f"Applying mitigations to avoid SIGSEGV 11 errors in cargo builds...") + # rustflags = os.getenv('RUSTFLAGS', '') + # rustflags += ' -C linker-plugin-lto=no' + # rustflags += ' -C opt-level=0' + + # set_env_var('RUST_MIN_STACK', '167772160') + # set_env_var('RUSTFLAGS', rustflags) + # set_env_var('CARGO_BUILD_JOBS', '1') + + # self.cfg.update('installopts', '-Csetup-args=-Doptimization=0') + # self.cfg.update('installopts', '-Csetup-args=-Db_lto=false') + + if env_vars: + new_opts = [f"export {k}='{v}'" for k, v in env_vars.items()] + [''] + new_opts_str = ' && '.join(new_opts) + self.cfg['preinstallopts'] = new_opts_str + self.cfg.get('preinstallopts', '') + + # On haswell with the 2025.06 EESSI release we have observed random SIGSEGV 11 errors in cargo builds + # here we add a preinstallopt to retry the build 10 times before failing, to mitigate this issue + # Other attempts like increasing the `RUST_MIN_STACK`, disabling LTO, or reducing the optimization level did + # not fix the issue + if cpu_target == CPU_TARGET_HASWELL: + self.cfg.update( + 'preinstallopts', + 'retry() { ' + 'for i in $(seq 1 10); do "$@" && return 0 || echo "Retry $i failed, retrying..."; done; ' + 'echo "All retries failed!"; return 1; ' + '}; retry ' + ) + + def pre_single_extension_hook(ext, *args, **kwargs): """Main pre-extension: trigger custom functions based on software name.""" if ext.name in PRE_SINGLE_EXTENSION_HOOKS: PRE_SINGLE_EXTENSION_HOOKS[ext.name](ext, *args, **kwargs) + pre_cargo_python_extension_2025(ext, *args, **kwargs) + def post_single_extension_hook(ext, *args, **kwargs): """Main post-extension hook: trigger custom functions based on software name.""" From 23b7701e629ccf1a6f0ad7ea1e4b25c1d9a5e684 Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 29 Oct 2026 15:00:09 +0100 Subject: [PATCH 2/3] Also run hook for non-extension `CargoPythonPackage`s --- eb_hooks.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index ff005679..1e02a6cb 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -1291,6 +1291,9 @@ def pre_configure_hook(self, *args, **kwargs): if self.name in PRE_CONFIGURE_HOOKS: PRE_CONFIGURE_HOOKS[self.name](self, *args, **kwargs) + # Run this both for extensions and pure CargoPythonPackages installs + pre_cargo_python_package_2025(self, *args, **kwargs) + # workaround for a Zlib macro being renamed in Gentoo, see https://bugs.gentoo.org/383179 # (solves "expected initializer before 'OF'" errors) if self.name in ['FreeXL', 'libspatialite', 'VSEARCH']: @@ -2087,14 +2090,14 @@ def pre_test_hook_Siesta_ignore_failure_with_crosscompilation(self, *args, **kwa -def pre_cargo_python_extension_2025(self, *args, **kwargs): +def pre_cargo_python_package_2025(self, *args, **kwargs): """Ensure that Cargo builds in python packages use the correct cpu-target for the current EESSI CPU family. Also mitigate the issue described in https://gitlab.com/eessi/support/-/work_items/270 where random SIGSEGV 11 errors are seen in cargo builds.""" eessi_version = get_eessi_envvar('EESSI_VERSION') if eessi_version != '2025.06': return - print_msg(f"Running pre_python_cargo_2025 for {self.name} {self.version}") + print_msg(f"Running pre_cargo_python_package_2025 for {self.name} {self.version}") from easybuild.easyblocks.generic.cargopythonpackage import CargoPythonPackage CPU_TARGET_MAP = { # CPU_TARGET_A64FX: 'a64fx', @@ -2170,7 +2173,7 @@ def pre_single_extension_hook(ext, *args, **kwargs): if ext.name in PRE_SINGLE_EXTENSION_HOOKS: PRE_SINGLE_EXTENSION_HOOKS[ext.name](ext, *args, **kwargs) - pre_cargo_python_extension_2025(ext, *args, **kwargs) + pre_cargo_python_package_2025(ext, *args, **kwargs) def post_single_extension_hook(ext, *args, **kwargs): From e099299b6992a3bc2f285c96bdb248c834c4f91f Mon Sep 17 00:00:00 2001 From: crivella Date: Thu, 29 Oct 2026 16:32:01 +0100 Subject: [PATCH 3/3] Cleanup comments --- eb_hooks.py | 38 ++++++++------------------------------ 1 file changed, 8 insertions(+), 30 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 1e02a6cb..5006f285 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -2097,24 +2097,22 @@ def pre_cargo_python_package_2025(self, *args, **kwargs): eessi_version = get_eessi_envvar('EESSI_VERSION') if eessi_version != '2025.06': return - print_msg(f"Running pre_cargo_python_package_2025 for {self.name} {self.version}") from easybuild.easyblocks.generic.cargopythonpackage import CargoPythonPackage - CPU_TARGET_MAP = { - # CPU_TARGET_A64FX: 'a64fx', + rust_cpu_target_map = { CPU_TARGET_HASWELL: 'haswell', } - # print_msg(f"Checking {self.name} for Cargo instances to set cpu-target") if isinstance(self, CargoPythonPackage): + print_msg(f"Running pre_cargo_python_package_2025 for {self.name} {self.version}") cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') env_vars = {} def set_env_var(var_name, var_value): env.setvar(var_name, var_value) env_vars[var_name] = var_value - if cpu_target in CPU_TARGET_MAP: + if cpu_target in rust_cpu_target_map: print_msg( f"Setting RUSTFLAGS for Cargo build of {self.name} {self.version} to use cpu-target {cpu_target}" ) - rust_target = CPU_TARGET_MAP[cpu_target] + rust_target = rust_cpu_target_map[cpu_target] old_rustflags = os.getenv('RUSTFLAGS', '') rustflags = old_rustflags if '-C target-cpu=' not in rustflags: @@ -2125,30 +2123,6 @@ def set_env_var(var_name, var_value): self.cfg.update('installopts', '-Cbuild-args=--verbose') - - # Avoid SIGSEGV 11 errors on Haswell + EESSI/2025.06 seen randomly with Cargo builds ran trhough maturin - # for python packages: - # - https://github.com/EESSI/software-layer/pull/1543#issuecomment-5328338503 - # - https://github.com/EESSI/software-layer/pull/1871#issuecomment-6441762518 - # Mitigate this by: - # - increase the available stack size for Rust to 160MB (RUST_MIN_STACK=167772160) - # - disable LTO (linker-plugin-lto=no) to reduce memory usage - # - Force optimization level to 2 (-C opt-level=2) usually set to 3 by maturin/pip - # if cpu_target in [ - # CPU_TARGET_HASWELL - # ]: - # print_msg(f"Applying mitigations to avoid SIGSEGV 11 errors in cargo builds...") - # rustflags = os.getenv('RUSTFLAGS', '') - # rustflags += ' -C linker-plugin-lto=no' - # rustflags += ' -C opt-level=0' - - # set_env_var('RUST_MIN_STACK', '167772160') - # set_env_var('RUSTFLAGS', rustflags) - # set_env_var('CARGO_BUILD_JOBS', '1') - - # self.cfg.update('installopts', '-Csetup-args=-Doptimization=0') - # self.cfg.update('installopts', '-Csetup-args=-Db_lto=false') - if env_vars: new_opts = [f"export {k}='{v}'" for k, v in env_vars.items()] + [''] new_opts_str = ' && '.join(new_opts) @@ -2159,6 +2133,10 @@ def set_env_var(var_name, var_value): # Other attempts like increasing the `RUST_MIN_STACK`, disabling LTO, or reducing the optimization level did # not fix the issue if cpu_target == CPU_TARGET_HASWELL: + print_msg( + f"Adding retry() function to preinstallopts for Cargo build of {self.name} {self.version} " + f"on {cpu_target} to mitigate random SIGSEGV 11 errors" + ) self.cfg.update( 'preinstallopts', 'retry() { '