Merge pull request #3456 from easybuilders/4.9.x

release EasyBuild v4.9.4

Author: boegel

RELEASE_NOTES

@@ -6,10 +6,25 @@ These release notes can also be consulted at http://easybuild.readthedocs.org/en
 The latest version of easybuild-easyblocks provides 259 software-specific easyblocks and 43 generic easyblocks.
 
 
+v4.9.4 (22 september 2024)
+--------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - allow Python bundles and packages to specify a maximum Python version for the system toolchain (#3431)
+  - copy EasyConfig instance in constructor of Bundle and Cargo easyblocks before making changes to it (#3448)
+  - fix crash in GCC easyblock when cuda-compute-capabilities EasyBuild configuration option is not set (#3449)
+- various bug fixes, including:
+  - ignore Python from virtualenvs in GROMACS configure via -DPython3_FIND_VIRTUALENV=STANDARD (#3283)
+  - enhance custom easyblock for NCCL: add licence to NCCL installation (#3451)
+
+
 v4.9.3 (14 September 2024)
 --------------------------
 
 update/bugfix release
+
 - minor updates, including:
   - update custom easyblock for Tensorflow for versions 2.14 + 2.15 (#3303)
   - add support for versions >= 2024a to MCR easyblock (#3369)

easybuild/easyblocks/__init__.py

@@ -43,7 +43,7 @@
 # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like
 #   UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0'
 # This causes problems further up the dependency chain...
-VERSION = LooseVersion('4.9.3')
+VERSION = LooseVersion('4.9.4')
 UNKNOWN = 'UNKNOWN'
 
 

easybuild/easyblocks/g/gcc.py

@@ -383,7 +383,7 @@ def map_nvptx_capability(self):
         architecture_mappings_replacement = "misa=,"
 
         # Determine which compute capabilities are configured. If there are none, return immediately.
-        if cuda_cc_list is None:
+        if not cuda_cc_list:
             return None
         cuda_sm_list = [f"sm_{cc.replace('.', '')}" for cc in cuda_cc_list]
 

easybuild/easyblocks/g/gromacs.py

@@ -167,7 +167,9 @@ def prepare_step(self, *args, **kwargs):
     def configure_step(self):
         """Custom configuration procedure for GROMACS: set configure options for configure or cmake."""
 
-        if LooseVersion(self.version) >= LooseVersion('4.6'):
+        gromacs_version = LooseVersion(self.version)
+
+        if gromacs_version >= '4.6':
             cuda = get_software_root('CUDA')
             if cuda:
                 # CUDA with double precision is currently not supported in GROMACS yet
@@ -188,10 +190,11 @@ def configure_step(self):
                     self.log.info("skipping configure step")
                     return
 
-                if LooseVersion(self.version) >= LooseVersion('2021'):
-                    self.cfg.update('configopts', "-DGMX_GPU=CUDA -DCUDA_TOOLKIT_ROOT_DIR=%s" % cuda)
+                if gromacs_version >= '2021':
+                    self.cfg.update('configopts', "-DGMX_GPU=CUDA")
                 else:
-                    self.cfg.update('configopts', "-DGMX_GPU=ON -DCUDA_TOOLKIT_ROOT_DIR=%s" % cuda)
+                    self.cfg.update('configopts', "-DGMX_GPU=ON")
+                self.cfg.update('configopts', "-DCUDA_TOOLKIT_ROOT_DIR=%s" % cuda)
 
                 # Set CUDA capabilities based on template value.
                 if '-DGMX_CUDA_TARGET_SM' not in self.cfg['configopts']:
@@ -236,14 +239,14 @@ def configure_step(self):
         # Ensure that the GROMACS log files report how the code was patched
         # during the build, so that any problems are easier to diagnose.
         # The GMX_VERSION_STRING_OF_FORK feature is available since 2020.
-        if (LooseVersion(self.version) >= LooseVersion('2020') and
+        if (gromacs_version >= '2020' and
                 '-DGMX_VERSION_STRING_OF_FORK=' not in self.cfg['configopts']):
             gromacs_version_string_suffix = 'EasyBuild-%s' % EASYBUILD_VERSION
             if plumed_root:
                 gromacs_version_string_suffix += '-PLUMED-%s' % get_software_version('PLUMED')
             self.cfg.update('configopts', '-DGMX_VERSION_STRING_OF_FORK=%s' % gromacs_version_string_suffix)
 
-        if LooseVersion(self.version) < LooseVersion('4.6'):
+        if gromacs_version < '4.6':
             self.log.info("Using configure script for configuring GROMACS build.")
 
             if self.cfg['build_shared_libs']:
@@ -259,7 +262,7 @@ def configure_step(self):
             self.cfg.update('configopts', "--without-x")
 
             # OpenMP is not supported for versions older than 4.5.
-            if LooseVersion(self.version) >= LooseVersion('4.5'):
+            if gromacs_version >= '4.5':
                 # enable OpenMP support if desired
                 if self.toolchain.options.get('openmp', None):
                     self.cfg.update('configopts', "--enable-threads")
@@ -310,22 +313,26 @@ def configure_step(self):
                               mpiexec_path, self.cfg.get('mpiexec_numproc_flag'),
                               mpi_numprocs)
 
-            if LooseVersion(self.version) >= LooseVersion('2019'):
+            if gromacs_version >= '2019':
                 # Building the gmxapi interface requires shared libraries,
                 # this is handled in the class initialisation so --module-only works
                 self.cfg.update('configopts', "-DGMXAPI=ON")
 
-                if LooseVersion(self.version) >= LooseVersion('2020'):
+                if gromacs_version >= '2020':
                     # build Python bindings if Python is loaded as a dependency
                     python_root = get_software_root('Python')
                     if python_root:
+                        self.cfg.update('configopts', "-DGMX_PYTHON_PACKAGE=ON")
                         bin_python = os.path.join(python_root, 'bin', 'python')
+                        # For find_package(PythonInterp)
                         self.cfg.update('configopts', "-DPYTHON_EXECUTABLE=%s" % bin_python)
-                        self.cfg.update('configopts', "-DGMX_PYTHON_PACKAGE=ON")
+                        if gromacs_version >= '2021':
+                            # For find_package(Python3) - Ignore virtual envs
+                            self.cfg.update('configopts', "-DPython3_FIND_VIRTUALENV=STANDARD")
 
             # Now patch GROMACS for PLUMED before cmake
             if plumed_root:
-                if LooseVersion(self.version) >= LooseVersion('5.1'):
+                if gromacs_version >= '5.1':
                     # Use shared or static patch depending on
                     # setting of self.cfg['build_shared_libs']
                     # and adapt cmake flags accordingly as per instructions
@@ -355,7 +362,7 @@ def configure_step(self):
             if self.toolchain.toolchain_family() != toolchain.CRAYPE:
                 gmx_simd = self.get_gromacs_arch()
                 if gmx_simd:
-                    if LooseVersion(self.version) < LooseVersion('5.0'):
+                    if gromacs_version < '5.0':
                         self.cfg.update('configopts', "-DGMX_CPU_ACCELERATION=%s" % gmx_simd)
                     else:
                         self.cfg.update('configopts', "-DGMX_SIMD=%s" % gmx_simd)
@@ -404,7 +411,7 @@ def configure_step(self):
                             env.setvar('LDFLAGS', "%s -lgfortran -lm" % os.environ.get('LDFLAGS', ''))
 
             # no more GSL support in GROMACS 5.x, see http://redmine.gromacs.org/issues/1472
-            if LooseVersion(self.version) < LooseVersion('5.0'):
+            if gromacs_version < '5.0':
                 # enable GSL when it's provided
                 if get_software_root('GSL'):
                     self.cfg.update('configopts', "-DGMX_GSL=ON")
@@ -424,7 +431,7 @@ def configure_step(self):
             out = super(EB_GROMACS, self).configure_step()
 
             # for recent GROMACS versions, make very sure that a decent BLAS, LAPACK and FFT is found and used
-            if LooseVersion(self.version) >= LooseVersion('4.6.5'):
+            if gromacs_version >= '4.6.5':
                 patterns = [
                     r"Using external FFT library - \S*",
                     r"Looking for dgemm_ - found",

easybuild/easyblocks/generic/bundle.py

@@ -85,6 +85,10 @@ def __init__(self, *args, **kwargs):
             if self.cfg['patches']:
                 raise EasyBuildError("List of patches for bundle itself must be empty, found %s", self.cfg['patches'])
 
+        # copy EasyConfig instance before we make changes to it
+        # (like adding component sources to top-level sources easyconfig parameter)
+        self.cfg = self.cfg.copy()
+
         # disable templating to avoid premature resolving of template values
         self.cfg.enable_templating = False
 

easybuild/easyblocks/generic/cargo.py

@@ -151,6 +151,9 @@ def __init__(self, *args, **kwargs):
                     'filename': self.crate_src_filename(crate, version),
                 })
 
+        # copy EasyConfig instance before we make changes to it
+        self.cfg = self.cfg.copy()
+
         self.cfg.update('sources', sources)
 
     @property

easybuild/easyblocks/generic/pythonbundle.py

@@ -107,15 +107,21 @@ def prepare_step(self, *args, **kwargs):
             if req_py_minver is None:
                 req_py_minver = sys.version_info[1]
 
-            python_cmd = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver)
+            # Get the max_py_majver and max_py_minver from the config
+            max_py_majver = self.cfg['max_py_majver']
+            max_py_minver = self.cfg['max_py_minver']
+
+            python_cmd = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver,
+                                         max_py_majver=max_py_majver, max_py_minver=max_py_minver)
 
             # If pick_python_cmd didn't find a (system) Python command, we should raise an error
             if python_cmd:
                 self.log.info("Python command being used: %s", python_cmd)
             else:
                 raise EasyBuildError(
                     "Failed to pick Python command that satisfies requirements in the easyconfig "
-                    "(req_py_majver = %s, req_py_minver = %s)", req_py_majver, req_py_minver
+                    "(req_py_majver = %s, req_py_minver = %s, max_py_majver = %s, max_py_minver = %s)",
+                    req_py_majver, req_py_minver, max_py_majver, max_py_minver
                 )
 
         self.all_pylibdirs = get_pylibdirs(python_cmd=python_cmd)

easybuild/easyblocks/generic/pythonpackage.py

@@ -83,7 +83,7 @@ def det_python_version(python_cmd):
     return out.strip()
 
 
-def pick_python_cmd(req_maj_ver=None, req_min_ver=None):
+def pick_python_cmd(req_maj_ver=None, req_min_ver=None, max_py_majver=None, max_py_minver=None):
     """
     Pick 'python' command to use, based on specified version requirements.
     If the major version is specified, it must be an exact match (==).
@@ -130,6 +130,20 @@ def check_python_cmd(python_cmd):
                 log.debug("Minimal requirement for minor Python version not satisfied: %s vs %s", pyver, req_majmin_ver)
                 return False
 
+        if max_py_majver is not None:
+            if max_py_minver is None:
+                max_majmin_ver = '%s.0' % max_py_majver
+            else:
+                max_majmin_ver = '%s.%s' % (max_py_majver, max_py_minver)
+
+            pyver = det_python_version(python_cmd)
+
+            if LooseVersion(pyver) > LooseVersion(max_majmin_ver):
+                log.debug("Python version (%s) on the system is newer than the maximum supported "
+                          "Python version specified in the easyconfig (%s)",
+                          pyver, max_majmin_ver)
+                return False
+
         # all check passed
         log.debug("All check passed for Python command '%s'!", python_cmd)
         return True
@@ -348,6 +362,8 @@ def extra_options(extra_vars=None):
                                   "Enabled by default if the EB option --debug is used.", CUSTOM],
             'req_py_majver': [None, "Required major Python version (only relevant when using system Python)", CUSTOM],
             'req_py_minver': [None, "Required minor Python version (only relevant when using system Python)", CUSTOM],
+            'max_py_majver': [None, "Maximum major Python version (only relevant when using system Python)", CUSTOM],
+            'max_py_minver': [None, "Maximum minor Python version (only relevant when using system Python)", CUSTOM],
             'sanity_pip_check': [False, "Run 'python -m pip check' to ensure all required Python packages are "
                                         "installed and check for any package with an invalid (0.0.0) version.", CUSTOM],
             'runtest': [True, "Run unit tests.", CUSTOM],  # overrides default
@@ -508,18 +524,25 @@ def prepare_python(self):
             if req_py_minver is None:
                 req_py_minver = sys.version_info[1]
 
+            # Get the max_py_majver and max_py_minver from the config
+            max_py_majver = self.cfg['max_py_majver']
+            max_py_minver = self.cfg['max_py_minver']
+
             # if using system Python, go hunting for a 'python' command that satisfies the requirements
-            python = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver)
+            python = pick_python_cmd(req_maj_ver=req_py_majver, req_min_ver=req_py_minver,
+                                     max_py_majver=max_py_majver, max_py_minver=max_py_minver)
 
             # Check if we have Python by now. If not, and if self.require_python, raise a sensible error
             if python:
                 self.python_cmd = python
                 self.log.info("Python command being used: %s", self.python_cmd)
             elif self.require_python:
-                if req_py_majver is not None or req_py_minver is not None:
+                if (req_py_majver is not None or req_py_minver is not None
+                        or max_py_majver is not None or max_py_minver is not None):
                     raise EasyBuildError(
                         "Failed to pick Python command that satisfies requirements in the easyconfig "
-                        "(req_py_majver = %s, req_py_minver = %s)", req_py_majver, req_py_minver
+                        "(req_py_majver = %s, req_py_minver = %s, max_py_majver = %s, max_py_minver = %s)",
+                        req_py_majver, req_py_minver, max_py_majver, max_py_minver
                     )
                 else:
                     raise EasyBuildError("Failed to pick Python command to use")

easybuild/easyblocks/n/nccl.py

@@ -26,10 +26,14 @@
 EasyBuild support for building NCCL, implemented as an easyblock
 
 @author: Simon Branford (University of Birmingham)
+@author: Lara Peeters (Gent University)
 """
+import os
+
 from easybuild.easyblocks.generic.configuremake import ConfigureMake
 from easybuild.tools.config import build_option
 from easybuild.tools.systemtools import get_shared_lib_ext
+from easybuild.tools.filetools import copy_file
 
 
 class EB_NCCL(ConfigureMake):
@@ -67,6 +71,8 @@ def install_step(self):
         """Install NCCL"""
         self.cfg.update('installopts', "PREFIX=%s" % self.installdir)
 
+        copy_file(os.path.join(self.cfg['start_dir'], 'LICENSE.txt'), os.path.join(self.installdir, 'LICENSE.txt'))
+
         super(EB_NCCL, self).install_step()
 
     def sanity_check_step(self):

test/easyblocks/module.py

@@ -231,6 +231,8 @@ def test_pythonpackage_pick_python_cmd(self):
         self.assertTrue(pick_python_cmd(2) is not None)
         self.assertTrue(pick_python_cmd(2, 6) is not None)
         self.assertTrue(pick_python_cmd(123, 456) is None)
+        self.assertTrue(pick_python_cmd(2, 6, 123, 456) is not None)
+        self.assertTrue(pick_python_cmd(2, 6, 1, 1) is None)
 
 
 def template_module_only_test(self, easyblock, name, version='1.3.2', extra_txt='', tmpdir=None):