Change build_enstools.sh to build_python_environment.sh

conf/jobs.yaml

@@ -61,8 +61,9 @@ JOBS:
     WALLCLOCK: 04:00
     PROCESSORS: 16
 
-  BUILD_ENSTOOLS:
-    FILE: templates/build_enstools.sh
+  BUILD_PYTHON_ENVIRONMENT:
+    FILE: templates/build_python_environment.sh
+    # Right now we rely on spack for building icon and having a python interpreter, so we need this dependency:
     DEPENDENCIES: BUILD_ICON
     WALLCLOCK: 01:00
     PROCESSORS: 16
@@ -90,7 +91,7 @@ JOBS:
 
   COMPRESS:
     FILE: templates/compress.py
-    DEPENDENCIES: RUN_ICON BUILD_ENSTOOLS COMPRESS-1
+    DEPENDENCIES: RUN_ICON BUILD_PYTHON_ENVIRONMENT COMPRESS-1
     RUNNING: chunk
     TYPE: python
     EXECUTABLE: "%HPCROOTDIR%/venv/bin/python3"

templates/build_enstools.sh → templates/build_python_environment.sh

@@ -17,16 +17,15 @@ cd ${WORKDIR}
 # Check if experiment's spack installation already exists, if it doesn't, clone it.
 SPACK_ENV=spack/share/spack/setup-env.sh
 if [ ! -f ${SPACK_ENV} ]; then
-  git clone ${SPACK_URL} -b ${SPACK_BRANCH}
-  #TODO: Would be good to enable the re-utilization of existing spack packages (via packages.yaml or upstreams.yaml)
+  echo "Spack folder not found!"
+  exit 1
 fi
 
-# Setup the environment
+# Setup the spack environment
 source ${SPACK_ENV}
 
 # Use spack to get a recent enough version of python3
-if [ $(
-  spack find python@3.8: &>/dev/null
+if [ $( spack find python@3.8: &>/dev/null
   echo $?
 ) -ne 0 ]; then
   echo "Installing a version of python3"
@@ -36,22 +35,29 @@ else
   echo "python@3.8: already installed!"
 fi
 
-# Link the python binary into the main folder so can be easily used later:
-
+# Load the python module
 spack load python@3.8:
 
+PYTHON_ENVIRONMENT_FOLDER=%python_environment.folder_name%
+
 if [ ! -d venv ]; then
   # Create virtual environment
-  python3 -m venv --prompt AS venv
+  python3 -m venv --prompt AS ${PYTHON_ENVIRONMENT_FOLDER}
 fi
 
 # Load environment
-source venv/bin/activate
+source ${PYTHON_ENVIRONMENT_FOLDER}/bin/activate
 
 # Create a link to the binary
 ln -sf $(which python3) ${WORKDIR}/python3
 
-# Install enstools-compression via pip
+# Install the requirements via pip
 # TODO: Due to a incompatibility issue between latest numba and numpy I have to add this here. Hopefully removable soon.
-pip install numpy==1.23
-pip install enstools-compression
+requirements=%python_environment.requirements%
+
+echo ${requirements}
+
+# Install requirements.
+for requirement in ${requirements} ; do
+  python -m pip install ${requirement}
+done