From 1fb8aa71ee13ef9338e257c948afee50d969b283 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Oriol=20Tint=C3=B3=20Prims?= <oriol.tinto@lmu.de>
Date: Thu, 16 Mar 2023 15:15:07 +0100
Subject: [PATCH] Change build_enstools.sh to build_python_environment.sh
---
conf/jobs.yaml | 7 +++--
...nstools.sh => build_python_environment.sh} | 30 +++++++++++--------
2 files changed, 22 insertions(+), 15 deletions(-)
rename templates/{build_enstools.sh => build_python_environment.sh} (68%)
diff --git a/conf/jobs.yaml b/conf/jobs.yaml
index fb79047..4eda6e7 100644
--- a/conf/jobs.yaml
+++ b/conf/jobs.yaml
@@ -61,8 +61,9 @@ JOBS:
WALLCLOCK: 04:00
PROCESSORS: 16
- BUILD_ENSTOOLS:
- FILE: templates/build_enstools.sh
+ BUILD_PYTHON_ENVIRONMENT:
+ FILE: templates/build_python_environment.sh
+ # Right now we rely on spack for building icon and having a python interpreter, so we need this dependency:
DEPENDENCIES: BUILD_ICON
WALLCLOCK: 01:00
PROCESSORS: 16
@@ -90,7 +91,7 @@ JOBS:
COMPRESS:
FILE: templates/compress.py
- DEPENDENCIES: RUN_ICON BUILD_ENSTOOLS COMPRESS-1
+ DEPENDENCIES: RUN_ICON BUILD_PYTHON_ENVIRONMENT COMPRESS-1
RUNNING: chunk
TYPE: python
EXECUTABLE: "%HPCROOTDIR%/venv/bin/python3"
diff --git a/templates/build_enstools.sh b/templates/build_python_environment.sh
similarity index 68%
rename from templates/build_enstools.sh
rename to templates/build_python_environment.sh
index 71a5684..5a7b226 100644
--- a/templates/build_enstools.sh
+++ b/templates/build_python_environment.sh
@@ -17,16 +17,15 @@ cd ${WORKDIR}
# Check if experiment's spack installation already exists, if it doesn't, clone it.
SPACK_ENV=spack/share/spack/setup-env.sh
if [ ! -f ${SPACK_ENV} ]; then
- git clone ${SPACK_URL} -b ${SPACK_BRANCH}
- #TODO: Would be good to enable the re-utilization of existing spack packages (via packages.yaml or upstreams.yaml)
+ echo "Spack folder not found!"
+ exit 1
fi
-# Setup the environment
+# Setup the spack environment
source ${SPACK_ENV}
# Use spack to get a recent enough version of python3
-if [ $(
- spack find python@3.8: &>/dev/null
+if [ $( spack find python@3.8: &>/dev/null
echo $?
) -ne 0 ]; then
echo "Installing a version of python3"
@@ -36,22 +35,29 @@ else
echo "python@3.8: already installed!"
fi
-# Link the python binary into the main folder so can be easily used later:
-
+# Load the python module
spack load python@3.8:
+PYTHON_ENVIRONMENT_FOLDER=%python_environment.folder_name%
+
if [ ! -d venv ]; then
# Create virtual environment
- python3 -m venv --prompt AS venv
+ python3 -m venv --prompt AS ${PYTHON_ENVIRONMENT_FOLDER}
fi
# Load environment
-source venv/bin/activate
+source ${PYTHON_ENVIRONMENT_FOLDER}/bin/activate
# Create a link to the binary
ln -sf $(which python3) ${WORKDIR}/python3
-# Install enstools-compression via pip
+# Install the requirements via pip
# TODO: Due to a incompatibility issue between latest numba and numpy I have to add this here. Hopefully removable soon.
-pip install numpy==1.23
-pip install enstools-compression
+requirements=%python_environment.requirements%
+
+echo ${requirements}
+
+# Install requirements.
+for requirement in ${requirements} ; do
+ python -m pip install ${requirement}
+done
--
GitLab