diff --git a/conf/jobs.yaml b/conf/jobs.yaml index fb790474ba821f00ca7324f7802e49247c99533c..4eda6e78dbaa3b586b5a5d37feb5106776cbd584 100644 --- a/conf/jobs.yaml +++ b/conf/jobs.yaml @@ -61,8 +61,9 @@ JOBS: WALLCLOCK: 04:00 PROCESSORS: 16 - BUILD_ENSTOOLS: - FILE: templates/build_enstools.sh + BUILD_PYTHON_ENVIRONMENT: + FILE: templates/build_python_environment.sh + # Right now we rely on spack for building icon and having a python interpreter, so we need this dependency: DEPENDENCIES: BUILD_ICON WALLCLOCK: 01:00 PROCESSORS: 16 @@ -90,7 +91,7 @@ JOBS: COMPRESS: FILE: templates/compress.py - DEPENDENCIES: RUN_ICON BUILD_ENSTOOLS COMPRESS-1 + DEPENDENCIES: RUN_ICON BUILD_PYTHON_ENVIRONMENT COMPRESS-1 RUNNING: chunk TYPE: python EXECUTABLE: "%HPCROOTDIR%/venv/bin/python3" diff --git a/templates/build_enstools.sh b/templates/build_python_environment.sh similarity index 68% rename from templates/build_enstools.sh rename to templates/build_python_environment.sh index 71a568460f50a1ed122a0ed93220a9234823ebda..5a7b226b2bb51a3283374173a0550f9986125376 100644 --- a/templates/build_enstools.sh +++ b/templates/build_python_environment.sh @@ -17,16 +17,15 @@ cd ${WORKDIR} # Check if experiment's spack installation already exists, if it doesn't, clone it. SPACK_ENV=spack/share/spack/setup-env.sh if [ ! -f ${SPACK_ENV} ]; then - git clone ${SPACK_URL} -b ${SPACK_BRANCH} - #TODO: Would be good to enable the re-utilization of existing spack packages (via packages.yaml or upstreams.yaml) + echo "Spack folder not found!" + exit 1 fi -# Setup the environment +# Setup the spack environment source ${SPACK_ENV} # Use spack to get a recent enough version of python3 -if [ $( - spack find python@3.8: &>/dev/null +if [ $( spack find python@3.8: &>/dev/null echo $? ) -ne 0 ]; then echo "Installing a version of python3" @@ -36,22 +35,29 @@ else echo "python@3.8: already installed!" fi -# Link the python binary into the main folder so can be easily used later: - +# Load the python module spack load python@3.8: +PYTHON_ENVIRONMENT_FOLDER=%python_environment.folder_name% + if [ ! -d venv ]; then # Create virtual environment - python3 -m venv --prompt AS venv + python3 -m venv --prompt AS ${PYTHON_ENVIRONMENT_FOLDER} fi # Load environment -source venv/bin/activate +source ${PYTHON_ENVIRONMENT_FOLDER}/bin/activate # Create a link to the binary ln -sf $(which python3) ${WORKDIR}/python3 -# Install enstools-compression via pip +# Install the requirements via pip # TODO: Due to a incompatibility issue between latest numba and numpy I have to add this here. Hopefully removable soon. -pip install numpy==1.23 -pip install enstools-compression +requirements=%python_environment.requirements% + +echo ${requirements} + +# Install requirements. +for requirement in ${requirements} ; do + python -m pip install ${requirement} +done