Merge remote-tracking branch 'origin/minimal-configuration' into FJ/spack_icon_install

conf/common/build.yml

@@ -18,6 +18,7 @@ icon:
 python_environment:
   # Name of the virtual environment in the remote platform experiment folder
   folder_name: python_environment
+  python_version: "3.8:"   # In spack notation use x: to indicate at least x
   requirements:
     # Because there's an issue with numba, for now we need to keep a specific version of numpy
     - numpy==1.23
@@ -25,3 +26,5 @@ python_environment:
     # Just to try a library from a git repository.
     - git+https://gitlab.physik.uni-muenchen.de/Oriol.Tinto/otils.git
     - f90nml
+
+LOGIN: "LOGIN"
\ No newline at end of file

conf/real-from-ideal/jobs.yml

@@ -79,14 +79,12 @@ JOBS:
     DEPENDENCIES: BUILD_ICON
     RUNNING: once
     WALLCLOCK: 00:10
-    PLATFORM: LOGIN
 
   PREPARE_IDEAL_DIRECTORY:
     FILE: templates/real-from-ideal/prepare_ideal_directory.sh
     DEPENDENCIES: PREPARE_EXPERIMENT
     RUNNING: date
     WALLCLOCK: 00:10
-    PLATFORM: LOGIN
 
   PREPARE_IDEAL_NAMELIST:
     FILE: templates/real-from-ideal/prepare_ideal_namelist.py
@@ -95,7 +93,6 @@ JOBS:
     WALLCLOCK: 00:10
     TYPE: python
     EXECUTABLE: "%HPCROOTDIR%/%python_environment.folder_name%/bin/python3"
-    PLATFORM: LOGIN
 
   RUN_IDEAL:
     FILE: templates/real-from-ideal/run_ideal.sh
@@ -125,7 +122,6 @@ JOBS:
     DEPENDENCIES: FG_ANA_FROM_IDEALIZED EXTPAR_FROM_IDEALIZED
     RUNNING: member
     WALLCLOCK: 01:00
-    PLATFORM: LOGIN
 
   PREPARE_CHUNK:
     FILE: templates/real-from-ideal/prepare_chunk.py
@@ -134,7 +130,6 @@ JOBS:
     RUNNING: chunk
     TYPE: python
     EXECUTABLE: "%HPCROOTDIR%/%python_environment.folder_name%/bin/python3"
-    PLATFORM: LOGIN
 
   RUN_ICON:
     FILE: templates/common/run_icon.sh
@@ -162,4 +157,3 @@ JOBS:
     DEPENDENCIES: TRANSFER
     WALLCLOCK: 00:10
     RUNNING: member
-    PLATFORM: LOGIN

templates/common/build_python_environment.sh

@@ -2,41 +2,29 @@
 # TODO: What do we do to ensure that these variables are defined in the proj file?
 WORKDIR=%HPCROOTDIR%
 ICON_VERSION=%ICON_VERSION%
-SPACK_URL=%spack.url%
-SPACK_BRANCH=%spack.branch%
-SPACK_COMPILER=%spack.compiler%
-
-# If the workdir directory does not exist create it
-if [ ! -d ${WORKDIR} ]; then
-  mkdir -p ${WORKDIR}
-fi
+SPACK_URL=%SPACK.URL%
+SPACK_BRANCH=%SPACK.BRANCH%
+SPACK_COMPILER=%SPACK.COMPILER%
 
+PYTHON_VERSION=%PYTHON_ENVIRONMENT.PYTHON_VERSION%
 # Go to the working directory
 cd ${WORKDIR}
 
-# Check if experiment's spack installation already exists, if it doesn't, clone it.
-SPACK_ENV=spack/share/spack/setup-env.sh
-if [ ! -f ${SPACK_ENV} ]; then
-  echo "Spack folder not found!"
-  exit 1
-fi
+. ${WORKDIR}/proj/platforms/common/spack_utils.sh
+load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%"
 
-# Setup the spack environment
-source ${SPACK_ENV}
 
 # Use spack to get a recent enough version of python3
-if [ $( spack find python@3.8: &>/dev/null
-  echo $?
-) -ne 0 ]; then
+if [ $(spack find python@${PYTHON_VERSION}: &>/dev/null ) ]; then
+  echo "python@${PYTHON_VERSION} already installed!"
+else
   echo "Installing a version of python3"
   # Compile openmpi with schedulers=slurm
-  spack install python@3.8:
-else
-  echo "python@3.8: already installed!"
+  spack install python@${PYTHON_VERSION}
 fi
 
 # Load the python module
-spack load python@3.8:
+spack load --first python@${PYTHON_VERSION}
 
 PYTHON_ENVIRONMENT_FOLDER=%python_environment.folder_name%
 
@@ -56,7 +44,7 @@ ln -sf $(which python3) ${WORKDIR}/python3
 requirements="%python_environment.requirements%"
 
 # Convert list with python format to a bash array
-requirements=($( echo ${requirements} | sed "s/'//g" | tr -d '[],'))
+requirements=($(echo ${requirements} | sed "s/'//g" | tr -d '[],'))
 
 #TODO: Shouldn't be necessary but it is for now to overcome an issue with a repetition of the requirements.
 # Use sort and uniq to get the unique elements
@@ -66,6 +54,6 @@ unique_requirements=($(printf "%s\n" "${requirements[@]}" | sort -u))
 echo "${unique_requirements[@]}"
 
 # Install requirements.
-for requirement in ${unique_requirements[@]} ; do
+for requirement in "${unique_requirements[@]}"; do
   python -m pip install ${requirement}
 done

templates/common/run_icon.sh

@@ -12,8 +12,9 @@ RUNDIR=${WORKDIR}/${STARTDATE}/${MEMBER}
 cd ${RUNDIR}
 
 # Activate spack
-SPACK_ENV=${WORKDIR}/spack/share/spack/setup-env.sh
-source ${SPACK_ENV}
+. ${WORKDIR}/proj/platforms/common/spack_utils.sh
+load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%"
+
 # Load icon module
 spack load icon-nwp@%ICON_VERSION%
 

templates/real-from-dwd-ana/prepare_experiment.sh

@@ -8,8 +8,8 @@ EXTERNAL_PARAMETERS_FILE=%simulation.external_parameters_filename%
 
 
 # Activate spack
-SPACK_ENV=${WORKDIR}/spack/share/spack/setup-env.sh
-source ${SPACK_ENV}
+. ${WORKDIR}/proj/platforms/common/spack_utils.sh
+load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%"
 
 # Load icon module needed to retrieve some data
 spack load icon-nwp@%ICON_VERSION%

templates/real-from-ideal/prepare_experiment.sh

@@ -6,8 +6,8 @@ DYNAMICS_GRID_FILENAME=%simulation.dynamics_grid_filename%
 RADIATION_GRID_FILE=%simulation.radiation_grid_filename%
 
 # Activate spack
-SPACK_ENV=${WORKDIR}/spack/share/spack/setup-env.sh
-source ${SPACK_ENV}
+. ${WORKDIR}/proj/platforms/common/spack_utils.sh
+load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%"
 
 # Load icon module needed to retrieve some data
 spack load icon-nwp@%ICON_VERSION%