From 2026f6254c3810b6ec5c85c75ce56c63b5841345 Mon Sep 17 00:00:00 2001
From: "Fabian.Jakub" <fabian@jakub.com>
Date: Mon, 27 Mar 2023 17:09:15 +0200
Subject: [PATCH] add spack config var to allow to search for externals
---
conf/common/build.yml | 1 +
platforms/common/spack_utils.sh | 16 ++++++++++++----
templates/common/build_icon.sh | 2 +-
templates/common/build_python_environment.sh | 2 +-
templates/common/run_icon.sh | 2 +-
.../real-from-dwd-ana/prepare_experiment.sh | 2 +-
templates/real-from-ideal/prepare_experiment.sh | 2 +-
templates/real-from-ideal/run_ideal.sh | 2 +-
8 files changed, 19 insertions(+), 10 deletions(-)
diff --git a/conf/common/build.yml b/conf/common/build.yml
index b41438a..8f4e190 100644
--- a/conf/common/build.yml
+++ b/conf/common/build.yml
@@ -5,6 +5,7 @@ spack:
init: "" # command to load spack environment, e.g. module load spack, use spack/setup-env.sh if empty
url: https://github.com/spack/spack.git # url to download spack if necessary
branch: develop # if downloaded, branch name to use
+ externals: "" # list of packages we try to find with spack external find
compiler: "gcc@12.2.0" # desired compiler for spack
root: "%HPCROOTDIR%/spack" # path to a spack install, will be downloaded to if not present
user_cache_path: "%HPCROOTDIR%/spack_user_cache_path" # spack puts data here when bootstrapping, leave empty to use home folder
diff --git a/platforms/common/spack_utils.sh b/platforms/common/spack_utils.sh
index f336447..17e3e23 100644
--- a/platforms/common/spack_utils.sh
+++ b/platforms/common/spack_utils.sh
@@ -6,6 +6,12 @@ function _install_spack() {
_init_spack "$SPACK_INIT_CMD" "$SPACK_ROOT"
+ if [ ! -z "${SPACK_EXTERNALS}" ] ; then
+ for ext in ${SPACK_EXTERNALS}; do
+ spack external find $ext
+ done
+ fi
+
if [[ $(spack compiler info ${SPACK_COMPILER}) ]]; then
echo "Found Compiler $(spack compiler info ${SPACK_COMPILER})"
else
@@ -32,10 +38,11 @@ function load_spack() {
export SPACK_ROOT=$2 # i.e.: spack
export SPACK_URL=$3 # i.e.: https://github.com/spack/spack.git
export SPACK_BRANCH=$4 # i.e.: develop
- export SPACK_COMPILER=$5 # i.e.: gcc@12.2.0
- export SPACK_DISABLE_LOCAL_CONFIG=$6 # i.e.: true
- export SPACK_USER_CACHE_PATH=$7 # i.e.: ${SPACK_ROOT}/spack_user_cache_path
- export SPACK_USER_CONFIG_PATH=$8 # i.e.: ${SPACK_ROOT}/spack_user_config_path
+ export SPACK_EXTERNALS=$5 # i.e.: slurm
+ export SPACK_COMPILER=$6 # i.e.: gcc@12.2.0
+ export SPACK_DISABLE_LOCAL_CONFIG=$7 # i.e.: true
+ export SPACK_USER_CACHE_PATH=$8 # i.e.: ${SPACK_ROOT}/spack_user_cache_path
+ export SPACK_USER_CONFIG_PATH=$9 # i.e.: ${SPACK_ROOT}/spack_user_config_path
if [ "$SPACK_DISABLE_LOCAL_CONFIG" != "True" ]; then
unset SPACK_DISABLE_LOCAL_CONFIG
@@ -45,6 +52,7 @@ function load_spack() {
echo "SPACK_ROOT = $SPACK_ROOT"
echo "SPACK_URL = $SPACK_URL"
echo "SPACK_BRANCH = $SPACK_BRANCH"
+ echo "SPACK_EXTERNALS = $SPACK_EXTERNALS"
echo "SPACK_COMPILER = $SPACK_COMPILER"
echo "SPACK_DISABLE_LOCAL_CONFIG = ${SPACK_DISABLE_LOCAL_CONFIG:-False}"
echo "SPACK_USER_CACHE_PATH = $SPACK_USER_CACHE_PATH"
diff --git a/templates/common/build_icon.sh b/templates/common/build_icon.sh
index 51dd449..62729b3 100644
--- a/templates/common/build_icon.sh
+++ b/templates/common/build_icon.sh
@@ -14,7 +14,7 @@ cd ${WORKDIR} || exit
. ${WORKDIR}/proj/platforms/common/spack_utils.sh
-load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%"
+load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%"
if [ ! $(rpm -qa | grep bzip2) ]; then
spack install bzip2
diff --git a/templates/common/build_python_environment.sh b/templates/common/build_python_environment.sh
index 3220707..102f900 100644
--- a/templates/common/build_python_environment.sh
+++ b/templates/common/build_python_environment.sh
@@ -11,7 +11,7 @@ PYTHON_VERSION=%PYTHON_ENVIRONMENT.PYTHON_VERSION%
cd ${WORKDIR}
. ${WORKDIR}/proj/platforms/common/spack_utils.sh
-load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%"
+load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%"
# Use spack to get a recent enough version of python3
diff --git a/templates/common/run_icon.sh b/templates/common/run_icon.sh
index a1ceaf1..5af49ea 100644
--- a/templates/common/run_icon.sh
+++ b/templates/common/run_icon.sh
@@ -13,7 +13,7 @@ cd ${RUNDIR}
# Activate spack
. ${WORKDIR}/proj/platforms/common/spack_utils.sh
-load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%"
+load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%"
# Load icon module
spack load icon-nwp@%ICON_VERSION%
diff --git a/templates/real-from-dwd-ana/prepare_experiment.sh b/templates/real-from-dwd-ana/prepare_experiment.sh
index 677cc05..c35bc45 100644
--- a/templates/real-from-dwd-ana/prepare_experiment.sh
+++ b/templates/real-from-dwd-ana/prepare_experiment.sh
@@ -9,7 +9,7 @@ EXTERNAL_PARAMETERS_FILE=%simulation.external_parameters_filename%
# Activate spack
. ${WORKDIR}/proj/platforms/common/spack_utils.sh
-load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%"
+load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%"
# Load icon module needed to retrieve some data
spack load icon-nwp@%ICON_VERSION%
diff --git a/templates/real-from-ideal/prepare_experiment.sh b/templates/real-from-ideal/prepare_experiment.sh
index d7f3791..70dd86e 100644
--- a/templates/real-from-ideal/prepare_experiment.sh
+++ b/templates/real-from-ideal/prepare_experiment.sh
@@ -7,7 +7,7 @@ RADIATION_GRID_FILE=%simulation.radiation_grid_filename%
# Activate spack
. ${WORKDIR}/proj/platforms/common/spack_utils.sh
-load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%"
+load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%"
# Load icon module needed to retrieve some data
spack load icon-nwp@%ICON_VERSION%
diff --git a/templates/real-from-ideal/run_ideal.sh b/templates/real-from-ideal/run_ideal.sh
index b808cae..9bf0a58 100644
--- a/templates/real-from-ideal/run_ideal.sh
+++ b/templates/real-from-ideal/run_ideal.sh
@@ -12,7 +12,7 @@ cd ${RUNDIR}
# Activate spack
. ${WORKDIR}/proj/platforms/common/spack_utils.sh
-load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%"
+load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%"
# Load icon module
spack load icon-nwp@%ICON_VERSION%
--
GitLab