From b123ed1777f6eb1bfea83c51c48436e984970027 Mon Sep 17 00:00:00 2001
From: "Fabian.Jakub" <fabian@jakub.com>
Date: Mon, 15 May 2023 14:38:24 +0200
Subject: [PATCH] add option to use spack upstream repos
---
conf/common/build.yml | 1 +
examples/01_real-from-ideal.sh | 3 +-
platforms/common/spack_utils.sh | 30 +++++++++++++++++--
templates/common/build_icon.sh | 3 +-
templates/common/build_python_environment.sh | 3 +-
templates/common/run_icon.sh | 3 +-
.../real-from-dwd-ana/prepare_experiment.sh | 2 +-
.../real-from-ideal/prepare_experiment.sh | 3 +-
templates/real-from-ideal/run_ideal.sh | 3 +-
9 files changed, 42 insertions(+), 9 deletions(-)
diff --git a/conf/common/build.yml b/conf/common/build.yml
index 9875df3..650ab99 100644
--- a/conf/common/build.yml
+++ b/conf/common/build.yml
@@ -10,6 +10,7 @@ spack:
user_cache_path: "%HPCROOTDIR%/spack_user_cache_path" # spack puts data here when bootstrapping, leave empty to use home folder
user_config_path: "%HPCROOTDIR%/spack_user_config_path" # spack puts data here when bootstrapping, leave empty to use home folder
disable_local_config: false # if true, spack installs into spack source dir
+ upstreams: "" # path to an spack upstream repo, e.g. to use a pre-existing system wide spack repo
icon:
# The command that will be used to build icon-nwp with spack
diff --git a/examples/01_real-from-ideal.sh b/examples/01_real-from-ideal.sh
index 553a083..2d3abc0 100644
--- a/examples/01_real-from-ideal.sh
+++ b/examples/01_real-from-ideal.sh
@@ -120,7 +120,7 @@ EOF
cat > autosubmit/${EXPID}/conf/myconf.yml <<EOF
spack:
- init: "module load spack/2023.02 gcc/11.3.0 openmpi/4.1.4-gcc-11.3.0 netcdf-fortran/4.6.0-gcc-11.3.0-openmpi" # command to load spack environment, e.g. module load spack, use spack/setup-env.sh if empty
+ init: "" # command to load spack environment, e.g. module load spack, use spack/setup-env.sh if empty
url: https://github.com/spack/spack.git # url to download spack if necessary
branch: develop # if downloaded, branch name to use
compiler: "gcc@11.3.0" # desired compiler for spack
@@ -129,6 +129,7 @@ spack:
user_cache_path: "%HPCROOTDIR%/../spack_user_cache_path" # spack puts data here when bootstrapping, leave empty to use home folder
user_config_path: "%HPCROOTDIR%/../spack_user_config_path" # spack puts data here when bootstrapping, leave empty to use home folder
disable_local_config: false # if true, spack installs into spack source dir
+ upstreams: "/software/opt/focal/x86_64/spack/2023.02"
icon:
#build_cmd: "icon-nwp@%ICON.VERSION%% %SPACK.COMPILER%+debug+petsc target=x86_64_v2 ^openmpi+pmi+legacylaunchers schedulers=slurm fabrics=ucx ucx+dc+dm+ib_hw_tm+mlx5_dv+rc+rdmacm+thread_multiple+ud+verbs"
diff --git a/platforms/common/spack_utils.sh b/platforms/common/spack_utils.sh
index 02ec602..9f45ee8 100644
--- a/platforms/common/spack_utils.sh
+++ b/platforms/common/spack_utils.sh
@@ -4,6 +4,26 @@ function _install_spack() {
git clone --depth 1 ${SPACK_URL} -b ${SPACK_BRANCH} ${SPACK_ROOT}
fi
+
+ if [ ! -z "${SPACK_UPSTREAMS}" ]; then
+ UPSTREAMS_CONFIG=${SPACK_ROOT}/etc/spack/upstreams.yaml
+ if [ ! -z $SPACK_USER_CONFIG_PATH ]; then
+ UPSTREAMS_CONFIG=${SPACK_USER_CONFIG_PATH}/upstreams.yaml
+ fi
+ echo "upstreams:" > $UPSTREAMS_CONFIG
+ let i=0
+ for UP in ${SPACK_UPSTREAMS}; do
+ echo "Adding upstream repo to config $UPSTREAMS_CONFIG <= $UP"
+ let i=$((i+1))
+ cat << EOF >> $UPSTREAMS_CONFIG
+ spack-instance-$i:
+ install_tree: $UP/spack/opt/spack
+ modules:
+ tcl: $UP/spack/share/spack/modules
+EOF
+ done
+ fi
+
_init_spack "$SPACK_INIT_CMD" "$SPACK_ROOT"
if [ ! -z "${SPACK_EXTERNALS}" ] ; then
@@ -15,8 +35,12 @@ function _install_spack() {
if [[ $(spack compiler info ${SPACK_COMPILER}) ]]; then
echo "Found Compiler $(spack compiler info ${SPACK_COMPILER})"
else
- echo "could not find compiler, will now try to install it... this may take a while"
- spack install --reuse ${SPACK_COMPILER}
+ if [[ $(spack load ${SPACK_COMPILER}) ]]; then
+ echo "found the compiler ${SPACK_COMPILER} as a spack package at $(spack location --install-dir $SPACK_COMPILER)"
+ else
+ echo "could not find compiler, will now try to install it... this may take a while"
+ spack install --reuse ${SPACK_COMPILER}
+ fi
spack compiler add $(spack location --install-dir $SPACK_COMPILER)
fi
}
@@ -43,6 +67,7 @@ function load_spack() {
export SPACK_DISABLE_LOCAL_CONFIG=$7 # i.e.: true
export SPACK_USER_CACHE_PATH=$8 # i.e.: ${SPACK_ROOT}/spack_user_cache_path
export SPACK_USER_CONFIG_PATH=$9 # i.e.: ${SPACK_ROOT}/spack_user_config_path
+ export SPACK_UPSTREAMS=${10} # i.e.: pre-existing spack installs to use as upstream repos
if [ "$SPACK_DISABLE_LOCAL_CONFIG" != "True" ]; then
unset SPACK_DISABLE_LOCAL_CONFIG
@@ -57,6 +82,7 @@ function load_spack() {
echo "SPACK_DISABLE_LOCAL_CONFIG = ${SPACK_DISABLE_LOCAL_CONFIG:-False}"
echo "SPACK_USER_CACHE_PATH = $SPACK_USER_CACHE_PATH"
echo "SPACK_USER_CONFIG_PATH = $SPACK_USER_CONFIG_PATH"
+ echo "SPACK_UPSTREAMS = $SPACK_UPSTREAMS"
if [ -z "$SPACK_USER_CACHE_PATH" ]; then
unset SPACK_USER_CACHE_PATH
diff --git a/templates/common/build_icon.sh b/templates/common/build_icon.sh
index 2f53a0a..e8ff67c 100644
--- a/templates/common/build_icon.sh
+++ b/templates/common/build_icon.sh
@@ -14,7 +14,8 @@ cd ${WORKDIR} || exit
. ${WORKDIR}/proj/platforms/common/spack_utils.sh
-load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%"
+load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%" "%spack.upstreams%"
+
if [ ! $(rpm -qa | grep bzip2) ]; then
spack install --reuse bzip2
diff --git a/templates/common/build_python_environment.sh b/templates/common/build_python_environment.sh
index 102f900..e671ea8 100644
--- a/templates/common/build_python_environment.sh
+++ b/templates/common/build_python_environment.sh
@@ -11,7 +11,8 @@ PYTHON_VERSION=%PYTHON_ENVIRONMENT.PYTHON_VERSION%
cd ${WORKDIR}
. ${WORKDIR}/proj/platforms/common/spack_utils.sh
-load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%"
+load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%" "%spack.upstreams%"
+
# Use spack to get a recent enough version of python3
diff --git a/templates/common/run_icon.sh b/templates/common/run_icon.sh
index a907581..3215e74 100644
--- a/templates/common/run_icon.sh
+++ b/templates/common/run_icon.sh
@@ -13,7 +13,8 @@ cd ${RUNDIR}
# Activate spack
. ${WORKDIR}/proj/platforms/common/spack_utils.sh
-load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%"
+load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%" "%spack.upstreams%"
+
# Load icon module
spack load --first icon-nwp@%ICON_VERSION%
diff --git a/templates/real-from-dwd-ana/prepare_experiment.sh b/templates/real-from-dwd-ana/prepare_experiment.sh
index c1c1b44..36ba043 100644
--- a/templates/real-from-dwd-ana/prepare_experiment.sh
+++ b/templates/real-from-dwd-ana/prepare_experiment.sh
@@ -9,7 +9,7 @@ EXTERNAL_PARAMETERS_FILE=%simulation.external_parameters_filename%
# Activate spack
. ${WORKDIR}/proj/platforms/common/spack_utils.sh
-load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%"
+load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%" "%spack.upstreams%"
# Load icon module needed to retrieve some data
spack load --first icon-nwp@%ICON_VERSION%
diff --git a/templates/real-from-ideal/prepare_experiment.sh b/templates/real-from-ideal/prepare_experiment.sh
index 7458bf9..34e419c 100644
--- a/templates/real-from-ideal/prepare_experiment.sh
+++ b/templates/real-from-ideal/prepare_experiment.sh
@@ -7,7 +7,8 @@ RADIATION_GRID_FILE=%simulation.radiation_grid_filename%
# Activate spack
. ${WORKDIR}/proj/platforms/common/spack_utils.sh
-load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%"
+load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%" "%spack.upstreams%"
+
# Load icon module needed to retrieve some data
spack load --first icon-nwp@%ICON_VERSION%
diff --git a/templates/real-from-ideal/run_ideal.sh b/templates/real-from-ideal/run_ideal.sh
index e9959aa..1949c28 100644
--- a/templates/real-from-ideal/run_ideal.sh
+++ b/templates/real-from-ideal/run_ideal.sh
@@ -12,7 +12,8 @@ cd ${RUNDIR}
# Activate spack
. ${WORKDIR}/proj/platforms/common/spack_utils.sh
-load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%"
+load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%" "%spack.upstreams%"
+
# Load icon module
spack load --first icon-nwp@%ICON_VERSION%
--
GitLab