diff --git a/conf/common/build.yml b/conf/common/build.yml index 9875df32417fb2fc99f5dd38a437bc3945d3eccf..650ab99452f6cf99f93ebff970369561ead0aa28 100644 --- a/conf/common/build.yml +++ b/conf/common/build.yml @@ -10,6 +10,7 @@ spack: user_cache_path: "%HPCROOTDIR%/spack_user_cache_path" # spack puts data here when bootstrapping, leave empty to use home folder user_config_path: "%HPCROOTDIR%/spack_user_config_path" # spack puts data here when bootstrapping, leave empty to use home folder disable_local_config: false # if true, spack installs into spack source dir + upstreams: "" # path to an spack upstream repo, e.g. to use a pre-existing system wide spack repo icon: # The command that will be used to build icon-nwp with spack diff --git a/examples/01_real-from-ideal.sh b/examples/01_real-from-ideal.sh index 553a083802e7423324826cb2078afd792bef91f6..2d3abc0e26415163674613ce56e99fe7a4cddf1d 100644 --- a/examples/01_real-from-ideal.sh +++ b/examples/01_real-from-ideal.sh @@ -120,7 +120,7 @@ EOF cat > autosubmit/${EXPID}/conf/myconf.yml <<EOF spack: - init: "module load spack/2023.02 gcc/11.3.0 openmpi/4.1.4-gcc-11.3.0 netcdf-fortran/4.6.0-gcc-11.3.0-openmpi" # command to load spack environment, e.g. module load spack, use spack/setup-env.sh if empty + init: "" # command to load spack environment, e.g. module load spack, use spack/setup-env.sh if empty url: https://github.com/spack/spack.git # url to download spack if necessary branch: develop # if downloaded, branch name to use compiler: "gcc@11.3.0" # desired compiler for spack @@ -129,6 +129,7 @@ spack: user_cache_path: "%HPCROOTDIR%/../spack_user_cache_path" # spack puts data here when bootstrapping, leave empty to use home folder user_config_path: "%HPCROOTDIR%/../spack_user_config_path" # spack puts data here when bootstrapping, leave empty to use home folder disable_local_config: false # if true, spack installs into spack source dir + upstreams: "/software/opt/focal/x86_64/spack/2023.02" icon: #build_cmd: "icon-nwp@%ICON.VERSION%% %SPACK.COMPILER%+debug+petsc target=x86_64_v2 ^openmpi+pmi+legacylaunchers schedulers=slurm fabrics=ucx ucx+dc+dm+ib_hw_tm+mlx5_dv+rc+rdmacm+thread_multiple+ud+verbs" diff --git a/platforms/common/spack_utils.sh b/platforms/common/spack_utils.sh index 02ec602587a49823bce10091fbd4933741a37c02..9f45ee8da70dceaca1368ceb481cf1e6226137cb 100644 --- a/platforms/common/spack_utils.sh +++ b/platforms/common/spack_utils.sh @@ -4,6 +4,26 @@ function _install_spack() { git clone --depth 1 ${SPACK_URL} -b ${SPACK_BRANCH} ${SPACK_ROOT} fi + + if [ ! -z "${SPACK_UPSTREAMS}" ]; then + UPSTREAMS_CONFIG=${SPACK_ROOT}/etc/spack/upstreams.yaml + if [ ! -z $SPACK_USER_CONFIG_PATH ]; then + UPSTREAMS_CONFIG=${SPACK_USER_CONFIG_PATH}/upstreams.yaml + fi + echo "upstreams:" > $UPSTREAMS_CONFIG + let i=0 + for UP in ${SPACK_UPSTREAMS}; do + echo "Adding upstream repo to config $UPSTREAMS_CONFIG <= $UP" + let i=$((i+1)) + cat << EOF >> $UPSTREAMS_CONFIG + spack-instance-$i: + install_tree: $UP/spack/opt/spack + modules: + tcl: $UP/spack/share/spack/modules +EOF + done + fi + _init_spack "$SPACK_INIT_CMD" "$SPACK_ROOT" if [ ! -z "${SPACK_EXTERNALS}" ] ; then @@ -15,8 +35,12 @@ function _install_spack() { if [[ $(spack compiler info ${SPACK_COMPILER}) ]]; then echo "Found Compiler $(spack compiler info ${SPACK_COMPILER})" else - echo "could not find compiler, will now try to install it... this may take a while" - spack install --reuse ${SPACK_COMPILER} + if [[ $(spack load ${SPACK_COMPILER}) ]]; then + echo "found the compiler ${SPACK_COMPILER} as a spack package at $(spack location --install-dir $SPACK_COMPILER)" + else + echo "could not find compiler, will now try to install it... this may take a while" + spack install --reuse ${SPACK_COMPILER} + fi spack compiler add $(spack location --install-dir $SPACK_COMPILER) fi } @@ -43,6 +67,7 @@ function load_spack() { export SPACK_DISABLE_LOCAL_CONFIG=$7 # i.e.: true export SPACK_USER_CACHE_PATH=$8 # i.e.: ${SPACK_ROOT}/spack_user_cache_path export SPACK_USER_CONFIG_PATH=$9 # i.e.: ${SPACK_ROOT}/spack_user_config_path + export SPACK_UPSTREAMS=${10} # i.e.: pre-existing spack installs to use as upstream repos if [ "$SPACK_DISABLE_LOCAL_CONFIG" != "True" ]; then unset SPACK_DISABLE_LOCAL_CONFIG @@ -57,6 +82,7 @@ function load_spack() { echo "SPACK_DISABLE_LOCAL_CONFIG = ${SPACK_DISABLE_LOCAL_CONFIG:-False}" echo "SPACK_USER_CACHE_PATH = $SPACK_USER_CACHE_PATH" echo "SPACK_USER_CONFIG_PATH = $SPACK_USER_CONFIG_PATH" + echo "SPACK_UPSTREAMS = $SPACK_UPSTREAMS" if [ -z "$SPACK_USER_CACHE_PATH" ]; then unset SPACK_USER_CACHE_PATH diff --git a/templates/common/build_icon.sh b/templates/common/build_icon.sh index 2f53a0ada2a5b07207e15d977e6a7135f722d30b..e8ff67cffc0b9f52224e50679094b7d4d5666ed2 100644 --- a/templates/common/build_icon.sh +++ b/templates/common/build_icon.sh @@ -14,7 +14,8 @@ cd ${WORKDIR} || exit . ${WORKDIR}/proj/platforms/common/spack_utils.sh -load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%" +load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%" "%spack.upstreams%" + if [ ! $(rpm -qa | grep bzip2) ]; then spack install --reuse bzip2 diff --git a/templates/common/build_python_environment.sh b/templates/common/build_python_environment.sh index 102f9008c686c63814091696edbcf8f409323fee..e671ea8b7442105ba20c4719acbdc6855d55ded7 100644 --- a/templates/common/build_python_environment.sh +++ b/templates/common/build_python_environment.sh @@ -11,7 +11,8 @@ PYTHON_VERSION=%PYTHON_ENVIRONMENT.PYTHON_VERSION% cd ${WORKDIR} . ${WORKDIR}/proj/platforms/common/spack_utils.sh -load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%" +load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%" "%spack.upstreams%" + # Use spack to get a recent enough version of python3 diff --git a/templates/common/run_icon.sh b/templates/common/run_icon.sh index a9075818bfa3ab9683ff65ba3b687268b1d98f6a..3215e74460d810cb3878feed8c15368bf1bc7017 100644 --- a/templates/common/run_icon.sh +++ b/templates/common/run_icon.sh @@ -13,7 +13,8 @@ cd ${RUNDIR} # Activate spack . ${WORKDIR}/proj/platforms/common/spack_utils.sh -load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%" +load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%" "%spack.upstreams%" + # Load icon module spack load --first icon-nwp@%ICON_VERSION% diff --git a/templates/real-from-dwd-ana/prepare_experiment.sh b/templates/real-from-dwd-ana/prepare_experiment.sh index c1c1b4401e89ff0d94a5708fdd8093b913b49f72..36ba043d7bd4f41b78b2a92f7e20127cb4e1bf07 100644 --- a/templates/real-from-dwd-ana/prepare_experiment.sh +++ b/templates/real-from-dwd-ana/prepare_experiment.sh @@ -9,7 +9,7 @@ EXTERNAL_PARAMETERS_FILE=%simulation.external_parameters_filename% # Activate spack . ${WORKDIR}/proj/platforms/common/spack_utils.sh -load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%" +load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%" "%spack.upstreams%" # Load icon module needed to retrieve some data spack load --first icon-nwp@%ICON_VERSION% diff --git a/templates/real-from-ideal/prepare_experiment.sh b/templates/real-from-ideal/prepare_experiment.sh index 7458bf96668ff85d190cea43d44add67ee26f878..34e419cb897542bf87134dc9bf817048c41be303 100644 --- a/templates/real-from-ideal/prepare_experiment.sh +++ b/templates/real-from-ideal/prepare_experiment.sh @@ -7,7 +7,8 @@ RADIATION_GRID_FILE=%simulation.radiation_grid_filename% # Activate spack . ${WORKDIR}/proj/platforms/common/spack_utils.sh -load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%" +load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%" "%spack.upstreams%" + # Load icon module needed to retrieve some data spack load --first icon-nwp@%ICON_VERSION% diff --git a/templates/real-from-ideal/run_ideal.sh b/templates/real-from-ideal/run_ideal.sh index e9959aa2bf6a5e727a1dd259486f302884b94eaf..1949c28d7a3c4dcf2428f3486358ac196eb5c477 100644 --- a/templates/real-from-ideal/run_ideal.sh +++ b/templates/real-from-ideal/run_ideal.sh @@ -12,7 +12,8 @@ cd ${RUNDIR} # Activate spack . ${WORKDIR}/proj/platforms/common/spack_utils.sh -load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%" +load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%" "%spack.upstreams%" + # Load icon module spack load --first icon-nwp@%ICON_VERSION%