diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c9db163d6b95637012d72e42577828ef81ef1b9d..fc61f9857a6c59cbf6f5f0550ab895105b85040b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,11 @@ stages: - clean + - build - run_tests +variables: + PROJECT_BRANCH: $CI_COMMIT_REF_NAME + clean-spack: stage: clean script: @@ -17,6 +21,22 @@ clean-spack: - when: never +build-only: + stage: build + script: + - ./examples/00_build_only.sh + tags: + - slurm.meteo.physik.lmu.de + variables: + SRUN_OPTIONS: "--time 04:00:00 --mem 18G -n 4 -c 2" + + artifacts: + name: build-only-logs + when: always + paths: + - ./autosubmit/**/tmp + expire_in: 7 days + real-from-ideal: stage: run_tests @@ -62,6 +82,22 @@ real-from-dwd-ana: artifacts: name: real-from-dwd-ana-logs when: always + paths: + - ./autosubmit/**/tmp + expire_in: 7 days + +real-from-dwd-ana-B6: + stage: run_tests + script: + - ./examples/02_real-from-dwd-ana_B6.sh + tags: + - slurm.meteo.physik.lmu.de + variables: + SRUN_OPTIONS: "--time 04:00:00 --mem 64G -n 16 -c 2" + + artifacts: + name: real-from-dwd-ana-B6-logs + when: always paths: - ./autosubmit/**/tmp expire_in: 7 days \ No newline at end of file diff --git a/conf/build_only.yml b/conf/build_only.yml new file mode 100644 index 0000000000000000000000000000000000000000..da9eff21a5a49c4b9214e8f0cab59a9e5a20973e --- /dev/null +++ b/conf/build_only.yml @@ -0,0 +1,5 @@ +DEFAULT: + CUSTOM_CONFIG: + PRE: + - "%PROJDIR%/conf/common" + - "%PROJDIR%/conf/build_only" \ No newline at end of file diff --git a/conf/common/build.yml b/conf/common/build.yml index baa307ef9732bdbde1a063f797887183b4fba19d..0c5b556e04b839c83faa35503c5cf418cb4de830 100644 --- a/conf/common/build.yml +++ b/conf/common/build.yml @@ -2,7 +2,7 @@ spack: init: "" # command to load spack environment, e.g. module load spack, # use spack/setup-env.sh if empty url: https://github.com/spack/spack.git # url to download spack if necessary - branch: develop # if downloaded, branch name to use + branch: v0.20.1 # if downloaded, branch name to use externals: "" # list of packages we try to find with spack external find # In LMU we need to include slurm to this list. compiler: "gcc@12.2.0" # desired compiler for spack diff --git a/conf/real-from-dwd-ana.yml b/conf/real-from-dwd-ana.yml new file mode 100644 index 0000000000000000000000000000000000000000..0fb4a9be01160b6182f13a29dcd04bd8aba0ce76 --- /dev/null +++ b/conf/real-from-dwd-ana.yml @@ -0,0 +1,5 @@ +DEFAULT: + CUSTOM_CONFIG: + PRE: + - "%PROJDIR%/conf/common" + - "%PROJDIR%/conf/real-from-dwd-ana" \ No newline at end of file diff --git a/conf/real-from-ideal.yml b/conf/real-from-ideal.yml new file mode 100644 index 0000000000000000000000000000000000000000..6a1e8b69b5b066a9d1820f4b6349b01c0c8ddbea --- /dev/null +++ b/conf/real-from-ideal.yml @@ -0,0 +1,5 @@ +DEFAULT: + CUSTOM_CONFIG: + PRE: + - "%PROJDIR%/conf/common" + - "%PROJDIR%/conf/real-from-ideal" \ No newline at end of file diff --git a/examples/00_build_only.sh b/examples/00_build_only.sh new file mode 100755 index 0000000000000000000000000000000000000000..320ef660fd37823df8c322bc5934b50337363cf1 --- /dev/null +++ b/examples/00_build_only.sh @@ -0,0 +1,21 @@ +#!/bin/bash +####################################################################################################################### +# Variables that are required to define the test case +EXPID="bo000" +ICON_CASE="build_only" +ICON_VERSION="2.6.5-nwp0" +PLATFORM="LOCAL" +EXTRA_NAMELIST="" +EXTRA_CONFIGURATION="" + +####################################################################################################################### +set -euo pipefail + +# Get path to the script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" + +# Import the function run_autoicon_example +source ${SCRIPT_DIR}/example_utils.sh + +# Setup and run the experiment +run_autoicon_example "$EXPID" "$ICON_CASE" "$ICON_VERSION" "$PLATFORM" "${EXTRA_NAMELIST}" "${EXTRA_CONFIGURATION}" diff --git a/examples/02_real-from-dwd-ana_B6.sh b/examples/02_real-from-dwd-ana_B6.sh new file mode 100755 index 0000000000000000000000000000000000000000..23da15417a2162883fd33a8cbda8b2384d40081d --- /dev/null +++ b/examples/02_real-from-dwd-ana_B6.sh @@ -0,0 +1,21 @@ +#!/bin/bash +####################################################################################################################### +# Variables that are required to define the test case +EXPID="rfdaB6" +ICON_CASE="real-from-dwd-ana" +ICON_VERSION="w2w-B6" +PLATFORM="LOCAL" +EXTRA_NAMELIST="" +EXTRA_CONFIGURATION="" + +####################################################################################################################### +set -euo pipefail + +# Get path to the script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" + +# Import the function run_autoicon_example +source ${SCRIPT_DIR}/example_utils.sh + +# Setup and run the experiment +run_autoicon_example "$EXPID" "$ICON_CASE" "$ICON_VERSION" "$PLATFORM" "${EXTRA_NAMELIST}" "${EXTRA_CONFIGURATION}" diff --git a/examples/example_utils.sh b/examples/example_utils.sh index 491fba642fc050b4ddff0bba18b2b6dad07d0cbb..7aed35c3d410bec17adc70dda53942f87b8928b7 100644 --- a/examples/example_utils.sh +++ b/examples/example_utils.sh @@ -18,7 +18,9 @@ function run_autoicon_example() { configure_and_install_autosubmit # Set up project - setup_project "$EXPID" + EXPID=$( setup_project ) + + echo "NewEXPID=${EXPID}" # Create and run the experiment create_and_run_experiment "$EXPID" @@ -27,7 +29,7 @@ function run_autoicon_example() { function setup_environment() { # Set Autosubmit version and define some directories - AUTOSUBMIT_VERSION="4.0.76" + AUTOSUBMIT_VERSION="4.0.79" # Define the path of the directory where this script is located SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" # Define the root path of the project @@ -66,49 +68,15 @@ function configure_and_install_autosubmit() { } function setup_project() { - local EXPID=$1 PROJECT_ORIGIN="https://gitlab.physik.uni-muenchen.de/w2w/autoicon.git" : ${PROJECT_BRANCH:="master"} # If experiment directory doesn't exist, create it and set up the experiment - if [ ! -e autosubmit/$EXPID/ ]; then - mkdir -p autosubmit/$EXPID/ - autosubmit expid -min -H lmu -d myiconsim -repo $PROJECT_ORIGIN -b $PROJECT_BRANCH &> >(tee autosubmit/$EXPID/log.autosubmit.expid) - AUTOID=$(grep Experiment autosubmit/$EXPID/log.autosubmit.expid | awk '{print $2}') - mv -v autosubmit/$AUTOID/* autosubmit/$EXPID - rmdir -v autosubmit/$AUTOID - fi - - # Create and write the minimal config file - cat >autosubmit/$EXPID/conf/minimal.yml <<EOF -ICON_CASE: "${ICON_CASE}" - -CONFIG: - AUTOSUBMIT_VERSION: "$AUTOSUBMIT_VERSION" - TOTALJOBS: 20 - MAXWAITINGJOBS: 20 - RETRIALS: 0 -DEFAULT: - EXPID: "$EXPID" - HPCARCH: "${PLATFORM}" # use LMU to run on cluster - CUSTOM_CONFIG: - PRE: - - "%PROJDIR%/conf/common" - - "%PROJDIR%/conf/%ICON_CASE%" - -PROJECT: - PROJECT_TYPE: "git" - PROJECT_DESTINATION: "autoicon" -GIT: - PROJECT_ORIGIN: "$PROJECT_ORIGIN" - PROJECT_BRANCH: "$PROJECT_BRANCH" - PROJECT_COMMIT: "" - PROJECT_SUBMODULES: "" - FETCH_SINGLE_BRANCH: True -EOF + autosubmit expid -min -H LOCAL -d myiconsim -repo $PROJECT_ORIGIN -b $PROJECT_BRANCH -conf "conf/${ICON_CASE}.yml" &> autosubmit/log.autosubmit.expid + EXPID=$(grep "Experiment.*created" autosubmit/log.autosubmit.expid | awk '{print $2}') # Create and write the additional config file - cat >autosubmit/${EXPID}/conf/01_myconf.yml <<EOF + cat > autosubmit/${EXPID}/conf/01_myconf.yml <<EOF spack: init: "" # command to load spack environment, e.g. module load spack, use spack/setup-env.sh if empty url: https://github.com/spack/spack.git # url to download spack if necessary @@ -152,6 +120,7 @@ $EXTRA_NAMELIST EOF fi +echo "${EXPID}" } function create_and_run_experiment() { diff --git a/spack_repo/packages/dwd-icon-tools/package.py b/spack_repo/packages/dwd-icon-tools/package.py index b0f92bb63aa5bc66e9082073d0b21eacea8552e4..ed7a8784a359cbbdeca2c291f1f5ce5bfb6a04ee 100644 --- a/spack_repo/packages/dwd-icon-tools/package.py +++ b/spack_repo/packages/dwd-icon-tools/package.py @@ -70,7 +70,7 @@ class DwdIconTools(Package): "FCFLAGS": "-I/usr/include -stand f08 -O2 -g -fpp -qopenmp -check bounds -Wunused -DNOMPI", "CFLAGS": "-std=gnu99 -O3 -DHAVE_LIBNETCDF -DHAVE_NETCDF4 -DHAVE_CF_INTERFACE -DHAVE_LIBGRIB " "-DHAVE_LIBGRIB_API -DpgiFortran -D__ICON__ -DHAVE_CONFIG_H -traceback", - "LIBS": f"-leccodes -lgfortran -lhdf5 -lxml2", + "LIBS": f"-leccodes -lhdf5 -lxml2", "V": "1", "VERBOSE": "1", "LDFLAGS": "-Wl,--copy-dt-needed-entries",