diff --git a/conf/build_only/jobs.yml b/conf/build_only/jobs.yml new file mode 100644 index 0000000000000000000000000000000000000000..f6a5526eaa0b2ed350847ee1a66b22863813380a --- /dev/null +++ b/conf/build_only/jobs.yml @@ -0,0 +1,12 @@ +JOBS: + TRANSFER_PROJECT: + FILE: templates/common/transfer_project.sh + PLATFORM: LOCAL + + BUILD_ICON: + FILE: templates/common/build_icon.sh + DEPENDENCIES: TRANSFER_PROJECT + WALLCLOCK: 04:00 + PROCESSORS: 16 + RETRIALS: 2 # retry because spack downloads sometimes timeout + NODES: 1 \ No newline at end of file diff --git a/conf/common/jobs.yml b/conf/common/jobs.generic similarity index 94% rename from conf/common/jobs.yml rename to conf/common/jobs.generic index 472a8c9a26b3476750df764f7b0f6dac550de474..b4a030c79e7b82bd95c5c3d093535bb23d2539e4 100644 --- a/conf/common/jobs.yml +++ b/conf/common/jobs.generic @@ -1,3 +1,6 @@ +## This file contains the template of the generic workflow +## It can be used as a reference to create a custom workflow. + JOBS: TRANSFER_PROJECT: FILE: templates/common/transfer_project.sh diff --git a/conf/real-from-dwd-ana/jobs.yml b/conf/real-from-dwd-ana/jobs.yml index 2ce9baeaeac5577faf1126e2c3b6d4ca6d86755d..c625691c296bf13af4b63f715fea9998c7211709 100644 --- a/conf/real-from-dwd-ana/jobs.yml +++ b/conf/real-from-dwd-ana/jobs.yml @@ -1,6 +1,29 @@ JOBS: + TRANSFER_PROJECT: + FILE: templates/common/transfer_project.sh + PLATFORM: LOCAL + + BUILD_ICON: + FILE: templates/common/build_icon.sh + DEPENDENCIES: TRANSFER_PROJECT + WALLCLOCK: 04:00 + PROCESSORS: 16 + RETRIALS: 2 # retry because spack downloads sometimes timeout + NODES: 1 + + BUILD_PYTHON_ENVIRONMENT: + FILE: templates/common/build_python_environment.sh + # Right now we rely on spack for building icon and having a python interpreter, so we need this dependency: + DEPENDENCIES: BUILD_ICON + WALLCLOCK: 01:00 + PROCESSORS: 16 + NODES: 1 + PREPARE_EXPERIMENT: FILE: templates/real-from-dwd-ana/prepare_experiment.sh + DEPENDENCIES: BUILD_ICON + RUNNING: once + WALLCLOCK: 01:00 PREPARE_DATE_LOCAL: FILE: templates/real-from-dwd-ana/prepare_date_local.sh @@ -16,6 +39,46 @@ JOBS: PREPARE_MEMBER: FILE: templates/real-from-dwd-ana/prepare_member.sh DEPENDENCIES: PREPARE_EXPERIMENT PREPARE_DATE_REMOTE PREPARE_DATE_LOCAL + RUNNING: member + WALLCLOCK: 01:00 PREPARE_NAMELIST: - FILE: templates/real-from-dwd-ana/prepare_namelist.py \ No newline at end of file + FILE: templates/real-from-dwd-ana/prepare_namelist.py + DEPENDENCIES: TRANSFER_PROJECT BUILD_PYTHON_ENVIRONMENT PREPARE_MEMBER RUN_ICON-1 + WALLCLOCK: 00:05 + RUNNING: chunk + TYPE: python + EXECUTABLE: "%HPCROOTDIR%/%python_environment.folder_name%/bin/python3" + + RUN_ICON: + FILE: templates/common/run_icon.sh + DEPENDENCIES: PREPARE_NAMELIST + WALLCLOCK: 04:00 + RUNNING: chunk + PROCESSORS: 64 + MEMORY: 81920 + CUSTOM_DIRECTIVES: [ "#SBATCH --exclusive" ] + + COMPRESS: + FILE: templates/common/compress.py + DEPENDENCIES: RUN_ICON BUILD_PYTHON_ENVIRONMENT COMPRESS-1 + RUNNING: chunk + TYPE: python + EXECUTABLE: "%HPCROOTDIR%/%python_environment.folder_name%/bin/python3" + PROCESSORS: 16 + MEMORY: 16384 + WALLCLOCK: 01:00 + + TRANSFER: + FILE: templates/common/transfer.sh + DEPENDENCIES: COMPRESS + # Since this is running locally, can simply leave a long wallclock. + WALLCLOCK: 24:00 + RUNNING: member + PLATFORM: LOCAL + + CLEAN: + FILE: templates/common/clean.sh + DEPENDENCIES: TRANSFER + WALLCLOCK: 00:10 + RUNNING: member diff --git a/conf/real-from-ideal/jobs.yml b/conf/real-from-ideal/jobs.yml index ddeb81723dc1fd5493e714cb8ce02f05eb34adfd..6b3149c08eef274cd8cf012817bbe9de86d551da 100644 --- a/conf/real-from-ideal/jobs.yml +++ b/conf/real-from-ideal/jobs.yml @@ -1,6 +1,29 @@ JOBS: + TRANSFER_PROJECT: + FILE: templates/common/transfer_project.sh + PLATFORM: LOCAL + + BUILD_ICON: + FILE: templates/common/build_icon.sh + DEPENDENCIES: TRANSFER_PROJECT + WALLCLOCK: 04:00 + PROCESSORS: 16 + RETRIALS: 2 # retry because spack downloads sometimes timeout + NODES: 1 + + BUILD_PYTHON_ENVIRONMENT: + FILE: templates/common/build_python_environment.sh + # Right now we rely on spack for building icon and having a python interpreter, so we need this dependency: + DEPENDENCIES: BUILD_ICON + WALLCLOCK: 01:00 + PROCESSORS: 16 + NODES: 1 + PREPARE_EXPERIMENT: FILE: templates/real-from-ideal/prepare_experiment.sh + DEPENDENCIES: BUILD_ICON + RUNNING: once + WALLCLOCK: 01:00 PREPARE_IDEAL_DIRECTORY: FILE: templates/real-from-ideal/prepare_ideal_directory.sh @@ -41,6 +64,46 @@ JOBS: PREPARE_MEMBER: FILE: templates/real-from-ideal/prepare_member.sh DEPENDENCIES: FG_ANA_FROM_IDEALIZED EXTPAR_FROM_IDEALIZED + RUNNING: member + WALLCLOCK: 01:00 PREPARE_NAMELIST: FILE: templates/real-from-ideal/prepare_namelist.py + DEPENDENCIES: TRANSFER_PROJECT BUILD_PYTHON_ENVIRONMENT PREPARE_MEMBER RUN_ICON-1 + WALLCLOCK: 00:05 + RUNNING: chunk + TYPE: python + EXECUTABLE: "%HPCROOTDIR%/%python_environment.folder_name%/bin/python3" + + RUN_ICON: + FILE: templates/common/run_icon.sh + DEPENDENCIES: PREPARE_NAMELIST + WALLCLOCK: 04:00 + RUNNING: chunk + PROCESSORS: 64 + MEMORY: 81920 + CUSTOM_DIRECTIVES: [ "#SBATCH --exclusive" ] + + COMPRESS: + FILE: templates/common/compress.py + DEPENDENCIES: RUN_ICON BUILD_PYTHON_ENVIRONMENT COMPRESS-1 + RUNNING: chunk + TYPE: python + EXECUTABLE: "%HPCROOTDIR%/%python_environment.folder_name%/bin/python3" + PROCESSORS: 16 + MEMORY: 16384 + WALLCLOCK: 01:00 + + TRANSFER: + FILE: templates/common/transfer.sh + DEPENDENCIES: COMPRESS + # Since this is running locally, can simply leave a long wallclock. + WALLCLOCK: 24:00 + RUNNING: member + PLATFORM: LOCAL + + CLEAN: + FILE: templates/common/clean.sh + DEPENDENCIES: TRANSFER + WALLCLOCK: 00:10 + RUNNING: member diff --git a/examples/example_utils.sh b/examples/example_utils.sh index ef4aa473927b021d394c733eb1fe69853482c150..491fba642fc050b4ddff0bba18b2b6dad07d0cbb 100644 --- a/examples/example_utils.sh +++ b/examples/example_utils.sh @@ -68,7 +68,7 @@ function configure_and_install_autosubmit() { function setup_project() { local EXPID=$1 PROJECT_ORIGIN="https://gitlab.physik.uni-muenchen.de/w2w/autoicon.git" - PROJECT_BRANCH="master" + : ${PROJECT_BRANCH:="master"} # If experiment directory doesn't exist, create it and set up the experiment if [ ! -e autosubmit/$EXPID/ ]; then @@ -108,7 +108,7 @@ GIT: EOF # Create and write the additional config file - cat >autosubmit/${EXPID}/conf/myconf.yml <<EOF + cat >autosubmit/${EXPID}/conf/01_myconf.yml <<EOF spack: init: "" # command to load spack environment, e.g. module load spack, use spack/setup-env.sh if empty url: https://github.com/spack/spack.git # url to download spack if necessary @@ -140,7 +140,7 @@ EOF # If EXTRA_NAMELIST is provided, create and write the extra namelist file if [ ! -z "$EXTRA_CONFIGURATION" ]; then - cat >autosubmit/${EXPID}/conf/extra_conf.yml <<EOF + cat >autosubmit/${EXPID}/conf/00_extra_conf.yml <<EOF $EXTRA_CONFIGURATION EOF fi diff --git a/spack_repo/packages/dwd-icon-tools/package.py b/spack_repo/packages/dwd-icon-tools/package.py index effad57d9dbb1a93a38ab6b6396f0e5a743866b4..b0f92bb63aa5bc66e9082073d0b21eacea8552e4 100644 --- a/spack_repo/packages/dwd-icon-tools/package.py +++ b/spack_repo/packages/dwd-icon-tools/package.py @@ -56,13 +56,28 @@ class DwdIconTools(Package): git("submodule", "update", "--init", "--recursive") def setup_build_environment(self, env): - spec = self.spec # Some environment variables to set - env_variables_to_set = { - "CXXFLAGS": "-O2 -g -fopenmp -Wunused -DNOMPI", - "FCFLAGS": "-I/usr/include --std=f2008 -O2 -g -cpp -fopenmp -fbounds-check -Wunused -DNOMPI", - "LIBS": f"-leccodes -lgfortran -lhdf5 -lxml2", - } + if self.spec.satisfies("%gcc"): + env_variables_to_set = { + "CXXFLAGS": "-O2 -g -fopenmp -Wunused -DNOMPI", + "FCFLAGS": "-I/usr/include --std=f2008 -O2 -g -cpp -fopenmp -fbounds-check -Wunused -DNOMPI", + "LIBS": f"-leccodes -lgfortran -lhdf5 -lxml2", + } + elif self.spec.satisfies("%intel"): + # Intel + env_variables_to_set = { + "CXXFLAGS": "-O2 -g -qopenmp -Wunused -DNOMPI", + "FCFLAGS": "-I/usr/include -stand f08 -O2 -g -fpp -qopenmp -check bounds -Wunused -DNOMPI", + "CFLAGS": "-std=gnu99 -O3 -DHAVE_LIBNETCDF -DHAVE_NETCDF4 -DHAVE_CF_INTERFACE -DHAVE_LIBGRIB " + "-DHAVE_LIBGRIB_API -DpgiFortran -D__ICON__ -DHAVE_CONFIG_H -traceback", + "LIBS": f"-leccodes -lgfortran -lhdf5 -lxml2", + "V": "1", + "VERBOSE": "1", + "LDFLAGS": "-Wl,--copy-dt-needed-entries", + } + else: + raise AssertionError("Compiler does not match gcc nor intel") + for variable, value in env_variables_to_set.items(): env.set(variable, value) @@ -72,6 +87,8 @@ class DwdIconTools(Package): "--with-eccodes=yes", "--enable-grib2", "--enable-iso-c-interface", + "--enable-static", + "--disable-shared", f"--with-netcdf={spec['netcdf-c'].prefix}" ] diff --git a/spack_repo/packages/icon-nwp/package.py b/spack_repo/packages/icon-nwp/package.py index 172de511a4cb5f64460e7a7762fc519e1ce498ee..a1e3cd05b29a17cae5517a3ced9870cb8f88d092 100644 --- a/spack_repo/packages/icon-nwp/package.py +++ b/spack_repo/packages/icon-nwp/package.py @@ -19,8 +19,7 @@ class IconNwp(Package): homepage = "https://code.mpimet.mpg.de/projects/iconpublic" # maintainers("oriol.tinto") - # Version w2w-B6 points to a different repository and branch) - version("w2w-B6", git="ssh://git@gitlab.physik.uni-muenchen.de/w2w/icon-w2w.git", branch="icon-w2w/icon-nwp-B6") + version("w2w-B6", branch="w2w/B6_MCRPH") # FIXME: The ugly configuration system in older icon versions prevents us from using this package with it. # version("2.5.0-nwp3", branch="icon-nwp/op-release-2.5.0-nwp3") version("2.6.4-nwp3", branch="icon-nwp/op-release-2.6.4-nwp3") @@ -70,13 +69,6 @@ class IconNwp(Package): depends_on("openblas", when="%gcc") depends_on("intel-mkl", when="%intel") - # Extra dependencies for B6, including yaml, and the hdf5 filters for compression. - depends_on("libyaml", when="@w2w-B6") - depends_on("hdf5-blosc", when="@w2w-B6") - depends_on("h5z-zfp", when="@w2w-B6") - depends_on("sz~hdf5", when="@w2w-B6") - depends_on("sz3~hdf5", when="@w2w-B6") - phases = ["configure", "build", "install"] def do_fetch(self, mirror_only=False): @@ -99,80 +91,8 @@ class IconNwp(Package): def patch(self): # Run git submodule update git = which("git") - if self.spec.variants['source'].value == 'lmu': - self.patch_submodules() git("submodule", "update", "--init", "--recursive") - def patch_submodules(self): - """ - Because of the lack of access rights to the original submodule repositories, - we patch the gitmodules file to point to a different mirror. - """ - git_submodules_file = Path().cwd() / ".gitmodules" - - git_mirror = "git@gitlab.lrz.de:dkrz-mirror" - - git_modules_patch = f""" - [submodule "externals/mtime"] - path = externals/mtime - url = {git_mirror}/libmtime.git - [submodule "externals/jsbach"] - path = externals/jsbach - url = {git_mirror}/jsbach.git - [submodule "externals/yac"] - path = externals/yac - url = {git_mirror}/YAC.git - [submodule "externals/self"] - path = externals/self - url = {git_mirror}/libself.git - [submodule "externals/tixi"] - path = externals/tixi - url = {git_mirror}/libtixi.git - [submodule "externals/yaxt"] - path = externals/yaxt - url = {git_mirror}/yaxt.git - [submodule "externals/rte-rrtmgp"] - path = externals/rte-rrtmgp - url = https://github.com/earth-system-radiation/rte-rrtmgp.git - [submodule "externals/cub"] - path = externals/cub - url = https://github.com/NVlabs/cub.git - [submodule "externals/omni-xmod-pool"] - path = externals/omni-xmod-pool - url = https://github.com/claw-project/omni-xmod-pool.git - [submodule "externals/cdi"] - path = externals/cdi - url = {git_mirror}/libcdi.git - [submodule "externals/sct"] - path = externals/sct - url = {git_mirror}/sct.git - [submodule "externals/ecrad"] - path = externals/ecrad - url = {git_mirror}/libecrad.git - [submodule "externals/dace_icon"] - path = externals/dace_icon - url = {git_mirror}/dace-icon-interface.git - [submodule "externals/emvorado"] - path = externals/emvorado - url = {git_mirror}/emvorado-for-icon.git - [submodule "utils/mkexp"] - path = utils/mkexp - url = https://git.mpimet.mpg.de/public/mkexp - [submodule "externals/art"] - path = externals/art - url = {git_mirror}/art.git - [submodule "externals/ppm"] - path = externals/ppm - url = https://gitlab.dkrz.de/jahns/ppm.git - [submodule "externals/probtest"] - path = externals/probtest - url = {git_mirror}/cscs-sw_probtest.git - """ - - # Replace the content of the original file with the patch - with git_submodules_file.open("w") as out_f: - out_f.write(git_modules_patch) - def setup_build_environment(self, env): spec = self.spec @@ -187,7 +107,7 @@ class IconNwp(Package): def configure(self, spec, prefix): spec = self.spec - print(spec["mpi"].mpifc) + INCLUDES = [f"-I{spec['libxml2'].prefix}/include/libxml2", ] libs = [ f"-L{spec['netcdf-c'].prefix.lib} -lnetcdf ", # netcdf-c libs @@ -201,11 +121,7 @@ class IconNwp(Package): elif self.spec.satisfies("%intel"): libs.append("-qmkl=sequential") - if self.spec.version == Version("w2w-B6"): - libs.append("-lyaml") - mtune = "generic" - INCLUDES = [f"-I{spec['libxml2'].prefix}/include/libxml2", ] CPPFLAGS = [] if "+plexrt" in self.spec: @@ -218,24 +134,40 @@ class IconNwp(Package): INCLUDES.append(f"-I{spec['petsc'].prefix}/include") libs.append(f"-L{spec['petsc'].prefix.lib} -lpetsc") + if self.spec.satisfies("%gcc"): + FCFLAGS = "-std=legacy -fmodule-private -fimplicit-none -fmax-identifier-length=63 -Wall " \ + "-Wcharacter-truncation -Wconversion -Wunderflow -Wunused-parameter " \ + "-Wno-surprising -fall-intrinsics -g -mpc64 -w" + ARCH_FLAG = f"-mtune={mtune}" + ICON_FCFLAGS = "-std=f2008 -fallow-argument-mismatch -fmodule-private -fimplicit-none -Wall" + elif self.spec.satisfies("%intel"): + FCFLAGS = "-stand none -implicitnone -warn all -warn unused -fall-intrinsics -g -w" + ARCH_FLAG = f"-xHost" + ICON_FCFLAGS = "-stand f08 -implicitnone -warn all" + else: + raise AssertionError options = [ f"CC={spec['mpi'].mpicc}", f"FC={spec['mpi'].mpifc}", f"CFLAGS=-g -mpc64 {' '.join(INCLUDES)}", - f"ICON_CFLAGS=-O3 -g -mtune={mtune}", + f"ICON_CFLAGS=-O3 -g {ARCH_FLAG}", f"ICON_BUNDLED_CFLAGS=-O3 -mtune={mtune}", f"CPPFLAGS={' '.join(CPPFLAGS)}", - "FCFLAGS=-std=legacy -fmodule-private -fimplicit-none -fmax-identifier-length=63 -Wall -Wcharacter-truncation -Wconversion -Wunderflow -Wunused-parameter -Wno-surprising -fall-intrinsics -g -mpc64 -w", - f"ICON_OCEAN_FCFLAGS=-O3 -mtune={mtune}", + f"FCFLAGS={FCFLAGS}", + f"ICON_OCEAN_FCFLAGS=-O3 {ARCH_FLAG}", f"LDFLAGS={' '.join(libs)}", f"LIBS={' '.join(libs)}", f"--prefix={prefix}", ] - ICON_FCFLAGS="-std=f2008 -fallow-argument-mismatch -fmodule-private -fimplicit-none -Wall" if "+debug" in self.spec: - options.append(f"ICON_FCFLAGS={ICON_FCFLAGS} -fbacktrace -fbounds-check -fstack-protector-all -finit-real=nan -finit-integer=-2147483648 -finit-character=127 -w -O2") + if self.spec.satisfies("%gcc"): + options.append(f"ICON_FCFLAGS={ICON_FCFLAGS} -fbacktrace -fbounds-check -fstack-protector-all -finit-real=nan -finit-integer=-2147483648 -finit-character=127 -w -O2") + elif self.spec.satisfies("%intel"): + options.append(f"ICON_FCFLAGS={ICON_FCFLAGS} -traceback -check bounds -init=snan -O2") + else: + raise AssertionError("Compiler does not satisfy gcc nor intel.") else: options.append(f"ICON_FCFLAGS={ICON_FCFLAGS} -O3") diff --git a/spack_repo/packages/sz3/package.py b/spack_repo/packages/sz3/package.py deleted file mode 100644 index b2bcbd3e9b4c15a4a641e974539e08db4723daa3..0000000000000000000000000000000000000000 --- a/spack_repo/packages/sz3/package.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -from spack.package import * - - -class Sz3(CMakePackage): - """SZ3 is the next generation of the SZ compressor framework""" - - homepage = "https://github.com/szcompressor/SZ3" - git = "https://github.com/szcompressor/SZ3" - - # maintainers("disheng222") - tags = ["e4s"] - - version("master") - version("3.1.7", commit="c49fd17f2d908835c41000c1286c510046c0480e") - version("3.1.5.4", commit="4c6ddf628f27d36b28d1bbda02174359cd05573d") - version("3.1.5.1", commit="5736a63b917e439dd62248b4ff6234e96726af5d") - version("3.1.3.1", commit="323cb17b412d657c4be681b52c34beaf933fe7af") - version("3.1.3", commit="695dff8dc326f3b165f6676d810f46add088a585") - - variant("hdf5", default=False, description="enable hdf5 filter support") - variant("mdz", default=True, description="build mdz executable") - - depends_on("zstd") - depends_on("gsl") - depends_on("pkgconfig") - depends_on("hdf5", when="+hdf5") - - def setup_run_environment(self, env): - if "+hdf5" in self.spec: - env.prepend_path("HDF5_PLUGIN_PATH", self.prefix.lib64) - - def cmake_args(self): - return [ - "-DSZ3_USE_BUNDLED_ZSTD=OFF", - "-DSZ3_DEBUG_TIMINGS=OFF", - self.define_from_variant("BUILD_MDZ", "mdz"), - self.define_from_variant("BUILD_H5Z_FILTER", "hdf5"), - ] - - def test(self): - if self.spec.satisfies("@:3.1.6"): - print("smoke tests are only supported on 3.1.7 and later, skipping") - return - - self.run_test(self.prefix.bin.sz3_smoke_test, purpose="sz3 works") - - if "+mdz" in self.spec: - self.run_test(self.prefix.bin.mdz_smoke_test, purpose="mdz works") diff --git a/templates/common/run_icon.sh b/templates/common/run_icon.sh index 1d7ecc6a51a96a2cd0391e44bfe5e3a6335fa8ae..a72b0c827d9c2fe15010b2bb8e5dd16e8e4ed7d9 100644 --- a/templates/common/run_icon.sh +++ b/templates/common/run_icon.sh @@ -16,8 +16,15 @@ cd ${RUNDIR} load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%" "%spack.upstreams%" +# Get proper load command. +SPACK_BUILD_ICON="%ICON.BUILD_CMD%" +SPACK_LOAD_ICON="%ICON.LOAD_CMD%" + +if [ "${SPACK_LOAD_ICON}" == "build_cmd" ]; then + SPACK_LOAD_ICON=${SPACK_BUILD_ICON} +fi # Load icon module -spack load --first icon-nwp@%ICON_VERSION% +spack load --first ${SPACK_LOAD_ICON} # Set environment variable for eccodes-dwd definitions: source ${WORKDIR}/eccodes_defs.env