diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..eeb8a6ec40875917aba8a4d6a6a335d2011e6253
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+**/__pycache__
diff --git a/README.md b/README.md
index d6f99021d7e60043816aeda416dfd57e8ad9c081..73b08f4116aebd847e10db0364929797b4701593 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,127 @@
# Autosubmit-Icon-Today
-Set of templates and scripts to run Icon's forecast for today.
+Available examples:
+- **real-from-ideal**: Start with a test simulation (ltestcase=.TRUE.) to produce a first-guess file and an analysis file to start a second simulation.
+- **real-from-dwd-ana**: Start with a first-guess and analysis from DWD (relying on the local archive to do that.)
+> **_NOTE:_** The current implementation of **real-from-dwd-ana** relies on the local archives at LMU. The templates to provide the initial conditions have to be modified to run from a different local system.
+
+With autosubmit installed, we can run these ICON workflows with **autosubmit** we need to do the following:
+
+
+1. Create an experiment:
+
+ ```bash
+ autosubmit expid -min -repo https://gitlab.physik.uni-muenchen.de/w2w/autosubmit-icon-today.git -d "Launching an ICON experiment with autosubmit."
+ ```
+
+ This will create a new experiment folder and will give us the corresponding experiment id.
+
+2. Modify the **minimal.yml** configuration file
+
+ If we go to the **conf** folder corresponding to our experiment, we will find a **minimal.yml** file there:
+```yaml
+CONFIG:
+ AUTOSUBMIT_VERSION: "4.0.76"
+ TOTALJOBS: 20
+ MAXWAITINGJOBS: 20
+DEFAULT:
+ EXPID: "a00p"
+ HPCARCH: "local"
+ #hint: use %PROJDIR% to point to the project folder (where the project is cloned)
+ CUSTOM_CONFIG: "%PROJDIR%/"
+PROJECT:
+ PROJECT_TYPE: "git"
+ PROJECT_DESTINATION: "git_project"
+GIT:
+ PROJECT_ORIGIN: "https://gitlab.physik.uni-muenchen.de/w2w/autosubmit-icon-today.git"
+ PROJECT_BRANCH: ""
+ PROJECT_COMMIT: ""
+ PROJECT_SUBMODULES: ""
+ FETCH_SINGLE_BRANCH: True
+
+```
+
+We will modify this file as follows:
+ - We will add a new variable to select which example we do want to run: `ICON_CASE: "real-from-ideal"`
+ - We will modify **CUSTOM_CONFIG** to point to the desired configurations.
+ - We will add a path in the local machine at which the output data will be transferred.
+ - We will modify **HPCARCH** to point to our desired platform.
+```yaml
+# Select case between real-from-ideal, real-from-dwd-ana
+ICON_CASE: "real-from-ideal"
+
+CONFIG:
+ AUTOSUBMIT_VERSION: "4.0.76"
+ TOTALJOBS: 20
+ MAXWAITINGJOBS: 20
+DEFAULT:
+ EXPID: "a00p" # Will depend on the actual experiment id.
+ HPCARCH: "LRZ"
+ #hint: use %PROJDIR% to point to the project folder (where the project is cloned)
+ CUSTOM_CONFIG:
+ PRE:
+ - "%PROJDIR%/conf/common"
+ - "%PROJDIR%/conf/%ICON_CASE%"
+
+PROJECT:
+ PROJECT_TYPE: "git"
+ PROJECT_DESTINATION: "git_project"
+GIT:
+ PROJECT_ORIGIN: "https://gitlab.physik.uni-muenchen.de/w2w/autosubmit-icon-today.git"
+ PROJECT_BRANCH: ""
+ PROJECT_COMMIT: ""
+ PROJECT_SUBMODULES: ""
+ FETCH_SINGLE_BRANCH: True
+
+data_management:
+ # Where do we put the output files afterwards?
+ local_destination_folder: /Path/to/output/folder
+
+```
+
+3. Modify build command.
+
+We are using spack to build icon. The default command that would be used to build icon would be:
+
+```bash
+spack install icon-nwp@version%compiler
+```
+
+> **_NOTE:_** Since in the LMU example we are building everything from scratch, probably it is the best starting point to try to build icon with spack in a new platform.
+
+However, in some systems it is necessary to link to specific libraries or use existing packages.
+Some additional parameters need to be changed in the configuration to allow that.
+Here we have two examples:
+- At **LMU** we need to compile openmpi with a specific list of fabrics, and we need to use the external slurm as well. To do that we provide this additional configuration file:
+
+```yaml
+spack:
+ externals: "slurm"
+icon:
+ build_cmd: "icon-nwp@%ICON.VERSION%% %SPACK.COMPILER% ^openmpi+pmi+legacylaunchers schedulers=slurm fabrics=ucx ucx+dc+dm+ib_hw_tm+mlx5_dv+rc+rdmacm+thread_multiple+ud+verbs"
+```
+
+- At **LRZ** we rely on the spack package already available in the system that allows us to reuse most of the packages. To do so we add the following configuration:
+
+```yaml
+spack:
+ init: "module load user_spack" # command to load spack environment, e.g. module load spack, use spack/setup-env.sh if empty
+ root: "" # Setting this to an empty string to use the existing module.
+ compiler: gcc@11.2.0 # desired compiler for spack. Matching the one used for the existing packages in LRZ
+
+icon:
+ build_cmd: "icon-nwp@%ICON.VERSION%% %SPACK.COMPILER% ^openmpi/amct7nx"
+```
+
+4. Platforms
+
+The user needs to provide a platforms file. It can be done in different ways but the easiest is to put the yaml file in the configuration folder.
+Because we wanted to execute some of the lightweight runs on the login nodes of the remote platform, it is necessary to define not only the desired platform but also an additional login.
+For example, if we use the platform **LMU**, we need to define **LMU_LOGIN** as well.
+
+5. Github password-less access
+In order to get icon-sources through autosubmit, the password-less through ssh keys should be configured for dkrz gitlab.
+
+6. Create and run
+With this changes in the configuration should be already possible to create the experiment and run it.
\ No newline at end of file
diff --git a/spack_repo/packages/icon-nwp/package.py b/spack_repo/packages/icon-nwp/package.py
index e97a858e9c82f407ebf4e74760c9c266b3447377..40a28488e32f652712703af23b6f8dfbc9bd1209 100644
--- a/spack_repo/packages/icon-nwp/package.py
+++ b/spack_repo/packages/icon-nwp/package.py
@@ -23,10 +23,13 @@ class IconNwp(Package):
# version("2.5.0-nwp3", branch="icon-nwp/op-release-2.5.0-nwp3")
version("2.6.4-nwp3", branch="icon-nwp/op-release-2.6.4-nwp3")
version("2.6.5-nwp0", branch="icon-nwp/op-release-2.6.5-nwp0")
- version("master", branch="master")
version("psp", branch="icon-nwp/icon-nwp-psp")
+ version("plexrt", branch="icon-nwp/icon-nwp-plexrt")
+ version("master", branch="master")
variant("lmu", default=False, description="if git.url and submodules should be patched to use the LMU mirrors")
+ variant("plexrt", default=False, description="add 3D radiation support with TenStream")
+ variant("petsc", default=False, description="add PETSc support")
# Dependencies
depends_on("mpi")
@@ -35,6 +38,11 @@ class IconNwp(Package):
depends_on("eccodes+fortran")
depends_on("libxml2")
+ depends_on("petsc", when="@psp")
+ depends_on("petsc", when="+petsc")
+ depends_on("tenstream", when="@plexrt")
+ depends_on("tenstream", when="+plexrt")
+
# Openblas? best way of doing it?
depends_on("openblas", when="%gcc")
depends_on("intel-mkl", when="%intel")
@@ -162,13 +170,27 @@ class IconNwp(Package):
libs.append("-lyaml")
mtune = "generic"
- INCLUDES = f"-I{spec['libxml2'].prefix}/include/libxml2"
+ INCLUDES = [f"-I{spec['libxml2'].prefix}/include/libxml2", ]
+ CPPFLAGS = []
+
+ if "+plexrt" in self.spec:
+ CPPFLAGS.append("-DHAVE_PLEXRT")
+ INCLUDES.append(f"-I{spec['tenstream'].prefix}/include")
+ libs.append(f"-L{spec['tenstream'].prefix.lib} -ltenstream")
+
+ if "+petsc" in self.spec:
+ CPPFLAGS.append("-DHAVE_PETSC")
+ INCLUDES.append(f"-I{spec['petsc'].prefix}/include")
+ libs.append(f"-L{spec['petsc'].prefix.lib} -lpetsc")
+
+
options = [
f"CC={spec['mpi'].mpicc}",
f"FC={spec['mpi'].mpifc}",
- f"CFLAGS=-g -mpc64 {INCLUDES}",
+ f"CFLAGS=-g -mpc64 {' '.join(INCLUDES)}",
f"ICON_CFLAGS=-O3 -g -mtune={mtune}",
f"ICON_BUNDLED_CFLAGS=-O3 -mtune={mtune}",
+ f"CPPFLAGS={' '.join(CPPFLAGS)}",
"FCFLAGS=-std=legacy -fmodule-private -fimplicit-none -fmax-identifier-length=63 -Wall -Wcharacter-truncation -Wconversion -Wunderflow -Wunused-parameter -Wno-surprising -fall-intrinsics -g -mpc64 -w",
"ICON_FCFLAGS=-fbacktrace -fbounds-check -fstack-protector-all -finit-real=nan -finit-integer=-2147483648 -finit-character=127 -w -O2",
f"ICON_OCEAN_FCFLAGS=-O3 -mtune={mtune}",
diff --git a/spack_repo/packages/tenstream/package.py b/spack_repo/packages/tenstream/package.py
new file mode 100644
index 0000000000000000000000000000000000000000..56ad0f21392663c2aa0c25eb37244e77b9c3a64f
--- /dev/null
+++ b/spack_repo/packages/tenstream/package.py
@@ -0,0 +1,51 @@
+# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+class Tenstream(CMakePackage):
+ """
+ Recipe to build TenStream radiative transfer lib.
+ """
+
+ homepage = "https://gitlab.com/jakubfabian/tenstream.git"
+
+ git = "https://gitlab.com/jakubfabian/tenstream.git"
+
+ version("master", branch="master")
+ version("next", branch="next")
+
+ variant("debug", default=False, description="use debug flags to build")
+ variant("rayli", default=False, description="download and build with RayLI - MonteCarlo raytracer for unstructured meshes")
+
+ # Dependencies
+ depends_on("mpi")
+ depends_on("cmake")
+ depends_on("netcdf-fortran")
+ depends_on("petsc")
+
+ def setup_build_environment(self, env):
+ spec = self.spec
+
+ # Some environment variables to set
+ env_variables_to_set = {
+ "CC": spec["mpi"].mpicc,
+ "CXX": spec["mpi"].mpicxx,
+ "FC": spec["mpi"].mpifc,
+ "F77": spec["mpi"].mpif77,
+ }
+ for variable, value in env_variables_to_set.items():
+ env.set(variable, value)
+
+ def cmake_args(self):
+ args = []
+
+ if "+debug" in self.spec:
+ args.append('-DCMAKE_BUILD_TYPE=DEBUG')
+
+ if "+rayli" in self.spec:
+ args.append("-DBUILD_RAYLI=ON")
+
+ return args