From 13b1330c95df6aedd402cf5cc146e3c99af4ff3e Mon Sep 17 00:00:00 2001
From: "Oriol.Tinto" <oriol.tinto@physik.uni-muenchen.de>
Date: Wed, 10 May 2023 09:53:21 +0000
Subject: [PATCH] Implementation to allow custom namelist modification through
configuration files.
---
conf/real-from-dwd-ana/simulation.yml | 2 +-
conf/real-from-ideal/simulation.yml | 2 +-
.../icon_master.namelist | 3 -
.../icon_atmosphere.namelist | 20 +--
.../icon_atmosphere_ideal.namelist | 17 +-
.../icon_atmosphere_real.namelist | 30 +---
.../real-from-ideal/icon_master.namelist | 16 --
.../real-from-dwd-ana/prepare_namelist.py | 167 ++++++++++++------
.../real-from-ideal/prepare_ideal_namelist.py | 149 +++++++++++-----
templates/real-from-ideal/prepare_namelist.py | 163 ++++++++++++-----
10 files changed, 360 insertions(+), 209 deletions(-)
rename namelists/{real-from-dwd-ana => common}/icon_master.namelist (70%)
delete mode 100644 namelists/real-from-ideal/icon_master.namelist
diff --git a/conf/real-from-dwd-ana/simulation.yml b/conf/real-from-dwd-ana/simulation.yml
index c3c274f..1a2f526 100644
--- a/conf/real-from-dwd-ana/simulation.yml
+++ b/conf/real-from-dwd-ana/simulation.yml
@@ -5,7 +5,7 @@ simulation:
date_format: '%Y-%m-%dT%H:%M:%SZ'
namelist_paths:
# Path to the namelists
- master: "%HPCROOTDIR%/proj/namelists/real-from-dwd-ana/icon_master.namelist"
+ master: "%HPCROOTDIR%/proj/namelists/common/icon_master.namelist"
atmosphere: "%HPCROOTDIR%/proj/namelists/real-from-dwd-ana/icon_atmosphere.namelist"
# List of output file names that will be copied (Wildcards * allowed)
diff --git a/conf/real-from-ideal/simulation.yml b/conf/real-from-ideal/simulation.yml
index ebafd90..c490054 100644
--- a/conf/real-from-ideal/simulation.yml
+++ b/conf/real-from-ideal/simulation.yml
@@ -5,7 +5,7 @@ simulation:
date_format: '%Y-%m-%dT%H:%M:%SZ'
namelist_paths:
# Path to the name lists
- master: "%HPCROOTDIR%/proj/namelists/real-from-ideal/icon_master.namelist"
+ master: "%HPCROOTDIR%/proj/namelists/common/icon_master.namelist"
atmosphere:
ideal: "%HPCROOTDIR%/proj/namelists/real-from-ideal/icon_atmosphere_ideal.namelist"
real: "%HPCROOTDIR%/proj/namelists/real-from-ideal/icon_atmosphere_real.namelist"
diff --git a/namelists/real-from-dwd-ana/icon_master.namelist b/namelists/common/icon_master.namelist
similarity index 70%
rename from namelists/real-from-dwd-ana/icon_master.namelist
rename to namelists/common/icon_master.namelist
index fe401c4..6ec9910 100644
--- a/namelists/real-from-dwd-ana/icon_master.namelist
+++ b/namelists/common/icon_master.namelist
@@ -1,5 +1,4 @@
&master_nml
- lrestart = "%is_restart%"
lrestart_write_last = .true.
/
@@ -11,6 +10,4 @@
&master_time_control_nml
calendar = "proleptic gregorian"
- experimentStartDate = '%Chunk_START_DATE%'
- experimentStopDate = '%Chunk_END_DATE%'
/
\ No newline at end of file
diff --git a/namelists/real-from-dwd-ana/icon_atmosphere.namelist b/namelists/real-from-dwd-ana/icon_atmosphere.namelist
index d05040f..de70193 100644
--- a/namelists/real-from-dwd-ana/icon_atmosphere.namelist
+++ b/namelists/real-from-dwd-ana/icon_atmosphere.namelist
@@ -11,13 +11,6 @@
iforcing = 3
/
-&time_nml
- dt_restart = '%checkpoint_time%'
-/
-
-&io_nml
- dt_checkpoint = '%checkpoint_time%'
-/
&nwp_phy_nml
lupatmo_phy = .FALSE.
@@ -25,20 +18,15 @@
&grid_nml
dynamics_parent_grid_id = 0
- dynamics_grid_filename = '%dynamics_grid_filename%'
- radiation_grid_filename = '%radiation_grid_filename%'
lredgrid_phys = .true.
/
&extpar_nml
itopo = 1
- extpar_filename = '%external_parameters_filename%'
/
&initicon_nml
init_mode = 1,
- dwdfg_filename = '%first_guess_filename%'
- dwdana_filename = '%analysis_filename%'
lconsistency_checks = .false.
ana_varnames_map_file = 'ana_varnames_map_file.txt'
/
@@ -69,8 +57,8 @@
! LATBC files, these files will be used as input for the next example.
&output_nml
file_interval = 'PT3600S'
- output_start = '%Chunk_START_DATE%'
- output_end = '%Chunk_END_DATE%'
+ output_start = '%OUTPUT_START%'
+ output_end = '%OUTPUT_END%'
output_filename = "latbc"
output_interval = 'PT3600S'
include_last = .true.
@@ -80,8 +68,8 @@
! First Guess file
&output_nml
file_interval = 'PT3600S'
- output_start = '%Chunk_START_DATE%'
- output_end = '%Chunk_END_DATE%'
+ output_start = '%OUTPUT_START%'
+ output_end = '%OUTPUT_END%'
output_filename = "init"
output_interval = 'PT3600S'
include_last = .true.
diff --git a/namelists/real-from-ideal/icon_atmosphere_ideal.namelist b/namelists/real-from-ideal/icon_atmosphere_ideal.namelist
index 094401d..8099750 100644
--- a/namelists/real-from-ideal/icon_atmosphere_ideal.namelist
+++ b/namelists/real-from-ideal/icon_atmosphere_ideal.namelist
@@ -13,8 +13,6 @@
&grid_nml
dynamics_parent_grid_id = 0
- dynamics_grid_filename = '%dynamics_grid_filename%'
- radiation_grid_filename = '%radiation_grid_filename%'
lredgrid_phys = .true.
/
@@ -34,13 +32,6 @@
/
-&time_nml
- dt_restart = '%checkpoint_time%'
-/
-
-&io_nml
- dt_checkpoint = '%checkpoint_time%'
-/
¶llel_nml
nproma = 16
@@ -49,8 +40,8 @@
! the following two output files are used to initialize the next run
&output_nml
file_interval = 'PT3600S'
- output_start = '%Chunk_START_DATE%'
- output_end = '%Chunk_END_DATE%'
+ output_start = '%OUTPUT_START%'
+ output_end = '%OUTPUT_END%'
output_filename = "init-test"
output_interval = 'PT3600S'
include_last = .true.
@@ -60,8 +51,8 @@
/
&output_nml
steps_per_file = 1
- output_start = '%Chunk_START_DATE%'
- output_end = '%Chunk_START_DATE%'
+ output_start = '%OUTPUT_START%'
+ output_end = '%OUTPUT_START%'
output_filename = "init-test-ext"
include_last = .true.
output_interval = 'PT3600S'
diff --git a/namelists/real-from-ideal/icon_atmosphere_real.namelist b/namelists/real-from-ideal/icon_atmosphere_real.namelist
index 35be207..56e8e93 100644
--- a/namelists/real-from-ideal/icon_atmosphere_real.namelist
+++ b/namelists/real-from-ideal/icon_atmosphere_real.namelist
@@ -11,22 +11,22 @@
iforcing = 3
/
+
+&nwp_phy_nml
+ lupatmo_phy = .FALSE.
+/
+
&grid_nml
dynamics_parent_grid_id = 0
- dynamics_grid_filename = '%dynamics_grid_filename%'
- radiation_grid_filename = '%radiation_grid_filename%'
lredgrid_phys = .true.
/
&extpar_nml
itopo = 1
- extpar_filename = 'extpar_DOM01.nc'
/
&initicon_nml
init_mode = 1,
- dwdfg_filename = 'init-test-fg_DOM01_ML_0001.nc'
- dwdana_filename = 'init-test-ana_DOM01_ML_0001.nc'
lconsistency_checks = .false.
/
@@ -42,8 +42,8 @@
! LATBC files
&output_nml
file_interval = 'PT3600S'
- output_start = '%Chunk_START_DATE%'
- output_end = '%Chunk_END_DATE%'
+ output_start = '%OUTPUT_START%'
+ output_end = '%OUTPUT_END%'
output_filename = "latbc"
output_interval = 'PT3600S'
include_last = .true.
@@ -53,22 +53,10 @@
! First Guess file
&output_nml
file_interval = 'PT3600S'
- output_start = '%Chunk_START_DATE%'
- output_end = '%Chunk_START_DATE%'
+ output_start = '%OUTPUT_START%'
+ output_end = '%OUTPUT_START%'
output_filename = "init"
output_interval = 'PT3600S'
include_last = .true.
ml_varlist = 'group:dwd_fg_atm_vars', 'group:dwd_fg_sfc_vars'
/
-
-&time_nml
- dt_restart = '%checkpoint_time%'
-/
-
-&io_nml
- dt_checkpoint = '%checkpoint_time%'
-/
-
-&nwp_phy_nml
- lupatmo_phy = .FALSE.
-/
\ No newline at end of file
diff --git a/namelists/real-from-ideal/icon_master.namelist b/namelists/real-from-ideal/icon_master.namelist
deleted file mode 100644
index fe401c4..0000000
--- a/namelists/real-from-ideal/icon_master.namelist
+++ /dev/null
@@ -1,16 +0,0 @@
-&master_nml
- lrestart = "%is_restart%"
- lrestart_write_last = .true.
-/
-
-&master_model_nml
- model_type = 1 ! atmospheric model
- model_name = "ATMO" ! name of this model component
- model_namelist_filename = "icon_atmosphere.namelist"
-/
-
-&master_time_control_nml
- calendar = "proleptic gregorian"
- experimentStartDate = '%Chunk_START_DATE%'
- experimentStopDate = '%Chunk_END_DATE%'
-/
\ No newline at end of file
diff --git a/templates/real-from-dwd-ana/prepare_namelist.py b/templates/real-from-dwd-ana/prepare_namelist.py
index 6656e4e..1ee82f8 100644
--- a/templates/real-from-dwd-ana/prepare_namelist.py
+++ b/templates/real-from-dwd-ana/prepare_namelist.py
@@ -1,9 +1,9 @@
import logging
-import re
from datetime import datetime, timedelta
from pathlib import Path
import f90nml
+import yaml
logger = logging.getLogger("prepare_chunk")
logger.setLevel(logging.INFO)
@@ -13,6 +13,14 @@ WORKDIR = "%HPCROOTDIR%"
STARTDATE = "%SDATE%"
MEMBER = "%MEMBER%"
CHUNK = "%CHUNK%"
+# Get run directory
+RUNDIR = Path(f"{WORKDIR}/{STARTDATE}/{MEMBER}")
+ATMOSPHERE_NAMELIST_PATH = Path("%simulation.namelist_paths.atmosphere%")
+MASTER_NAMELIST_PATH = Path("%simulation.namelist_paths.master%")
+# TODO: This is a bit ugly
+# Read first-guess and analysis filenames from files:
+first_guess_filename = (RUNDIR / "fg_file.txt").read_text().strip()
+analysis_filename = (RUNDIR / "an_file.txt").read_text().strip()
# Example of date format "2018-06-01T00:00:00Z"
date_format = "%simulation.date_format%"
@@ -30,66 +38,125 @@ END_HOUR = "%Chunk_END_HOUR%"
Chunk_START_DATE = datetime(year=int(START_YEAR), month=int(START_MONTH), day=int(START_DAY), hour=int(START_HOUR))
Chunk_END_DATE = datetime(year=int(END_YEAR), month=int(END_MONTH), day=int(END_DAY), hour=int(END_HOUR))
+# Read custom namelist parameters from configuration
+atmosphere_namelist_string = """
+%atmosphere_namelist%
+"""
+
+master_namelist_string = """
+%master_namelist%
+"""
+
# Compute difference in seconds
checkpoint_time = int((Chunk_END_DATE - Chunk_START_DATE).total_seconds())
# TODO: Is that really necessary?
# Add 10 minutes to allow the model to write the restarts
Chunk_END_DATE = Chunk_END_DATE + timedelta(minutes=10)
-# Get run directory
-RUNDIR = Path(f"{WORKDIR}/{STARTDATE}/{MEMBER}")
-# TODO: This is a bit ugly
-# Read first-guess and analysis filenames from files:
-first_guess_filename = (RUNDIR / "fg_file.txt").read_text().strip()
-analysis_filename = (RUNDIR / "an_file.txt").read_text().strip()
-
-# Get some variable replacements from the proj.yml file through autosubmit
-variable_replacements = {
- "dynamics_grid_filename": "%simulation.dynamics_grid_filename%",
- "radiation_grid_filename": "%simulation.radiation_grid_filename%",
- "external_parameters_filename": "%simulation.external_parameters_filename%",
- "first_guess_filename": first_guess_filename,
- "analysis_filename": analysis_filename,
- "Chunk_START_DATE": Chunk_START_DATE.strftime(date_format),
- "Chunk_END_DATE": Chunk_END_DATE.strftime(date_format),
- "is_restart": False if "%CHUNK%" == "1" else True,
- "checkpoint_time": checkpoint_time,
+atmosphere_namelist_replacements = {
+ "time_nml": {
+ "dt_restart": checkpoint_time
+ },
+ "io_nml": {
+ "dt_checkpoint": checkpoint_time
+ },
+
+ "grid_nml": {
+ "dynamics_grid_filename": "%simulation.dynamics_grid_filename%",
+ "radiation_grid_filename": "%simulation.radiation_grid_filename%",
+ },
+
+ "extpar_nml": {
+ "extpar_filename": "%simulation.external_parameters_filename%",
+ },
+
+ "initicon_nml": {
+ "dwdfg_filename": first_guess_filename,
+ "dwdana_filename": analysis_filename,
+ }
}
+master_namelist_replacements = {
+ "master_nml": {
+ "lrestart": False if "%CHUNK%" == "1" else True,
+ },
+ "master_time_control_nml": {
+ "experimentStartDate": Chunk_START_DATE.strftime(date_format),
+ "experimentStopDate": Chunk_END_DATE.strftime(date_format),
+ }
+}
-def adapt_namelist(input_namelist: str, output_namelist: str):
- input_namelist = Path(input_namelist)
- output_namelist = Path(output_namelist)
-
- namelist = f90nml.read(input_namelist.as_posix())
- group_keys = [gk for gk in namelist]
-
- for group in group_keys:
- variable_keys = [vk for vk in namelist[group]]
- for variable in variable_keys:
- value = namelist[group][variable]
- m = re.match(r"%(.*)%", str(value))
- if m:
- key = m.group(1)
-
- if key not in variable_replacements:
- raise AssertionError(f"The namelist {input_namelist.as_posix()!r} contains the variable {key!r} "
- f"which is not in the list of provided replacements:\n"
- f"{[v for v in variable_replacements]}")
- logger.info(f"Replacing {group}>{variable}:{key} with {variable_replacements[key]!r}")
- namelist[group][variable] = variable_replacements[key]
- f90nml.write(nml=namelist, nml_path=output_namelist.as_posix(), force=True)
+def read_namelist(namelist_string: str) -> dict:
+ """
+ Function to read the custom namelist specifications provided in the configuration files.
+ It accepts both yaml and f90nml format.
+ :param namelist_string:
+ :return:
+ """
+ parameters = yaml.safe_load(namelist_string)
+ if isinstance(parameters, str):
+ parameters = f90nml.reads(nml_string=namelist_string).todict()
+ return parameters
+
+
+def patch_output_entries(namelist: f90nml.Namelist) -> f90nml.Namelist:
+ output_entries = [entry for entry in namelist["output_nml"]]
+ for entry in output_entries:
+ for key in entry:
+ if entry[key] == "%OUTPUT_START%":
+ entry[key] = Chunk_START_DATE.strftime(date_format)
+ elif entry[key] == "%OUTPUT_END%":
+ entry[key] = Chunk_END_DATE.strftime(date_format)
+
+ return namelist
+
+
+def main():
+ """
+ Main function that processes both atmosphere and master namelists and adds the necessary patches
+ :return:
+ """
+ # Process atmosphere namelist
+ atmosphere_namelist = f90nml.read(ATMOSPHERE_NAMELIST_PATH.as_posix())
+ print("Original atmosphere namelist:")
+ print(atmosphere_namelist)
+ atmosphere_namelist.patch(atmosphere_namelist_replacements)
+
+ # Read custom namelist parameters from configuration file
+ atmosphere_custom_namelist = read_namelist(atmosphere_namelist_string)
+
+ if atmosphere_custom_namelist is not None:
+ try:
+ atmosphere_namelist.patch(atmosphere_custom_namelist)
+ except AttributeError:
+ raise AssertionError("Problem applying the namelist patch! Probably related with the output section.")
+
+ # Patch output entries:
+ atmosphere_namelist = patch_output_entries(atmosphere_namelist)
+
+ print("Patched atmosphere namelist:")
+ print(atmosphere_namelist)
+
+ atmosphere_output_namelist = (RUNDIR / "icon_atmosphere.namelist")
+ f90nml.write(nml=atmosphere_namelist, nml_path=atmosphere_output_namelist.as_posix(), force=True)
+
+ master_namelist = f90nml.read(MASTER_NAMELIST_PATH.as_posix())
+ print("Original master namelist:")
+ print(master_namelist)
+ # Read custom namelist parameters from configuration file
+ master_custom_namelist = read_namelist(master_namelist_string)
+ # Process atmosphere namelist
+ master_namelist.patch(master_namelist_replacements)
+ if master_custom_namelist is not None:
+ master_namelist.patch(master_custom_namelist)
+ print("Patched master namelist:")
+ print(master_namelist)
+ master_output_namelist = (RUNDIR / "icon_master.namelist")
+ f90nml.write(nml=master_namelist, nml_path=master_output_namelist.as_posix(), force=True)
if __name__ == '__main__':
- atmosphere_namelist_path = "%simulation.namelist_paths.atmosphere%"
- master_namelist_path = "%simulation.namelist_paths.master%"
-
- # Adapt atmosphere namelist
- adapt_namelist(input_namelist=atmosphere_namelist_path,
- output_namelist=(RUNDIR / "icon_atmosphere.namelist").as_posix())
- # Adapt master namelist
- adapt_namelist(input_namelist=master_namelist_path,
- output_namelist=(RUNDIR / "icon_master.namelist").as_posix())
+ main()
+
diff --git a/templates/real-from-ideal/prepare_ideal_namelist.py b/templates/real-from-ideal/prepare_ideal_namelist.py
index e05e8d6..1474fc4 100644
--- a/templates/real-from-ideal/prepare_ideal_namelist.py
+++ b/templates/real-from-ideal/prepare_ideal_namelist.py
@@ -1,9 +1,9 @@
import logging
-import re
from datetime import datetime, timedelta
from pathlib import Path
import f90nml
+import yaml
logger = logging.getLogger("prepare_chunk")
logger.setLevel(logging.INFO)
@@ -11,6 +11,10 @@ logger.setLevel(logging.INFO)
# Get some autosubmit variables
WORKDIR = "%HPCROOTDIR%"
STARTDATE = "%SDATE%"
+# Get run directory
+RUNDIR = Path(f"{WORKDIR}/{STARTDATE}/ideal")
+ATMOSPHERE_NAMELIST_PATH = Path("%simulation.namelist_paths.atmosphere.ideal%")
+MASTER_NAMELIST_PATH = Path("%simulation.namelist_paths.master%")
# Example of date format "2018-06-01T00:00:00Z"
date_format = "%simulation.date_format%"
@@ -28,59 +32,122 @@ END_HOUR = "%Chunk_END_HOUR%"
Chunk_START_DATE = datetime(year=int(START_YEAR), month=int(START_MONTH), day=int(START_DAY), hour=int(START_HOUR))
Chunk_END_DATE = datetime(year=int(END_YEAR), month=int(END_MONTH), day=int(END_DAY), hour=int(END_HOUR))
+
+# Read custom namelist parameters from configuration
+atmosphere_namelist_string = """
+%atmosphere_namelist_ideal%
+"""
+
+master_namelist_string = """
+%master_namelist_ideal%
+"""
+
# Compute difference in seconds
checkpoint_time = int((Chunk_END_DATE - Chunk_START_DATE).total_seconds())
# TODO: Is that really necessary?
# Add 10 minutes to allow the model to write the restarts
Chunk_END_DATE = Chunk_END_DATE + timedelta(minutes=10)
-# Get run directory
-RUNDIR = Path(f"{WORKDIR}/{STARTDATE}/ideal")
-# Get some variable replacements from the proj.yml file through autosubmit
-variable_replacements = {
- "dynamics_grid_filename": "%simulation.dynamics_grid_filename%",
- "radiation_grid_filename": "%simulation.radiation_grid_filename%",
- "external_parameters_filename": "%simulation.external_parameters_filename%",
- "Chunk_START_DATE": Chunk_START_DATE.strftime(date_format),
- "Chunk_END_DATE": Chunk_END_DATE.strftime(date_format),
- "is_restart": False if "%CHUNK%" == "1" else True,
- "checkpoint_time": checkpoint_time,
-}
+atmosphere_namelist_replacements = {
+ "time_nml": {
+ "dt_restart": checkpoint_time
+ },
+ "io_nml": {
+ "dt_checkpoint": checkpoint_time
+ },
+ "grid_nml": {
+ "dynamics_grid_filename": "%simulation.dynamics_grid_filename%",
+ "radiation_grid_filename": "%simulation.radiation_grid_filename%",
+ },
-def adapt_namelist(input_namelist: str, output_namelist: str):
- input_namelist = Path(input_namelist)
- output_namelist = Path(output_namelist)
+ "extpar_nml": {
+ "extpar_filename": "%simulation.external_parameters_filename%",
+ },
- namelist = f90nml.read(input_namelist.as_posix())
- group_keys = [gk for gk in namelist]
+}
- for group in group_keys:
- variable_keys = [vk for vk in namelist[group]]
- for variable in variable_keys:
- value = namelist[group][variable]
- m = re.match(r"%(.*)%", str(value))
- if m:
- key = m.group(1)
+master_namelist_replacements = {
+ "master_nml": {
+ "lrestart": False if "%CHUNK%" == "1" else True,
+ },
+ "master_time_control_nml": {
+ "experimentStartDate": Chunk_START_DATE.strftime(date_format),
+ "experimentStopDate": Chunk_END_DATE.strftime(date_format),
+ }
+}
- if key not in variable_replacements:
- raise AssertionError(f"The namelist {input_namelist.as_posix()!r} contains the variable {key!r} "
- f"which is not in the list of provided replacements:\n"
- f"{[v for v in variable_replacements]}")
- logger.info(f"Replacing {group}>{variable}:{key} with {variable_replacements[key]!r}")
- namelist[group][variable] = variable_replacements[key]
- f90nml.write(nml=namelist, nml_path=output_namelist.as_posix(), force=True)
+def read_namelist(namelist_string: str) -> dict:
+ """
+ Function to read the custom namelist specifications provided in the configuration files.
+ It accepts both yaml and f90nml format.
+ :param namelist_string:
+ :return:
+ """
+ parameters = yaml.safe_load(namelist_string)
+ if isinstance(parameters, str):
+ parameters = f90nml.reads(nml_string=namelist_string).todict()
+ return parameters
+
+
+def patch_output_entries(namelist: f90nml.Namelist) -> f90nml.Namelist:
+ output_entries = [entry for entry in namelist["output_nml"]]
+ for entry in output_entries:
+ for key in entry:
+ if entry[key] == "%OUTPUT_START%":
+ entry[key] = Chunk_START_DATE.strftime(date_format)
+ elif entry[key] == "%OUTPUT_END%":
+ entry[key] = Chunk_END_DATE.strftime(date_format)
+
+ return namelist
+
+
+def main():
+ """
+ Main function that processes both atmosphere and master namelists and adds the necessary patches
+ :return:
+ """
+ # Process atmosphere namelist
+ atmosphere_namelist = f90nml.read(ATMOSPHERE_NAMELIST_PATH.as_posix())
+ print("Original atmosphere namelist:")
+ print(atmosphere_namelist)
+ atmosphere_namelist.patch(atmosphere_namelist_replacements)
+
+ # Read custom namelist parameters from configuration file
+ atmosphere_custom_namelist = read_namelist(atmosphere_namelist_string)
+
+ if atmosphere_custom_namelist is not None:
+ try:
+ atmosphere_namelist.patch(atmosphere_custom_namelist)
+ except AttributeError:
+ raise AssertionError("Problem applying the namelist patch! Probably related with the output section.")
+
+ # Patch output entries:
+ atmosphere_namelist = patch_output_entries(atmosphere_namelist)
+
+ print("Patched atmosphere namelist:")
+ print(atmosphere_namelist)
+
+ atmosphere_output_namelist = (RUNDIR / "icon_atmosphere.namelist")
+ f90nml.write(nml=atmosphere_namelist, nml_path=atmosphere_output_namelist.as_posix(), force=True)
+
+ master_namelist = f90nml.read(MASTER_NAMELIST_PATH.as_posix())
+ print("Original master namelist:")
+ print(master_namelist)
+ # Read custom namelist parameters from configuration file
+ master_custom_namelist = read_namelist(master_namelist_string)
+ # Process atmosphere namelist
+ master_namelist.patch(master_namelist_replacements)
+ if master_custom_namelist is not None:
+ master_namelist.patch(master_custom_namelist)
+ print("Patched master namelist:")
+ print(master_namelist)
+ master_output_namelist = (RUNDIR / "icon_master.namelist")
+ f90nml.write(nml=master_namelist, nml_path=master_output_namelist.as_posix(), force=True)
if __name__ == '__main__':
- atmosphere_namelist_path = "%simulation.namelist_paths.atmosphere.ideal%"
- master_namelist_path = "%simulation.namelist_paths.master%"
-
- # Adapt atmosphere namelist
- adapt_namelist(input_namelist=atmosphere_namelist_path,
- output_namelist=(RUNDIR / "icon_atmosphere.namelist").as_posix())
- # Adapt master namelist
- adapt_namelist(input_namelist=master_namelist_path,
- output_namelist=(RUNDIR / "icon_master.namelist").as_posix())
+ main()
+
diff --git a/templates/real-from-ideal/prepare_namelist.py b/templates/real-from-ideal/prepare_namelist.py
index 2322efd..dd24930 100644
--- a/templates/real-from-ideal/prepare_namelist.py
+++ b/templates/real-from-ideal/prepare_namelist.py
@@ -1,9 +1,9 @@
import logging
-import re
from datetime import datetime, timedelta
from pathlib import Path
import f90nml
+import yaml
logger = logging.getLogger("prepare_chunk")
logger.setLevel(logging.INFO)
@@ -13,6 +13,13 @@ WORKDIR = "%HPCROOTDIR%"
STARTDATE = "%SDATE%"
MEMBER = "%MEMBER%"
CHUNK = "%CHUNK%"
+# Get run directory
+RUNDIR = Path(f"{WORKDIR}/{STARTDATE}/{MEMBER}")
+ATMOSPHERE_NAMELIST_PATH = Path("%simulation.namelist_paths.atmosphere.real%")
+MASTER_NAMELIST_PATH = Path("%simulation.namelist_paths.master%")
+# Set first-guess and analysis filenames:
+first_guess_filename = "init-test-fg_DOM01_ML_0001.nc"
+analysis_filename = "init-test-ana_DOM01_ML_0001.nc"
# Example of date format "2018-06-01T00:00:00Z"
date_format = "%simulation.date_format%"
@@ -30,63 +37,125 @@ END_HOUR = "%Chunk_END_HOUR%"
Chunk_START_DATE = datetime(year=int(START_YEAR), month=int(START_MONTH), day=int(START_DAY), hour=int(START_HOUR))
Chunk_END_DATE = datetime(year=int(END_YEAR), month=int(END_MONTH), day=int(END_DAY), hour=int(END_HOUR))
+# Read custom namelist parameters from configuration
+atmosphere_namelist_string = """
+%atmosphere_namelist%
+"""
+
+master_namelist_string = """
+%master_namelist%
+"""
+
# Compute difference in seconds
checkpoint_time = int((Chunk_END_DATE - Chunk_START_DATE).total_seconds())
# TODO: Is that really necessary?
# Add 10 minutes to allow the model to write the restarts
Chunk_END_DATE = Chunk_END_DATE + timedelta(minutes=10)
-# Get run directory
-RUNDIR = Path(f"{WORKDIR}/{STARTDATE}/{MEMBER}")
-# TODO: This is a bit ugly
-
-# Get some variable replacements from the proj.yml file through autosubmit
-variable_replacements = {
- "dynamics_grid_filename": "%simulation.dynamics_grid_filename%",
- "radiation_grid_filename": "%simulation.radiation_grid_filename%",
- "external_parameters_filename": "%simulation.external_parameters_filename%",
- "first_guess_filename": "init-test-fg_DOM01_ML_0001.nc",
- "analysis_filename": "init-test-ana_DOM01_ML_0001.nc",
- "Chunk_START_DATE": Chunk_START_DATE.strftime(date_format),
- "Chunk_END_DATE": Chunk_END_DATE.strftime(date_format),
- "is_restart": False if "%CHUNK%" == "1" else True,
- "checkpoint_time": checkpoint_time,
+atmosphere_namelist_replacements = {
+ "time_nml": {
+ "dt_restart": checkpoint_time
+ },
+ "io_nml": {
+ "dt_checkpoint": checkpoint_time
+ },
+
+ "grid_nml": {
+ "dynamics_grid_filename": "%simulation.dynamics_grid_filename%",
+ "radiation_grid_filename": "%simulation.radiation_grid_filename%",
+ },
+
+ "extpar_nml": {
+ "extpar_filename": "%simulation.external_parameters_filename%",
+ },
+
+ "initicon_nml": {
+ "dwdfg_filename": first_guess_filename,
+ "dwdana_filename": analysis_filename,
+ }
}
+master_namelist_replacements = {
+ "master_nml": {
+ "lrestart": False if "%CHUNK%" == "1" else True,
+ },
+ "master_time_control_nml": {
+ "experimentStartDate": Chunk_START_DATE.strftime(date_format),
+ "experimentStopDate": Chunk_END_DATE.strftime(date_format),
+ }
+}
-def adapt_namelist(input_namelist: str, output_namelist: str):
- input_namelist = Path(input_namelist)
- output_namelist = Path(output_namelist)
-
- namelist = f90nml.read(input_namelist.as_posix())
- group_keys = [gk for gk in namelist]
-
- for group in group_keys:
- variable_keys = [vk for vk in namelist[group]]
- for variable in variable_keys:
- value = namelist[group][variable]
- m = re.match(r"%(.*)%", str(value))
- if m:
- key = m.group(1)
-
- if key not in variable_replacements:
- raise AssertionError(f"The namelist {input_namelist.as_posix()!r} contains the variable {key!r} "
- f"which is not in the list of provided replacements:\n"
- f"{[v for v in variable_replacements]}")
- logger.info(f"Replacing {group}>{variable}:{key} with {variable_replacements[key]!r}")
- namelist[group][variable] = variable_replacements[key]
- f90nml.write(nml=namelist, nml_path=output_namelist.as_posix(), force=True)
+def read_namelist(namelist_string: str) -> dict:
+ """
+ Function to read the custom namelist specifications provided in the configuration files.
+ It accepts both yaml and f90nml format.
+ :param namelist_string:
+ :return:
+ """
+ parameters = yaml.safe_load(namelist_string)
+ if isinstance(parameters, str):
+ parameters = f90nml.reads(nml_string=namelist_string).todict()
+ return parameters
+
+
+def patch_output_entries(namelist: f90nml.Namelist) -> f90nml.Namelist:
+ output_entries = [entry for entry in namelist["output_nml"]]
+ for entry in output_entries:
+ for key in entry:
+ if entry[key] == "%OUTPUT_START%":
+ entry[key] = Chunk_START_DATE.strftime(date_format)
+ elif entry[key] == "%OUTPUT_END%":
+ entry[key] = Chunk_END_DATE.strftime(date_format)
+
+ return namelist
+
+
+def main():
+ """
+ Main function that processes both atmosphere and master namelists and adds the necessary patches
+ :return:
+ """
+ # Process atmosphere namelist
+ atmosphere_namelist = f90nml.read(ATMOSPHERE_NAMELIST_PATH.as_posix())
+ print("Original atmosphere namelist:")
+ print(atmosphere_namelist)
+ atmosphere_namelist.patch(atmosphere_namelist_replacements)
+
+ # Read custom namelist parameters from configuration file
+ atmosphere_custom_namelist = read_namelist(atmosphere_namelist_string)
+
+ if atmosphere_custom_namelist is not None:
+ try:
+ atmosphere_namelist.patch(atmosphere_custom_namelist)
+ except AttributeError:
+ raise AssertionError("Problem applying the namelist patch! Probably related with the output section.")
+
+ # Patch output entries:
+ atmosphere_namelist = patch_output_entries(atmosphere_namelist)
+
+ print("Patched atmosphere namelist:")
+ print(atmosphere_namelist)
+
+ atmosphere_output_namelist = (RUNDIR / "icon_atmosphere.namelist")
+ f90nml.write(nml=atmosphere_namelist, nml_path=atmosphere_output_namelist.as_posix(), force=True)
+
+ master_namelist = f90nml.read(MASTER_NAMELIST_PATH.as_posix())
+ print("Original master namelist:")
+ print(master_namelist)
+ # Read custom namelist parameters from configuration file
+ master_custom_namelist = read_namelist(master_namelist_string)
+ # Process atmosphere namelist
+ master_namelist.patch(master_namelist_replacements)
+ if master_custom_namelist is not None:
+ master_namelist.patch(master_custom_namelist)
+ print("Patched master namelist:")
+ print(master_namelist)
+ master_output_namelist = (RUNDIR / "icon_master.namelist")
+ f90nml.write(nml=master_namelist, nml_path=master_output_namelist.as_posix(), force=True)
if __name__ == '__main__':
- atmosphere_namelist_path = "%simulation.namelist_paths.atmosphere.real%"
- master_namelist_path = "%simulation.namelist_paths.master%"
-
- # Adapt atmosphere namelist
- adapt_namelist(input_namelist=atmosphere_namelist_path,
- output_namelist=(RUNDIR / "icon_atmosphere.namelist").as_posix())
- # Adapt master namelist
- adapt_namelist(input_namelist=master_namelist_path,
- output_namelist=(RUNDIR / "icon_master.namelist").as_posix())
+ main()
+
--
GitLab