Replace prepare_chunk.sh with prepare_chunk.py. Adapt configuration files.

conf/jobs_real-from-dwd-ana.yaml

@@ -68,6 +68,10 @@ JOBS:
     WALLCLOCK: 01:00
     PROCESSORS: 16
 
+  TRANSFER_NAMELISTS:
+    FILE: templates/common/transfer_namelists.sh
+
+
   PREPARE_EXPERIMENT:
     FILE: templates/common/prepare_experiment.sh
     DEPENDENCIES: BUILD_ICON
@@ -86,10 +90,13 @@ JOBS:
     WALLCLOCK: 01:00
 
   PREPARE_CHUNK:
-    FILE: templates/common/prepare_chunk.sh
-    DEPENDENCIES: PREPARE_MEMBER RUN_ICON-1
+    FILE: templates/common/prepare_chunk.py
+    DEPENDENCIES: TRANSFER_NAMELISTS BUILD_PYTHON_ENVIRONMENT PREPARE_MEMBER RUN_ICON-1
     WALLCLOCK: 00:05
     RUNNING: chunk
+    TYPE: python
+    EXECUTABLE: "%HPCROOTDIR%/%python_environment.folder_name%/bin/python3"
+
 
   RUN_ICON:
     FILE: templates/common/run_icon.sh

conf/proj_real-from-dwd-ana.yaml

@@ -21,10 +21,10 @@ simulation:
   dynamics_grid_filename: icon_grid_0016_R02B06_G.nc
   radiation_grid_filename: icon_grid_0015_R02B05_R.nc
   external_parameters_filename: icon_extpar_0016_R02B06_G_20131206.nc
-  namelist:
+  namelist_paths:
     # Path to the namelists
-    master_path: "%PROJDIR%/namelists/icon_master_real-from-dwd-ana.namelist"
-    atmosphere_path: "%PROJDIR%/namelists/icon_atmosphere_real-from-dwd-ana.namelist"
+    master: "%HPCROOTDIR%/namelists/icon_master_real-from-dwd-ana.namelist"
+    atmosphere: "%HPCROOTDIR%/namelists/icon_atmosphere_real-from-dwd-ana.namelist"
 
   # List of output file names that will be copied (Wildcards * allowed)
   output_file_names: "init_DOM01_ML_*.nc latbc_DOM01_ML_*.nc"

templates/common/prepare_chunk.py

+import logging
+import re
+from pathlib import Path
+
+import f90nml
+
+logger = logging.getLogger("prepare_chunk")
+logger.setLevel(logging.INFO)
+
+# Get some autosubmit variables
+WORKDIR = "%HPCROOTDIR%"
+STARTDATE = "%SDATE%"
+MEMBER = "%MEMBER%"
+CHUNK = "%CHUNK%"
+
+# Get run directory
+RUNDIR = Path(f"{WORKDIR}/{STARTDATE}/{MEMBER}")
+
+# Get some variable replacements from the proj.yml file through autosubmit
+variable_replacements = {
+    "dynamics_grid_filename": "%simulation.dynamics_grid_filename%",
+    "radiation_grid_filename": "%simulation.radiation_grid_filename%",
+    "external_parameters_filename": "%simulation.external_parameters_filename%",
+    "first_guess_filename": "%simulation.first_guess_filename%",
+    "analysis_filename": "%simulation.analysis_filename%",
+    "Chunk_START_DATE": "%Chunk_START_DATE%",
+    "Chunk_END_DATE": "%Chunk_END_DATE%",
+    "is_restart": False if "%CHUNK%" == 1 else True,
+}
+
+
+def adapt_namelist(input_namelist: str, output_namelist: str):
+    input_namelist = Path(input_namelist)
+    output_namelist = Path(output_namelist)
+
+    namelist = f90nml.read(input_namelist.as_posix())
+    group_keys = [gk for gk in namelist]
+
+    for group in group_keys:
+        variable_keys = [vk for vk in namelist[group]]
+        for variable in variable_keys:
+            value = namelist[group][variable]
+            # print(f"{variable}:{value}")
+            m = re.match(r"%(.*)%", str(value))
+            if m:
+                key = m.group(1)
+
+                if key not in variable_replacements:
+                    raise AssertionError(f"The namelist {input_namelist.as_posix()!r} contains the variable {key!r} "
+                                         f"which is not in the list of provided replacements:\n"
+                                         f"{[v for v in variable_replacements]}")
+                logger.info(f"Replacing {group}>{variable}:{key} with {variable_replacements[key]!r}")
+                namelist[group][variable] = variable_replacements[key]
+
+    f90nml.write(nml=namelist, nml_path=output_namelist.as_posix(), force=True)
+
+
+if __name__ == '__main__':
+    atmosphere_namelist_path = "%simulation.namelist_paths.atmosphere%"
+    master_namelist_path = "%simulation.namelist_paths.master%"
+
+    # Adapt atmosphere namelist
+    adapt_namelist(input_namelist=atmosphere_namelist_path,
+                   output_namelist=(RUNDIR / "icon_atmosphere.namelist").as_posix())
+    # Adapt master namelist
+    adapt_namelist(input_namelist=master_namelist_path,
+                   output_namelist=(RUNDIR / "icon_master.namelist").as_posix())