initialise the branch for ICON-D2 from DWD ana

.gitignore

 **/__pycache__
+.ipynb_checkpoints
\ No newline at end of file

conf/real-from-d2-ana/jobs.yml

+JOBS:
+  PREPARE_EXPERIMENT:
+    FILE: templates/real-from-dwd-ana/prepare_experiment.sh
+
+  PREPARE_DATE_LOCAL:
+    FILE: templates/real-from-dwd-ana/prepare_date_local.sh
+    RUNNING: date
+    WALLCLOCK: 01:00
+    PLATFORM: LOCAL
+
+  PREPARE_DATE_REMOTE:
+    FILE: templates/real-from-dwd-ana/prepare_date_remote.sh
+    RUNNING: date
+    WALLCLOCK: 01:00
+
+  PREPARE_MEMBER:
+    FILE: templates/real-from-dwd-ana/prepare_member.sh
+    DEPENDENCIES: PREPARE_EXPERIMENT PREPARE_DATE_REMOTE PREPARE_DATE_LOCAL
+
+  PREPARE_NAMELIST:
+    FILE: templates/real-from-dwd-ana/prepare_namelist.py
\ No newline at end of file

templates/real-from-d2-ana/prepare_date_local.sh

+#!/bin/bash -l
+
+# This script is executed on the machine at which autosubmit is executed.
+# and will be used if the initial conditions are in this same system.
+
+# Because it can happen that the initial conditions as well and the execution happens in the local
+# system we need to define these two variables:
+DATA_IS_LOCAL=%SIMULATION.INITIAL_CONDITIONS.LOCAL%
+
+if [ "x%HPCARCH%" == "xlocal" ]; then
+  RUN_MACHINE_IS_LOCAL="True"
+else
+  RUN_MACHINE_IS_LOCAL="False"
+fi
+
+
+
+if [ "${DATA_IS_LOCAL}" == "True" ]; then
+  # Get some variables provided by autosubmit.
+  WORKDIR=%HPCROOTDIR%
+  STARTDATE=%SDATE%
+  HPCUSER=%HPCUSER%
+  HPCHOST=%HPCHOST%
+
+  # Define date directory, create it and go there
+  COMMON_DATE_FOLDER=${WORKDIR}/${STARTDATE}/inidata
+
+  AN_MEMBER=$(printf "%03d" %SIMULATION.INITIAL_CONDITIONS.MEMBER%)
+  INITIAL_CONDITIONS_PARENT_FOLDER=%SIMULATION.INITIAL_CONDITIONS.PARENT_FOLDER%
+  INITIAL_CONDITIONS_PATH=${INITIAL_CONDITIONS_PARENT_FOLDER}/${STARTDATE:0:6}/${STARTDATE:0:8}T00
+
+  AN_SOURCE=$(find ${INITIAL_CONDITIONS_PATH} -name "igaf*00.m${AN_MEMBER}.grb" | sort | tail -n 1)
+  FG_SOURCE=$(find ${INITIAL_CONDITIONS_PATH} -name "igfff00030000.m${AN_MEMBER}.grb" | sort | tail -n 1)
+
+  AN_FILE=$(basename "${AN_SOURCE}")
+  FG_FILE=$(basename "${FG_SOURCE}")
+
+  # Find files
+  if [ ! -f "${AN_SOURCE}" ]; then
+    echo "Analysis file for date ${STARTDATE} not found!"
+    exit 1
+  fi
+
+  if [ ! -f "${FG_SOURCE}" ]; then
+    echo "FG file for date ${STARTDATE} not found!"
+    exit 1
+  fi
+
+
+  # Check if we copy the initial conditions from the local system or the remote one
+  if [ "${RUN_MACHINE_IS_LOCAL}" != "True" ]; then
+    # Create member folder
+    ssh "${HPCUSER}@${HPCHOST}" mkdir -p ${COMMON_DATE_FOLDER}
+
+    # Save filenames to be used later by other scripts.
+    echo "${AN_FILE}" > an_file.txt
+    echo "${FG_FILE}" > fg_file.txt
+    rsync -v an_file.txt "${HPCUSER}@${HPCHOST}":"${COMMON_DATE_FOLDER}/an_file.txt"
+    rsync -v fg_file.txt "${HPCUSER}@${HPCHOST}":"${COMMON_DATE_FOLDER}/fg_file.txt"
+
+    # Remove temporary files.
+    rm an_file.txt
+    rm fg_file.txt
+
+    # Copy the first-guess and analysis files.
+    rsync -v "${FG_SOURCE}" "${HPCUSER}@${HPCHOST}":"${COMMON_DATE_FOLDER}/${FG_FILE}"
+    rsync -v "${AN_SOURCE}" "${HPCUSER}@${HPCHOST}":"${COMMON_DATE_FOLDER}/${AN_FILE}"
+
+    # Change permissions to read only.
+    ssh "${HPCUSER}@${HPCHOST}" chmod 440 "${COMMON_DATE_FOLDER}/*"
+  else
+    # Create member folder and go there
+    mkdir -p ${COMMON_DATE_FOLDER}
+    cd ${COMMON_DATE_FOLDER} || exit
+
+    # Save filenames to be used later by other scripts.
+    echo "${AN_FILE}" > an_file.txt
+    echo "${FG_FILE}" > fg_file.txt
+
+    # Copy the first-guess and analysis files.
+    cp "${FG_SOURCE}" "${FG_FILE}"
+    cp "${AN_SOURCE}" "${AN_FILE}"
+
+    # Change permissions to read only.
+    chmod 440 ./*
+  fi
+fi
\ No newline at end of file

templates/real-from-d2-ana/prepare_date_remote.sh

+#!/bin/bash -l
+
+# This script is executed on the remote system at which the simulation will happen
+# and will be used if the initial conditions are in this same remote system.
+DATA_IS_LOCAL=%SIMULATION.INITIAL_CONDITIONS.LOCAL%
+
+
+if [ "${DATA_IS_LOCAL}" != "True" ]; then
+  # Get some variables provided by autosubmit.
+  WORKDIR=%HPCROOTDIR%
+  STARTDATE=%SDATE%
+  HPCUSER=%HPCUSER%
+  HPCHOST=%HPCHOST%
+  # Define date directory, create it and go there
+  COMMON_DATE_FOLDER=${WORKDIR}/${STARTDATE}/inidata
+
+  AN_MEMBER=$(printf "%03d" %SIMULATION.INITIAL_CONDITIONS.MEMBER%)
+  INITIAL_CONDITIONS_PARENT_FOLDER=%SIMULATION.INITIAL_CONDITIONS.PARENT_FOLDER%
+  INITIAL_CONDITIONS_PATH=${INITIAL_CONDITIONS_PARENT_FOLDER}/${STARTDATE:0:6}/${STARTDATE:0:8}T00
+
+  AN_SOURCE=$(find ${INITIAL_CONDITIONS_PATH} -name "igaf*00.m${AN_MEMBER}.grb" | sort | tail -n 1)
+  FG_SOURCE=$(find ${INITIAL_CONDITIONS_PATH} -name "igfff00030000.m${AN_MEMBER}.grb" | sort | tail -n 1)
+
+  AN_FILE=$(basename "${AN_SOURCE}")
+  FG_FILE=$(basename "${FG_SOURCE}")
+
+  # Find files
+  if [ ! -f "${AN_SOURCE}" ]; then
+    echo "Analysis file for date ${STARTDATE} not found!"
+    exit 1
+  fi
+
+  if [ ! -f "${FG_SOURCE}" ]; then
+    echo "FG file for date ${STARTDATE} not found!"
+    exit 1
+  fi
+
+
+  # Check if we copy the initial conditions from the local system or the remote one
+  # Create member folder and go there
+  mkdir -p ${COMMON_DATE_FOLDER}
+  cd ${COMMON_DATE_FOLDER} || exit
+
+  # Save filenames to be used later by other scripts.
+  echo "${AN_FILE}" > an_file.txt
+  echo "${FG_FILE}" > fg_file.txt
+
+  # Copy the first-guess and analysis files.
+  cp "${FG_SOURCE}" "${FG_FILE}"
+  cp "${AN_SOURCE}" "${AN_FILE}"
+
+  # Change permissions to read only.
+  chmod 440 ./*
+fi
\ No newline at end of file

templates/real-from-d2-ana/prepare_experiment.sh

+#!/bin/bash -l
+
+# Get some variables provided by autosubmit.
+WORKDIR=%HPCROOTDIR%
+DYNAMICS_GRID_FILENAME=%simulation.dynamics_grid_filename%
+RADIATION_GRID_FILE=%simulation.radiation_grid_filename%
+EXTERNAL_PARAMETERS_FILE=%simulation.external_parameters_filename%
+
+
+# Activate spack
+. ${WORKDIR}/proj/platforms/common/spack_utils.sh
+load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%" "%spack.upstreams%"
+
+# Load icon module needed to retrieve some data
+spack load --first icon-nwp@%ICON_VERSION%
+
+# Create a folder for the common inidata and go there
+COMMON_INIDATA_FOLDER=${WORKDIR}/inidata
+mkdir -p "${COMMON_INIDATA_FOLDER}"
+cd "${COMMON_INIDATA_FOLDER}" || exit
+
+# Download or copy required input files
+function download_file() {
+  URL=$1
+  FILE=${2:-$(basename "$URL")}
+  if [ ! -e "$FILE" ]; then
+    echo "Download $URL => $FILE"
+    wget -q "$URL" -O "$FILE"
+  fi
+}
+
+# Download grid files and external parameters
+BASEURL=http://icon-downloads.mpimet.mpg.de/grids/public/edzw
+download_file $BASEURL/${DYNAMICS_GRID_FILENAME}
+download_file $BASEURL/${RADIATION_GRID_FILE}
+download_file $BASEURL/${EXTERNAL_PARAMETERS_FILE}
+
+# Link input for radiation
+ln -sf "${ICON_DATA_PATH}/rrtmg_lw.nc" .
+ln -sf "${ICON_DATA_PATH}/ECHAM6_CldOptProps.nc" .
+ln -sf "${ICON_BASE_PATH}/run/ana_varnames_map_file.txt" .
+
+
+# Change permissions to read only.
+chmod 440 ./*
\ No newline at end of file

templates/real-from-d2-ana/prepare_member.sh

+#!/bin/bash -l
+
+# Get some variables provided by autosubmit.
+WORKDIR=%HPCROOTDIR%
+STARTDATE=%SDATE%
+MEMBER=%MEMBER%
+
+# Common folder with data needed for all simulations
+COMMON_INIDATA_FOLDER=${WORKDIR}/inidata
+# Common folder for the same start date
+COMMON_DATE_FOLDER=${WORKDIR}/${STARTDATE}/inidata
+
+# Member folder
+MEMBER_DIR=${WORKDIR}/${STARTDATE}/${MEMBER}
+
+# Create member folder and go there
+mkdir -p ${MEMBER_DIR}
+
+cd ${MEMBER_DIR} || exit
+
+
+# Link all files from the common inidata folder and the common date folder
+ln -sf ${COMMON_INIDATA_FOLDER}/* .
+ln -sf ${COMMON_DATE_FOLDER}/* .

templates/real-from-d2-ana/prepare_namelist.py

+import logging
+from datetime import datetime, timedelta
+from pathlib import Path
+
+import f90nml
+import yaml
+
+logger = logging.getLogger("prepare_chunk")
+logger.setLevel(logging.INFO)
+
+# Get some autosubmit variables
+WORKDIR = "%HPCROOTDIR%"
+STARTDATE = "%SDATE%"
+MEMBER = "%MEMBER%"
+CHUNK = "%CHUNK%"
+# Get run directory
+RUNDIR = Path(f"{WORKDIR}/{STARTDATE}/{MEMBER}")
+ATMOSPHERE_NAMELIST_PATH = Path("%simulation.namelist_paths.atmosphere%")
+MASTER_NAMELIST_PATH = Path("%simulation.namelist_paths.master%")
+# TODO: This is a bit ugly
+# Read first-guess and analysis filenames from files:
+first_guess_filename = (RUNDIR / "fg_file.txt").read_text().strip()
+analysis_filename = (RUNDIR / "an_file.txt").read_text().strip()
+
+# Example of date format "2018-06-01T00:00:00Z"
+date_format = "%simulation.date_format%"
+
+START_YEAR = "%Chunk_START_YEAR%"
+START_MONTH = "%Chunk_START_MONTH%"
+START_DAY = "%Chunk_START_DAY%"
+START_HOUR = "%Chunk_START_HOUR%"
+
+END_YEAR = "%Chunk_END_YEAR%"
+END_MONTH = "%Chunk_END_MONTH%"
+END_DAY = "%Chunk_END_DAY%"
+END_HOUR = "%Chunk_END_HOUR%"
+
+Chunk_START_DATE = datetime(year=int(START_YEAR), month=int(START_MONTH), day=int(START_DAY), hour=int(START_HOUR))
+Chunk_END_DATE = datetime(year=int(END_YEAR), month=int(END_MONTH), day=int(END_DAY), hour=int(END_HOUR))
+
+# Read custom namelist parameters from configuration
+atmosphere_namelist_string = """
+%atmosphere_namelist%
+"""
+
+master_namelist_string = """
+%master_namelist%
+"""
+
+# Compute difference in seconds
+checkpoint_time = int((Chunk_END_DATE - Chunk_START_DATE).total_seconds())
+
+# TODO: Is that really necessary?
+# Add 10 minutes to allow the model to write the restarts
+Chunk_END_DATE = Chunk_END_DATE + timedelta(minutes=10)
+
+atmosphere_namelist_replacements = {
+    "time_nml": {
+        "dt_restart": checkpoint_time
+    },
+    "io_nml": {
+        "dt_checkpoint": checkpoint_time
+    },
+
+    "grid_nml": {
+        "dynamics_grid_filename": "%simulation.dynamics_grid_filename%",
+        "radiation_grid_filename": "%simulation.radiation_grid_filename%",
+    },
+
+    "extpar_nml": {
+        "extpar_filename": "%simulation.external_parameters_filename%",
+    },
+
+    "initicon_nml": {
+        "dwdfg_filename": first_guess_filename,
+        "dwdana_filename": analysis_filename,
+    }
+}
+
+master_namelist_replacements = {
+    "master_nml": {
+        "lrestart": False if "%CHUNK%" == "1" else True,
+    },
+    "master_time_control_nml": {
+        "experimentStartDate": Chunk_START_DATE.strftime(date_format),
+        "experimentStopDate": Chunk_END_DATE.strftime(date_format),
+    }
+}
+
+
+def read_namelist(namelist_string: str) -> dict:
+    """
+    Function to read the custom namelist specifications provided in the configuration files.
+    It accepts both yaml and f90nml format.
+    :param namelist_string:
+    :return:
+    """
+    parameters = yaml.safe_load(namelist_string)
+    if isinstance(parameters, str):
+        parameters = f90nml.reads(nml_string=namelist_string).todict()
+    return parameters
+
+
+def patch_output_entries(namelist: f90nml.Namelist) -> f90nml.Namelist:
+    output_entries = [entry for entry in namelist["output_nml"]]
+    for entry in output_entries:
+        for key in entry:
+            if entry[key] == "#OUTPUT_START#":
+                entry[key] = Chunk_START_DATE.strftime(date_format)
+            elif entry[key] == "#OUTPUT_END#":
+                entry[key] = Chunk_END_DATE.strftime(date_format)
+
+    return namelist
+
+
+def main():
+    """
+    Main function that processes both atmosphere and master namelists and adds the necessary patches
+    :return:
+    """
+    # Process atmosphere namelist
+    atmosphere_namelist = f90nml.read(ATMOSPHERE_NAMELIST_PATH.as_posix())
+    # Convert output_nml to a co-group.
+    atmosphere_namelist.create_cogroup("output_nml")
+    print("Original atmosphere namelist:")
+    print(atmosphere_namelist)
+    atmosphere_namelist.patch(atmosphere_namelist_replacements)
+
+    # Read custom namelist parameters from configuration file
+    atmosphere_custom_namelist = read_namelist(atmosphere_namelist_string)
+
+    if atmosphere_custom_namelist is not None:
+        try:
+            atmosphere_namelist.patch(atmosphere_custom_namelist)
+        except AttributeError:
+            raise AssertionError("Problem applying the namelist patch! Probably related with the output section.")
+
+    # Patch output entries:
+    atmosphere_namelist = patch_output_entries(atmosphere_namelist)
+
+    print("Patched atmosphere namelist:")
+    print(atmosphere_namelist)
+
+    atmosphere_output_namelist = (RUNDIR / "icon_atmosphere.namelist")
+    f90nml.write(nml=atmosphere_namelist, nml_path=atmosphere_output_namelist.as_posix(), force=True)
+
+    master_namelist = f90nml.read(MASTER_NAMELIST_PATH.as_posix())
+    print("Original master namelist:")
+    print(master_namelist)
+    # Read custom namelist parameters from configuration file
+    master_custom_namelist = read_namelist(master_namelist_string)
+    # Process atmosphere namelist
+    master_namelist.patch(master_namelist_replacements)
+    if master_custom_namelist is not None:
+        master_namelist.patch(master_custom_namelist)
+    print("Patched master namelist:")
+    print(master_namelist)
+    master_output_namelist = (RUNDIR / "icon_master.namelist")
+    f90nml.write(nml=master_namelist, nml_path=master_output_namelist.as_posix(), force=True)
+
+
+if __name__ == '__main__':
+    main()
+