diff --git a/conf/jobs.yaml b/conf/jobs_real-from-dwd-ana.yaml
similarity index 92%
rename from conf/jobs.yaml
rename to conf/jobs_real-from-dwd-ana.yaml
index 82ff3dd18b8adda9786a2954341a59bdb051ef1b..34edcd1fedeb5010289a0793911bf5f7f6fb927d 100644
--- a/conf/jobs.yaml
+++ b/conf/jobs_real-from-dwd-ana.yaml
@@ -57,42 +57,42 @@ JOBS:
# EXECUTABLE: /my_python_env/python3
BUILD_ICON:
- FILE: templates/build_icon.sh
+ FILE: templates/common/build_icon.sh
WALLCLOCK: 04:00
PROCESSORS: 16
BUILD_PYTHON_ENVIRONMENT:
- FILE: templates/build_python_environment.sh
+ FILE: templates/common/build_python_environment.sh
# Right now we rely on spack for building icon and having a python interpreter, so we need this dependency:
DEPENDENCIES: BUILD_ICON
WALLCLOCK: 01:00
PROCESSORS: 16
PREPARE_EXPERIMENT:
- FILE: templates/prepare_experiment.sh
+ FILE: templates/common/prepare_experiment.sh
DEPENDENCIES: BUILD_ICON
RUNNING: once
WALLCLOCK: 01:00
PREPARE_DATE:
- FILE: templates/prepare_date.sh
+ FILE: templates/common/prepare_date.sh
RUNNING: date
WALLCLOCK: 01:00
PREPARE_MEMBER:
- FILE: templates/prepare_member.sh
+ FILE: templates/common/prepare_member.sh
DEPENDENCIES: PREPARE_EXPERIMENT PREPARE_DATE
RUNNING: member
WALLCLOCK: 01:00
PREPARE_CHUNK:
- FILE: templates/prepare_chunk.sh
+ FILE: templates/common/prepare_chunk.sh
DEPENDENCIES: PREPARE_MEMBER RUN_ICON-1
WALLCLOCK: 00:05
RUNNING: chunk
RUN_ICON:
- FILE: templates/run_icon.sh
+ FILE: templates/common/run_icon.sh
DEPENDENCIES: PREPARE_CHUNK
WALLCLOCK: 04:00
RUNNING: chunk
@@ -101,7 +101,7 @@ JOBS:
CUSTOM_DIRECTIVES: [ "#SBATCH --exclusive", "export OMPI_MCA_btl_tcp_if_include=10.0.0.0/8"]
COMPRESS:
- FILE: templates/compress.py
+ FILE: templates/common/compress.py
DEPENDENCIES: RUN_ICON BUILD_PYTHON_ENVIRONMENT COMPRESS-1
RUNNING: chunk
TYPE: python
@@ -110,7 +110,7 @@ JOBS:
MEMORY: 16384
TRANSFER:
- FILE: templates/transfer.sh
+ FILE: templates/common/transfer.sh
DEPENDENCIES: COMPRESS
# Since this is running locally, can simply leave a long wallclock.
WALLCLOCK: 24:00
@@ -118,7 +118,7 @@ JOBS:
PLATFORM: LOCAL
CLEAN:
- FILE: templates/clean.sh
+ FILE: templates/common/clean.sh
DEPENDENCIES: TRANSFER
WALLCLOCK: 00:10
RUNNING: member
diff --git a/conf/proj.yaml b/conf/proj.yaml
deleted file mode 100644
index 3e21878b42ac34783fabd5d7ca4062eed84c603f..0000000000000000000000000000000000000000
--- a/conf/proj.yaml
+++ /dev/null
@@ -1,38 +0,0 @@
-spack:
- url: git@gitlab.physik.uni-muenchen.de:LDAP_rbg/spack.git
- branch: lmu/ubuntu20.04-icon
- compiler: gcc@11.3.0
-
-icon:
- version: 2.6.5-nwp0
-
-python_environment:
- folder_name: python_environment
- requirements:
- - numpy==1.23
- - enstools-compression
- # Just to try a library from a git repository.
- - git+https://gitlab.physik.uni-muenchen.de/Oriol.Tinto/otils.git
-
-simulation:
- BGRID_GLOBAL: icon_grid_0012_R02B04_G.nc
- RGRID_GLOBAL: icon_grid_0011_R02B03_R.nc
- BGRID: icon_grid_0016_R02B06_G.nc
- RGRID: icon_grid_0015_R02B05_R.nc
- BEXTPAR: icon_extpar_0016_R02B06_G_20131206.nc
- BGRID_NEST: icon_grid_nest_G_DOM01.nc
- RGRID_NEST: icon_grid_nest_R_DOM01.nc
- BGRID_NEST_LATBC: icon_grid_nest_G_DOM01_latbc.nc
-
- # List of output files
- OUTPUT_FILES: "init_DOM01_ML_*.nc latbc_DOM01_ML_*.nc"
- FILES_TO_CLEAN: "*.nc"
-
-
-data_management:
- local_folder: /scratch/o/Oriol.Tinto/tmp/
-
-
-initial_conditions:
- parent_folder: /archive/meteo/external-models/dwd/icon/oper/icon_oper_eps_gridded-global_rolling/
- member: 1
\ No newline at end of file
diff --git a/conf/proj_real-from-dwd-ana.yaml b/conf/proj_real-from-dwd-ana.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..34870249164b06a5c03d9f10546ccf427a2bfa27
--- /dev/null
+++ b/conf/proj_real-from-dwd-ana.yaml
@@ -0,0 +1,38 @@
+spack:
+ url: git@gitlab.physik.uni-muenchen.de:LDAP_rbg/spack.git
+ branch: lmu/ubuntu20.04-icon
+ compiler: gcc@11.3.0
+
+icon:
+ version: 2.6.5-nwp0
+
+python_environment:
+ folder_name: python_environment
+ requirements:
+ - numpy==1.23
+ - enstools-compression
+ # Just to try a library from a git repository.
+ - git+https://gitlab.physik.uni-muenchen.de/Oriol.Tinto/otils.git
+
+simulation:
+ dynamics_grid_filename: icon_grid_0016_R02B06_G.nc
+ radiation_grid_filename: icon_grid_0015_R02B05_R.nc
+ external_parameters_filename: icon_extpar_0016_R02B06_G_20131206.nc
+
+ # List of output file names that will be copied (Wildcards * allowed)
+ output_file_names: "init_DOM01_ML_*.nc latbc_DOM01_ML_*.nc"
+ files_to_clean: "*.nc"
+
+data_management:
+ # Where do we put the output files afterwards?
+ local_folder: /scratch/o/Oriol.Tinto/tmp/
+
+initial_conditions:
+ # Where are we getting our initial data from?
+ parent_folder: /archive/meteo/external-models/dwd/icon/oper/icon_oper_eps_gridded-global_rolling/
+ member: 1
+
+namelist:
+ # Path to the namelist
+ master_path: "%PROJDIR%/namelists/icon_master_real-from-dwd-ana.namelist"
+ atmosphere_path: "%PROJDIR%/namelists/icon_atmosphere_real-from-dwd-ana.namelist"
\ No newline at end of file
diff --git a/templates/build_icon.sh b/templates/common/build_icon.sh
similarity index 100%
rename from templates/build_icon.sh
rename to templates/common/build_icon.sh
diff --git a/templates/build_python_environment.sh b/templates/common/build_python_environment.sh
similarity index 100%
rename from templates/build_python_environment.sh
rename to templates/common/build_python_environment.sh
diff --git a/templates/clean.sh b/templates/common/clean.sh
similarity index 100%
rename from templates/clean.sh
rename to templates/common/clean.sh
diff --git a/templates/compress.py b/templates/common/compress.py
similarity index 100%
rename from templates/compress.py
rename to templates/common/compress.py
diff --git a/templates/common/prepare_chunk.sh b/templates/common/prepare_chunk.sh
new file mode 100644
index 0000000000000000000000000000000000000000..f99dfcd65100425227cbae0123dd3ea7c75d6ce3
--- /dev/null
+++ b/templates/common/prepare_chunk.sh
@@ -0,0 +1,58 @@
+#!/bin/bash -l
+
+# Get some variables provided by autosubmit.
+WORKDIR=%HPCROOTDIR%
+
+dynamics_grid_filename=%simulation.dynamics_grid_filename%
+radiation_grid_filename=%simulation.radiation_grid_filename%
+external_parameters_filename=%simulation.external_parameters_filename%
+
+
+STARTDATE=%SDATE%
+
+# Example of date format "2018-06-01T00:00:00Z"
+START_YEAR=%Chunk_START_YEAR%
+START_MONTH=%Chunk_START_MONTH%
+START_DAY=%Chunk_START_DAY%
+START_HOUR=%Chunk_START_HOUR%
+
+END_YEAR=%Chunk_END_YEAR%
+END_MONTH=%Chunk_END_MONTH%
+END_DAY=%Chunk_END_DAY%
+END_HOUR=%Chunk_END_HOUR%
+
+Chunk_START_DATE="${START_YEAR}-${START_MONTH}-${START_DAY}T${START_HOUR}:00:00Z"
+Chunk_END_DATE="${END_YEAR}-${END_MONTH}-${END_DAY}T${END_HOUR}:00:00Z"
+
+# Convert dates to Unix timestamps
+t1_unix=$(date -d "$Chunk_START_DATE" +%s)
+t2_unix=$(date -d "$Chunk_END_DATE" +%s)
+
+# Compute difference in seconds
+checkpoint_time=$(((t2_unix - t1_unix)))
+
+# Compute number of steps
+dtime=180
+
+# Extend chunk 10 minutes to ensure checkpoint creation at the proper time
+# TODO: Works but it is a bit ugly.
+Chunk_END_DATE="${END_YEAR}-${END_MONTH}-${END_DAY}T${END_HOUR}:10:00Z"
+
+MEMBER=%MEMBER%
+CHUNK=%CHUNK%
+
+# If the chunk is not the first one, start from a restart file.
+if [[ "${CHUNK}" -eq "1" ]]; then
+ is_restart=.false.
+else
+ is_restart=.true.
+fi
+
+# Define rundir
+RUNDIR=${WORKDIR}/${STARTDATE}/${MEMBER}
+
+cd ${RUNDIR} || exit
+
+# Get AN and FG file names from file that was saved during prepare_rundir
+AN_FILE=$( cat an_file.txt )
+FG_FILE=$( cat fg_file.txt )
diff --git a/templates/prepare_date.sh b/templates/common/prepare_date.sh
similarity index 100%
rename from templates/prepare_date.sh
rename to templates/common/prepare_date.sh
diff --git a/templates/prepare_experiment.sh b/templates/common/prepare_experiment.sh
similarity index 100%
rename from templates/prepare_experiment.sh
rename to templates/common/prepare_experiment.sh
diff --git a/templates/prepare_member.sh b/templates/common/prepare_member.sh
similarity index 100%
rename from templates/prepare_member.sh
rename to templates/common/prepare_member.sh
diff --git a/templates/run_icon.sh b/templates/common/run_icon.sh
similarity index 100%
rename from templates/run_icon.sh
rename to templates/common/run_icon.sh
diff --git a/templates/transfer.sh b/templates/common/transfer.sh
similarity index 100%
rename from templates/transfer.sh
rename to templates/common/transfer.sh
diff --git a/templates/prepare_chunk.sh b/templates/prepare_chunk.sh
deleted file mode 100644
index 1a433745083121d016740d99a19c7c92baee6b66..0000000000000000000000000000000000000000
--- a/templates/prepare_chunk.sh
+++ /dev/null
@@ -1,271 +0,0 @@
-#!/bin/bash -l
-
-# Get some variables provided by autosubmit.
-WORKDIR=%HPCROOTDIR%
-ICON_VERSION=%ICON_VERSION%
-
-BGRID_GLOBAL=%simulation.BGRID_GLOBAL%
-RGRID_GLOBAL=%simulation.RGRID_GLOBAL%
-BGRID=%simulation.BGRID%
-RGRID=%simulation.RGRID%
-BEXTPAR=%simulation.BEXTPAR%
-
-
-
-STARTDATE=%SDATE%
-
-# Example of date format "2018-06-01T00:00:00Z"
-START_YEAR=%Chunk_START_YEAR%
-START_MONTH=%Chunk_START_MONTH%
-START_DAY=%Chunk_START_DAY%
-START_HOUR=%Chunk_START_HOUR%
-
-END_YEAR=%Chunk_END_YEAR%
-END_MONTH=%Chunk_END_MONTH%
-END_DAY=%Chunk_END_DAY%
-END_HOUR=%Chunk_END_HOUR%
-
-Chunk_START_DATE="${START_YEAR}-${START_MONTH}-${START_DAY}T${START_HOUR}:00:00Z"
-Chunk_END_DATE="${END_YEAR}-${END_MONTH}-${END_DAY}T${END_HOUR}:00:00Z"
-
-# Convert dates to Unix timestamps
-t1_unix=$(date -d "$Chunk_START_DATE" +%s)
-t2_unix=$(date -d "$Chunk_END_DATE" +%s)
-
-# Compute difference in seconds
-checkpoint_time=$(((t2_unix - t1_unix)))
-
-# Compute number of steps
-dtime=180
-nsteps=$((checkpoint_time / ${dtime}))
-
-# Extend chunk 10 minutes to ensure checkpoint creation at the proper time
-# TODO: Works but it is a bit ugly.
-Chunk_END_DATE="${END_YEAR}-${END_MONTH}-${END_DAY}T${END_HOUR}:10:00Z"
-
-MEMBER=%MEMBER%
-CHUNK=%CHUNK%
-
-# If the chunk is not the first one, start from a restart file.
-if [[ "${CHUNK}" -eq "1" ]]; then
- is_restart=.false.
-else
- is_restart=.true.
-fi
-
-# Define rundir
-RUNDIR=${WORKDIR}/${STARTDATE}/${MEMBER}
-
-cd ${RUNDIR}
-
-# Get AN and FG file names from file that was saved during prepare_rundir
-AN_FILE=$( cat an_file.txt )
-FG_FILE=$( cat fg_file.txt )
-
-
-# cat >icon_master.namelist <<EOF
-# &master_nml
-# lrestart = ${is_restart}
-# lrestart_write_last = .true.
-# /
-
-# &master_model_nml
-# model_type = 1 ! atmospheric model
-# model_name = "ATMO" ! name of this model component
-# model_namelist_filename = "icon_atmosphere.namelist"
-# /
-
-# &master_time_control_nml
-# calendar = "proleptic gregorian"
-# experimentStartDate = '${Chunk_START_DATE}'
-# experimentStopDate = '${Chunk_END_DATE}'
-# /
-# EOF
-
-# cat >icon_atmosphere.namelist <<EOF
-# &run_nml
-# ltestcase = .TRUE. ! idealized testcase runs
-# dtime = 300 ! time step of 300 seconds
-# output = 'nml' ! use output nameslists
-# msg_level = 15
-# nsteps = ${nsteps}
-# num_lev = 31
-# lvert_nest = .false.
-# ldynamics = .true.
-# ltransport = .true.
-# ntracer = 5
-# iforcing = 3
-# /
-
-# &time_nml
-# dt_restart = ${checkpoint_time}
-# /
-
-# &io_nml
-# dt_checkpoint = ${checkpoint_time}
-# /
-
-# &grid_nml
-# dynamics_grid_filename = '$BGRID_GLOBAL'
-# radiation_grid_filename = '$RGRID_GLOBAL'
-# dynamics_parent_grid_id = 0
-# lredgrid_phys = .true.
-# /
-
-# &nh_testcase_nml
-# nh_test_name = 'APE_nwp' ! testcase selection
-# ape_sst_case = 'sst_qobs'
-# /
-
-# &nonhydrostatic_nml
-# damp_height = 18000
-# rayleigh_coeff = 0.75
-# /
-
-# &nwp_phy_nml
-# inwp_surface = 0
-# /
-
-# ¶llel_nml
-# nproma = 16
-# /
-
-# ! the following two output files are used to initialize the next run
-# &output_nml
-# file_interval = 'PT3600S'
-# output_start = '${Chunk_START_DATE}'
-# output_end = '${Chunk_END_DATE}'
-# output_filename = "init-test"
-# output_interval = 'PT3600S'
-# include_last = .true.
-# mode = 1
-# taxis_tunit = 1
-# ml_varlist = 'group:dwd_fg_atm_vars', 'group:dwd_fg_sfc_vars'
-# remap = 1
-# reg_lon_def = -30.,0.5,30.
-# reg_lat_def = 90.,-0.5, -90.
-# /
-# &output_nml
-# steps_per_file = 1
-# output_start = '${Chunk_START_DATE}'
-# output_end = '${Chunk_START_DATE}'
-# output_filename = "init-test-ext"
-# include_last = .true.
-# output_interval = 'PT3600S'
-# ml_varlist = 'depth_lk', 'emis_rad', 'fr_lake', 'fr_land', 'topography_c', 'soiltyp', 'sso_stdh', 'sso_theta', 'sso_gamma', 'sso_sigma'
-# /
-# EOF
-
-
-# TODO: The namelist shouldn't be hardcoded into the template files.
-cat > icon_atmosphere.namelist << EOF
-&run_nml
- ltestcase = .false.
- dtime = ${dtime}
- output = 'nml'
- msg_level = 15
- num_lev = 90
- lvert_nest = .false.
- ldynamics = .true.
- ltransport = .true.
- ntracer = 5
- iforcing = 3
-/
-
-&time_nml
- dt_restart = ${checkpoint_time}
-/
-
-&io_nml
- dt_checkpoint = ${checkpoint_time}
-/
-
-&nwp_phy_nml
- lupatmo_phy = .FALSE.
-/
-
-&grid_nml
- dynamics_parent_grid_id = 0
- dynamics_grid_filename = '$BGRID'
- radiation_grid_filename = '$RGRID'
- lredgrid_phys = .true.
-/
-
-&extpar_nml
- itopo = 1
- extpar_filename = '$BEXTPAR'
-/
-
-&initicon_nml
- init_mode = 1,
- dwdfg_filename = '$FG_FILE'
- dwdana_filename = '$AN_FILE'
- lconsistency_checks = .false.
- ana_varnames_map_file = 'ana_varnames_map_file.txt'
-/
-
-! settings from operational setup for vertical coordinate
-&sleve_nml
- min_lay_thckn = 20.
- max_lay_thckn = 400.
- htop_thcknlimit = 14000.
- top_height = 75000.
- stretch_fac = 0.9
- decay_scale_1 = 4000.
- decay_scale_2 = 2500.
- decay_exp = 1.2
- flat_height = 16000.
-/
-
-&nonhydrostatic_nml
- damp_height = 22000.
- rayleigh_coeff = 1
-/
-
-
-¶llel_nml
- nproma = 16
-/
-
-! LATBC files, these files will be used as input for the next example.
-&output_nml
- file_interval = 'PT3600S'
- output_start = '${Chunk_START_DATE}'
- output_end = '${Chunk_END_DATE}'
- output_filename = "latbc"
- output_interval = 'PT3600S'
- include_last = .true.
- ml_varlist = 'u', 'v', 'w', 'theta_v', 'rho', 'qv', 'qc', 'qi', 'qr', 'qs', 'z_ifc'
-/
-
-! First Guess file
-&output_nml
- file_interval = 'PT3600S'
- output_start = '${Chunk_START_DATE}'
- output_end = '${Chunk_END_DATE}'
- output_filename = "init"
- output_interval = 'PT3600S'
- include_last = .true.
- ml_varlist = 'group:dwd_fg_atm_vars', 'group:dwd_fg_sfc_vars'
-/
-EOF
-
-
-cat > icon_master.namelist << EOF
-&master_nml
- lrestart = ${is_restart}
- lrestart_write_last = .true.
-/
-
-&master_model_nml
- model_type = 1 ! atmospheric model
- model_name = "ATMO" ! name of this model component
- model_namelist_filename = "icon_atmosphere.namelist"
-/
-
-&master_time_control_nml
- calendar = "proleptic gregorian"
- experimentStartDate = '${Chunk_START_DATE}'
- experimentStopDate = '${Chunk_END_DATE}'
-/
-EOF