Adaption of preparation for an ensemble IC/LBC

conf/common/expdef.yml

@@ -3,6 +3,6 @@ experiment:
   MEMBERS: "fc0"
   CHUNKSIZEUNIT: hour
   CHUNKSIZE: 1
-  NUMCHUNKS: 2
+  NUMCHUNKS: 1
   CHUNKINI: 0
   CALENDAR: standard
\ No newline at end of file

conf/real-from-d2-ana/simulation.yml

@@ -20,13 +20,11 @@ simulation:
     local: false
     #parent_folder: /archive/meteo/w2w-p2/B3/ICON-D2_oper_ICLBC/
     parent_folder: /dss/dsskcsfs01/pn34ca/pn34ca-dss-0007/DATA-OP/
-    member: 1
 
   boundary_conditions:
-    # Where are we getting our initial data from?
+    # Where are we getting our boundary data from?
     local: false
     #parent_folder: /archive/meteo/w2w-p2/B3/ICON-D2_oper_ICLBC/
     parent_folder: /dss/dsskcsfs01/pn34ca/pn34ca-dss-0007/DATA-OP/
-    member: 1
 
     

templates/real-from-d2-ana/icon-remap-helper.py

@@ -153,7 +153,10 @@ def remap_one_file(in_grid, out_grid, one_file, dst_fodler, rename=None, src_vgr
         vinp = False
 
     # rename the file if requested
-    if rename is not None and args.output_format != "grb":
+    if rename is not None:
+        if args.output_format != "grb":
+            logging.error("--rename option is only supported for netCDF output.")
+            exit(-1)
         # read the time stamp
         if content["time"].size != 1:
             logging.error("more then one timestep, unable to rename the file!")

templates/real-from-d2-ana/prepare_date_local.sh

@@ -25,6 +25,7 @@ if [ "${IC_DATA_IS_LOCAL}" == "True" ]; then
   # Define date directory, create it and go there
   COMMON_DATE_FOLDER=${WORKDIR}/${STARTDATE}/inidata
 
+  #AN_MEMBER=$(printf "%03d" %SIMULATION.INITIAL_CONDITIONS.MEMBER%)
   AN_MEMBER=$(printf "%03d" %SIMULATION.INITIAL_CONDITIONS.MEMBER%)
   INITIAL_CONDITIONS_PARENT_FOLDER=%SIMULATION.INITIAL_CONDITIONS.PARENT_FOLDER%
   INITIAL_CONDITIONS_PATH=${INITIAL_CONDITIONS_PARENT_FOLDER}/${STARTDATE:0:8}00

templates/real-from-d2-ana/prepare_date_remote.sh

@@ -11,16 +11,18 @@ if [ "${IC_DATA_IS_LOCAL}" != "True" ]; then
   STARTDATE=%SDATE%
   HPCUSER=%HPCUSER%
   HPCHOST=%HPCHOST%
+
   # Define date directory, create it and go there
   COMMON_DATE_FOLDER=${WORKDIR}/${STARTDATE}/inidata
 
-  AN_MEMBER=$(printf "%03d" %SIMULATION.INITIAL_CONDITIONS.MEMBER%)
+  # Exploit the number from the member name
+  AN_MEMBER=$(printf "%03d" ${MEMBER:1})
   INITIAL_CONDITIONS_PARENT_FOLDER=%SIMULATION.INITIAL_CONDITIONS.PARENT_FOLDER%
   INITIAL_CONDITIONS_PATH=${INITIAL_CONDITIONS_PARENT_FOLDER}/${STARTDATE:0:8}00
 
-  AN_SOURCE=$(find ${INITIAL_CONDITIONS_PATH} -name "an_R19B07.*00_an.${AN_MEMBER}" | sort | tail -n 1)
-  AN_INC_SOURCE=$(find ${INITIAL_CONDITIONS_PATH} -name "an_R19B07.*00_inc.${AN_MEMBER}" | sort | tail -n 1)
-  FG_SOURCE=$(find ${INITIAL_CONDITIONS_PATH} -name "fc_R19B07.*5500.${AN_MEMBER}" | sort | tail -n 1)
+  AN_SOURCE=$(find ${INITIAL_CONDITIONS_PATH} -name "an_R19B07.*00_an.*" | sort -n )
+  AN_INC_SOURCE=$(find ${INITIAL_CONDITIONS_PATH} -name "an_R19B07.*00_inc.*" | sort -n )
+  FG_SOURCE=$(find ${INITIAL_CONDITIONS_PATH} -name "fc_R19B07.*5500.*" | sort -n )
 
   AN_FILE=$(basename "${AN_SOURCE}")
   AN_INC_FILE=$(basename "${AN_INC_SOURCE}")

templates/real-from-d2-ana/prepare_lbc_remote.sh

@@ -19,6 +19,7 @@ if [ "${LBC_DATA_IS_LOCAL}" != "True" ]; then
   STARTDATE=%SDATE%
   HPCUSER=%HPCUSER%
   HPCHOST=%HPCHOST%
+  MEMBER=%MEMBER%
 
   . ${WORKDIR}/proj/platforms/common/spack_utils.sh
   load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%" "%spack.upstreams%"
@@ -33,13 +34,15 @@ if [ "${LBC_DATA_IS_LOCAL}" != "True" ]; then
   # Define date directory, create it and go there
   COMMON_DATE_FOLDER=${WORKDIR}/${STARTDATE}/inidata
   COMMON_GRID_FOLDER=${WORKDIR}/inidata
+  MEMBER_DIR=${WORKDIR}/${STARTDATE}/${MEMBER^}
 
-  AN_MEMBER=$(printf "%03d" %SIMULATION.BOUNDARY_CONDITIONS.MEMBER%)
+  # Leave out the character part
   BOUNDARY_CONDITIONS_PARENT_FOLDER=%SIMULATION.BOUNDARY_CONDITIONS.PARENT_FOLDER%
   BOUNDARY_CONDITIONS_PATH=${BOUNDARY_CONDITIONS_PARENT_FOLDER}/${STARTDATE:0:8}00
 
+
   # Fetch tar.gz file of ICON-EU-EPS outputs necessary for the lateral boundary conditions
-  FC_TAR_SOURCE=$(find ${BOUNDARY_CONDITIONS_PATH} -name "iefff.m${AN_MEMBER}.tar.gz" | sort | tail -n 1)
+  FC_TAR_SOURCE=$(find ${BOUNDARY_CONDITIONS_PATH} -name "iefff.${MEMBER,}.tar.gz" | sort | tail -n 1)
 
   FC_TAR_FILE=$(basename "${FC_TAR_SOURCE}")
 
@@ -50,7 +53,7 @@ if [ "${LBC_DATA_IS_LOCAL}" != "True" ]; then
   fi
 
   # make a working directory for converting boundary conditions
-  mkdir -p work
+  mkdir -p work/${MEMBER} || exit
 
   tar -zxvf ${FC_TAR_SOURCE} -C work
 
@@ -76,14 +79,22 @@ if [ "${LBC_DATA_IS_LOCAL}" != "True" ]; then
     ${WORKDIR}/%python_environment.folder_name%/bin/python3 ${WORKDIR}/proj/templates/real-from-d2-ana/icon-remap-helper.py \
         --src-grid ${COMMON_GRID_FOLDER}/${PARENT_GRID_FILE} \
         --dst-grid ${COMMON_GRID_FOLDER}/${BOUNDARY_GRID_FILE} \
-        --source   work/iefff${EU_FC_TIME}0000.m${AN_MEMBER} \
-        --dest ${COMMON_DATE_FOLDER} \
-        --rename latbc_${D2_FC_TIME}00.m${AN_MEMBER} \
+        --source   work/iefff${EU_FC_TIME}0000.${MEMBER,} \
+        --dest work/${MEMBER} \
         --output-format grb
+
+
+    # rename to the comfortable style for ICON
+    mv work/${MEMBER}/iefff${EU_FC_TIME}0000.grb ${MEMBER_DIR}/latbc_${D2_FC_TIME}.${MEMBER,}.grib
+
   done
 
+  # clean up the working directory
+  rm -r work/${MEMBER}
+
+
   # Fetch tar.gz file of ICON-EU-EPS outputs necessary for the lateral boundary conditions
-  LBC_SOURCE=$(find ${COMMON_DATE_FOLDER} -name "latbc_*00.m${AN_MEMBER}" | sort )
+  LBC_SOURCE=$(find ${MEMBER_DIR} -name "latbc_*00.${MEMBER,}.grib" | sort )
 
   LBC_FILE=$(basename "${LBC_SOURCE}")
 
@@ -99,7 +110,7 @@ if [ "${LBC_DATA_IS_LOCAL}" != "True" ]; then
   cd ${COMMON_DATE_FOLDER} || exit
 
   # Save filenames to be used later by other scripts.
-  echo "${LBC_SOURCE}" > lbc_file.txt
+  echo "${LBC_SOURCE}" >> lbc_file.txt
 
   # Copy the first-guess and analysis files.
   #cp "${LBC_SOURCE}" "${LBC_FILE}"

templates/real-from-d2-ana/prepare_member.sh

@@ -11,7 +11,7 @@ COMMON_INIDATA_FOLDER=${WORKDIR}/inidata
 COMMON_DATE_FOLDER=${WORKDIR}/${STARTDATE}/inidata
 
 # Member folder
-MEMBER_DIR=${WORKDIR}/${STARTDATE}/${MEMBER}
+MEMBER_DIR=${WORKDIR}/${STARTDATE}/${MEMBER^}
 
 # Create member folder and go there
 mkdir -p ${MEMBER_DIR}
@@ -21,4 +21,5 @@ cd ${MEMBER_DIR} || exit
 
 # Link all files from the common inidata folder and the common date folder
 ln -sf ${COMMON_INIDATA_FOLDER}/* .
-ln -sf ${COMMON_DATE_FOLDER}/* .
+
+ln -sf ${COMMON_DATE_FOLDER}/*${MEMBER:1}* .