From ae66624f118c06010c6f53b7a27c2462b98504ea Mon Sep 17 00:00:00 2001
From: "Takumi.Matsunobu" <Takumi.Matsunobu@physik.uni-muenchen.de>
Date: Thu, 15 Jun 2023 13:45:16 +0200
Subject: [PATCH] Adaption of preparation for an ensemble IC/LBC
---
conf/common/expdef.yml | 2 +-
conf/real-from-d2-ana/simulation.yml | 4 +--
.../real-from-d2-ana/icon-remap-helper.py | 5 +++-
.../real-from-d2-ana/prepare_date_local.sh | 1 +
.../real-from-d2-ana/prepare_date_remote.sh | 10 ++++---
.../real-from-d2-ana/prepare_lbc_remote.sh | 27 +++++++++++++------
templates/real-from-d2-ana/prepare_member.sh | 5 ++--
7 files changed, 35 insertions(+), 19 deletions(-)
diff --git a/conf/common/expdef.yml b/conf/common/expdef.yml
index fd7b15e..3e1f911 100644
--- a/conf/common/expdef.yml
+++ b/conf/common/expdef.yml
@@ -3,6 +3,6 @@ experiment:
MEMBERS: "fc0"
CHUNKSIZEUNIT: hour
CHUNKSIZE: 1
- NUMCHUNKS: 2
+ NUMCHUNKS: 1
CHUNKINI: 0
CALENDAR: standard
\ No newline at end of file
diff --git a/conf/real-from-d2-ana/simulation.yml b/conf/real-from-d2-ana/simulation.yml
index 2441731..72cb077 100644
--- a/conf/real-from-d2-ana/simulation.yml
+++ b/conf/real-from-d2-ana/simulation.yml
@@ -20,13 +20,11 @@ simulation:
local: false
#parent_folder: /archive/meteo/w2w-p2/B3/ICON-D2_oper_ICLBC/
parent_folder: /dss/dsskcsfs01/pn34ca/pn34ca-dss-0007/DATA-OP/
- member: 1
boundary_conditions:
- # Where are we getting our initial data from?
+ # Where are we getting our boundary data from?
local: false
#parent_folder: /archive/meteo/w2w-p2/B3/ICON-D2_oper_ICLBC/
parent_folder: /dss/dsskcsfs01/pn34ca/pn34ca-dss-0007/DATA-OP/
- member: 1
diff --git a/templates/real-from-d2-ana/icon-remap-helper.py b/templates/real-from-d2-ana/icon-remap-helper.py
index d930894..785deb2 100644
--- a/templates/real-from-d2-ana/icon-remap-helper.py
+++ b/templates/real-from-d2-ana/icon-remap-helper.py
@@ -153,7 +153,10 @@ def remap_one_file(in_grid, out_grid, one_file, dst_fodler, rename=None, src_vgr
vinp = False
# rename the file if requested
- if rename is not None and args.output_format != "grb":
+ if rename is not None:
+ if args.output_format != "grb":
+ logging.error("--rename option is only supported for netCDF output.")
+ exit(-1)
# read the time stamp
if content["time"].size != 1:
logging.error("more then one timestep, unable to rename the file!")
diff --git a/templates/real-from-d2-ana/prepare_date_local.sh b/templates/real-from-d2-ana/prepare_date_local.sh
index 16ec5f7..9a62be3 100644
--- a/templates/real-from-d2-ana/prepare_date_local.sh
+++ b/templates/real-from-d2-ana/prepare_date_local.sh
@@ -25,6 +25,7 @@ if [ "${IC_DATA_IS_LOCAL}" == "True" ]; then
# Define date directory, create it and go there
COMMON_DATE_FOLDER=${WORKDIR}/${STARTDATE}/inidata
+ #AN_MEMBER=$(printf "%03d" %SIMULATION.INITIAL_CONDITIONS.MEMBER%)
AN_MEMBER=$(printf "%03d" %SIMULATION.INITIAL_CONDITIONS.MEMBER%)
INITIAL_CONDITIONS_PARENT_FOLDER=%SIMULATION.INITIAL_CONDITIONS.PARENT_FOLDER%
INITIAL_CONDITIONS_PATH=${INITIAL_CONDITIONS_PARENT_FOLDER}/${STARTDATE:0:8}00
diff --git a/templates/real-from-d2-ana/prepare_date_remote.sh b/templates/real-from-d2-ana/prepare_date_remote.sh
index cb3e59e..e3b3dbe 100644
--- a/templates/real-from-d2-ana/prepare_date_remote.sh
+++ b/templates/real-from-d2-ana/prepare_date_remote.sh
@@ -11,16 +11,18 @@ if [ "${IC_DATA_IS_LOCAL}" != "True" ]; then
STARTDATE=%SDATE%
HPCUSER=%HPCUSER%
HPCHOST=%HPCHOST%
+
# Define date directory, create it and go there
COMMON_DATE_FOLDER=${WORKDIR}/${STARTDATE}/inidata
- AN_MEMBER=$(printf "%03d" %SIMULATION.INITIAL_CONDITIONS.MEMBER%)
+ # Exploit the number from the member name
+ AN_MEMBER=$(printf "%03d" ${MEMBER:1})
INITIAL_CONDITIONS_PARENT_FOLDER=%SIMULATION.INITIAL_CONDITIONS.PARENT_FOLDER%
INITIAL_CONDITIONS_PATH=${INITIAL_CONDITIONS_PARENT_FOLDER}/${STARTDATE:0:8}00
- AN_SOURCE=$(find ${INITIAL_CONDITIONS_PATH} -name "an_R19B07.*00_an.${AN_MEMBER}" | sort | tail -n 1)
- AN_INC_SOURCE=$(find ${INITIAL_CONDITIONS_PATH} -name "an_R19B07.*00_inc.${AN_MEMBER}" | sort | tail -n 1)
- FG_SOURCE=$(find ${INITIAL_CONDITIONS_PATH} -name "fc_R19B07.*5500.${AN_MEMBER}" | sort | tail -n 1)
+ AN_SOURCE=$(find ${INITIAL_CONDITIONS_PATH} -name "an_R19B07.*00_an.*" | sort -n )
+ AN_INC_SOURCE=$(find ${INITIAL_CONDITIONS_PATH} -name "an_R19B07.*00_inc.*" | sort -n )
+ FG_SOURCE=$(find ${INITIAL_CONDITIONS_PATH} -name "fc_R19B07.*5500.*" | sort -n )
AN_FILE=$(basename "${AN_SOURCE}")
AN_INC_FILE=$(basename "${AN_INC_SOURCE}")
diff --git a/templates/real-from-d2-ana/prepare_lbc_remote.sh b/templates/real-from-d2-ana/prepare_lbc_remote.sh
index 965eb20..1db5b56 100644
--- a/templates/real-from-d2-ana/prepare_lbc_remote.sh
+++ b/templates/real-from-d2-ana/prepare_lbc_remote.sh
@@ -19,6 +19,7 @@ if [ "${LBC_DATA_IS_LOCAL}" != "True" ]; then
STARTDATE=%SDATE%
HPCUSER=%HPCUSER%
HPCHOST=%HPCHOST%
+ MEMBER=%MEMBER%
. ${WORKDIR}/proj/platforms/common/spack_utils.sh
load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.externals%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%" "%spack.upstreams%"
@@ -33,13 +34,15 @@ if [ "${LBC_DATA_IS_LOCAL}" != "True" ]; then
# Define date directory, create it and go there
COMMON_DATE_FOLDER=${WORKDIR}/${STARTDATE}/inidata
COMMON_GRID_FOLDER=${WORKDIR}/inidata
+ MEMBER_DIR=${WORKDIR}/${STARTDATE}/${MEMBER^}
- AN_MEMBER=$(printf "%03d" %SIMULATION.BOUNDARY_CONDITIONS.MEMBER%)
+ # Leave out the character part
BOUNDARY_CONDITIONS_PARENT_FOLDER=%SIMULATION.BOUNDARY_CONDITIONS.PARENT_FOLDER%
BOUNDARY_CONDITIONS_PATH=${BOUNDARY_CONDITIONS_PARENT_FOLDER}/${STARTDATE:0:8}00
+
# Fetch tar.gz file of ICON-EU-EPS outputs necessary for the lateral boundary conditions
- FC_TAR_SOURCE=$(find ${BOUNDARY_CONDITIONS_PATH} -name "iefff.m${AN_MEMBER}.tar.gz" | sort | tail -n 1)
+ FC_TAR_SOURCE=$(find ${BOUNDARY_CONDITIONS_PATH} -name "iefff.${MEMBER,}.tar.gz" | sort | tail -n 1)
FC_TAR_FILE=$(basename "${FC_TAR_SOURCE}")
@@ -50,7 +53,7 @@ if [ "${LBC_DATA_IS_LOCAL}" != "True" ]; then
fi
# make a working directory for converting boundary conditions
- mkdir -p work
+ mkdir -p work/${MEMBER} || exit
tar -zxvf ${FC_TAR_SOURCE} -C work
@@ -76,14 +79,22 @@ if [ "${LBC_DATA_IS_LOCAL}" != "True" ]; then
${WORKDIR}/%python_environment.folder_name%/bin/python3 ${WORKDIR}/proj/templates/real-from-d2-ana/icon-remap-helper.py \
--src-grid ${COMMON_GRID_FOLDER}/${PARENT_GRID_FILE} \
--dst-grid ${COMMON_GRID_FOLDER}/${BOUNDARY_GRID_FILE} \
- --source work/iefff${EU_FC_TIME}0000.m${AN_MEMBER} \
- --dest ${COMMON_DATE_FOLDER} \
- --rename latbc_${D2_FC_TIME}00.m${AN_MEMBER} \
+ --source work/iefff${EU_FC_TIME}0000.${MEMBER,} \
+ --dest work/${MEMBER} \
--output-format grb
+
+
+ # rename to the comfortable style for ICON
+ mv work/${MEMBER}/iefff${EU_FC_TIME}0000.grb ${MEMBER_DIR}/latbc_${D2_FC_TIME}.${MEMBER,}.grib
+
done
+ # clean up the working directory
+ rm -r work/${MEMBER}
+
+
# Fetch tar.gz file of ICON-EU-EPS outputs necessary for the lateral boundary conditions
- LBC_SOURCE=$(find ${COMMON_DATE_FOLDER} -name "latbc_*00.m${AN_MEMBER}" | sort )
+ LBC_SOURCE=$(find ${MEMBER_DIR} -name "latbc_*00.${MEMBER,}.grib" | sort )
LBC_FILE=$(basename "${LBC_SOURCE}")
@@ -99,7 +110,7 @@ if [ "${LBC_DATA_IS_LOCAL}" != "True" ]; then
cd ${COMMON_DATE_FOLDER} || exit
# Save filenames to be used later by other scripts.
- echo "${LBC_SOURCE}" > lbc_file.txt
+ echo "${LBC_SOURCE}" >> lbc_file.txt
# Copy the first-guess and analysis files.
#cp "${LBC_SOURCE}" "${LBC_FILE}"
diff --git a/templates/real-from-d2-ana/prepare_member.sh b/templates/real-from-d2-ana/prepare_member.sh
index ff7ebc1..c6a57df 100644
--- a/templates/real-from-d2-ana/prepare_member.sh
+++ b/templates/real-from-d2-ana/prepare_member.sh
@@ -11,7 +11,7 @@ COMMON_INIDATA_FOLDER=${WORKDIR}/inidata
COMMON_DATE_FOLDER=${WORKDIR}/${STARTDATE}/inidata
# Member folder
-MEMBER_DIR=${WORKDIR}/${STARTDATE}/${MEMBER}
+MEMBER_DIR=${WORKDIR}/${STARTDATE}/${MEMBER^}
# Create member folder and go there
mkdir -p ${MEMBER_DIR}
@@ -21,4 +21,5 @@ cd ${MEMBER_DIR} || exit
# Link all files from the common inidata folder and the common date folder
ln -sf ${COMMON_INIDATA_FOLDER}/* .
-ln -sf ${COMMON_DATE_FOLDER}/* .
+
+ln -sf ${COMMON_DATE_FOLDER}/*${MEMBER:1}* .
--
GitLab