From c1e230713028ea4c1a2658141bedf64dfd387ff4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Oriol=20Tint=C3=B3=20Prims?= <oriol.tinto@lmu.de>
Date: Fri, 24 Mar 2023 15:21:25 +0100
Subject: [PATCH] Typo in HPCARCH. Split prepare_date.sh in
prepare_date_local.sh and prepare_date_remote.sh.
---
conf/real-from-dwd-ana/jobs.yml | 18 +++--
conf/real-from-dwd-ana/simulation.yml | 10 +--
conf/real-from-ideal/jobs.yml | 12 ++--
templates/real-from-dwd-ana/prepare_date.sh | 44 -------------
.../real-from-dwd-ana/prepare_date_local.sh | 65 +++++++++++++++++++
.../real-from-dwd-ana/prepare_date_remote.sh | 65 +++++++++++++++++++
6 files changed, 153 insertions(+), 61 deletions(-)
delete mode 100644 templates/real-from-dwd-ana/prepare_date.sh
create mode 100644 templates/real-from-dwd-ana/prepare_date_local.sh
create mode 100644 templates/real-from-dwd-ana/prepare_date_remote.sh
diff --git a/conf/real-from-dwd-ana/jobs.yml b/conf/real-from-dwd-ana/jobs.yml
index afc7b95..cc04a30 100644
--- a/conf/real-from-dwd-ana/jobs.yml
+++ b/conf/real-from-dwd-ana/jobs.yml
@@ -79,15 +79,21 @@ JOBS:
RUNNING: once
WALLCLOCK: 01:00
- PREPARE_DATE:
- FILE: templates/real-from-dwd-ana/prepare_date.sh
+ PREPARE_DATE_LOCAL:
+ FILE: templates/real-from-dwd-ana/prepare_date_local.sh
RUNNING: date
WALLCLOCK: 01:00
- PLATFORM: "%DEFAULT.HPCARH%_LOGIN"
+ PLATFORM: LOCAL
+
+ PREPARE_DATE_REMOTE:
+ FILE: templates/real-from-dwd-ana/prepare_date_remote.sh
+ RUNNING: date
+ WALLCLOCK: 01:00
+ PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"
PREPARE_MEMBER:
FILE: templates/real-from-dwd-ana/prepare_member.sh
- DEPENDENCIES: PREPARE_EXPERIMENT PREPARE_DATE
+ DEPENDENCIES: PREPARE_EXPERIMENT PREPARE_DATE_REMOTE PREPARE_DATE_LOCAL
RUNNING: member
WALLCLOCK: 01:00
@@ -98,7 +104,7 @@ JOBS:
RUNNING: chunk
TYPE: python
EXECUTABLE: "%HPCROOTDIR%/%python_environment.folder_name%/bin/python3"
- PLATFORM: "%DEFAULT.HPCARH%_LOGIN"
+ PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"
RUN_ICON:
@@ -133,4 +139,4 @@ JOBS:
DEPENDENCIES: TRANSFER
WALLCLOCK: 00:10
RUNNING: member
- PLATFORM: "%DEFAULT.HPCARH%_LOGIN"
\ No newline at end of file
+ PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"
\ No newline at end of file
diff --git a/conf/real-from-dwd-ana/simulation.yml b/conf/real-from-dwd-ana/simulation.yml
index c423062..c3c274f 100644
--- a/conf/real-from-dwd-ana/simulation.yml
+++ b/conf/real-from-dwd-ana/simulation.yml
@@ -12,9 +12,9 @@ simulation:
output_file_names: "init_DOM01_ML_*.nc latbc_DOM01_ML_*.nc"
files_to_clean: "*.nc"
-initial_conditions:
- # Where are we getting our initial data from?
- local: True
- parent_folder: /archive/meteo/external-models/dwd/icon/oper/icon_oper_eps_gridded-global_rolling/
- member: 1
+ initial_conditions:
+ # Where are we getting our initial data from?
+ local: true
+ parent_folder: /archive/meteo/external-models/dwd/icon/oper/icon_oper_eps_gridded-global_rolling/
+ member: 1
diff --git a/conf/real-from-ideal/jobs.yml b/conf/real-from-ideal/jobs.yml
index 8a0840a..27d09d3 100644
--- a/conf/real-from-ideal/jobs.yml
+++ b/conf/real-from-ideal/jobs.yml
@@ -77,14 +77,14 @@ JOBS:
DEPENDENCIES: BUILD_ICON
RUNNING: once
WALLCLOCK: 00:10
- PLATFORM: "%DEFAULT.HPCARH%_LOGIN"
+ PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"
PREPARE_IDEAL_DIRECTORY:
FILE: templates/real-from-ideal/prepare_ideal_directory.sh
DEPENDENCIES: PREPARE_EXPERIMENT
RUNNING: date
WALLCLOCK: 00:10
- PLATFORM: "%DEFAULT.HPCARH%_LOGIN"
+ PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"
PREPARE_IDEAL_NAMELIST:
FILE: templates/real-from-ideal/prepare_ideal_namelist.py
@@ -93,7 +93,7 @@ JOBS:
WALLCLOCK: 00:10
TYPE: python
EXECUTABLE: "%HPCROOTDIR%/%python_environment.folder_name%/bin/python3"
- PLATFORM: "%DEFAULT.HPCARH%_LOGIN"
+ PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"
RUN_IDEAL:
FILE: templates/real-from-ideal/run_ideal.sh
@@ -123,7 +123,7 @@ JOBS:
DEPENDENCIES: FG_ANA_FROM_IDEALIZED EXTPAR_FROM_IDEALIZED
RUNNING: member
WALLCLOCK: 01:00
- PLATFORM: "%DEFAULT.HPCARH%_LOGIN"
+ PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"
PREPARE_CHUNK:
FILE: templates/real-from-ideal/prepare_chunk.py
@@ -132,7 +132,7 @@ JOBS:
RUNNING: chunk
TYPE: python
EXECUTABLE: "%HPCROOTDIR%/%python_environment.folder_name%/bin/python3"
- PLATFORM: "%DEFAULT.HPCARH%_LOGIN"
+ PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"
RUN_ICON:
FILE: templates/common/run_icon.sh
@@ -160,4 +160,4 @@ JOBS:
DEPENDENCIES: TRANSFER
WALLCLOCK: 00:10
RUNNING: member
- PLATFORM: "%DEFAULT.HPCARH%_LOGIN"
+ PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"
diff --git a/templates/real-from-dwd-ana/prepare_date.sh b/templates/real-from-dwd-ana/prepare_date.sh
deleted file mode 100644
index 4152601..0000000
--- a/templates/real-from-dwd-ana/prepare_date.sh
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/bin/bash -l
-
-# Get some variables provided by autosubmit.
-WORKDIR=%HPCROOTDIR%
-STARTDATE=%SDATE%
-
-# Define date directory, create it and go there
-COMMON_DATE_FOLDER=${WORKDIR}/${STARTDATE}/inidata
-# Create member folder and go there
-mkdir -p ${COMMON_DATE_FOLDER}
-cd ${COMMON_DATE_FOLDER} || exit
-
-# some settings
-AN_MEMBER=$(printf "%03d" %initial_conditions.member%)
-INITIAL_CONDITIONS_PARENT_FOLDER=%initial_conditions.parent_folder%
-
-INITIAL_CONDITIONS_PATH=${INITIAL_CONDITIONS_PARENT_FOLDER}/${STARTDATE:0:6}/${STARTDATE:0:8}T00
-
-AN_SOURCE=$(find ${INITIAL_CONDITIONS_PATH} -name "igaf*00.m${AN_MEMBER}.grb" | sort | tail -n 1)
-FG_SOURCE=$(find ${INITIAL_CONDITIONS_PATH} -name "igfff00030000.m${AN_MEMBER}.grb" | sort | tail -n 1)
-
-if [ ! -f "${AN_SOURCE}" ]; then
- echo "Analysis file for date ${STARTDATE} not found!"
- exit 1
-fi
-
-if [ ! -f "${FG_SOURCE}" ]; then
- echo "FG file for date ${STARTDATE} not found!"
- exit 1
-fi
-
-AN_FILE=$(basename "${AN_SOURCE}")
-FG_FILE=$(basename "${FG_SOURCE}")
-
-# Save filenames to be used later by other scripts.
-echo "${AN_FILE}" > an_file.txt
-echo "${FG_FILE}" > fg_file.txt
-
-# Copy the first-guess and analysis files.
-cp "${FG_SOURCE}" "${FG_FILE}"
-cp "${AN_SOURCE}" "${AN_FILE}"
-
-# Change permissions to read only.
-chmod 440 ./*
diff --git a/templates/real-from-dwd-ana/prepare_date_local.sh b/templates/real-from-dwd-ana/prepare_date_local.sh
new file mode 100644
index 0000000..1aa323c
--- /dev/null
+++ b/templates/real-from-dwd-ana/prepare_date_local.sh
@@ -0,0 +1,65 @@
+#!/bin/bash -l
+
+IS_LOCAL=%SIMULATION.INITIAL_CONDITIONS.MEMBER%
+
+if [ "${IS_LOCAL}" == "True" ]; then
+ # Get some variables provided by autosubmit.
+ WORKDIR=%HPCROOTDIR%
+ STARTDATE=%SDATE%
+ # Define date directory, create it and go there
+ COMMON_DATE_FOLDER=${WORKDIR}/${STARTDATE}/inidata
+
+ AN_MEMBER=$(printf "%03d" %SIMULATION.INITIAL_CONDITIONS.MEMBER%)
+ INITIAL_CONDITIONS_PARENT_FOLDER=%SIMULATION.INITIAL_CONDITIONS.PARENT_FOLDER%
+ INITIAL_CONDITIONS_PATH=${INITIAL_CONDITIONS_PARENT_FOLDER}/${STARTDATE:0:6}/${STARTDATE:0:8}T00
+
+ AN_SOURCE=$(find ${INITIAL_CONDITIONS_PATH} -name "igaf*00.m${AN_MEMBER}.grb" | sort | tail -n 1)
+ FG_SOURCE=$(find ${INITIAL_CONDITIONS_PATH} -name "igfff00030000.m${AN_MEMBER}.grb" | sort | tail -n 1)
+
+ AN_FILE=$(basename "${AN_SOURCE}")
+ FG_FILE=$(basename "${FG_SOURCE}")
+
+ # Find files
+ if [ ! -f "${AN_SOURCE}" ]; then
+ echo "Analysis file for date ${STARTDATE} not found!"
+ exit 1
+ fi
+
+ if [ ! -f "${FG_SOURCE}" ]; then
+ echo "FG file for date ${STARTDATE} not found!"
+ exit 1
+ fi
+
+
+ # Check if we copy the initial conditions from the local system or the remote one
+ if [ "${IS_LOCAL}" == "True" ]; then
+ # Create member folder
+ ssh "${HPCUSER}@${HPCHOST}" mkdir -p ${COMMON_DATE_FOLDER}
+
+ # Save filenames to be used later by other scripts.
+ ssh "${HPCUSER}@${HPCHOST}" echo "${AN_FILE}" > ${COMMON_DATE_FOLDER}/an_file.txt
+ ssh "${HPCUSER}@${HPCHOST}" echo "${FG_FILE}" > ${COMMON_DATE_FOLDER}/fg_file.txt
+
+ # Copy the first-guess and analysis files.
+ rsync "${FG_SOURCE}" "${HPCUSER}@${HPCHOST}":"${COMMON_DATE_FOLDER}/${FG_FILE}"
+ rsync "${AN_SOURCE}" "${HPCUSER}@${HPCHOST}":"${COMMON_DATE_FOLDER}/${AN_FILE}"
+
+ # Change permissions to read only.
+ ssh "${HPCUSER}@${HPCHOST}" chmod 440 ./*
+ else
+ # Create member folder and go there
+ mkdir -p ${COMMON_DATE_FOLDER}
+ cd ${COMMON_DATE_FOLDER} || exit
+
+ # Save filenames to be used later by other scripts.
+ echo "${AN_FILE}" > an_file.txt
+ echo "${FG_FILE}" > fg_file.txt
+
+ # Copy the first-guess and analysis files.
+ cp "${FG_SOURCE}" "${FG_FILE}"
+ cp "${AN_SOURCE}" "${AN_FILE}"
+
+ # Change permissions to read only.
+ chmod 440 ./*
+ fi
+fi
\ No newline at end of file
diff --git a/templates/real-from-dwd-ana/prepare_date_remote.sh b/templates/real-from-dwd-ana/prepare_date_remote.sh
new file mode 100644
index 0000000..8d5ca31
--- /dev/null
+++ b/templates/real-from-dwd-ana/prepare_date_remote.sh
@@ -0,0 +1,65 @@
+#!/bin/bash -l
+
+IS_LOCAL=%SIMULATION.INITIAL_CONDITIONS.MEMBER%
+
+if [ "${IS_LOCAL}" == "False" ]; then
+ # Get some variables provided by autosubmit.
+ WORKDIR=%HPCROOTDIR%
+ STARTDATE=%SDATE%
+ # Define date directory, create it and go there
+ COMMON_DATE_FOLDER=${WORKDIR}/${STARTDATE}/inidata
+
+ AN_MEMBER=$(printf "%03d" %SIMULATION.INITIAL_CONDITIONS.MEMBER%)
+ INITIAL_CONDITIONS_PARENT_FOLDER=%SIMULATION.INITIAL_CONDITIONS.PARENT_FOLDER%
+ INITIAL_CONDITIONS_PATH=${INITIAL_CONDITIONS_PARENT_FOLDER}/${STARTDATE:0:6}/${STARTDATE:0:8}T00
+
+ AN_SOURCE=$(find ${INITIAL_CONDITIONS_PATH} -name "igaf*00.m${AN_MEMBER}.grb" | sort | tail -n 1)
+ FG_SOURCE=$(find ${INITIAL_CONDITIONS_PATH} -name "igfff00030000.m${AN_MEMBER}.grb" | sort | tail -n 1)
+
+ AN_FILE=$(basename "${AN_SOURCE}")
+ FG_FILE=$(basename "${FG_SOURCE}")
+
+ # Find files
+ if [ ! -f "${AN_SOURCE}" ]; then
+ echo "Analysis file for date ${STARTDATE} not found!"
+ exit 1
+ fi
+
+ if [ ! -f "${FG_SOURCE}" ]; then
+ echo "FG file for date ${STARTDATE} not found!"
+ exit 1
+ fi
+
+
+ # Check if we copy the initial conditions from the local system or the remote one
+ if [ "${IS_LOCAL}" == "True" ]; then
+ # Create member folder
+ ssh "${HPCUSER}@${HPCHOST}" mkdir -p ${COMMON_DATE_FOLDER}
+
+ # Save filenames to be used later by other scripts.
+ ssh "${HPCUSER}@${HPCHOST}" echo "${AN_FILE}" > ${COMMON_DATE_FOLDER}/an_file.txt
+ ssh "${HPCUSER}@${HPCHOST}" echo "${FG_FILE}" > ${COMMON_DATE_FOLDER}/fg_file.txt
+
+ # Copy the first-guess and analysis files.
+ rsync "${FG_SOURCE}" "${HPCUSER}@${HPCHOST}":"${COMMON_DATE_FOLDER}/${FG_FILE}"
+ rsync "${AN_SOURCE}" "${HPCUSER}@${HPCHOST}":"${COMMON_DATE_FOLDER}/${AN_FILE}"
+
+ # Change permissions to read only.
+ ssh "${HPCUSER}@${HPCHOST}" chmod 440 ./*
+ else
+ # Create member folder and go there
+ mkdir -p ${COMMON_DATE_FOLDER}
+ cd ${COMMON_DATE_FOLDER} || exit
+
+ # Save filenames to be used later by other scripts.
+ echo "${AN_FILE}" > an_file.txt
+ echo "${FG_FILE}" > fg_file.txt
+
+ # Copy the first-guess and analysis files.
+ cp "${FG_SOURCE}" "${FG_FILE}"
+ cp "${AN_SOURCE}" "${AN_FILE}"
+
+ # Change permissions to read only.
+ chmod 440 ./*
+ fi
+fi
\ No newline at end of file
--
GitLab