From e24da65951ca4820e20cf9edfc8b4a0496600950 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Oriol=20Tint=C3=B3?= <oriol.tinto@lmu.de>
Date: Wed, 17 May 2023 13:46:01 +0200
Subject: [PATCH] Refactor job files to get most of the workflow defined in the
common folder.
---
conf/common/jobs.yml | 88 +++++++++++++++++++++++++++++++++
conf/real-from-dwd-ana/jobs.yml | 75 +---------------------------
conf/real-from-ideal/jobs.yml | 59 ----------------------
3 files changed, 89 insertions(+), 133 deletions(-)
create mode 100644 conf/common/jobs.yml
diff --git a/conf/common/jobs.yml b/conf/common/jobs.yml
new file mode 100644
index 0000000..261d115
--- /dev/null
+++ b/conf/common/jobs.yml
@@ -0,0 +1,88 @@
+JOBS:
+ TRANSFER_PROJECT:
+ FILE: templates/common/transfer_project.sh
+ PLATFORM: LOCAL
+
+ BUILD_ICON:
+ FILE: templates/common/build_icon.sh
+ DEPENDENCIES: TRANSFER_PROJECT
+ WALLCLOCK: 04:00
+ PROCESSORS: 16
+ RETRIALS: 2 # retry because spack downloads sometimes timeout
+ NODES: 1
+
+ BUILD_PYTHON_ENVIRONMENT:
+ FILE: templates/common/build_python_environment.sh
+ # Right now we rely on spack for building icon and having a python interpreter, so we need this dependency:
+ DEPENDENCIES: BUILD_ICON
+ WALLCLOCK: 01:00
+ PROCESSORS: 16
+ NODES: 1
+
+ PREPARE_EXPERIMENT:
+ # FILE: How the experiment folder is prepared should be defined for each specific case.
+ DEPENDENCIES: BUILD_ICON
+ RUNNING: once
+ WALLCLOCK: 01:00
+
+ PREPARE_DATE_LOCAL:
+ # FILE: How the date folder is prepared should be defined for each specific case.
+ RUNNING: date
+ WALLCLOCK: 01:00
+ PLATFORM: LOCAL
+
+ PREPARE_DATE_REMOTE:
+ # FILE: How the date folder is prepared should be defined for each specific case.
+ RUNNING: date
+ WALLCLOCK: 01:00
+ PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"
+
+ PREPARE_MEMBER:
+ # FILE: How the member folder is prepared should be defined for each specific case.
+ DEPENDENCIES: PREPARE_EXPERIMENT PREPARE_DATE_REMOTE PREPARE_DATE_LOCAL
+ RUNNING: member
+ WALLCLOCK: 01:00
+
+ PREPARE_NAMELIST:
+ # FILE: How the namelist is prepared should be defined for each specific case.
+ DEPENDENCIES: TRANSFER_PROJECT BUILD_PYTHON_ENVIRONMENT PREPARE_MEMBER RUN_ICON-1
+ WALLCLOCK: 00:05
+ RUNNING: chunk
+ TYPE: python
+ EXECUTABLE: "%HPCROOTDIR%/%python_environment.folder_name%/bin/python3"
+ PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"
+
+
+ RUN_ICON:
+ FILE: templates/common/run_icon.sh
+ DEPENDENCIES: PREPARE_NAMELIST
+ WALLCLOCK: 04:00
+ RUNNING: chunk
+ PROCESSORS: 64
+ MEMORY: 81920
+ CUSTOM_DIRECTIVES: [ "#SBATCH --exclusive" ]
+
+ COMPRESS:
+ FILE: templates/common/compress.py
+ DEPENDENCIES: RUN_ICON BUILD_PYTHON_ENVIRONMENT COMPRESS-1
+ RUNNING: chunk
+ TYPE: python
+ EXECUTABLE: "%HPCROOTDIR%/%python_environment.folder_name%/bin/python3"
+ PROCESSORS: 16
+ MEMORY: 16384
+ WALLCLOCK: 01:00
+
+ TRANSFER:
+ FILE: templates/common/transfer.sh
+ DEPENDENCIES: COMPRESS
+ # Since this is running locally, can simply leave a long wallclock.
+ WALLCLOCK: 24:00
+ RUNNING: member
+ PLATFORM: LOCAL
+
+ CLEAN:
+ FILE: templates/common/clean.sh
+ DEPENDENCIES: TRANSFER
+ WALLCLOCK: 00:10
+ RUNNING: member
+ PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"
diff --git a/conf/real-from-dwd-ana/jobs.yml b/conf/real-from-dwd-ana/jobs.yml
index 04e57d4..2c9aa02 100644
--- a/conf/real-from-dwd-ana/jobs.yml
+++ b/conf/real-from-dwd-ana/jobs.yml
@@ -1,88 +1,15 @@
JOBS:
- TRANSFER_PROJECT:
- FILE: templates/common/transfer_project.sh
- PLATFORM: LOCAL
-
- BUILD_ICON:
- FILE: templates/common/build_icon.sh
- DEPENDENCIES: TRANSFER_PROJECT
- WALLCLOCK: 04:00
- PROCESSORS: 16
- RETRIALS: 2 # retry because spack downloads sometimes timeout
- NODES: 1
-
- BUILD_PYTHON_ENVIRONMENT:
- FILE: templates/common/build_python_environment.sh
- # Right now we rely on spack for building icon and having a python interpreter, so we need this dependency:
- DEPENDENCIES: BUILD_ICON
- WALLCLOCK: 01:00
- PROCESSORS: 16
- NODES: 1
-
PREPARE_EXPERIMENT:
FILE: templates/real-from-dwd-ana/prepare_experiment.sh
- DEPENDENCIES: BUILD_ICON
- RUNNING: once
- WALLCLOCK: 01:00
PREPARE_DATE_LOCAL:
FILE: templates/real-from-dwd-ana/prepare_date_local.sh
- RUNNING: date
- WALLCLOCK: 01:00
- PLATFORM: LOCAL
PREPARE_DATE_REMOTE:
FILE: templates/real-from-dwd-ana/prepare_date_remote.sh
- RUNNING: date
- WALLCLOCK: 01:00
- PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"
PREPARE_MEMBER:
FILE: templates/real-from-dwd-ana/prepare_member.sh
- DEPENDENCIES: PREPARE_EXPERIMENT PREPARE_DATE_REMOTE PREPARE_DATE_LOCAL
- RUNNING: member
- WALLCLOCK: 01:00
PREPARE_NAMELIST:
- FILE: templates/real-from-dwd-ana/prepare_namelist.py
- DEPENDENCIES: TRANSFER_PROJECT BUILD_PYTHON_ENVIRONMENT PREPARE_MEMBER RUN_ICON-1
- WALLCLOCK: 00:05
- RUNNING: chunk
- TYPE: python
- EXECUTABLE: "%HPCROOTDIR%/%python_environment.folder_name%/bin/python3"
- PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"
-
-
- RUN_ICON:
- FILE: templates/common/run_icon.sh
- DEPENDENCIES: PREPARE_NAMELIST
- WALLCLOCK: 04:00
- RUNNING: chunk
- PROCESSORS: 64
- MEMORY: 81920
- CUSTOM_DIRECTIVES: [ "#SBATCH --exclusive" ]
-
- COMPRESS:
- FILE: templates/common/compress.py
- DEPENDENCIES: RUN_ICON BUILD_PYTHON_ENVIRONMENT COMPRESS-1
- RUNNING: chunk
- TYPE: python
- EXECUTABLE: "%HPCROOTDIR%/%python_environment.folder_name%/bin/python3"
- PROCESSORS: 16
- MEMORY: 16384
- WALLCLOCK: 01:00
-
- TRANSFER:
- FILE: templates/common/transfer.sh
- DEPENDENCIES: COMPRESS
- # Since this is running locally, can simply leave a long wallclock.
- WALLCLOCK: 24:00
- RUNNING: member
- PLATFORM: LOCAL
-
- CLEAN:
- FILE: templates/common/clean.sh
- DEPENDENCIES: TRANSFER
- WALLCLOCK: 00:10
- RUNNING: member
- PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"
+ FILE: templates/real-from-dwd-ana/prepare_namelist.py
\ No newline at end of file
diff --git a/conf/real-from-ideal/jobs.yml b/conf/real-from-ideal/jobs.yml
index 1fc83dc..ddeb817 100644
--- a/conf/real-from-ideal/jobs.yml
+++ b/conf/real-from-ideal/jobs.yml
@@ -1,29 +1,6 @@
JOBS:
- TRANSFER_PROJECT:
- FILE: templates/common/transfer_project.sh
- PLATFORM: LOCAL
-
- BUILD_ICON:
- FILE: templates/common/build_icon.sh
- DEPENDENCIES: TRANSFER_PROJECT
- WALLCLOCK: 04:00
- PROCESSORS: 16
- RETRIALS: 2 # retry because spack downloads sometimes timeout
- NODES: 1
-
- BUILD_PYTHON_ENVIRONMENT:
- FILE: templates/common/build_python_environment.sh
- # Right now we rely on spack for building icon and having a python interpreter, so we need this dependency:
- DEPENDENCIES: BUILD_ICON
- WALLCLOCK: 01:00
- PROCESSORS: 16
- NODES: 1
-
PREPARE_EXPERIMENT:
FILE: templates/real-from-ideal/prepare_experiment.sh
- DEPENDENCIES: BUILD_ICON
- RUNNING: once
- WALLCLOCK: 00:10
PREPARE_IDEAL_DIRECTORY:
FILE: templates/real-from-ideal/prepare_ideal_directory.sh
@@ -61,45 +38,9 @@ JOBS:
TYPE: python
EXECUTABLE: "%HPCROOTDIR%/%python_environment.folder_name%/bin/python3"
-
PREPARE_MEMBER:
FILE: templates/real-from-ideal/prepare_member.sh
DEPENDENCIES: FG_ANA_FROM_IDEALIZED EXTPAR_FROM_IDEALIZED
- RUNNING: member
- WALLCLOCK: 01:00
PREPARE_NAMELIST:
FILE: templates/real-from-ideal/prepare_namelist.py
- DEPENDENCIES: TRANSFER_PROJECT BUILD_PYTHON_ENVIRONMENT PREPARE_MEMBER RUN_ICON-1
- WALLCLOCK: 00:05
- RUNNING: chunk
- TYPE: python
- EXECUTABLE: "%HPCROOTDIR%/%python_environment.folder_name%/bin/python3"
-
- RUN_ICON:
- FILE: templates/common/run_icon.sh
- DEPENDENCIES: PREPARE_NAMELIST
- WALLCLOCK: 01:00
- RUNNING: chunk
- PROCESSORS: 16
- NODES: 1
-
- COMPRESS:
- FILE: templates/common/compress.py
- DEPENDENCIES: RUN_ICON COMPRESS-1 BUILD_PYTHON_ENVIRONMENT
- RUNNING: chunk
- TYPE: python
- EXECUTABLE: "%HPCROOTDIR%/%python_environment.folder_name%/bin/python3"
-
- TRANSFER:
- FILE: templates/common/transfer.sh
- DEPENDENCIES: COMPRESS
- WALLCLOCK: 00:10
- RUNNING: member
- PLATFORM: LOCAL
-
- CLEAN:
- FILE: templates/common/clean.sh
- DEPENDENCIES: TRANSFER
- WALLCLOCK: 00:10
- RUNNING: member
--
GitLab