Skip to content
Snippets Groups Projects
Commit e24da659 authored by Oriol Tintó's avatar Oriol Tintó
Browse files

Refactor job files to get most of the workflow defined in the common folder.

parent ff2b9c15
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
conf/common/jobs.yml 0 → 100644
+ 88
− 0
View file @ e24da659
JOBS:
TRANSFER_PROJECT:
FILE: templates/common/transfer_project.sh
PLATFORM: LOCAL
BUILD_ICON:
FILE: templates/common/build_icon.sh
DEPENDENCIES: TRANSFER_PROJECT
WALLCLOCK: 04:00
PROCESSORS: 16
RETRIALS: 2 # retry because spack downloads sometimes timeout
NODES: 1
BUILD_PYTHON_ENVIRONMENT:
FILE: templates/common/build_python_environment.sh
# Right now we rely on spack for building icon and having a python interpreter, so we need this dependency:
DEPENDENCIES: BUILD_ICON
WALLCLOCK: 01:00
PROCESSORS: 16
NODES: 1
PREPARE_EXPERIMENT:
# FILE: How the experiment folder is prepared should be defined for each specific case.
DEPENDENCIES: BUILD_ICON
RUNNING: once
WALLCLOCK: 01:00
PREPARE_DATE_LOCAL:
# FILE: How the date folder is prepared should be defined for each specific case.
RUNNING: date
WALLCLOCK: 01:00
PLATFORM: LOCAL
PREPARE_DATE_REMOTE:
# FILE: How the date folder is prepared should be defined for each specific case.
RUNNING: date
WALLCLOCK: 01:00
PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"
PREPARE_MEMBER:
# FILE: How the member folder is prepared should be defined for each specific case.
DEPENDENCIES: PREPARE_EXPERIMENT PREPARE_DATE_REMOTE PREPARE_DATE_LOCAL
RUNNING: member
WALLCLOCK: 01:00
PREPARE_NAMELIST:
# FILE: How the namelist is prepared should be defined for each specific case.
DEPENDENCIES: TRANSFER_PROJECT BUILD_PYTHON_ENVIRONMENT PREPARE_MEMBER RUN_ICON-1
WALLCLOCK: 00:05
RUNNING: chunk
TYPE: python
EXECUTABLE: "%HPCROOTDIR%/%python_environment.folder_name%/bin/python3"
PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"
RUN_ICON:
FILE: templates/common/run_icon.sh
DEPENDENCIES: PREPARE_NAMELIST
WALLCLOCK: 04:00
RUNNING: chunk
PROCESSORS: 64
MEMORY: 81920
CUSTOM_DIRECTIVES: [ "#SBATCH --exclusive" ]
COMPRESS:
FILE: templates/common/compress.py
DEPENDENCIES: RUN_ICON BUILD_PYTHON_ENVIRONMENT COMPRESS-1
RUNNING: chunk
TYPE: python
EXECUTABLE: "%HPCROOTDIR%/%python_environment.folder_name%/bin/python3"
PROCESSORS: 16
MEMORY: 16384
WALLCLOCK: 01:00
TRANSFER:
FILE: templates/common/transfer.sh
DEPENDENCIES: COMPRESS
# Since this is running locally, can simply leave a long wallclock.
WALLCLOCK: 24:00
RUNNING: member
PLATFORM: LOCAL
CLEAN:
FILE: templates/common/clean.sh
DEPENDENCIES: TRANSFER
WALLCLOCK: 00:10
RUNNING: member
PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"
conf/real-from-dwd-ana/jobs.yml
+ 1
− 74
View file @ e24da659
JOBS: JOBS:
TRANSFER_PROJECT:
FILE: templates/common/transfer_project.sh
PLATFORM: LOCAL
BUILD_ICON:
FILE: templates/common/build_icon.sh
DEPENDENCIES: TRANSFER_PROJECT
WALLCLOCK: 04:00
PROCESSORS: 16
RETRIALS: 2 # retry because spack downloads sometimes timeout
NODES: 1
BUILD_PYTHON_ENVIRONMENT:
FILE: templates/common/build_python_environment.sh
# Right now we rely on spack for building icon and having a python interpreter, so we need this dependency:
DEPENDENCIES: BUILD_ICON
WALLCLOCK: 01:00
PROCESSORS: 16
NODES: 1
PREPARE_EXPERIMENT: PREPARE_EXPERIMENT:
FILE: templates/real-from-dwd-ana/prepare_experiment.sh FILE: templates/real-from-dwd-ana/prepare_experiment.sh
DEPENDENCIES: BUILD_ICON
RUNNING: once
WALLCLOCK: 01:00
PREPARE_DATE_LOCAL: PREPARE_DATE_LOCAL:
FILE: templates/real-from-dwd-ana/prepare_date_local.sh FILE: templates/real-from-dwd-ana/prepare_date_local.sh
RUNNING: date
WALLCLOCK: 01:00
PLATFORM: LOCAL
PREPARE_DATE_REMOTE: PREPARE_DATE_REMOTE:
FILE: templates/real-from-dwd-ana/prepare_date_remote.sh FILE: templates/real-from-dwd-ana/prepare_date_remote.sh
RUNNING: date
WALLCLOCK: 01:00
PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"
PREPARE_MEMBER: PREPARE_MEMBER:
FILE: templates/real-from-dwd-ana/prepare_member.sh FILE: templates/real-from-dwd-ana/prepare_member.sh
DEPENDENCIES: PREPARE_EXPERIMENT PREPARE_DATE_REMOTE PREPARE_DATE_LOCAL
RUNNING: member
WALLCLOCK: 01:00
PREPARE_NAMELIST: PREPARE_NAMELIST:
FILE: templates/real-from-dwd-ana/prepare_namelist.py FILE: templates/real-from-dwd-ana/prepare_namelist.py
DEPENDENCIES: TRANSFER_PROJECT BUILD_PYTHON_ENVIRONMENT PREPARE_MEMBER RUN_ICON-1 \ No newline at end of file
WALLCLOCK: 00:05
RUNNING: chunk
TYPE: python
EXECUTABLE: "%HPCROOTDIR%/%python_environment.folder_name%/bin/python3"
PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"
RUN_ICON:
FILE: templates/common/run_icon.sh
DEPENDENCIES: PREPARE_NAMELIST
WALLCLOCK: 04:00
RUNNING: chunk
PROCESSORS: 64
MEMORY: 81920
CUSTOM_DIRECTIVES: [ "#SBATCH --exclusive" ]
COMPRESS:
FILE: templates/common/compress.py
DEPENDENCIES: RUN_ICON BUILD_PYTHON_ENVIRONMENT COMPRESS-1
RUNNING: chunk
TYPE: python
EXECUTABLE: "%HPCROOTDIR%/%python_environment.folder_name%/bin/python3"
PROCESSORS: 16
MEMORY: 16384
WALLCLOCK: 01:00
TRANSFER:
FILE: templates/common/transfer.sh
DEPENDENCIES: COMPRESS
# Since this is running locally, can simply leave a long wallclock.
WALLCLOCK: 24:00
RUNNING: member
PLATFORM: LOCAL
CLEAN:
FILE: templates/common/clean.sh
DEPENDENCIES: TRANSFER
WALLCLOCK: 00:10
RUNNING: member
PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"
conf/real-from-ideal/jobs.yml
+ 0
− 59
View file @ e24da659
JOBS: JOBS:
TRANSFER_PROJECT:
FILE: templates/common/transfer_project.sh
PLATFORM: LOCAL
BUILD_ICON:
FILE: templates/common/build_icon.sh
DEPENDENCIES: TRANSFER_PROJECT
WALLCLOCK: 04:00
PROCESSORS: 16
RETRIALS: 2 # retry because spack downloads sometimes timeout
NODES: 1
BUILD_PYTHON_ENVIRONMENT:
FILE: templates/common/build_python_environment.sh
# Right now we rely on spack for building icon and having a python interpreter, so we need this dependency:
DEPENDENCIES: BUILD_ICON
WALLCLOCK: 01:00
PROCESSORS: 16
NODES: 1
PREPARE_EXPERIMENT: PREPARE_EXPERIMENT:
FILE: templates/real-from-ideal/prepare_experiment.sh FILE: templates/real-from-ideal/prepare_experiment.sh
DEPENDENCIES: BUILD_ICON
RUNNING: once
WALLCLOCK: 00:10
PREPARE_IDEAL_DIRECTORY: PREPARE_IDEAL_DIRECTORY:
FILE: templates/real-from-ideal/prepare_ideal_directory.sh FILE: templates/real-from-ideal/prepare_ideal_directory.sh
...@@ -61,45 +38,9 @@ JOBS: ...@@ -61,45 +38,9 @@ JOBS:
TYPE: python TYPE: python
EXECUTABLE: "%HPCROOTDIR%/%python_environment.folder_name%/bin/python3" EXECUTABLE: "%HPCROOTDIR%/%python_environment.folder_name%/bin/python3"
PREPARE_MEMBER: PREPARE_MEMBER:
FILE: templates/real-from-ideal/prepare_member.sh FILE: templates/real-from-ideal/prepare_member.sh
DEPENDENCIES: FG_ANA_FROM_IDEALIZED EXTPAR_FROM_IDEALIZED DEPENDENCIES: FG_ANA_FROM_IDEALIZED EXTPAR_FROM_IDEALIZED
RUNNING: member
WALLCLOCK: 01:00
PREPARE_NAMELIST: PREPARE_NAMELIST:
FILE: templates/real-from-ideal/prepare_namelist.py FILE: templates/real-from-ideal/prepare_namelist.py
DEPENDENCIES: TRANSFER_PROJECT BUILD_PYTHON_ENVIRONMENT PREPARE_MEMBER RUN_ICON-1
WALLCLOCK: 00:05
RUNNING: chunk
TYPE: python
EXECUTABLE: "%HPCROOTDIR%/%python_environment.folder_name%/bin/python3"
RUN_ICON:
FILE: templates/common/run_icon.sh
DEPENDENCIES: PREPARE_NAMELIST
WALLCLOCK: 01:00
RUNNING: chunk
PROCESSORS: 16
NODES: 1
COMPRESS:
FILE: templates/common/compress.py
DEPENDENCIES: RUN_ICON COMPRESS-1 BUILD_PYTHON_ENVIRONMENT
RUNNING: chunk
TYPE: python
EXECUTABLE: "%HPCROOTDIR%/%python_environment.folder_name%/bin/python3"
TRANSFER:
FILE: templates/common/transfer.sh
DEPENDENCIES: COMPRESS
WALLCLOCK: 00:10
RUNNING: member
PLATFORM: LOCAL
CLEAN:
FILE: templates/common/clean.sh
DEPENDENCIES: TRANSFER
WALLCLOCK: 00:10
RUNNING: member
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment