Skip to content
Snippets Groups Projects
Commit 69558664 authored by Oriol Tintó's avatar Oriol Tintó
Browse files

Refactor examples to improve code maintainability.

parent 1bc72701
No related branches found
No related tags found
No related merge requests found
Pipeline #18794 passed
Stage: everything
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
+ 3
− 22
View file @ 69558664
......@@ -4,14 +4,7 @@ stages:
real-from-ideal:
stage: everything
script:
# Build
- ./examples/01_real-from-ideal.sh 0
# Run Ideal
- ./examples/01_real-from-ideal.sh 1
# Run Real
- ./examples/01_real-from-ideal.sh 2
# Post-process and clean
- ./examples/01_real-from-ideal.sh 3
- ./examples/01_real-from-ideal.sh
tags:
- slurm.meteo.physik.lmu.de
variables:
......@@ -27,14 +20,7 @@ real-from-ideal:
real-from-ideal-psp:
stage: everything
script:
# Build
- ./examples/01_real-from-ideal+psp.sh 0
# Run Ideal
- ./examples/01_real-from-ideal+psp.sh 1
# Run Real
- ./examples/01_real-from-ideal+psp.sh 2
# Post-process and clean
- ./examples/01_real-from-ideal+psp.sh 3
- ./examples/01_real-from-ideal+psp.sh
tags:
- slurm.meteo.physik.lmu.de
variables:
......@@ -50,12 +36,7 @@ real-from-ideal-psp:
real-from-dwd-ana:
stage: everything
script:
# Build
- ./examples/02_real-from-dwd-ana.sh 0
# Run
- ./examples/02_real-from-dwd-ana.sh 1
# Post-process and clean
- ./examples/02_real-from-dwd-ana.sh 2
- ./examples/02_real-from-dwd-ana.sh
tags:
- slurm.meteo.physik.lmu.de
variables:
......
examples/01_real-from-ideal+psp.sh
+ 23
− 153
View file @ 69558664
#!/bin/bash
set -euo pipefail
STAGE=${1:-}
function help() {
echo "Need to provide a integer for the stage we want to run."
echo " e.g."
echo " - 0: Build Icon and python environments"
echo " - 1: Run the ideal case used to initialize a real run from"
echo " - 2: Run the real case"
echo " - 3: post-process outputs"
}
if [ -z "$STAGE" ]; then
help
exit 1
fi
if [[ "$STAGE" -lt 0 ]] || [[ "$STAGE" -gt 3 ]]; then
echo "invalid stage id: $STAGE"
help
exit 2
fi
AUTOSUBMIT_VERSION="4.0.76"
SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
PROJECT_ROOT=$(readlink -f $SCRIPTDIR/../)
OUTPUTDIR=$PROJECT_ROOT/output
mkdir -p $OUTPUTDIR
WORKDIR=$SCRATCH/autosubmit_rundir
PROJECTNAME='ls-mayer'
mkdir -p $WORKDIR/$PROJECTNAME/$(whoami)
virtualenv -p python3 pyenv
. pyenv/bin/activate
cat > requirements.txt <<EOF
autosubmit==$AUTOSUBMIT_VERSION
EOF
pip install -r requirements.txt
if [ ! -e ./.autosubmitrc ] || [ ! -e autosubmit ] ; then
# Note that we create autosubmit here with --local to create the .autosubmitrc not in the home dir
# This requires to run all autosubmit calls from within this root dir.
autosubmit configure --local -db $(pwd)/autosubmit -dbf autosubmit.db -lr $(pwd)/autosubmit
fi
if [ ! -e autosubmit/autosubmit.db ]; then
autosubmit install
fi
PROJECT_ORIGIN="https://gitlab.physik.uni-muenchen.de/w2w/autoicon.git"
PROJECT_BRANCH="master"
EXPID=rfi001
if [ ! -e autosubmit/$EXPID/ ]; then
mkdir -p autosubmit/$EXPID/
autosubmit expid -min -H lmu -d myiconsim -repo $PROJECT_ORIGIN -b $PROJECT_BRANCH &> >(tee autosubmit/$EXPID/log.autosubmit.expid)
AUTOID=$(grep Experiment autosubmit/$EXPID/log.autosubmit.expid | awk '{print $2}')
mv -v autosubmit/$AUTOID/* autosubmit/$EXPID
rmdir -v autosubmit/$AUTOID
fi
cat > autosubmit/$EXPID/conf/minimal.yml <<EOF
ICON_CASE: "real-from-ideal"
CONFIG:
AUTOSUBMIT_VERSION: "$AUTOSUBMIT_VERSION"
TOTALJOBS: 20
MAXWAITINGJOBS: 20
RETRIALS: 0
DEFAULT:
EXPID: "$EXPID"
HPCARCH: "LOCAL" # use LMU to run on cluster
CUSTOM_CONFIG:
PRE:
- "%PROJDIR%/conf/common"
- "%PROJDIR%/conf/%ICON_CASE%"
PROJECT:
PROJECT_TYPE: "git"
PROJECT_DESTINATION: "autoicon"
GIT:
PROJECT_ORIGIN: "$PROJECT_ORIGIN"
PROJECT_BRANCH: "$PROJECT_BRANCH"
PROJECT_COMMIT: ""
PROJECT_SUBMODULES: ""
FETCH_SINGLE_BRANCH: True
Platforms:
DUMMY:
TYPE: ps
EOF
cat > autosubmit/${EXPID}/conf/myconf.yml <<EOF
spack:
init: "" # command to load spack environment, e.g. module load spack, use spack/setup-env.sh if empty
url: https://github.com/spack/spack.git # url to download spack if necessary
branch: develop # if downloaded, branch name to use
compiler: "gcc@11.3.0" # desired compiler for spack
root: "$SCRATCH/autoicon-spack" # path to a spack install, will be downloaded to if not present
externals: "slurm"
user_cache_path: "$SCRATCH/autoicon-spackcache" # spack puts data here when bootstrapping, leave empty to use home folder
user_config_path: "$SCRATCH/autoicon-spackconfig" # spack puts data here when bootstrapping, leave empty to use home folder
disable_local_config: false # if true, spack installs into spack source dir
upstreams: "/software/opt/focal/x86_64/spack/2023.02/spack/opt/spack"
icon:
#build_cmd: "icon-nwp@%ICON.VERSION%% %SPACK.COMPILER%+debug+petsc target=x86_64_v2 ^openmpi+pmi+legacylaunchers schedulers=slurm fabrics=ucx ucx+dc+dm+ib_hw_tm+mlx5_dv+rc+rdmacm+thread_multiple+ud+verbs"
build_cmd: "icon-nwp@%ICON.VERSION%% %SPACK.COMPILER%+debug~mpichecks target=x86_64_v2 source=dkrz_https"
version: psp
data_management:
# Where do we put the output files afterwards?
local_destination_folder: $OUTPUTDIR/
#######################################################################################################################
# Three variables are required to define the test case
ICON_CASE="real-from-ideal"
ICON_VERSION="psp"
EXTRA_NAMELIST=$(
cat <<-END
atmosphere_namelist: |
nwp_phy_nml:
inwp_psp: 1
......@@ -127,41 +13,25 @@ atmosphere_namelist_ideal: |
nwp_phy_nml:
inwp_psp: 1
psp_scale: 5.0
END
)
EOF
#######################################################################################################################
set -euo pipefail
if [ "$STAGE" -eq 0 ]; then
echo "Running Stage 0"
autosubmit create ${EXPID} -np
autosubmit refresh ${EXPID}
autosubmit setstatus ${EXPID} -ft PREPARE_EXPERIMENT -t SUSPENDED -s -np
autosubmit run ${EXPID}
# Check if there are failed jobs
[ ! -s autosubmit/${EXPID}/tmp/ASLOGS/jobs_failed_status.log ] || exit 1
fi
# Get path to the script directory
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
if [ "$STAGE" -eq 1 ]; then
echo "Running Stage 1"
autosubmit setstatus ${EXPID} -fs SUSPENDED -t WAITING -s -np
autosubmit setstatus ${EXPID} -ft "EXTPAR_FROM_IDEALIZED FG_ANA_FROM_IDEALIZED" -t SUSPENDED -s -np
autosubmit run ${EXPID}
# Check if there are failed jobs
[ ! -s autosubmit/${EXPID}/tmp/ASLOGS/jobs_failed_status.log ] || exit 1
fi
# Import the function setup_autoicon_example
source ${SCRIPT_DIR}/example_utils.sh
if [ "$STAGE" -eq 2 ]; then
echo "Running Stage 2"
autosubmit setstatus ${EXPID} -fs SUSPENDED -t WAITING -s -np
autosubmit setstatus ${EXPID} -ft TRANSFER -t SUSPENDED -s -np
autosubmit run ${EXPID}
# Check if there are failed jobs
[ ! -s autosubmit/${EXPID}/tmp/ASLOGS/jobs_failed_status.log ] || exit 1
fi
# Setup the experiment
# setup_autoicon_example takes the following arguments:
# case: right now it can be real-from-ideal or real-from-dwd-ana
# ICON_VERSION: it can be master, psp, plexrt or a specific version like 2.6.5-nwp0 (the latest at that moment)
# EXTRA_NAMELIST: In case we want to provide extra parameters we can use it
setup_autoicon_example "$ICON_CASE" "$ICON_VERSION" "${EXTRA_NAMELIST}"
if [ "$STAGE" -eq 3 ]; then
echo "Running Stage 3"
autosubmit setstatus ${EXPID} -fs SUSPENDED -t WAITING -s -np
autosubmit run ${EXPID}
# Check if there are failed jobs
[ ! -s autosubmit/${EXPID}/tmp/ASLOGS/jobs_failed_status.log ] || exit 1
fi
# Create and run experiment
autosubmit create ${EXPID} -np
autosubmit run ${EXPID}
examples/01_real-from-ideal.sh
+ 20
− 152
View file @ 69558664
#!/bin/bash
set -euo pipefail
STAGE=${1:-}
function help() {
echo "Need to provide a integer for the stage we want to run."
echo " e.g."
echo " - 0: Build Icon and python environments"
echo " - 1: Run the ideal case used to initialize a real run from"
echo " - 2: Run the real case"
echo " - 3: post-process outputs"
}
if [ -z "$STAGE" ]; then
help
exit 1
fi
if [[ "$STAGE" -lt 0 ]] || [[ "$STAGE" -gt 3 ]]; then
echo "invalid stage id: $STAGE"
help
exit 2
fi
AUTOSUBMIT_VERSION="4.0.76"
SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
PROJECT_ROOT=$(readlink -f $SCRIPTDIR/../)
OUTPUTDIR=$PROJECT_ROOT/output
mkdir -p $OUTPUTDIR
WORKDIR=$SCRATCH/autosubmit_rundir
PROJECTNAME='ls-mayer'
mkdir -p $WORKDIR/$PROJECTNAME/$(whoami)
virtualenv -p python3 pyenv
. pyenv/bin/activate
cat > requirements.txt <<EOF
autosubmit==$AUTOSUBMIT_VERSION
EOF
pip install -r requirements.txt
#######################################################################################################################
# Three variables are required to define the test case
ICON_CASE="real-from-ideal"
ICON_VERSION="2.6.5-nwp0"
EXTRA_NAMELIST=""
if [ ! -e ./.autosubmitrc ] || [ ! -e autosubmit ] ; then
# Note that we create autosubmit here with --local to create the .autosubmitrc not in the home dir
# This requires to run all autosubmit calls from within this root dir.
autosubmit configure --local -db $(pwd)/autosubmit -dbf autosubmit.db -lr $(pwd)/autosubmit
fi
if [ ! -e autosubmit/autosubmit.db ]; then
autosubmit install
fi
PROJECT_ORIGIN="https://gitlab.physik.uni-muenchen.de/w2w/autoicon.git"
PROJECT_BRANCH="master"
EXPID=rfi001
if [ ! -e autosubmit/$EXPID/ ]; then
mkdir -p autosubmit/$EXPID/
autosubmit expid -min -H lmu -d myiconsim -repo $PROJECT_ORIGIN -b $PROJECT_BRANCH &> >(tee autosubmit/$EXPID/log.autosubmit.expid)
AUTOID=$(grep Experiment autosubmit/$EXPID/log.autosubmit.expid | awk '{print $2}')
mv -v autosubmit/$AUTOID/* autosubmit/$EXPID
rmdir -v autosubmit/$AUTOID
fi
cat > autosubmit/$EXPID/conf/minimal.yml <<EOF
ICON_CASE: "real-from-ideal"
CONFIG:
AUTOSUBMIT_VERSION: "$AUTOSUBMIT_VERSION"
TOTALJOBS: 20
MAXWAITINGJOBS: 20
RETRIALS: 0
DEFAULT:
EXPID: "$EXPID"
HPCARCH: "LOCAL" # use LMU to run on cluster
CUSTOM_CONFIG:
PRE:
- "%PROJDIR%/conf/common"
- "%PROJDIR%/conf/%ICON_CASE%"
PROJECT:
PROJECT_TYPE: "git"
PROJECT_DESTINATION: "autoicon"
GIT:
PROJECT_ORIGIN: "$PROJECT_ORIGIN"
PROJECT_BRANCH: "$PROJECT_BRANCH"
PROJECT_COMMIT: ""
PROJECT_SUBMODULES: ""
FETCH_SINGLE_BRANCH: True
EOF
cat > autosubmit/${EXPID}/conf/myconf.yml <<EOF
spack:
init: "" # command to load spack environment, e.g. module load spack, use spack/setup-env.sh if empty
url: https://github.com/spack/spack.git # url to download spack if necessary
branch: develop # if downloaded, branch name to use
compiler: "gcc@11.3.0" # desired compiler for spack
root: "$SCRATCH/autoicon-spack" # path to a spack install, will be downloaded to if not present
externals: "slurm"
user_cache_path: "$SCRATCH/autoicon-spackcache" # spack puts data here when bootstrapping, leave empty to use home folder
user_config_path: "$SCRATCH/autoicon-spackconfig" # spack puts data here when bootstrapping, leave empty to use home folder
disable_local_config: false # if true, spack installs into spack source dir
upstreams: "/software/opt/focal/x86_64/spack/2023.02/spack/opt/spack"
icon:
#build_cmd: "icon-nwp@%ICON.VERSION%% %SPACK.COMPILER%+debug+petsc target=x86_64_v2 ^openmpi+pmi+legacylaunchers schedulers=slurm fabrics=ucx ucx+dc+dm+ib_hw_tm+mlx5_dv+rc+rdmacm+thread_multiple+ud+verbs"
build_cmd: "icon-nwp@%ICON.VERSION%% %SPACK.COMPILER%+debug~mpichecks target=x86_64_v2 source=dkrz_https"
version: 2.6.5-nwp0
data_management:
# Where do we put the output files afterwards?
local_destination_folder: $OUTPUTDIR/
Platforms:
DUMMY:
TYPE: ps
EOF
#######################################################################################################################
set -euo pipefail
if [ "$STAGE" -eq 0 ]; then
echo "Running Stage 0"
autosubmit create ${EXPID} -np
autosubmit refresh ${EXPID}
autosubmit setstatus ${EXPID} -ft PREPARE_EXPERIMENT -t SUSPENDED -s -np
autosubmit run ${EXPID}
# Check if there are failed jobs
[ ! -s autosubmit/${EXPID}/tmp/ASLOGS/jobs_failed_status.log ] || exit 1
fi
# Get path to the script directory
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
if [ "$STAGE" -eq 1 ]; then
echo "Running Stage 1"
autosubmit setstatus ${EXPID} -fs SUSPENDED -t WAITING -s -np
autosubmit setstatus ${EXPID} -ft "EXTPAR_FROM_IDEALIZED FG_ANA_FROM_IDEALIZED" -t SUSPENDED -s -np
autosubmit run ${EXPID}
# Check if there are failed jobs
[ ! -s autosubmit/${EXPID}/tmp/ASLOGS/jobs_failed_status.log ] || exit 1
fi
# Import the function setup_autoicon_example
source ${SCRIPT_DIR}/example_utils.sh
if [ "$STAGE" -eq 2 ]; then
echo "Running Stage 2"
autosubmit setstatus ${EXPID} -fs SUSPENDED -t WAITING -s -np
autosubmit setstatus ${EXPID} -ft TRANSFER -t SUSPENDED -s -np
autosubmit run ${EXPID}
# Check if there are failed jobs
[ ! -s autosubmit/${EXPID}/tmp/ASLOGS/jobs_failed_status.log ] || exit 1
fi
# Setup the experiment
# setup_autoicon_example takes the following arguments:
# case: right now it can be real-from-ideal or real-from-dwd-ana
# ICON_VERSION: it can be master, psp, plexrt or a specific version like 2.6.5-nwp0 (the latest at that moment)
# EXTRA_NAMELIST: In case we want to provide extra parameters we can use it
setup_autoicon_example "$ICON_CASE" "$ICON_VERSION" "${EXTRA_NAMELIST}"
if [ "$STAGE" -eq 3 ]; then
echo "Running Stage 3"
autosubmit setstatus ${EXPID} -fs SUSPENDED -t WAITING -s -np
autosubmit run ${EXPID}
# Check if there are failed jobs
[ ! -s autosubmit/${EXPID}/tmp/ASLOGS/jobs_failed_status.log ] || exit 1
fi
# Create and run experiment
autosubmit create ${EXPID} -np
autosubmit run ${EXPID}
examples/02_real-from-dwd-ana.sh
+ 20
− 143
View file @ 69558664
#!/bin/bash
set -euo pipefail
STAGE=${1:-}
function help() {
echo "Need to provide a integer for the stage we want to run."
echo " e.g."
echo " - 0: Build Icon and python environemnts"
echo " - 1: Run the ideal case used to initialize a real run from"
echo " - 2: Run the real case"
echo " - 3: postprocess outputs"
}
if [ -z "$STAGE" ]; then
help
exit 1
fi
if [[ "$STAGE" -lt 0 ]] || [[ "$STAGE" -gt 3 ]]; then
echo "invalid stage id: $STAGE"
help
exit 2
fi
AUTOSUBMIT_VERSION="4.0.76"
SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
PROJECT_ROOT=$(readlink -f $SCRIPTDIR/../)
OUTPUTDIR=$PROJECT_ROOT/output
mkdir -p $OUTPUTDIR
WORKDIR=$SCRATCH/autosubmit_rundir
PROJECTNAME='ls-mayer'
mkdir -p $WORKDIR/$PROJECTNAME/$(whoami)
virtualenv -p python3 pyenv
. pyenv/bin/activate
cat > requirements.txt <<EOF
autosubmit==$AUTOSUBMIT_VERSION
EOF
pip install -r requirements.txt
#######################################################################################################################
# Three variables are required to define the test case
ICON_CASE="real-from-dwd-ana"
ICON_VERSION="2.6.5-nwp0"
EXTRA_NAMELIST=""
if [ ! -e ./.autosubmitrc ] || [ ! -e autosubmit ] ; then
# Note that we create autosubmit here with --local to create the .autosubmitrc not in the home dir
# This requires to run all autosubmit calls from within this root dir.
autosubmit configure --local -db $(pwd)/autosubmit -dbf autosubmit.db -lr $(pwd)/autosubmit
fi
if [ ! -e autosubmit/autosubmit.db ]; then
autosubmit install
fi
PROJECT_ORIGIN="https://gitlab.physik.uni-muenchen.de/w2w/autoicon.git"
PROJECT_BRANCH="master"
EXPID=rfda001
if [ ! -e autosubmit/$EXPID/ ]; then
mkdir -p autosubmit/$EXPID/
autosubmit expid -min -H lmu -d myiconsim -repo $PROJECT_ORIGIN -b $PROJECT_BRANCH &> >(tee autosubmit/$EXPID/log.autosubmit.expid)
AUTOID=$(grep Experiment autosubmit/$EXPID/log.autosubmit.expid | awk '{print $2}')
mv -v autosubmit/$AUTOID/* autosubmit/$EXPID
rmdir -v autosubmit/$AUTOID
fi
cat > autosubmit/$EXPID/conf/minimal.yml <<EOF
ICON_CASE: "real-from-dwd-ana"
CONFIG:
AUTOSUBMIT_VERSION: "$AUTOSUBMIT_VERSION"
TOTALJOBS: 20
MAXWAITINGJOBS: 20
RETRIALS: 0
DEFAULT:
EXPID: "$EXPID"
HPCARCH: "LOCAL" # use LMU to run on cluster
CUSTOM_CONFIG:
PRE:
- "%PROJDIR%/conf/common"
- "%PROJDIR%/conf/%ICON_CASE%"
PROJECT:
PROJECT_TYPE: "git"
PROJECT_DESTINATION: "autoicon"
GIT:
PROJECT_ORIGIN: "$PROJECT_ORIGIN"
PROJECT_BRANCH: "$PROJECT_BRANCH"
PROJECT_COMMIT: ""
PROJECT_SUBMODULES: ""
FETCH_SINGLE_BRANCH: True
EOF
cat > autosubmit/${EXPID}/conf/myconf.yml <<EOF
spack:
init: "" # command to load spack environment, e.g. module load spack, use spack/setup-env.sh if empty
url: https://github.com/spack/spack.git # url to download spack if necessary
branch: develop # if downloaded, branch name to use
compiler: "gcc@11.3.0" # desired compiler for spack
root: "$SCRATCH/autoicon-spack" # path to a spack install, will be downloaded to if not present
externals: "slurm"
user_cache_path: "$SCRATCH/autoicon-spackcache" # spack puts data here when bootstrapping, leave empty to use home folder
user_config_path: "$SCRATCH/autoicon-spackconfig" # spack puts data here when bootstrapping, leave empty to use home folder
disable_local_config: false # if true, spack installs into spack source dir
upstreams: "/software/opt/focal/x86_64/spack/2023.02/spack/opt/spack"
icon:
#build_cmd: "icon-nwp@%ICON.VERSION%% %SPACK.COMPILER%+debug+petsc target=x86_64_v2 ^openmpi+pmi+legacylaunchers schedulers=slurm fabrics=ucx ucx+dc+dm+ib_hw_tm+mlx5_dv+rc+rdmacm+thread_multiple+ud+verbs"
build_cmd: "icon-nwp@%ICON.VERSION%% %SPACK.COMPILER%+debug target=x86_64_v2 source=dkrz_https"
version: 2.6.5-nwp0
data_management:
# Where do we put the output files afterwards?
local_destination_folder: $OUTPUTDIR/
Platforms:
DUMMY:
TYPE: ps
#######################################################################################################################
set -euo pipefail
EOF
# Get path to the script directory
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
if [ "$STAGE" -eq 0 ]; then
echo "Running Stage 0"
autosubmit create ${EXPID} -np
autosubmit refresh ${EXPID}
autosubmit setstatus ${EXPID} -ft PREPARE_EXPERIMENT -t SUSPENDED -s -np
autosubmit run ${EXPID}
# Check if there are failed jobs
[ ! -s autosubmit/${EXPID}/tmp/ASLOGS/jobs_failed_status.log ] || exit 1
fi
# Import the function setup_autoicon_example
source ${SCRIPT_DIR}/example_utils.sh
if [ "$STAGE" -eq 1 ]; then
echo "Running Stage 2"
autosubmit setstatus ${EXPID} -fs SUSPENDED -t WAITING -s -np
autosubmit setstatus ${EXPID} -ft TRANSFER -t SUSPENDED -s -np
autosubmit run ${EXPID}
# Check if there are failed jobs
[ ! -s autosubmit/${EXPID}/tmp/ASLOGS/jobs_failed_status.log ] || exit 1
fi
# Setup the experiment
# setup_autoicon_example takes the following arguments:
# case: right now it can be real-from-ideal or real-from-dwd-ana
# ICON_VERSION: it can be master, psp, plexrt or a specific version like 2.6.5-nwp0 (the latest at that moment)
# EXTRA_NAMELIST: In case we want to provide extra parameters we can use it
setup_autoicon_example "$ICON_CASE" "$ICON_VERSION" "${EXTRA_NAMELIST}"
if [ "$STAGE" -eq 2 ]; then
echo "Running Stage 3"
autosubmit setstatus ${EXPID} -fs SUSPENDED -t WAITING -s -np
autosubmit run ${EXPID}
# Check if there are failed jobs
[ ! -s autosubmit/${EXPID}/tmp/ASLOGS/jobs_failed_status.log ] || exit 1
fi
# Create and run experiment
autosubmit create ${EXPID} -np
autosubmit run ${EXPID}
examples/example_utils.sh 0 → 100644
+ 136
− 0
View file @ 69558664
# setup_autoicon_example function that sets up an AutoSub environment for the ICON model
function setup_autoicon_example() {
# Read the input parameters
local ICON_CASE=$1
local ICON_VERSION=$2
local EXTRA_NAMELIST=$3
# Setup environment
setup_environment
# Configure autosubmit if it hasn't been configured yet
configure_and_install_autosubmit
# Set up project
setup_project
}
function setup_environment() {
# Set Autosubmit version and define some directories
AUTOSUBMIT_VERSION="4.0.76"
# Define the path of the directory where this script is located
SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
# Define the root path of the project
PROJECT_ROOT=$(readlink -f $SCRIPTDIR/../)
# Define the output directory and create it
OUTPUTDIR=$PROJECT_ROOT/output
mkdir -p $OUTPUTDIR
# Define working directory and create it
WORKDIR=$SCRATCH/autosubmit_rundir
PROJECTNAME='ls-mayer'
mkdir -p $WORKDIR/$PROJECTNAME/$(whoami)
# Create a Python virtual environment and activate it
virtualenv -p python3 pyenv
. pyenv/bin/activate
# Define required Python packages
cat >requirements.txt <<EOF
autosubmit==$AUTOSUBMIT_VERSION
EOF
# Install required Python packages
pip install -r requirements.txt
}
function configure_and_install_autosubmit() {
# Configure autosubmit if it hasn't been configured yet
if [ ! -e ./.autosubmitrc ] || [ ! -e autosubmit ]; then
autosubmit configure --local -db $(pwd)/autosubmit -dbf autosubmit.db -lr $(pwd)/autosubmit
fi
# Install autosubmit if it hasn't been installed yet
if [ ! -e autosubmit/autosubmit.db ]; then
autosubmit install
fi
}
function setup_project() {
PROJECT_ORIGIN="https://gitlab.physik.uni-muenchen.de/w2w/autoicon.git"
PROJECT_BRANCH="master"
EXPID=rfi001
# If experiment directory doesn't exist, create it and set up the experiment
if [ ! -e autosubmit/$EXPID/ ]; then
mkdir -p autosubmit/$EXPID/
autosubmit expid -min -H lmu -d myiconsim -repo $PROJECT_ORIGIN -b $PROJECT_BRANCH &> >(tee autosubmit/$EXPID/log.autosubmit.expid)
AUTOID=$(grep Experiment autosubmit/$EXPID/log.autosubmit.expid | awk '{print $2}')
mv -v autosubmit/$AUTOID/* autosubmit/$EXPID
rmdir -v autosubmit/$AUTOID
fi
# Create and write the minimal config file
cat >autosubmit/$EXPID/conf/minimal.yml <<EOF
ICON_CASE: "${ICON_CASE}"
CONFIG:
AUTOSUBMIT_VERSION: "$AUTOSUBMIT_VERSION"
TOTALJOBS: 20
MAXWAITINGJOBS: 20
RETRIALS: 0
DEFAULT:
EXPID: "$EXPID"
HPCARCH: "LOCAL" # use LMU to run on cluster
CUSTOM_CONFIG:
PRE:
- "%PROJDIR%/conf/common"
- "%PROJDIR%/conf/%ICON_CASE%"
PROJECT:
PROJECT_TYPE: "git"
PROJECT_DESTINATION: "autoicon"
GIT:
PROJECT_ORIGIN: "$PROJECT_ORIGIN"
PROJECT_BRANCH: "$PROJECT_BRANCH"
PROJECT_COMMIT: ""
PROJECT_SUBMODULES: ""
FETCH_SINGLE_BRANCH: True
EOF
# Create and write the additional config file
cat >autosubmit/${EXPID}/conf/myconf.yml <<EOF
spack:
init: "" # command to load spack environment, e.g. module load spack, use spack/setup-env.sh if empty
url: https://github.com/spack/spack.git # url to download spack if necessary
branch: develop # if downloaded, branch name to use
compiler: "gcc@11.3.0" # desired compiler for spack
root: "$SCRATCH/autoicon-spack" # path to a spack install, will be downloaded to if not present
externals: "slurm"
user_cache_path: "$SCRATCH/autoicon-spackcache" # spack puts data here when bootstrapping, leave empty to use home folder
user_config_path: "$SCRATCH/autoicon-spackconfig" # spack puts data here when bootstrapping, leave empty to use home folder
disable_local_config: false # if true, spack installs into spack source dir
upstreams: "/software/opt/focal/x86_64/spack/2023.02/spack/opt/spack"
icon:
#build_cmd: "icon-nwp@%ICON.VERSION%% %SPACK.COMPILER%+debug+petsc target=x86_64_v2 ^openmpi+pmi+legacylaunchers schedulers=slurm fabrics=ucx ucx+dc+dm+ib_hw_tm+mlx5_dv+rc+rdmacm+thread_multiple+ud+verbs"
build_cmd: "icon-nwp@%ICON.VERSION%% %SPACK.COMPILER%+debug~mpichecks target=x86_64_v2 source=dkrz_https"
version: ${ICON_VERSION}
data_management:
# Where do we put the output files afterwards?
local_destination_folder: $OUTPUTDIR/
Platforms:
DUMMY:
TYPE: ps
EOF
# If EXTRA_NAMELIST is provided, create and write the extra namelist file
if [ -z "$EXTRA_NAMELIST" ]; then
cat >autosubmit/${EXPID}/conf/extra_namelist.yml <<EOF
$EXTRA_NAMELIST
EOF
fi
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment