Merge remote-tracking branch 'origin/minimal-configuration' into FJ/spack_icon_install

conf/real-from-dwd-ana/jobs.yml

@@ -82,14 +82,21 @@ JOBS:
     RUNNING: once
     WALLCLOCK: 01:00
 
-  PREPARE_DATE:
-    FILE: templates/real-from-dwd-ana/prepare_date.sh
+  PREPARE_DATE_LOCAL:
+    FILE: templates/real-from-dwd-ana/prepare_date_local.sh
     RUNNING: date
     WALLCLOCK: 01:00
+    PLATFORM: LOCAL
+
+  PREPARE_DATE_REMOTE:
+    FILE: templates/real-from-dwd-ana/prepare_date_remote.sh
+    RUNNING: date
+    WALLCLOCK: 01:00
+    PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"
 
   PREPARE_MEMBER:
     FILE: templates/real-from-dwd-ana/prepare_member.sh
-    DEPENDENCIES: PREPARE_EXPERIMENT PREPARE_DATE
+    DEPENDENCIES: PREPARE_EXPERIMENT PREPARE_DATE_REMOTE PREPARE_DATE_LOCAL
     RUNNING: member
     WALLCLOCK: 01:00
 
@@ -100,6 +107,7 @@ JOBS:
     RUNNING: chunk
     TYPE: python
     EXECUTABLE: "%HPCROOTDIR%/%python_environment.folder_name%/bin/python3"
+    PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"
 
 
   RUN_ICON:
@@ -134,3 +142,4 @@ JOBS:
     DEPENDENCIES: TRANSFER
     WALLCLOCK: 00:10
     RUNNING: member
+    PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"

conf/real-from-dwd-ana/simulation.yml

@@ -12,8 +12,9 @@ simulation:
   output_file_names: "init_DOM01_ML_*.nc latbc_DOM01_ML_*.nc"
   files_to_clean: "*.nc"
 
-initial_conditions:
-  # Where are we getting our initial data from?
-  parent_folder: /archive/meteo/external-models/dwd/icon/oper/icon_oper_eps_gridded-global_rolling/
-  member: 1
+  initial_conditions:
+    # Where are we getting our initial data from?
+    local: true
+    parent_folder: /archive/meteo/external-models/dwd/icon/oper/icon_oper_eps_gridded-global_rolling/
+    member: 1
 

conf/real-from-ideal/jobs.yml

@@ -80,12 +80,14 @@ JOBS:
     DEPENDENCIES: BUILD_ICON
     RUNNING: once
     WALLCLOCK: 00:10
+    PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"
 
   PREPARE_IDEAL_DIRECTORY:
     FILE: templates/real-from-ideal/prepare_ideal_directory.sh
     DEPENDENCIES: PREPARE_EXPERIMENT
     RUNNING: date
     WALLCLOCK: 00:10
+    PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"
 
   PREPARE_IDEAL_NAMELIST:
     FILE: templates/real-from-ideal/prepare_ideal_namelist.py
@@ -94,6 +96,7 @@ JOBS:
     WALLCLOCK: 00:10
     TYPE: python
     EXECUTABLE: "%HPCROOTDIR%/%python_environment.folder_name%/bin/python3"
+    PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"
 
   RUN_IDEAL:
     FILE: templates/real-from-ideal/run_ideal.sh
@@ -123,6 +126,7 @@ JOBS:
     DEPENDENCIES: FG_ANA_FROM_IDEALIZED EXTPAR_FROM_IDEALIZED
     RUNNING: member
     WALLCLOCK: 01:00
+    PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"
 
   PREPARE_CHUNK:
     FILE: templates/real-from-ideal/prepare_chunk.py
@@ -131,6 +135,7 @@ JOBS:
     RUNNING: chunk
     TYPE: python
     EXECUTABLE: "%HPCROOTDIR%/%python_environment.folder_name%/bin/python3"
+    PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"
 
   RUN_ICON:
     FILE: templates/common/run_icon.sh
@@ -159,3 +164,4 @@ JOBS:
     DEPENDENCIES: TRANSFER
     WALLCLOCK: 00:10
     RUNNING: member
+    PLATFORM: "%DEFAULT.HPCARCH%_LOGIN"

templates/common/build_icon.sh

 # Get some variables provided by autosubmit.
 # TODO: What do we do to ensure that these variables are defined in the proj file?
 WORKDIR=%HPCROOTDIR%
-ICON_VERSION=%icon.version%
+ICON_VERSION=%ICON.VERSION%
+COMPILER=%SPACK.COMPILER%
 
 # If the workdir directory does not exist create it
 if [ ! -d ${WORKDIR} ]; then
@@ -9,12 +10,16 @@ if [ ! -d ${WORKDIR} ]; then
 fi
 
 # Go to the working directory
-cd ${WORKDIR}
+cd ${WORKDIR} || exit
 
 
 . ${WORKDIR}/proj/platforms/common/spack_utils.sh
 load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%"
 
+if [ ! $(rpm -qa | grep bzip2) ]; then
+  spack install bzip2
+  spack load --first bzip2
+fi
 
 SPACK_BUILD_ICON="%ICON.BUILD_CMD%"
 if [ ! -z "$SPACK_BUILD_ICON" ]; then
@@ -28,21 +33,23 @@ if [ ! -z "$SPACK_BUILD_ICON" ]; then
   fi
   spack spec $SPACK_BUILD_ICON
   spack install $SPACK_BUILD_ICON
-  spack load $SPACK_BUILD_ICON
+  #TODO: had some problems with spack load when more than one version is available, adding --first to overcome that
+  # although in principle we should not install the model if its already installed.
+  spack load --first "icon-nwp@${ICON_VERSION}%${COMPILER}"
 else
   echo "\%icon.build_cmd\% is not defined. If you want to compile icon with spack, please provide a spack compile instruction string in your build.yml"
 fi
 
 # Need to get ECCODES DWD definitions:
-eccodes_version=$(spack find eccodes | grep eccodes@ | cut -d "@" -f 2)
+eccodes_version=$(spack spec icon-nwp@${ICON_VERSION}%${COMPILER} | grep eccodes | grep -o "@.*%" | grep -o "[0-9\.]*")
 
 definitions_tar_file=eccodes_definitions.edzw-${eccodes_version}-1.tar.bz2
-if [ ! -f ${definitions_tar_file} ]; then
+if [ ! -f "${definitions_tar_file}" ]; then
   defs_url=https://opendata.dwd.de/weather/lib/grib/${definitions_tar_file}
-  wget ${defs_url}
+  wget "${defs_url}"
 
   # Decompress definitions file
-  tar -xf ${definitions_tar_file}
+  tar -xf "${definitions_tar_file}"
   # Create a file containing the environment variable that needs to be set in order to use DWD's definitions:
   echo "export ECCODES_DEFINITION_PATH=${WORKDIR}/definitions.edzw-${eccodes_version}-1" >eccodes_defs.env
 fi

templates/real-from-dwd-ana/prepare_date.sh

-#!/bin/bash -l
-
-# Get some variables provided by autosubmit.
-WORKDIR=%HPCROOTDIR%
-STARTDATE=%SDATE%
-
-# Define date directory, create it and go there
-COMMON_DATE_FOLDER=${WORKDIR}/${STARTDATE}/inidata
-# Create member folder and go there
-mkdir -p ${COMMON_DATE_FOLDER}
-cd ${COMMON_DATE_FOLDER} || exit
-
-# some settings
-AN_MEMBER=$(printf "%03d" %initial_conditions.member%)
-INITIAL_CONDITIONS_PARENT_FOLDER=%initial_conditions.parent_folder%
-
-INITIAL_CONDITIONS_PATH=${INITIAL_CONDITIONS_PARENT_FOLDER}/${STARTDATE:0:6}/${STARTDATE:0:8}T00
-
-AN_SOURCE=$(find ${INITIAL_CONDITIONS_PATH} -name "igaf*00.m${AN_MEMBER}.grb" | sort | tail -n 1)
-FG_SOURCE=$(find ${INITIAL_CONDITIONS_PATH} -name "igfff00030000.m${AN_MEMBER}.grb" | sort | tail -n 1)
-
-if [ ! -f "${AN_SOURCE}" ]; then
-  echo "Analysis file for date ${STARTDATE} not found!"
-  exit 1
-fi
-
-if [ ! -f "${FG_SOURCE}" ]; then
-  echo "FG file for date ${STARTDATE} not found!"
-  exit 1
-fi
-
-AN_FILE=$(basename "${AN_SOURCE}")
-FG_FILE=$(basename "${FG_SOURCE}")
-
-# Save filenames to be used later by other scripts.
-echo "${AN_FILE}" > an_file.txt
-echo "${FG_FILE}" > fg_file.txt
-
-# Copy the first-guess and analysis files.
-cp "${FG_SOURCE}" "${FG_FILE}"
-cp "${AN_SOURCE}" "${AN_FILE}"
-
-# Change permissions to read only.
-chmod 440 ./*

templates/real-from-dwd-ana/prepare_date_local.sh

+#!/bin/bash -l
+
+IS_LOCAL=%SIMULATION.INITIAL_CONDITIONS.LOCAL%
+
+if [ "${IS_LOCAL}" == "True" ]; then
+  # Get some variables provided by autosubmit.
+  WORKDIR=%HPCROOTDIR%
+  STARTDATE=%SDATE%
+  HPCUSER=%HPCUSER%
+  HPCHOST=%HPCHOST%
+
+  # Define date directory, create it and go there
+  COMMON_DATE_FOLDER=${WORKDIR}/${STARTDATE}/inidata
+
+  AN_MEMBER=$(printf "%03d" %SIMULATION.INITIAL_CONDITIONS.MEMBER%)
+  INITIAL_CONDITIONS_PARENT_FOLDER=%SIMULATION.INITIAL_CONDITIONS.PARENT_FOLDER%
+  INITIAL_CONDITIONS_PATH=${INITIAL_CONDITIONS_PARENT_FOLDER}/${STARTDATE:0:6}/${STARTDATE:0:8}T00
+
+  AN_SOURCE=$(find ${INITIAL_CONDITIONS_PATH} -name "igaf*00.m${AN_MEMBER}.grb" | sort | tail -n 1)
+  FG_SOURCE=$(find ${INITIAL_CONDITIONS_PATH} -name "igfff00030000.m${AN_MEMBER}.grb" | sort | tail -n 1)
+
+  AN_FILE=$(basename "${AN_SOURCE}")
+  FG_FILE=$(basename "${FG_SOURCE}")
+
+  # Find files
+  if [ ! -f "${AN_SOURCE}" ]; then
+    echo "Analysis file for date ${STARTDATE} not found!"
+    exit 1
+  fi
+
+  if [ ! -f "${FG_SOURCE}" ]; then
+    echo "FG file for date ${STARTDATE} not found!"
+    exit 1
+  fi
+
+
+  # Check if we copy the initial conditions from the local system or the remote one
+  if [ "${IS_LOCAL}" == "True" ]; then
+    # Create member folder
+    ssh "${HPCUSER}@${HPCHOST}" mkdir -p ${COMMON_DATE_FOLDER}
+
+    # Save filenames to be used later by other scripts.
+    echo "${AN_FILE}" > an_file.txt
+    echo "${FG_FILE}" > fg_file.txt
+    rsync -v an_file.txt "${HPCUSER}@${HPCHOST}":"${COMMON_DATE_FOLDER}/an_file.txt"
+    rsync -v fg_file.txt "${HPCUSER}@${HPCHOST}":"${COMMON_DATE_FOLDER}/fg_file.txt"
+
+    # Remove temporary files.
+    rm an_file.txt
+    rm fg_file.txt
+
+    # Copy the first-guess and analysis files.
+    rsync -v "${FG_SOURCE}" "${HPCUSER}@${HPCHOST}":"${COMMON_DATE_FOLDER}/${FG_FILE}"
+    rsync -v "${AN_SOURCE}" "${HPCUSER}@${HPCHOST}":"${COMMON_DATE_FOLDER}/${AN_FILE}"
+
+    # Change permissions to read only.
+    ssh "${HPCUSER}@${HPCHOST}" chmod 440 "${COMMON_DATE_FOLDER}/*"
+  else
+    # Create member folder and go there
+    mkdir -p ${COMMON_DATE_FOLDER}
+    cd ${COMMON_DATE_FOLDER} || exit
+
+    # Save filenames to be used later by other scripts.
+    echo "${AN_FILE}" > an_file.txt
+    echo "${FG_FILE}" > fg_file.txt
+
+    # Copy the first-guess and analysis files.
+    cp "${FG_SOURCE}" "${FG_FILE}"
+    cp "${AN_SOURCE}" "${AN_FILE}"
+
+    # Change permissions to read only.
+    chmod 440 ./*
+  fi
+fi
\ No newline at end of file

templates/real-from-dwd-ana/prepare_date_remote.sh

+#!/bin/bash -l
+
+IS_LOCAL=%SIMULATION.INITIAL_CONDITIONS.LOCAL%
+
+if [ "${IS_LOCAL}" == "0" ]; then
+  # Get some variables provided by autosubmit.
+  WORKDIR=%HPCROOTDIR%
+  STARTDATE=%SDATE%
+  HPCUSER=%HPCUSER%
+  HPCHOST=%HPCHOST%
+  # Define date directory, create it and go there
+  COMMON_DATE_FOLDER=${WORKDIR}/${STARTDATE}/inidata
+
+  AN_MEMBER=$(printf "%03d" %SIMULATION.INITIAL_CONDITIONS.MEMBER%)
+  INITIAL_CONDITIONS_PARENT_FOLDER=%SIMULATION.INITIAL_CONDITIONS.PARENT_FOLDER%
+  INITIAL_CONDITIONS_PATH=${INITIAL_CONDITIONS_PARENT_FOLDER}/${STARTDATE:0:6}/${STARTDATE:0:8}T00
+
+  AN_SOURCE=$(find ${INITIAL_CONDITIONS_PATH} -name "igaf*00.m${AN_MEMBER}.grb" | sort | tail -n 1)
+  FG_SOURCE=$(find ${INITIAL_CONDITIONS_PATH} -name "igfff00030000.m${AN_MEMBER}.grb" | sort | tail -n 1)
+
+  AN_FILE=$(basename "${AN_SOURCE}")
+  FG_FILE=$(basename "${FG_SOURCE}")
+
+  # Find files
+  if [ ! -f "${AN_SOURCE}" ]; then
+    echo "Analysis file for date ${STARTDATE} not found!"
+    exit 1
+  fi
+
+  if [ ! -f "${FG_SOURCE}" ]; then
+    echo "FG file for date ${STARTDATE} not found!"
+    exit 1
+  fi
+
+
+  # Check if we copy the initial conditions from the local system or the remote one
+  if [ "${IS_LOCAL}" == "True" ]; then
+    # Create member folder
+    ssh "${HPCUSER}@${HPCHOST}" mkdir -p ${COMMON_DATE_FOLDER}
+
+    # Save filenames to be used later by other scripts.
+    echo "${AN_FILE}" > an_file.txt
+    echo "${FG_FILE}" > fg_file.txt
+    rsync -v an_file.txt "${HPCUSER}@${HPCHOST}":"${COMMON_DATE_FOLDER}/an_file.txt"
+    rsync -v fg_file.txt "${HPCUSER}@${HPCHOST}":"${COMMON_DATE_FOLDER}/fg_file.txt"
+
+    # Remove temporary files.
+    rm an_file.txt
+    rm fg_file.txt
+
+    # Copy the first-guess and analysis files.
+    rsync -v "${FG_SOURCE}" "${HPCUSER}@${HPCHOST}":"${COMMON_DATE_FOLDER}/${FG_FILE}"
+    rsync -v "${AN_SOURCE}" "${HPCUSER}@${HPCHOST}":"${COMMON_DATE_FOLDER}/${AN_FILE}"
+
+    # Change permissions to read only.
+    ssh "${HPCUSER}@${HPCHOST}" chmod 440 "${COMMON_DATE_FOLDER}/*"
+  else
+    # Create member folder and go there
+    mkdir -p ${COMMON_DATE_FOLDER}
+    cd ${COMMON_DATE_FOLDER} || exit
+
+    # Save filenames to be used later by other scripts.
+    echo "${AN_FILE}" > an_file.txt
+    echo "${FG_FILE}" > fg_file.txt
+
+    # Copy the first-guess and analysis files.
+    cp "${FG_SOURCE}" "${FG_FILE}"
+    cp "${AN_SOURCE}" "${AN_FILE}"
+
+    # Change permissions to read only.
+    chmod 440 ./*
+  fi
+fi
\ No newline at end of file

templates/real-from-ideal/run_ideal.sh

@@ -11,8 +11,9 @@ RUNDIR=${WORKDIR}/${STARTDATE}/ideal
 cd ${RUNDIR}
 
 # Activate spack
-SPACK_ENV=${WORKDIR}/spack/share/spack/setup-env.sh
-source ${SPACK_ENV}
+. ${WORKDIR}/proj/platforms/common/spack_utils.sh
+load_spack "%spack.init%" "%spack.root%" "%spack.url%" "%spack.branch%" "%spack.compiler%" "%spack.disable_local_config%" "%spack.user_cache_path%" "%spack.user_config_path%"
+
 # Load icon module
 spack load icon-nwp@%ICON_VERSION%