Merge emc/develop

.gitignore

@@ -81,6 +81,7 @@ parm/ufs/MOM6_data_table.IN
 parm/ufs/ice_in.IN
 parm/ufs/ufs.configure.*.IN
 parm/ufs/post_itag_gfs
+parm/ufs/ww3_shel.nml.IN
 parm/wafs
 
 # Ignore sorc and logs folders from externals

CMakeLists.txt

@@ -0,0 +1,27 @@
+# ------------------------------------------------------------------------- #
+# Global Workflow
+# ------------------------------------------------------------------------- #
+
+# Check for minimum cmake requirement
+cmake_minimum_required( VERSION 3.20 FATAL_ERROR )
+
+project(global_workflow VERSION 1.0.0)
+
+include(GNUInstallDirs)
+enable_testing()
+
+# Build type.
+if(NOT CMAKE_BUILD_TYPE MATCHES "^(Debug|Release|RelWithDebInfo|MinSizeRel)$")
+  message(STATUS "Setting build type to 'Release' as none was specified.")
+  set(CMAKE_BUILD_TYPE
+      "Release"
+      CACHE STRING "Choose the type of build." FORCE)
+  set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release"
+                                               "MinSizeRel" "RelWithDebInfo")
+endif()
+
+# Build global-workflow source codes
+# add_subdirectory(sorc)
+
+# Setup tests
+add_subdirectory(ctests)

ci/platforms/config.hera

@@ -2,6 +2,8 @@
 
 export GFS_CI_ROOT=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT
 export ICSDIR_ROOT=/scratch1/NCEPDEV/global/glopara/data/ICSDIR
+
+export STAGED_TESTS_DIR=${GFS_CI_ROOT}/STAGED_TESTS_DIR
 export HPC_ACCOUNT=nems
 export max_concurrent_cases=5
 export max_concurrent_pr=4

ci/platforms/config.orion

@@ -2,6 +2,7 @@
 
 export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT/ORION
 export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR
+export STAGED_TESTS_DIR=${GFS_CI_ROOT}/STAGED_TESTS_DIR
 export HPC_ACCOUNT=nems
 export max_concurrent_cases=5
 export max_concurrent_pr=4

ctests/CMakeLists.txt

@@ -0,0 +1,106 @@
+# ------------------------------------------------------------------------- #
+# CTests for Global Workflow
+# ------------------------------------------------------------------------- #
+# These ctests correspond to JJOBs (individual Rocoto jobs) that can be
+# run independently, each requiring its own YAML definition of inputs
+# and configurations. By integrating with Rocoto, these jobs can be
+# validated, staged, and executed as self-contained tests using
+# their own data and test parameters.
+# ------------------------------------------------------------------------- #
+
+# Function to set a variable from an environment variable or default value
+function(set_from_env_or_default VAR_NAME ENV_VAR DEFAULT_VALUE)
+  if (DEFINED ENV{${ENV_VAR}} AND NOT DEFINED ${VAR_NAME})
+    set(${VAR_NAME} $ENV{${ENV_VAR}} CACHE STRING "Set from environment variable ${ENV_VAR}")
+  elseif(NOT DEFINED ${VAR_NAME} AND NOT ${DEFAULT_VALUE} STREQUAL "")
+    set(${VAR_NAME} ${DEFAULT_VALUE} CACHE STRING "Default value for ${VAR_NAME}")
+  endif()
+endfunction()
+
+# Set HOMEgfs
+if (NOT DEFINED HOMEgfs)
+  set(HOMEgfs ${PROJECT_SOURCE_DIR})
+endif()
+
+# Set RUNTESTS
+set_from_env_or_default(RUNTESTS RUNTESTS "${CMAKE_CURRENT_BINARY_DIR}/RUNTESTS")
+
+# Set HPC_ACCOUNT
+set_from_env_or_default(HPC_ACCOUNT HPC_ACCOUNT " ")
+if (NOT DEFINED HPC_ACCOUNT)
+  message(WARNING "HPC_ACCOUNT must be set. CTests will not be created.")
+  return()
+endif()
+
+# Set ICSDIR_ROOT
+set_from_env_or_default(ICSDIR_ROOT ICSDIR_ROOT "")
+if (NOT DEFINED ICSDIR_ROOT)
+  message(WARNING "ICSDIR_ROOT must be set. CTests will not be created.")
+  return()
+endif()
+
+# Set STAGED_TESTS_DIR
+set_from_env_or_default(STAGED_TESTS_DIR STAGED_TESTS_DIR "")
+if (NOT DEFINED STAGED_TESTS_DIR)
+  message(WARNING "STAGED_TESTS_DIR must be set. CTests will not be created.")
+  return()
+endif()
+
+message(STATUS "gw: global-workflow baselines will be used from: '${HOMEgfs}'")
+message(STATUS "gw: global-workflow tests will be run at: '${RUNTESTS}'")
+message(STATUS "gw: global-workflow tests will use the allocation: '${HPC_ACCOUNT}'")
+message(STATUS "gw: global-workflow tests will use ICSDIR_ROOT: '${ICSDIR_ROOT}'")
+message(STATUS "gw: global-workflow tests will use staged data from:  '${STAGED_TESTS_DIR}'")
+
+# Prepare test scripts
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/setup.sh.in
+               ${CMAKE_CURRENT_BINARY_DIR}/scripts/setup.sh @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/stage.sh.in
+               ${CMAKE_CURRENT_BINARY_DIR}/scripts/stage.sh @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/execute.sh.in
+               ${CMAKE_CURRENT_BINARY_DIR}/scripts/execute.sh @ONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/scripts/validate.sh.in
+                ${CMAKE_CURRENT_BINARY_DIR}/scripts/validate.sh @ONLY)
+
+function(AddJJOBTest)
+
+  set(prefix     ARG)
+  set(novals     NOTRAPFPE NOVALGRIND)
+  set(singlevals CASE JOB TEST_DATE)
+  set(multivals  TEST_DEPENDS)
+
+  cmake_parse_arguments(${prefix}
+                       "${novals}" "${singlevals}" "${multivals}"
+                        ${ARGN})
+
+  set(TEST_NAME ${ARG_CASE}_${ARG_JOB})
+  set(CASE_PATH ${HOMEgfs}/ci/cases/pr)
+  set(CASE_YAML ${CASE_PATH}/${ARG_CASE}.yaml)
+
+  add_test(NAME test_${TEST_NAME}_setup
+    COMMAND ./setup.sh ${TEST_NAME} ${CASE_YAML} ${ARG_TEST_DATE}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts)
+  set_tests_properties(test_${TEST_NAME}_setup PROPERTIES LABELS "${ARG_CASE};${ARG_JOB}")
+
+  add_test(NAME test_${TEST_NAME}_stage
+      COMMAND ./stage.sh ${TEST_NAME} ${ARG_TEST_DATE}
+      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts)
+  set_tests_properties(test_${TEST_NAME}_stage PROPERTIES DEPENDS test_${TEST_NAME}_setup LABELS "${ARG_CASE};${ARG_JOB}")
+
+  add_test(NAME test_${TEST_NAME}_execute
+      COMMAND ./execute.sh ${TEST_NAME} ${ARG_JOB} ${ARG_TEST_DATE}
+      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts)
+  set_tests_properties(test_${TEST_NAME}_execute PROPERTIES DEPENDS test_${TEST_NAME}_stage LABELS "${ARG_CASE};${ARG_JOB}")
+
+  # TODO - This is a stub for the validation step
+  add_test(NAME test_${TEST_NAME}_validate
+      COMMAND ./validate.sh ${TEST_NAME} ${CASE_YAML}
+      WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts)
+  set_tests_properties(test_${TEST_NAME}_validate PROPERTIES DEPENDS test_${TEST_NAME}_execute LABELS "${ARG_CASE};${ARG_JOB}")
+endfunction()
+
+AddJJOBTest(
+  CASE "C48_ATM"
+  JOB  "gfs_fcst_seg0"
+  TEST_DATE "2021032312"
+)

ctests/README.md

@@ -0,0 +1,58 @@
+# CTest Framework for NOAA Global Workflow
+
+This directory contains the CTest framework for testing Rocoto JJOBS. The framework allows you to stage, execute, and validate individual JJOBS independently from other jobs in the workflow. Each test requires its own YAML definition of inputs and configurations.
+
+## Overview
+
+The CTest framework consists of the following scripts:
+- **setup.sh.in**: Prepares the environment and creates the experiment.
+- **stage.sh.in**: Stages the input files needed to run a JJOB.
+- **execute.sh.in**: Executes the JJOB and monitors its status.
+- **validate.sh.in**: (TODO) Validates the results of the JJOB.
+
+## Usage
+
+### CMake Configuration
+
+To configure the CTest framework using CMake, you need to provide several environment variables or default values. Here is an example of how to configure and build the project:
+
+```bash
+# Set environment variables (may also be include at command line with -D)
+export HPC_ACCOUNT="your_hpc_account"
+export ICSDIR_ROOT="/path/to/icsdir_root"
+export STAGED_TESTS_DIR="/path/to/staged_tests_dir"
+
+# Run CMake to configure the ctest framework
+cmake -S /path/to/HOMEgfs -B /path/to/build -DRUNTESTS=/path/to/runtests
+
+```
+
+### Running Tests with CTest
+
+Once the project is configured, you can run the tests using CTest. Here are some examples:
+
+#### Run All Tests
+
+```bash
+cd /path/to/build
+ctest
+```
+
+#### Run Tests for a Specific Case
+
+You can use the `-L` option with CTest to run tests for a specific case. For example, to run tests for the `C48_ATM` case:
+
+```bash
+cd /path/to/build
+ctest -L C48_ATM
+```
+
+To add a new test use the **AddJJOBTest()** function at the end of the `$HOMEgfs/ctest/CMakeLists.txt` file as follows:
+```cmake
+AddJJOBTest(
+  CASE "C48_ATM"
+  JOB  "gfs_fcst_seg0"
+  TEST_DATE "2021032312"
+)
+```
+Then create a new YAML file with the required staged input files as is done with this example found in `$HOMEgfs/ctests/cases/C48_ATM_gfs_fcts_seg0.yaml`

ctests/cases/C48_ATM_gfs_fcst_seg0.yaml

@@ -0,0 +1,17 @@
+input_files:
+    mkdir:
+        - "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input"
+    copy:
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_ctrl.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_ctrl.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile1.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile1.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile2.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile2.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile3.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile3.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile4.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile4.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile5.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile5.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/gfs_data.tile6.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/gfs_data.tile6.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile1.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile1.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile2.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile2.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile3.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile3.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile4.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile4.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile5.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile5.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ TEST_DATE | to_YMD }}/{{ TEST_DATE | strftime('%H') }}/model/atmos/input/sfc_data.tile6.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{TEST_DATE | to_YMD}}/{{TEST_DATE | strftime('%H')}}/model/atmos/input/sfc_data.tile6.nc"]

ctests/scripts/execute.sh.in

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+
+set -xe
+
+TEST_NAME=${1:?"Name of the test is required"}
+JOB=${2:?"Job name is required"}
+# TODO - adding idate by hand for now, need to get this from the test somehow 
+idate=$3
+
+#TODO - add rocotoboot_dryrun to repo some how
+rocotoboot_dryrun=/work2/noaa/global/mterry/rocoto_dryrun/bin/rocotoboot
+CASEDIR="@CMAKE_CURRENT_BINARY_DIR@/RUNTESTS/EXPDIR/${TEST_NAME}"
+cd "${CASEDIR}"
+rm -f ./*.db
+rm -f ./jobcard                                                                             
+
+yes | "${rocotoboot_dryrun}" -d "${TEST_NAME}.db" -w "${TEST_NAME}.xml" -v 10 -c "${idate}00" -t "${JOB}" 2> jobcard || true
+sed '/^{{\|^}}/d' < jobcard | sed '1d'  > "${TEST_NAME}.sub" || true
+
+#TODO - Generalize for batch system (hard coded to slurm)
+
+output=$(sbatch "${TEST_NAME}.sub")
+job_id=$(echo "${output}" | awk '{print $4}')
+echo "Job ${job_id} submitted for test ${TEST_NAME} with job name ${JOB}"
+
+# First loop: wait until job appears
+lack_of_job_count=0
+LACK_OF_JOB_LIMIT=5
+
+while true; do
+    job_status=$(sacct -j "${job_id}" --format=State --noheader -n | head -1) || true
+    if [[ -n "${job_status}" ]]; then
+        echo "Job ${job_id} found in sacct."
+        break
+    fi
+    echo "Job ${job_id} not in sacct yet, attempt ${lack_of_job_count}/${LACK_OF_JOB_LIMIT}."
+    lack_of_job_count=$((lack_of_job_count + 1))
+    if [[ "${lack_of_job_count}" -ge "${LACK_OF_JOB_LIMIT}" ]]; then
+        echo "Job ${job_id} not found after ${lack_of_job_count} attempts. Exiting."
+        exit 1
+    fi
+    sleep 30
+done
+
+# Second loop: monitor job status until completion or failure
+timeout=0
+TIMEOUT=60
+while true; do
+    # Trim trailing spaces from job_status
+    job_status=$(sacct -j "${job_id}" --format=State --noheader -n | head -1 | xargs) || true
+    if [[ "${job_status}" == "COMPLETED" ]]; then
+        echo "Job ${job_id} completed successfully."
+        break
+    elif [[ "${job_status}" =~ ^(FAILED|CANCELLED|TIMEOUT)$ ]]; then
+        echo "Job ${job_id} failed with status: ${job_status}."
+        exit 1
+    else
+        echo "Job ${job_id} is still running with status: ${job_status}."
+        sleep 60
+        timeout=$((timeout + 1))
+        if [[ "${timeout}" -gt "${TIMEOUT}" ]]; then
+            echo "Job ${job_id} has been running for more than ${TIMEOUT} minutes. Exiting."
+            exit 1
+        fi
+    fi
+done

ctests/scripts/setup.sh.in

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -ux
+
+TEST_NAME=${1:?"Name of the test is required"}
+YAML_FILE=${2:?"Name of the CI yaml file for the test"}
+
+# CMake to fill these variables
+HOMEgfs="@PROJECT_SOURCE_DIR@"
+RUNTESTS="@RUNTESTS@"
+ICSDIR_ROOT="@ICSDIR_ROOT@"
+HPC_ACCOUNT="@HPC_ACCOUNT@"
+
+set +x
+source "${HOMEgfs}/workflow/gw_setup.sh"
+set -x
+
+pslot="${TEST_NAME}" \
+RUNTESTS="${RUNTESTS}" \
+ICSDIR_ROOT="${ICSDIR_ROOT}" \
+HPC_ACCOUNT="${HPC_ACCOUNT}" \
+"${HOMEgfs}/workflow/create_experiment.py" --yaml "${YAML_FILE}" --overwrite
+rc=$?
+if [[ "${rc}" -ne 0 ]]; then
+    set +x
+    echo "Failed to create test experiment for '${TEST_NAME}' with yaml file '${YAML_FILE}'"
+    set -x
+    exit "${rc}"
+fi
+
+exit 0