From ae455b0114c9495907964866ce84665bfb948699 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Fri, 17 Jan 2025 15:03:55 -0600
Subject: [PATCH] Add more templating, begin to unify

---
 parm/config/gfs/config.arch   |  2 +-
 parm/config/gfs/config.globus | 16 +++++++++
 parm/globus/dm.conf.j2        |  1 +
 parm/globus/places.inc.j2     | 11 ++++++
 parm/globus/run_doorman.sh.j2 | 68 +++++++++++++++++++++++++++++++++++
 workflow/hosts/hercules.yaml  |  1 +
 6 files changed, 98 insertions(+), 1 deletion(-)
 create mode 100644 parm/globus/places.inc.j2
 create mode 100644 parm/globus/run_doorman.sh.j2

diff --git a/parm/config/gfs/config.arch b/parm/config/gfs/config.arch
index 1f3149f7dc..d5d6f1883d 100644
--- a/parm/config/gfs/config.arch
+++ b/parm/config/gfs/config.arch
@@ -14,7 +14,7 @@ export ARCH_GAUSSIAN_FHINC=${FHOUT_GFS}
 
 # If we are running globus archiving, create tarballs in a temporary location
 if [[ "${GLOBUSARCH}" == "YES" ]]; then
-    export ATARDIR="${DATAROOT}/archive_rotdir/${pslot}/${RUN}"
+    export ATARDIR="${DATAROOT}/archive_rotdir/${RUN}"
     export LOCALARCH="YES"
     export DATASETS_YAML="backup_tarballs.yaml"
 fi
diff --git a/parm/config/gfs/config.globus b/parm/config/gfs/config.globus
index 89f7d72493..712c5f3d95 100644
--- a/parm/config/gfs/config.globus
+++ b/parm/config/gfs/config.globus
@@ -11,4 +11,20 @@ echo "BEGIN: config.globus"
 # Set the globus staging directory populated by the arch jobs
 export STAGE_DIR="${DATAROOT}/archive_rotdir/${PSLOT}"
 
+# Set variables used by the Sven and Doorman services
+# Niagara's globus UUID
+export SERVER_GLOBUS_UUID=1bfd8a79-52b2-4589-88b2-0648e0c0b35d
+# Client address
+export CLIENT_GLOBUS_UUID=@CLIENT_UUID@
+
+# General delivery location on Niagara (staging area for data)
+# This is tricky because user IDs don't match between Hercules and Niagara.
+# This will be a user input at setup_expt runtime.
+niagara_uid=@niagara_uid@
+# data_untrusted is a misnomer.  This just means the data is kept for 5 days instead of 60.
+export GENERAL_DELIVERY_ROOT="/collab1/data_untrusted/${niagara_uid}/GENERAL_DELIVERY"
+
+# Sven's dropbox
+export SVEN_DROPBOX_ROOT="${DATAROOT}/archive_rotdir/${RUN}/SVEN_DROPBOX"
+
 echo "END: config.globus"
diff --git a/parm/globus/dm.conf.j2 b/parm/globus/dm.conf.j2
index 5cea0258c3..39e7ec1943 100644
--- a/parm/globus/dm.conf.j2
+++ b/parm/globus/dm.conf.j2
@@ -1,3 +1,4 @@
 #!/usr/bin/env bash
 
+# The location on the sending client (e.g. Hercules) of Sven's dropbox.
 export dropbox="{{sven_dropbox}}"
diff --git a/parm/globus/places.inc.j2 b/parm/globus/places.inc.j2
new file mode 100644
index 0000000000..9329edecee
--- /dev/null
+++ b/parm/globus/places.inc.j2
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# This file will be pushed to Niagara and is used to initialize the Doorman server
+
+# This is the staging area on Niagara.
+# This is where tarballs will be received and confirmations are written and sent.
+export GENDEL={{general_delivery_dir}}
+
+# The globus UUID for the sending platform (e.g. Hercules)
+export CLIENT_ENDPOINT={{HERC_GLC}}
+# The location of the dropbox on the sending platform
+export CLIENT_DROPBOX={{sven_dropbox}}
diff --git a/parm/globus/run_doorman.sh.j2 b/parm/globus/run_doorman.sh.j2
new file mode 100644
index 0000000000..9afacaff80
--- /dev/null
+++ b/parm/globus/run_doorman.sh.j2
@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+# This script runs on Niagara to interact with the Doorman service
+{% set cycle_YMDH = current_cycle | to_YMDH %}
+subdir="{{RUN}}/{{cycle_YMDH}}"
+
+# Make the working directory
+doorman_dir="${HOME}/.doorman/${subdir}"
+mkdir -p "${doorman_dir}"
+cd "${doorman_dir}"
+rm -f dm.conf
+rm -f places.inc
+rm -f FLIST
+
+# Tell the doorman where Sven's dropbox is on the sending client (e.g. Hercules)
+echo 'export dropbox="{{sven_dropbox}}"' > dm.conf
+
+# Tell the doorman where the general delivery space is on Niagara (unique for each RUN/cycle)
+# This is where tarballs will be received and confirmations are written and sent.
+echo 'export GENDEL={{GENERAL_DELIVERY_ROOT}}/{{RUN}}/{{cycle_YMDH}}' > places.inc
+# Tell the doorman what the sender's UUID is
+echo 'export CLIENT_ENDPOINT={{CLIENT_GLOBUS_UUID}}' >> places.inc
+# Tell the doorman where the sending client's dropbox is (why twice??)
+echo 'export CLIENT_DROPBOX={{sven_dropbox}}' >> places.inc
+
+# Point to the doorman executable scripts
+export PATH="${PATH}:{{doorman_root}}/bin"
+
+# Create the general delivery space if it wasn't already
+initialize.sh
+
+# Transfer the data from the sender and execute the 'todo' script
+receive.sh --go
+
+# If receive didn't produce an FLIST file, then something went wrong
+if [[ ! -f FLIST ]]; then
+    echo "receive.sh failed!"
+    return 2
+fi
+
+# Parse the FLIST file created by receive.sh to get the transfer IDs
+IDs=""
+while IFS= read -r line; do
+    package_name=$(grep -o "package_location_.*\.tgz")
+    tmp="${package_name#package_location_}"
+    ID="${tmp%.tgz}"
+    IDs="${IDs} ${ID}"
+done < FLIST
+
+# Sleep for a minute to allow time for all globus artifacts to resolve
+sleep 1m
+
+# Validate and generate the acknowledgement for each transfer ID
+for ID in ${IDs}; do
+    ack.sh "${ID}"
+done
+
+# Send the acknowledgement back to the sender
+send.sh
+
+stat=$?
+
+if [[ ${stat} -ne 0 ]]; then
+    echo "Failed to send status back to client!"
+    exit 3
+fi
+
+exit 0
diff --git a/workflow/hosts/hercules.yaml b/workflow/hosts/hercules.yaml
index a2974377dd..37c21043e1 100644
--- a/workflow/hosts/hercules.yaml
+++ b/workflow/hosts/hercules.yaml
@@ -32,3 +32,4 @@ COMINecmwf: /work/noaa/global/glopara/data/external_gempak/ecmwf
 COMINnam: /work/noaa/global/glopara/data/external_gempak/nam
 COMINukmet: /work/noaa/global/glopara/data/external_gempak/ukmet
 AERO_INPUTS_DIR: /work2/noaa/global/wkolczyn/noscrub/global-workflow/gocart_emissions
+CLIENT_GLOBUS_UUID: '869912fe-f6de-46c0-af10-b22efd84a022'