From ec7975142ba3695b31fc0038dbb0e45c144b36a2 Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 4 Oct 2024 11:41:00 -0400
Subject: [PATCH 01/17] conda environment.yml for environment inwhich to run
 certain fre-workflow things. another workflow yaml gutted from fre-cli for
 pipeline that lints cylc files and runs data_lineage pytest to start.

---
 .github/workflows/create_test_conda_env.yml | 34 +++++++++++++++++++++
 environment.yml                             | 18 +++++++++++
 2 files changed, 52 insertions(+)
 create mode 100644 .github/workflows/create_test_conda_env.yml
 create mode 100644 environment.yml

diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml
new file mode 100644
index 0000000..6adb858
--- /dev/null
+++ b/.github/workflows/create_test_conda_env.yml
@@ -0,0 +1,34 @@
+name: create_test_conda_env
+
+on: [push]
+
+jobs:
+  build-linux:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '>=3.9'
+
+      - name: Add conda to system path
+        run: |
+          # $CONDA is an env var pointing to root of miniconda dir
+          echo $CONDA/bin >> $GITHUB_PATH
+
+      - name: Create fre-workflows environment
+        run: |
+          # create environment containing all dependencies
+          # the env cannot be explicitly activated in github CI/CD
+          conda env create -f environment.yml --name fre-workflows
+
+          # add conda env's executables to github's PATH equiv.
+          echo $CONDA/envs/fre-workflows/bin >> $GITHUB_PATH
+
+	  - name: Run unittest in fre-workflows environment
+		run: |
+		  # try to make sure the right things are in GITHUB_PATH
+		  echo $CONDA/envs/fre-workflows/bin >> $GITHUB_PATH
+		  cylc lint -v
+		  cd data_lineage && python -m unittest discover -s test
diff --git a/environment.yml b/environment.yml
new file mode 100644
index 0000000..a0f4185
--- /dev/null
+++ b/environment.yml
@@ -0,0 +1,18 @@
+name: fre-workflows
+channels:
+  - defaults
+  - conda-forge
+  - noaa-gfdl
+dependencies:
+  - python
+  - pip
+  - click
+  - pyyaml
+  - pylint
+  - jsonschema
+  - conda-forge::cylc-flow>=8.2.0
+  - conda-forge::cylc-rose
+  - conda-forge::metomi-rose
+  - conda-forge::cylc-uiserver
+  - conda-forge::pytest
+  - conda-forge::pytest-cov
\ No newline at end of file

From eaf3336545558b123232566d4bb7ee9dd0790f69 Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 4 Oct 2024 11:46:03 -0400
Subject: [PATCH 02/17] edit workflow file for evil tabs instead of spaces

---
 .github/workflows/create_test_conda_env.yml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml
index 6adb858..352e6b3 100644
--- a/.github/workflows/create_test_conda_env.yml
+++ b/.github/workflows/create_test_conda_env.yml
@@ -26,9 +26,9 @@ jobs:
           # add conda env's executables to github's PATH equiv.
           echo $CONDA/envs/fre-workflows/bin >> $GITHUB_PATH
 
-	  - name: Run unittest in fre-workflows environment
-		run: |
-		  # try to make sure the right things are in GITHUB_PATH
-		  echo $CONDA/envs/fre-workflows/bin >> $GITHUB_PATH
-		  cylc lint -v
-		  cd data_lineage && python -m unittest discover -s test
+      - name: Run unittest in fre-workflows environment
+        run: |
+          # try to make sure the right things are in GITHUB_PATH
+          echo $CONDA/envs/fre-workflows/bin >> $GITHUB_PATH
+          cylc lint -v
+          cd data_lineage && python -m unittest discover -s test; cd -

From 4f959bd972c20cf4404d5eee4d48982b38d7ddf1 Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 4 Oct 2024 11:51:30 -0400
Subject: [PATCH 03/17] guard against non-zero exit status of cylc lint call

---
 .github/workflows/create_test_conda_env.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml
index 352e6b3..f65a678 100644
--- a/.github/workflows/create_test_conda_env.yml
+++ b/.github/workflows/create_test_conda_env.yml
@@ -30,5 +30,5 @@ jobs:
         run: |
           # try to make sure the right things are in GITHUB_PATH
           echo $CONDA/envs/fre-workflows/bin >> $GITHUB_PATH
-          cylc lint -v
+          cylc lint -v || echo "cylc lint threw non-zero exit code"
           cd data_lineage && python -m unittest discover -s test; cd -

From 7ec266b248632ae4182d687830089b9f28e9fc48 Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 4 Oct 2024 11:55:39 -0400
Subject: [PATCH 04/17] verbose unit test

---
 .github/workflows/create_test_conda_env.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml
index f65a678..379cd4e 100644
--- a/.github/workflows/create_test_conda_env.yml
+++ b/.github/workflows/create_test_conda_env.yml
@@ -31,4 +31,4 @@ jobs:
           # try to make sure the right things are in GITHUB_PATH
           echo $CONDA/envs/fre-workflows/bin >> $GITHUB_PATH
           cylc lint -v || echo "cylc lint threw non-zero exit code"
-          cd data_lineage && python -m unittest discover -s test; cd -
+          cd data_lineage && python -m unittest discover -s test -v; cd -

From e54e41b02f6478a6e84cbb2249fffb58422a4cd1 Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 4 Oct 2024 12:01:18 -0400
Subject: [PATCH 05/17] fre-nctools dependency added in for make-timeseries
 tests. comments into workflow file and pytest call for make-timeseries

---
 .github/workflows/create_test_conda_env.yml | 4 ++++
 environment.yml                             | 1 +
 2 files changed, 5 insertions(+)

diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml
index 379cd4e..070e3bc 100644
--- a/.github/workflows/create_test_conda_env.yml
+++ b/.github/workflows/create_test_conda_env.yml
@@ -30,5 +30,9 @@ jobs:
         run: |
           # try to make sure the right things are in GITHUB_PATH
           echo $CONDA/envs/fre-workflows/bin >> $GITHUB_PATH
+          # lint .cylc workflow files
           cylc lint -v || echo "cylc lint threw non-zero exit code"
+          # unittests for data_lineage
           cd data_lineage && python -m unittest discover -s test -v; cd -
+          # pytest unittests for make-timeseries
+          cd app/make-timeseries && pytest -v -v -x ./tests/; cd -
\ No newline at end of file
diff --git a/environment.yml b/environment.yml
index a0f4185..514aa81 100644
--- a/environment.yml
+++ b/environment.yml
@@ -10,6 +10,7 @@ dependencies:
   - pyyaml
   - pylint
   - jsonschema
+  - noaa-gfdl::fre-nctools
   - conda-forge::cylc-flow>=8.2.0
   - conda-forge::cylc-rose
   - conda-forge::metomi-rose

From 316fd3e67a1c3cb02a17822d74d4548c406e1099 Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 4 Oct 2024 12:07:42 -0400
Subject: [PATCH 06/17] add more test calls to apps/scripts that have them.
 exit status guards for most. expecting to likely not have all the req deps

---
 .github/workflows/create_test_conda_env.yml | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml
index 070e3bc..5e58ffc 100644
--- a/.github/workflows/create_test_conda_env.yml
+++ b/.github/workflows/create_test_conda_env.yml
@@ -30,9 +30,18 @@ jobs:
         run: |
           # try to make sure the right things are in GITHUB_PATH
           echo $CONDA/envs/fre-workflows/bin >> $GITHUB_PATH
+
           # lint .cylc workflow files
           cylc lint -v || echo "cylc lint threw non-zero exit code"
+
           # unittests for data_lineage
-          cd data_lineage && python -m unittest discover -s test -v; cd -
+          cd data_lineage && python -m unittest discover -s test -v; cd -;
+
           # pytest unittests for make-timeseries
-          cd app/make-timeseries && pytest -v -v -x ./tests/; cd -
\ No newline at end of file
+          cd app/make-timeseries && pytest -v -v -x ./tests/ || echo "pytest for make-timeseries failed"; cd -;
+
+          # pytest unittests for remap-pp-components
+          cd app/remap-pp-components && pytest -v -v -x ./t/ || echo "pytest for remap-pp-components failed"; cd -;
+
+          # pytest unittests for other things in this repository
+          pytest -v -v -x ./tests;
\ No newline at end of file

From e85da5e750c1fd3e230136354e6408d7c198d54e Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 4 Oct 2024 12:10:38 -0400
Subject: [PATCH 07/17] exit code guard for data_lineage test. fix test call
 for make-timeseries

---
 .github/workflows/create_test_conda_env.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml
index 5e58ffc..f5fe97e 100644
--- a/.github/workflows/create_test_conda_env.yml
+++ b/.github/workflows/create_test_conda_env.yml
@@ -35,10 +35,10 @@ jobs:
           cylc lint -v || echo "cylc lint threw non-zero exit code"
 
           # unittests for data_lineage
-          cd data_lineage && python -m unittest discover -s test -v; cd -;
+          cd data_lineage && python -m unittest discover -s test -v || echo "data_lineage unittest failed."; cd -;
 
           # pytest unittests for make-timeseries
-          cd app/make-timeseries && pytest -v -v -x ./tests/ || echo "pytest for make-timeseries failed"; cd -;
+          cd app/make-timeseries && pytest -v -v -x ./test || echo "pytest for make-timeseries failed"; cd -;
 
           # pytest unittests for remap-pp-components
           cd app/remap-pp-components && pytest -v -v -x ./t/ || echo "pytest for remap-pp-components failed"; cd -;

From ae4e9e0bf88a44fd5ec445c0163e6acd82983924 Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 4 Oct 2024 12:13:35 -0400
Subject: [PATCH 08/17] the fre-cli conda package should have every dep i can
 think of...

---
 environment.yml | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/environment.yml b/environment.yml
index 514aa81..ee9af15 100644
--- a/environment.yml
+++ b/environment.yml
@@ -6,14 +6,15 @@ channels:
 dependencies:
   - python
   - pip
-  - click
-  - pyyaml
-  - pylint
-  - jsonschema
-  - noaa-gfdl::fre-nctools
-  - conda-forge::cylc-flow>=8.2.0
-  - conda-forge::cylc-rose
-  - conda-forge::metomi-rose
-  - conda-forge::cylc-uiserver
-  - conda-forge::pytest
-  - conda-forge::pytest-cov
\ No newline at end of file
+  - noaa-gfdl::fre-cli
+#  - click
+#  - pyyaml
+#  - pylint
+#  - jsonschema
+#  - noaa-gfdl::fre-nctools
+#  - conda-forge::cylc-flow>=8.2.0
+#  - conda-forge::cylc-rose
+#  - conda-forge::metomi-rose
+#  - conda-forge::cylc-uiserver
+#  - conda-forge::pytest
+#  - conda-forge::pytest-cov
\ No newline at end of file

From 63aecbe20dd9020142c47d6be14bd058004fa15a Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 4 Oct 2024 12:37:16 -0400
Subject: [PATCH 09/17] full printout report of failures and keep testing,
 please (create_test_conda_env.yml edit). ignore files ending in *tags created
 by papiex_tooler tests. adjust to some more pythonic conventions inother test
 scripts

---
 .github/workflows/create_test_conda_env.yml |  9 +++--
 .gitignore                                  |  1 +
 tests/test_PPANHandler.py                   | 22 ++++++------
 tests/test_papiex_tooler.py                 | 38 +++++++++------------
 4 files changed, 35 insertions(+), 35 deletions(-)

diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml
index f5fe97e..76e2496 100644
--- a/.github/workflows/create_test_conda_env.yml
+++ b/.github/workflows/create_test_conda_env.yml
@@ -37,11 +37,14 @@ jobs:
           # unittests for data_lineage
           cd data_lineage && python -m unittest discover -s test -v || echo "data_lineage unittest failed."; cd -;
 
+		  # pytest unittests for regrid-xy
+		  cd app/regrid-xy && pytest -v -v -rx ./t || echo "pytest unittests for regrid-xy failed"; cd -;
+
           # pytest unittests for make-timeseries
-          cd app/make-timeseries && pytest -v -v -x ./test || echo "pytest for make-timeseries failed"; cd -;
+          cd app/make-timeseries && pytest -v -v -rx ./test || echo "pytest for make-timeseries failed"; cd -;
 
           # pytest unittests for remap-pp-components
-          cd app/remap-pp-components && pytest -v -v -x ./t/ || echo "pytest for remap-pp-components failed"; cd -;
+          cd app/remap-pp-components && pytest -v -v -rx ./t/ || echo "pytest for remap-pp-components failed"; cd -;
 
           # pytest unittests for other things in this repository
-          pytest -v -v -x ./tests;
\ No newline at end of file
+          pytest -v -v -rx ./tests || echo "pytest unittests for root directory failed :-(";
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 8025bc0..edb5a21 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
 __pycache__
 *~
+tests/test_files_papiex_tooler/*tags
\ No newline at end of file
diff --git a/tests/test_PPANHandler.py b/tests/test_PPANHandler.py
index 821306f..621a2ae 100644
--- a/tests/test_PPANHandler.py
+++ b/tests/test_PPANHandler.py
@@ -1,30 +1,30 @@
 ''' tests for PP/AN specific job_runner_handler class '''
 
-def test_import():
+def test_import(capfd):
     ''' check that ppan_handler can be imported.'''
     #print(f'__name__=={__name__}')
     from lib.python.ppan_handler import PPANHandler
     test_handler=PPANHandler()
 
-    assert(test_handler.test_import() == 0)
+    assert test_handler.test_import() == 0
 
 
-def test_tool_ops_import_in_handler():
+def test_tool_ops_import_in_handler(capfd):
     ''' check that ppan_handler can import tool_ops_w_papiex'''
     #print(f'__name__=={__name__}')
     from lib.python.ppan_handler import PPANHandler
     test_handler=PPANHandler()
 
-    assert(test_handler.test_tool_ops_import() == 0)
+    assert test_handler.test_tool_ops_import() == 0
 
-def test_submit():
+def test_submit(capfd):
     ''' check ppan_handler submit behavior with dry_run=True '''
     #print(f'__name__=={__name__}')
     from lib.python.ppan_handler import PPANHandler
     test_handler=PPANHandler()
     
     from pathlib import Path
-    job_file_path='./tests/test_files_papiex_tooler/am5_c96L33_amip_job_stage-history'
+    job_file_path='./tests/test_files_papiex_tooler/am5_c96L33_amip_mask-atmos-plevel_atmos_scalar_job'
 
     # check for existing test output, remove in order to recreate
     if Path(job_file_path+'.notags').exists():
@@ -37,10 +37,10 @@ def test_submit():
                                    job_file_path = job_file_path,
                                    submit_opts = submit_opts,
                                    dry_run = True, tool_ops = True)
-    assert(all([ ret_code == 0,
-                 ret_out  == "HELLO\n",
-                 ret_err  == "",
-                 Path(job_file_path+'.notags').exists(),
-                 Path(job_file_path).exists()  ]))
+    assert all ( [ ret_code == 0,
+                   ret_out  == "HELLO\n",
+                   ret_err  == "",
+                   Path(job_file_path+'.notags').exists(),
+                   Path(job_file_path).exists() ] ) 
 
 
diff --git a/tests/test_papiex_tooler.py b/tests/test_papiex_tooler.py
index d8b7f75..9bb3f0d 100644
--- a/tests/test_papiex_tooler.py
+++ b/tests/test_papiex_tooler.py
@@ -1,29 +1,32 @@
 ''' tests for PP/AN specific ops-tooling of job scripts for PAPIEX'''
+from subprocess import Popen, PIPE, DEVNULL
+from pathlib import Path
+import filecmp as fc
+import difflib as dl
 import pytest
 
 def test_import():
     ''' check that tool_ops_w_papiex functions can be imported '''
     from  lib.python.tool_ops_w_papiex import test_import
-    assert(test_import() == 0)
+    assert test_import() == 0
 
 def test_import_papiex_ops():
     ''' check that op list data can be imported '''
     import lib.python.papiex_ops as po
-    assert ( all( [ po.op_list is not None,
-                       len(po.op_list) > 0  ] ) )
+    assert all( [ po.op_list is not None,
+                     len(po.op_list) > 0  ] ) 
 
 def test_simple_failing_command():
     ''' check that setting/unsetting PAPIEX_TAGS around an op
     will not touch the exit status of that op '''
-    from pathlib import Path
-    control_script_targ=str(Path.cwd())+'/tests/test_files_papiex_tooler/simple_failing_command.bash'
+    control_script_targ = str(Path.cwd()) + \
+        '/tests/test_files_papiex_tooler/simple_failing_command.bash'
     assert Path(control_script_targ).exists() #quick check
 
     # import subprocess and run "control" process for later comparison
-    from subprocess import Popen, PIPE, DEVNULL
-    control_proc=None
+    control_proc = None
     try:
-        control_proc=Popen( args=['/bin/bash' ,control_script_targ],
+        control_proc = Popen( args=['/bin/bash' ,control_script_targ],
                          bufsize=0,
                          executable=None,
                          stdin=DEVNULL, stdout=PIPE, stderr=PIPE,
@@ -33,13 +36,14 @@ def test_simple_failing_command():
         assert False
 
     # grab control output
-    control_out, control_err= None,None
-    control_ret_code=None
+    control_out, control_err = None, None
+    control_ret_code = None
     try:
         control_out, control_err = (
             f.decode() for f in control_proc.communicate(DEVNULL) )
         control_ret_code = control_proc.wait()
-        print(f'control_out, control_err, control_ret_code = \n {control_out}, \n {control_err}, \n {control_ret_code}')
+        print(f'control_out, control_err, control_ret_code = ' + \
+              '\n {control_out}, \n {control_err}, \n {control_ret_code}')
         assert all( [ control_out is not None,
                       control_err is not None,
                       control_ret_code is not None ] )        
@@ -94,20 +98,17 @@ def test_check_simple_failing_command_for_diff():
     prev test, which could in theory succeed if tool_ops_w_papiex copies 
     the script but without adding tooling.'''
     
-    from pathlib import Path
     control_script_targ=str(Path.cwd())+'/tests/test_files_papiex_tooler/simple_failing_command.bash'
     script_targ=control_script_targ+'.tags'
     assert Path(control_script_targ).exists() #quick check
     assert Path(script_targ).exists() #quick check
 
     # check quickly that they are different in some manner.
-    import filecmp as fc
     is_different=not fc.cmp( control_script_targ, script_targ,
                              shallow=False)
     print(f'different? {is_different}\n\n')
 
     # now we will explicitly check for those differencves
-    import difflib as dl
     the_infile = open(control_script_targ)
     infile_contents=the_infile.readlines()
     the_infile.close()
@@ -135,8 +136,8 @@ def test_check_simple_failing_command_for_diff():
     
 
 def test_rose_task_run_for_diff():    
-    from pathlib import Path
-    control_script_targ=str(Path.cwd())+'/tests/test_files_papiex_tooler/am5_c96L33_amip_mask-atmos-plevel_atmos_scalar_job'
+    control_script_targ = str(Path.cwd()) + \
+        '/tests/test_files_papiex_tooler/am5_c96L33_amip_mask-atmos-plevel_atmos_scalar_job'
 
 
     # if we're testing over and over and '.notags' version exists,
@@ -159,13 +160,11 @@ def test_rose_task_run_for_diff():
     assert Path(script_targ).exists() #quick check
 
     # check quickly that they are different in some manner.
-    import filecmp as fc
     is_different=not fc.cmp( control_script_targ, script_targ,
                              shallow=False)
     print(f'different? {is_different}\n\n')
 
     # now we will explicitly check for those differences
-    import difflib as dl
     the_infile = open(control_script_targ)
     infile_contents=the_infile.readlines()
     the_infile.close()
@@ -197,7 +196,6 @@ def test_rose_task_run_for_diff():
 
 
 def test_pp_starter_for_no_diff():    
-    from pathlib import Path
     control_script_targ=str(Path.cwd())+'/tests/test_files_papiex_tooler/test_pp-starter'
 
     # if we're testing over and over and '.notags' version exists,
@@ -220,13 +218,11 @@ def test_pp_starter_for_no_diff():
     assert Path(script_targ).exists() #quick check
 
     # check quickly that they are different in some manner.
-    import filecmp as fc
     is_different=not fc.cmp( control_script_targ, script_targ,
                              shallow=False)
     print(f'different? {is_different}\n\n')
 
     # now we will explicitly check for those differences
-    import difflib as dl
     the_infile = open(control_script_targ)
     infile_contents=the_infile.readlines()
     the_infile.close()

From a25b4dbd7dbf72b35d1aa1080b835f5006b729eb Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 4 Oct 2024 12:38:46 -0400
Subject: [PATCH 10/17] TABS!!!!

---
 .github/workflows/create_test_conda_env.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml
index 76e2496..bf2c96e 100644
--- a/.github/workflows/create_test_conda_env.yml
+++ b/.github/workflows/create_test_conda_env.yml
@@ -37,14 +37,14 @@ jobs:
           # unittests for data_lineage
           cd data_lineage && python -m unittest discover -s test -v || echo "data_lineage unittest failed."; cd -;
 
-		  # pytest unittests for regrid-xy
-		  cd app/regrid-xy && pytest -v -v -rx ./t || echo "pytest unittests for regrid-xy failed"; cd -;
-
+          # pytest unittests for regrid-xy
+          cd app/regrid-xy && pytest -v -v -rx ./t || echo "pytest unittests for regrid-xy failed"; cd -;
+        
           # pytest unittests for make-timeseries
           cd app/make-timeseries && pytest -v -v -rx ./test || echo "pytest for make-timeseries failed"; cd -;
-
+        
           # pytest unittests for remap-pp-components
           cd app/remap-pp-components && pytest -v -v -rx ./t/ || echo "pytest for remap-pp-components failed"; cd -;
-
+        
           # pytest unittests for other things in this repository
           pytest -v -v -rx ./tests || echo "pytest unittests for root directory failed :-(";
\ No newline at end of file

From 16d4c19e0cafd9e878167fb292e28a63147d42b8 Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 4 Oct 2024 12:44:23 -0400
Subject: [PATCH 11/17] will this syntax work? add pylint verbose calls

---
 .github/workflows/create_test_conda_env.yml | 109 ++++++++++++--------
 1 file changed, 65 insertions(+), 44 deletions(-)

diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml
index bf2c96e..393e011 100644
--- a/.github/workflows/create_test_conda_env.yml
+++ b/.github/workflows/create_test_conda_env.yml
@@ -4,47 +4,68 @@ on: [push]
 
 jobs:
   build-linux:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '>=3.9'
-
-      - name: Add conda to system path
-        run: |
-          # $CONDA is an env var pointing to root of miniconda dir
-          echo $CONDA/bin >> $GITHUB_PATH
-
-      - name: Create fre-workflows environment
-        run: |
-          # create environment containing all dependencies
-          # the env cannot be explicitly activated in github CI/CD
-          conda env create -f environment.yml --name fre-workflows
-
-          # add conda env's executables to github's PATH equiv.
-          echo $CONDA/envs/fre-workflows/bin >> $GITHUB_PATH
-
-      - name: Run unittest in fre-workflows environment
-        run: |
-          # try to make sure the right things are in GITHUB_PATH
-          echo $CONDA/envs/fre-workflows/bin >> $GITHUB_PATH
-
-          # lint .cylc workflow files
-          cylc lint -v || echo "cylc lint threw non-zero exit code"
-
-          # unittests for data_lineage
-          cd data_lineage && python -m unittest discover -s test -v || echo "data_lineage unittest failed."; cd -;
-
-          # pytest unittests for regrid-xy
-          cd app/regrid-xy && pytest -v -v -rx ./t || echo "pytest unittests for regrid-xy failed"; cd -;
-        
-          # pytest unittests for make-timeseries
-          cd app/make-timeseries && pytest -v -v -rx ./test || echo "pytest for make-timeseries failed"; cd -;
-        
-          # pytest unittests for remap-pp-components
-          cd app/remap-pp-components && pytest -v -v -rx ./t/ || echo "pytest for remap-pp-components failed"; cd -;
-        
-          # pytest unittests for other things in this repository
-          pytest -v -v -rx ./tests || echo "pytest unittests for root directory failed :-(";
\ No newline at end of file
+	runs-on: ubuntu-latest
+	steps:
+	  - uses: actions/checkout@v4
+	  - name: Set up Python
+		uses: actions/setup-python@v5
+		with:
+		  python-version: '>=3.9'
+
+	  - name: Add conda to system path
+		run: |
+		  # $CONDA is an env var pointing to root of miniconda dir
+		  echo $CONDA/bin >> $GITHUB_PATH
+
+	  - name: Create fre-workflows environment
+		run: |
+		  # create environment containing all dependencies
+		  # the env cannot be explicitly activated in github CI/CD
+		  conda env create -f environment.yml --name fre-workflows
+
+		  # add conda env's executables to github's PATH equiv.
+		  echo $CONDA/envs/fre-workflows/bin >> $GITHUB_PATH
+
+	  - name: Run unittest in fre-workflows environment
+		run: |
+		  # try to make sure the right things are in GITHUB_PATH
+		  echo $CONDA/envs/fre-workflows/bin >> $GITHUB_PATH
+
+		  # lint .cylc workflow files
+		  cylc lint -v || echo "cylc lint threw non-zero exit code"
+
+		  # unittests for data_lineage
+		  cd data_lineage && \
+			 python -m unittest discover -s test -v || \
+			 echo "data_lineage unittest failed."; \
+			 pylint -v || \
+			 echo "pylint threw non-zero exit code."; \
+			 cd -;
+
+		  # pytest unittests for regrid-xy
+		  cd app/regrid-xy && \
+			 pytest -v -v -rx ./t || \
+			 echo "pytest unittests for regrid-xy failed"; \
+			 pylint -v || \
+			 echo "pylint threw non-zero exit code."; \
+			 cd -;
+
+		  # pytest unittests for make-timeseries
+			 cd app/make-timeseries && \
+			 pytest -v -v -rx ./test || \
+			 echo "pytest for make-timeseries failed"; \
+			 pylint -v || \
+			 echo "pylint threw non-zero exit code."; \
+			 cd -;
+
+		  # pytest unittests for remap-pp-components
+		  cd app/remap-pp-components && \
+			 pytest -v -v -rx ./t/ || \
+			 echo "pytest for remap-pp-components failed"; \
+			 pylint -v || \
+			 echo "pylint threw non-zero exit code."; \
+			 cd -;
+
+		  # pytest unittests for other things in this repository
+		  pytest -v -v -rx ./tests || \
+			 echo "pytest unittests for root directory failed :-(";
\ No newline at end of file

From 39e03a8c5d09868171eb184a0e7ca4c701e348ef Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 4 Oct 2024 12:52:06 -0400
Subject: [PATCH 12/17] whitespace fixes again...

---
 .github/workflows/create_test_conda_env.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml
index 393e011..82fc83d 100644
--- a/.github/workflows/create_test_conda_env.yml
+++ b/.github/workflows/create_test_conda_env.yml
@@ -51,7 +51,7 @@ jobs:
 			 cd -;
 
 		  # pytest unittests for make-timeseries
-			 cd app/make-timeseries && \
+		  cd app/make-timeseries && \
 			 pytest -v -v -rx ./test || \
 			 echo "pytest for make-timeseries failed"; \
 			 pylint -v || \

From ec6b459528fef28b497a841a65de075ddc44eed3 Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ian.laflotte@noaa.gov>
Date: Fri, 4 Oct 2024 13:03:07 -0400
Subject: [PATCH 13/17] something weird about macOS-flavored emacs and tabs...

---
 .github/workflows/create_test_conda_env.yml | 110 ++++++++++----------
 1 file changed, 55 insertions(+), 55 deletions(-)

diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml
index 82fc83d..4c82e19 100644
--- a/.github/workflows/create_test_conda_env.yml
+++ b/.github/workflows/create_test_conda_env.yml
@@ -4,68 +4,68 @@ on: [push]
 
 jobs:
   build-linux:
-	runs-on: ubuntu-latest
-	steps:
-	  - uses: actions/checkout@v4
-	  - name: Set up Python
-		uses: actions/setup-python@v5
-		with:
-		  python-version: '>=3.9'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '>=3.9'
 
-	  - name: Add conda to system path
-		run: |
-		  # $CONDA is an env var pointing to root of miniconda dir
-		  echo $CONDA/bin >> $GITHUB_PATH
+      - name: Add conda to system path
+        run: |
+          # $CONDA is an env var pointing to root of miniconda dir
+          echo $CONDA/bin >> $GITHUB_PATH
 
-	  - name: Create fre-workflows environment
-		run: |
-		  # create environment containing all dependencies
-		  # the env cannot be explicitly activated in github CI/CD
-		  conda env create -f environment.yml --name fre-workflows
+      - name: Create fre-workflows environment
+        run: |
+          # create environment containing all dependencies
+          # the env cannot be explicitly activated in github CI/CD
+          conda env create -f environment.yml --name fre-workflows
 
-		  # add conda env's executables to github's PATH equiv.
-		  echo $CONDA/envs/fre-workflows/bin >> $GITHUB_PATH
+          # add conda env's executables to github's PATH equiv.
+          echo $CONDA/envs/fre-workflows/bin >> $GITHUB_PATH
 
-	  - name: Run unittest in fre-workflows environment
-		run: |
-		  # try to make sure the right things are in GITHUB_PATH
-		  echo $CONDA/envs/fre-workflows/bin >> $GITHUB_PATH
+      - name: Run unittest in fre-workflows environment
+        run: |
+          # try to make sure the right things are in GITHUB_PATH
+          echo $CONDA/envs/fre-workflows/bin >> $GITHUB_PATH
 
-		  # lint .cylc workflow files
-		  cylc lint -v || echo "cylc lint threw non-zero exit code"
+          # lint .cylc workflow files
+          cylc lint -v || echo "cylc lint threw non-zero exit code"
 
-		  # unittests for data_lineage
-		  cd data_lineage && \
-			 python -m unittest discover -s test -v || \
-			 echo "data_lineage unittest failed."; \
-			 pylint -v || \
-			 echo "pylint threw non-zero exit code."; \
-			 cd -;
+          # unittests for data_lineage
+          cd data_lineage && \
+             python -m unittest discover -s test -v || \
+             echo "data_lineage unittest failed."; \
+             pylint -v || \
+             echo "pylint threw non-zero exit code."; \
+             cd -;
 
-		  # pytest unittests for regrid-xy
-		  cd app/regrid-xy && \
-			 pytest -v -v -rx ./t || \
-			 echo "pytest unittests for regrid-xy failed"; \
-			 pylint -v || \
-			 echo "pylint threw non-zero exit code."; \
-			 cd -;
+          # pytest unittests for regrid-xy
+          cd app/regrid-xy && \
+             pytest -v -v -rx ./t || \
+             echo "pytest unittests for regrid-xy failed"; \
+             pylint -v || \
+             echo "pylint threw non-zero exit code."; \
+             cd -;
 
-		  # pytest unittests for make-timeseries
-		  cd app/make-timeseries && \
-			 pytest -v -v -rx ./test || \
-			 echo "pytest for make-timeseries failed"; \
-			 pylint -v || \
-			 echo "pylint threw non-zero exit code."; \
-			 cd -;
+          # pytest unittests for make-timeseries
+          cd app/make-timeseries && \
+             pytest -v -v -rx ./test || \
+             echo "pytest for make-timeseries failed"; \
+             pylint -v || \
+             echo "pylint threw non-zero exit code."; \
+             cd -;
 
-		  # pytest unittests for remap-pp-components
-		  cd app/remap-pp-components && \
-			 pytest -v -v -rx ./t/ || \
-			 echo "pytest for remap-pp-components failed"; \
-			 pylint -v || \
-			 echo "pylint threw non-zero exit code."; \
-			 cd -;
+          # pytest unittests for remap-pp-components
+          cd app/remap-pp-components && \
+             pytest -v -v -rx ./t/ || \
+             echo "pytest for remap-pp-components failed"; \
+             pylint -v || \
+             echo "pylint threw non-zero exit code."; \
+             cd -;
 
-		  # pytest unittests for other things in this repository
-		  pytest -v -v -rx ./tests || \
-			 echo "pytest unittests for root directory failed :-(";
\ No newline at end of file
+          # pytest unittests for other things in this repository
+          pytest -v -v -rx ./tests || \
+             echo "pytest unittests for root directory failed :-(";

From b26bc2837e0b495430967ea403d3b82ca2f2cc6d Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 4 Oct 2024 13:32:11 -0400
Subject: [PATCH 14/17] coverage reports for pytest calls where it makes sense
 possibly

---
 .github/workflows/create_test_conda_env.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml
index 4c82e19..f57503a 100644
--- a/.github/workflows/create_test_conda_env.yml
+++ b/.github/workflows/create_test_conda_env.yml
@@ -44,7 +44,7 @@ jobs:
 
           # pytest unittests for regrid-xy
           cd app/regrid-xy && \
-             pytest -v -v -rx ./t || \
+             pytest -v -v -rx --cov=regrid_xy ./t || \
              echo "pytest unittests for regrid-xy failed"; \
              pylint -v || \
              echo "pylint threw non-zero exit code."; \
@@ -52,7 +52,7 @@ jobs:
 
           # pytest unittests for make-timeseries
           cd app/make-timeseries && \
-             pytest -v -v -rx ./test || \
+             pytest -v -v -rx --cov=bin/make-timeseries ./test || \
              echo "pytest for make-timeseries failed"; \
              pylint -v || \
              echo "pylint threw non-zero exit code."; \
@@ -60,7 +60,7 @@ jobs:
 
           # pytest unittests for remap-pp-components
           cd app/remap-pp-components && \
-             pytest -v -v -rx ./t/ || \
+             pytest -v -v -rx --cov=bin/remap-pp-components ./t/ || \
              echo "pytest for remap-pp-components failed"; \
              pylint -v || \
              echo "pylint threw non-zero exit code."; \

From 2841da085ed51b4d9973ad6607da2aaa74f6b7c7 Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 4 Oct 2024 14:37:26 -0400
Subject: [PATCH 15/17] see if i can break up the workflow steps for a more
 enjoyable experience clicking through action logs

---
 .github/workflows/create_test_conda_env.yml | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml
index f57503a..79118c5 100644
--- a/.github/workflows/create_test_conda_env.yml
+++ b/.github/workflows/create_test_conda_env.yml
@@ -5,6 +5,7 @@ on: [push]
 jobs:
   build-linux:
     runs-on: ubuntu-latest
+    continue-on-error: true
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python
@@ -26,14 +27,13 @@ jobs:
           # add conda env's executables to github's PATH equiv.
           echo $CONDA/envs/fre-workflows/bin >> $GITHUB_PATH
 
-      - name: Run unittest in fre-workflows environment
+      - name: in fre-workflows environment, cylc lint
         run: |
-          # try to make sure the right things are in GITHUB_PATH
-          echo $CONDA/envs/fre-workflows/bin >> $GITHUB_PATH
-
           # lint .cylc workflow files
-          cylc lint -v || echo "cylc lint threw non-zero exit code"
+          cylc lint -v
 
+      - name: in fre-workflows environment, data_lineage unittest and pylint
+        run: |
           # unittests for data_lineage
           cd data_lineage && \
              python -m unittest discover -s test -v || \
@@ -42,6 +42,8 @@ jobs:
              echo "pylint threw non-zero exit code."; \
              cd -;
 
+      - name: in fre-workflows environment, regrid-xy pytest and pylint
+        run: |
           # pytest unittests for regrid-xy
           cd app/regrid-xy && \
              pytest -v -v -rx --cov=regrid_xy ./t || \
@@ -50,6 +52,8 @@ jobs:
              echo "pylint threw non-zero exit code."; \
              cd -;
 
+      - name: in fre-workflows environment, make-timeseries pytest and pylint
+        run: |
           # pytest unittests for make-timeseries
           cd app/make-timeseries && \
              pytest -v -v -rx --cov=bin/make-timeseries ./test || \
@@ -58,6 +62,8 @@ jobs:
              echo "pylint threw non-zero exit code."; \
              cd -;
 
+      - name: in fre-workflows environment, remap-pp-components pytest and pylint
+        run: |
           # pytest unittests for remap-pp-components
           cd app/remap-pp-components && \
              pytest -v -v -rx --cov=bin/remap-pp-components ./t/ || \
@@ -66,6 +72,8 @@ jobs:
              echo "pylint threw non-zero exit code."; \
              cd -;
 
+      - name: in fre-workflows environment, root directory pytest
+        run: |
           # pytest unittests for other things in this repository
           pytest -v -v -rx ./tests || \
              echo "pytest unittests for root directory failed :-(";

From 64d6570f521119a1921caa53c22eb54c4c9c9b5d Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 4 Oct 2024 14:41:59 -0400
Subject: [PATCH 16/17] ... love how complete and descriptive githubs ci/cd
 syntax docs are...

---
 .github/workflows/create_test_conda_env.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml
index 79118c5..f7aa5a6 100644
--- a/.github/workflows/create_test_conda_env.yml
+++ b/.github/workflows/create_test_conda_env.yml
@@ -28,6 +28,7 @@ jobs:
           echo $CONDA/envs/fre-workflows/bin >> $GITHUB_PATH
 
       - name: in fre-workflows environment, cylc lint
+        continue-on-error: true
         run: |
           # lint .cylc workflow files
           cylc lint -v

From 5e286ca22bf90062a17bfd8c484136459e139c40 Mon Sep 17 00:00:00 2001
From: Ian Laflotte <ilaflott@gmail.com>
Date: Fri, 4 Oct 2024 14:50:51 -0400
Subject: [PATCH 17/17] OK! now i think i know what we want from this

---
 .github/workflows/create_test_conda_env.yml | 40 +++++++++------------
 1 file changed, 17 insertions(+), 23 deletions(-)

diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml
index f7aa5a6..9eb606a 100644
--- a/.github/workflows/create_test_conda_env.yml
+++ b/.github/workflows/create_test_conda_env.yml
@@ -5,7 +5,6 @@ on: [push]
 jobs:
   build-linux:
     runs-on: ubuntu-latest
-    continue-on-error: true
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python
@@ -34,47 +33,42 @@ jobs:
           cylc lint -v
 
       - name: in fre-workflows environment, data_lineage unittest and pylint
+        continue-on-error: true
         run: |
           # unittests for data_lineage
           cd data_lineage && \
-             python -m unittest discover -s test -v || \
-             echo "data_lineage unittest failed."; \
-             pylint -v || \
-             echo "pylint threw non-zero exit code."; \
-             cd -;
+             python -m unittest discover -s test -v ; \
+             pylint -v ; \
+             cd - ;
 
       - name: in fre-workflows environment, regrid-xy pytest and pylint
+        continue-on-error: true
         run: |
           # pytest unittests for regrid-xy
           cd app/regrid-xy && \
-             pytest -v -v -rx --cov=regrid_xy ./t || \
-             echo "pytest unittests for regrid-xy failed"; \
-             pylint -v || \
-             echo "pylint threw non-zero exit code."; \
-             cd -;
+             pytest -v -v -rx --cov=regrid_xy ./t ; \
+             pylint -v ; \
+             cd - ;
 
       - name: in fre-workflows environment, make-timeseries pytest and pylint
+        continue-on-error: true
         run: |
           # pytest unittests for make-timeseries
           cd app/make-timeseries && \
-             pytest -v -v -rx --cov=bin/make-timeseries ./test || \
-             echo "pytest for make-timeseries failed"; \
-             pylint -v || \
-             echo "pylint threw non-zero exit code."; \
-             cd -;
+             pytest -v -v -rx --cov=bin/make-timeseries ./test ; \
+             pylint -v ; \
+             cd - ;
 
       - name: in fre-workflows environment, remap-pp-components pytest and pylint
+        continue-on-error: true
         run: |
           # pytest unittests for remap-pp-components
           cd app/remap-pp-components && \
-             pytest -v -v -rx --cov=bin/remap-pp-components ./t/ || \
-             echo "pytest for remap-pp-components failed"; \
-             pylint -v || \
-             echo "pylint threw non-zero exit code."; \
-             cd -;
+             pytest -v -v -rx --cov=bin/remap-pp-components ./t ; \
+             cd - ;
 
       - name: in fre-workflows environment, root directory pytest
+        continue-on-error: true
         run: |
           # pytest unittests for other things in this repository
-          pytest -v -v -rx ./tests || \
-             echo "pytest unittests for root directory failed :-(";
+          pytest -v -v -rx ./tests