Merge pull request #215 from NOAA-GFDL/CMORPytestUpdates

 Adding reduced ascii file + updating pytest to convert it to binary

fre/cmor/tests/test_cmor_run_subtool.py

@@ -8,6 +8,8 @@
 
 import fre
 
+import subprocess
+
 # where are we? we're running pytest from the base directory of this repo
 ROOTDIR = 'fre/tests/test_files'
 
@@ -43,22 +45,37 @@ def test_setup_cmor_cmip_table_repo():
 FULL_OUTPUTDIR = \
    f"{OUTDIR}/{CMOR_CREATES_DIR}/v{YYYYMMDD}"
 FULL_OUTPUTFILE = \
-f"{FULL_OUTPUTDIR}/sos_Omon_PCMDI-test-1-0_piControl-withism_r3i1p1f1_gn_199307-199807.nc"
+f"{FULL_OUTPUTDIR}/sos_Omon_PCMDI-test-1-0_piControl-withism_r3i1p1f1_gn_199307-199308.nc"
 
 # FYI but helpful for tests
-FILENAME = 'ocean_monthly_1x1deg.199301-199712.sos.nc' # unneeded, this is mostly for reference
-FULL_INPUTFILE=f"{INDIR}/{FILENAME}"
+FILENAME = 'reduced_ocean_monthly_1x1deg.199307-199308.sos' # unneeded, this is mostly for reference
+FULL_INPUTFILE=f"{INDIR}/{FILENAME}.nc"
 
 def test_setup_fre_cmor_run_subtool(capfd):
-    ''' checks for outputfile from prev pytest runs, removes it if it's present.
-    this routine also checks to make sure the desired input file is present'''
+    ''' The routine generates a netCDF file from an ascii (cdl) file. It also checks for a ncgen output file from prev pytest runs,removes it if it's present, and ensures the new file is created without error. '''
+
+    ''' set-up test: create binary test files from reduced ascii files in root dir '''
+
+    ncgen_input = f"{ROOTDIR}/reduced_ascii_files/{FILENAME}.cdl"
+    ncgen_output = f"{ROOTDIR}/ocean_sos_var_file/{FILENAME}.nc"
+
+    if Path(ncgen_output).exists():
+        Path(ncgen_output).unlink() 
+    assert Path(ncgen_input).exists()
+
+    ex = [ 'ncgen3', '-k', 'netCDF-4', '-o', ncgen_output, ncgen_input ]
+
+    sp = subprocess.run(ex, check = True)
+
+    assert all( [ sp.returncode == 0, Path(ncgen_output).exists() ] )
+
     if Path(FULL_OUTPUTFILE).exists():
         Path(FULL_OUTPUTFILE).unlink()
-    if Path(OUTDIR).exists():
-        shutil.rmtree(OUTDIR)
-    assert not any ( [ Path(FULL_OUTPUTFILE).exists(),
-                       Path(OUTDIR).exists()           ] )
-    assert Path(FULL_INPUTFILE).exists()
+
+    assert not Path(FULL_OUTPUTFILE).exists()
+
+    #assert not any ( [ Path(FULL_OUTPUTFILE).exists(),
+    #                   Path(OUTDIR).exists()           ] )
     _out, _err = capfd.readouterr()
 
 def test_fre_cmor_run_subtool_case1(capfd):
@@ -102,10 +119,11 @@ def test_fre_cmor_run_subtool_case1_output_compare_data(capfd):
                              check=False,
                              capture_output=True
     )
+
     # err_list has length two if end in newline
     err_list = result.stderr.decode().split('\n')
     expected_err = \
-        "DIFFER : FILE FORMATS : NC_FORMAT_64BIT <> NC_FORMAT_NETCDF4_CLASSIC"
+        "DIFFER : FILE FORMATS : NC_FORMAT_NETCDF4 <> NC_FORMAT_NETCDF4_CLASSIC"
     assert all( [result.returncode == 1,
                  len(err_list)==2,
                  '' in err_list,
@@ -132,7 +150,7 @@ def test_fre_cmor_run_subtool_case1_output_compare_metadata(capfd):
 
 # FYI, but again, helpful for tests
 FILENAME_DIFF = \
-    'ocean_monthly_1x1deg.199301-199712.sosV2.nc'
+    'ocean_monthly_1x1deg.199307-199308.sosV2.nc'
 FULL_INPUTFILE_DIFF = \
     f"{INDIR}/{FILENAME_DIFF}"
 VARLIST_DIFF = \
@@ -224,7 +242,7 @@ def test_fre_cmor_run_subtool_case2_output_compare_data(capfd):
                           )
 
     err_list = result.stderr.decode().split('\n')#length two if end in newline
-    expected_err="DIFFER : FILE FORMATS : NC_FORMAT_64BIT <> NC_FORMAT_NETCDF4_CLASSIC"
+    expected_err="DIFFER : FILE FORMATS : NC_FORMAT_NETCDF4 <> NC_FORMAT_NETCDF4_CLASSIC"
     assert all( [result.returncode == 1,
                  len(err_list)==2,
                  '' in err_list,

fre/tests/test_files/reduced_ascii_files/reduced_ocean_monthly_1x1deg.199307-199308.sos.cdl

@@ -0,0 +1,84 @@
+netcdf reduced_ocean_monthly_1x1deg.199301-199712.sos {
+dimensions:
+	lat = 2 ;
+	bnds = 2 ;
+	lon = 2 ;
+	time = UNLIMITED ; // (2 currently)
+variables:
+	double lat(lat) ;
+		lat:long_name = "latitude" ;
+		lat:units = "degrees_N" ;
+		lat:axis = "Y" ;
+		lat:bounds = "lat_bnds" ;
+	double lat_bnds(lat, bnds) ;
+		lat_bnds:long_name = "latitude bounds" ;
+		lat_bnds:units = "degrees_N" ;
+		lat_bnds:axis = "Y" ;
+	double lon(lon) ;
+		lon:long_name = "longitude" ;
+		lon:units = "degrees_E" ;
+		lon:axis = "X" ;
+		lon:bounds = "lon_bnds" ;
+	double lon_bnds(lon, bnds) ;
+		lon_bnds:long_name = "longitude bounds" ;
+		lon_bnds:units = "degrees_E" ;
+		lon_bnds:axis = "X" ;
+	float sos(time, lat, lon) ;
+		sos:_FillValue = 1.e+20f ;
+		sos:missing_value = 1.e+20f ;
+		sos:units = "psu" ;
+		sos:long_name = "Sea Surface Salinity" ;
+		sos:cell_methods = "area:mean yh:mean xh:mean time: mean" ;
+		sos:cell_measures = "area: areacello" ;
+		sos:standard_name = "sea_surface_salinity" ;
+		sos:interp_method = "conserve_order1" ;
+	double time(time) ;
+		time:units = "days since 1958-01-01 00:00:00" ;
+		time:long_name = "time" ;
+		time:axis = "T" ;
+		time:calendar_type = "JULIAN" ;
+		time:calendar = "julian" ;
+		time:bounds = "time_bnds" ;
+	double time_bnds(time, bnds) ;
+		time_bnds:units = "days since 1958-01-01 00:00:00" ;
+		time_bnds:long_name = "time axis boundaries" ;
+
+// global attributes:
+		:title = "om5_b05_noHiLatHenyey_55" ;
+		:associated_files = "areacello: 19930101.ocean_static.nc" ;
+		:grid_type = "regular" ;
+		:grid_tile = "N/A" ;
+		:code_release_version = "2024.02" ;
+		:git_hash = "b86d27037f755a82c586e55073dd575245c144b1" ;
+		:creationtime = "Mon Jun 17 18:57:22 2024" ;
+		:hostname = "pp337" ;
+		:history = "Wed Nov  6 19:17:52 2024: ncks -d lat,0,1 -d lon,0,1 -d time,0,1 ocean_monthly_1x1deg.199301-199712.sos.nc -o reduced_ocean_monthly_1x1deg.199301-199712.sos.nc\n",
+			"fregrid --standard_dimension --input_mosaic ocean_mosaic.nc --input_file all --interp_method conserve_order1 --remap_file .fregrid_remap_file_360_by_180.nc --nlon 360 --nlat 180 --scalar_field (**please see the field list in this file**) --output_file out.nc" ;
+		:external_variables = "areacello" ;
+		:NCO = "netCDF Operators version 5.1.5 (Homepage = http://nco.sf.net, Code = http://github.com/nco/nco)" ;
+data:
+
+ lat = -89.5, -88.5 ;
+
+ lat_bnds =
+  -90, -89,
+  -89, -88 ;
+
+ lon = 0.5, 1.5 ;
+
+ lon_bnds =
+  0, 1,
+  1, 2 ;
+
+ sos =
+  35.5, 36.1,
+  35.2, 33.8,
+  32.6, 34.2,
+  33.9, 35.7 ;
+
+ time = 12799.5, 12829 ;
+
+ time_bnds =
+  12784, 12815,
+  12815, 12843 ;
+}

fre/tests/test_fre_cmor_cli.py

@@ -2,14 +2,20 @@
 
 from datetime import date
 from pathlib import Path
+import shutil
 
 import click
 from click.testing import CliRunner
 
 from fre import fre
 
+import subprocess
+
 runner = CliRunner()
 
+# where are we? we're running pytest from the base directory of this repo
+rootdir = 'fre/tests/test_files'
+
 # fre cmor
 def test_cli_fre_cmor():
     ''' fre cmor '''
@@ -42,17 +48,32 @@ def test_cli_fre_cmor_run_opt_dne():
     result = runner.invoke(fre.fre, args=["cmor", "run", "optionDNE"])
     assert result.exit_code == 2
 
+copied_nc_filename = 'reduced_ocean_monthly_1x1deg.199307-199308.sosV2.nc'
+full_copied_nc_filepath = f'{rootdir}/ocean_sos_var_file/{copied_nc_filename}'
+original_nc_file = f'{rootdir}/ocean_sos_var_file/reduced_ocean_monthly_1x1deg.199307-199308.sos.nc'
+
+def test_setup_test_files(capfd):
+    "set-up test: copy and rename NetCDF file created in test_fre_cmor_run_subtool.py"
+
+    assert Path(original_nc_file).exists()
+
+    if Path(full_copied_nc_filepath).exists():
+        Path(full_copied_nc_filepath).unlink()
+    assert not Path(full_copied_nc_filepath).exists()
+
+    shutil.copy(Path(original_nc_file), Path(full_copied_nc_filepath))
+
+    assert (Path(full_copied_nc_filepath).exists())
+
+    out, err = capfd.readouterr()
 
 # these unit tests should be more about the cli, rather than the workload
 YYYYMMDD=date.today().strftime('%Y%m%d')
-def test_cli_fre_cmor_run_case1():
+def test_cli_fre_cmor_run_case1(capfd):
     ''' fre cmor run, test-use case '''
 
-    # where are we? we're running pytest from the base directory of this repo
-    rootdir = 'fre/tests/test_files'
-
     # explicit inputs to tool
-    indir = f'{rootdir}/ocean_sos_var_file'
+    indir = f'{rootdir}/ocean_sos_var_file/'
     varlist = f'{rootdir}/varlist'
     table_config = f'{rootdir}/cmip6-cmor-tables/Tables/CMIP6_Omon.json'
     exp_config = f'{rootdir}/CMOR_input_example.json'
@@ -64,10 +85,10 @@ def test_cli_fre_cmor_run_case1():
     full_outputdir = \
         f"{outdir}/{cmor_creates_dir}/v{YYYYMMDD}" # yay no more 'fre' where it shouldnt be
     full_outputfile = \
-        f"{full_outputdir}/sos_Omon_PCMDI-test-1-0_piControl-withism_r3i1p1f1_gn_199307-199807.nc"
+        f"{full_outputdir}/sos_Omon_PCMDI-test-1-0_piControl-withism_r3i1p1f1_gn_199307-199308.nc"
 
     # FYI
-    filename = 'ocean_monthly_1x1deg.199301-199712.sos.nc' # unneeded, this is mostly for reference
+    filename = 'reduced_ocean_monthly_1x1deg.199307-199308.sos.nc' # unneeded, this is mostly for reference
     full_inputfile=f"{indir}/{filename}"
 
     # clean up, lest we fool outselves
@@ -86,12 +107,9 @@ def test_cli_fre_cmor_run_case1():
     assert all ( [ result.exit_code == 0,
                    Path(full_outputfile).exists(),
                    Path(full_inputfile).exists() ] )
+    _out, _err = capfd.readouterr()
 
-
-
-
-
-def test_cli_fre_cmor_run_case2():
+def test_cli_fre_cmor_run_case2(capfd):
     ''' fre cmor run, test-use case '''
 
     # where are we? we're running pytest from the base directory of this repo
@@ -110,10 +128,10 @@ def test_cli_fre_cmor_run_case2():
     full_outputdir = \
         f"{outdir}/{cmor_creates_dir}/v{YYYYMMDD}" # yay no more 'fre' where it shouldnt be
     full_outputfile = \
-        f"{full_outputdir}/sos_Omon_PCMDI-test-1-0_piControl-withism_r3i1p1f1_gn_199307-199807.nc"
+        f"{full_outputdir}/sos_Omon_PCMDI-test-1-0_piControl-withism_r3i1p1f1_gn_199307-199308.nc"
 
     # FYI
-    filename = 'ocean_monthly_1x1deg.199301-199712.sosV2.nc' # unneeded, this is mostly for reference
+    filename = 'reduced_ocean_monthly_1x1deg.199307-199308.sosV2.nc' # unneeded, this is mostly for reference
     full_inputfile=f"{indir}/{filename}"
 
     # clean up, lest we fool outselves
@@ -132,3 +150,4 @@ def test_cli_fre_cmor_run_case2():
     assert all ( [ result.exit_code == 0,
                    Path(full_outputfile).exists(),
                    Path(full_inputfile).exists() ] )
+    _out, _err = capfd.readouterr()