diff --git a/fre/yamltools/data_table/__init__.py b/fre/yamltools/data_table/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/fre/yamltools/data_table/combine_data_table_yamls.py b/fre/yamltools/data_table/combine_data_table_yamls.py
deleted file mode 100644
index df1e128e..00000000
--- a/fre/yamltools/data_table/combine_data_table_yamls.py
+++ /dev/null
@@ -1,121 +0,0 @@
-#!/usr/bin/env python3
-# ***********************************************************************
-# * GNU Lesser General Public License
-# *
-# * This file is part of the GFDL Flexible Modeling System (FMS) YAML
-# * tools.
-# *
-# * FMS_yaml_tools is free software: you can redistribute it and/or
-# * modify it under the terms of the GNU Lesser General Public License
-# * as published by the Free Software Foundation, either version 3 of the
-# * License, or (at your option) any later version.
-# *
-# * FMS_yaml_tools is distributed in the hope that it will be useful, but
-# * WITHOUT ANY WARRANTY; without even the implied warranty of
-# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# * General Public License for more details.
-# *
-# * You should have received a copy of the GNU Lesser General Public
-# * License along with FMS. If not, see .
-# ***********************************************************************
-
-from os import path, strerror
-import errno
-import click
-import yaml
-from .. import __version__
-
-""" Combines a series of data_table.yaml files into one file
- Author: Uriel Ramirez 04/11/2023
-"""
-
-
-def is_duplicate(data_table, new_entry):
- """
- Check if a data_table entry was already defined in a different file
-
- Args:
- data_table: List of dictionaries containing all of the data_table
- entries that have been combined
- new_entry: Dictionary of the data_table entry to check
- """
-
- for entry in data_table:
- if entry == new_entry:
- is_duplicate = True
- return is_duplicate
- else:
- if entry['fieldname_code'] == new_entry['fieldname_code']:
- raise Exception("A data_table entry is defined twice for the "
- "field_name_code:" + entry['fieldname_code'] +
- " with different keys/values!")
- is_duplicate = False
- return is_duplicate
-
-
-def combine_yaml(files):
- """
- Combines a list of yaml files into one
-
- Args:
- files: List of yaml file names to combine
- """
- data_table = {}
- data_table['data_table'] = []
- for f in files:
- # Check if the file exists
- if not path.exists(f):
- raise FileNotFoundError(errno.ENOENT,
- strerror(errno.ENOENT),
- f)
-
- with open(f) as fl:
- my_table = yaml.safe_load(fl)
- entries = my_table['data_table']
- for entry in entries:
- if not is_duplicate(data_table['data_table'], entry):
- data_table['data_table'].append(entry)
- return data_table
-
-
-def main():
- #: parse user input
- @click.command()
- @click.option('-f',
- '--in_files',
- type=str,
- multiple=True,
- default=["data_table"],
- help='Space seperated list with the '
- 'Names of the data_table.yaml files to combine',
- required=True)
- @click.option('-o',
- '--out_file',
- type=str,
- default='data_table.yaml',
- help="Ouput file name of the converted YAML \
- (Default: 'diag_table.yaml')",
- required=True)
- @click.option('-F'
- '--force',
- is_flag=True,
- help="Overwrite the output data table yaml file.")
- @click.version_option(version=__version__,
- prog_name='combine_data_table_yaml')
- def combine_data_table_yaml(in_files, out_file, force):
- """
- Combines a list of data_table.yaml files into one file" + "Requires pyyaml (https://pyyaml.org/)
- """
- try:
- data_table = combine_yaml(in_files)
- out_file_op = "x" # Exclusive write
- if force:
- out_file_op = "w"
- with open(out_file, out_file_op) as myfile:
- yaml.dump(data_table, myfile, default_flow_style=False)
- except Exception as err:
- raise SystemExit(err)
-
-
-if __name__ == "__main__":
- main()
diff --git a/fre/yamltools/data_table/data_table_to_yaml.py b/fre/yamltools/data_table/data_table_to_yaml.py
deleted file mode 100755
index 41f15614..00000000
--- a/fre/yamltools/data_table/data_table_to_yaml.py
+++ /dev/null
@@ -1,183 +0,0 @@
-#!/usr/bin/env python3
-# ***********************************************************************
-# * GNU Lesser General Public License
-# *
-# * This file is part of the GFDL Flexible Modeling System (FMS) YAML
-# * tools.
-# *
-# * FMS_yaml_tools is free software: you can redistribute it and/or
-# * modify it under the terms of the GNU Lesser General Public License
-# * as published by the Free Software Foundation, either version 3 of the
-# * License, or (at your option) any later version.
-# *
-# * FMS_yaml_tools is distributed in the hope that it will be useful, but
-# * WITHOUT ANY WARRANTY; without even the implied warranty of
-# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# * General Public License for more details.
-# *
-# * You should have received a copy of the GNU Lesser General Public
-# * License along with FMS. If not, see .
-# ***********************************************************************
-
-from os import path, strerror
-import errno
-import click
-import yaml
-from .. import __version__, TableParseError
-
-
-""" Converts a legacy ascii data_table to a yaml data_table.
- Run `python3 data_table_to_yaml.py -h` for more details
- Author: Uriel Ramirez 05/27/2022
-"""
-
-
-class DataType:
- def __init__(self, data_table_file='data_table',
- yaml_table_file='data_table.yaml',
- force_write=False):
- """Initialize the DataType"""
- self.data_table_file = data_table_file
- self.yaml_table_file = yaml_table_file
- self.out_file_op = "x" # Exclusive write
- if force_write:
- self.out_file_op = "w"
-
- self.data_type = {}
- self.data_type_keys = ['gridname',
- 'fieldname_code',
- 'fieldname_file',
- 'file_name',
- 'interpol_method',
- 'factor',
- 'lon_start',
- 'lon_end',
- 'lat_start',
- 'lat_end',
- 'region_type']
- self.data_type_values = {'gridname': str,
- 'fieldname_code': str,
- 'fieldname_file': str,
- 'file_name': str,
- 'interpol_method': str,
- 'factor': float,
- 'lon_start': float,
- 'lon_end': float,
- 'lat_start': float,
- 'lat_end': float,
- 'region_type': str}
-
- self.data_table_content = []
-
- #: check if data_table file exists
- if not path.exists(self.data_table_file):
- raise FileNotFoundError(errno.ENOENT,
- strerror(errno.ENOENT),
- data_table_file)
-
- # Check if path to the output yaml file exists
- if not path.exists(path.abspath(path.dirname(self.yaml_table_file))):
- raise NotADirectoryError(errno.ENOTDIR,
- "Directory does not exist",
- path.abspath(
- path.dirname(self.yaml_table_file)))
-
- def read_data_table(self):
- """Open and read the legacy ascii data_table file"""
- with open(self.data_table_file, 'r') as myfile:
- self.data_table_content = myfile.readlines()
-
- def parse_data_table(self):
- """Loop through each line in the ascii data_Table file and fill in
- data_type class"""
- iline_count = 0
- self.data_type['data_table'] = []
- for iline in self.data_table_content:
- iline_count += 1
- if iline.strip() != '' and '#' not in iline.strip()[0]:
- # get rid of comment at the end of line
- iline_list = iline.split('#')[0].split(',')
- try:
- tmp_list = {}
- for i in range(len(iline_list)):
- mykey = self.data_type_keys[i]
- myfunct = self.data_type_values[mykey]
- myval = myfunct(
- iline_list[i].strip('"\' \n'))
- if i == 4:
- # If LIMA format convert to the regular format
- # #FUTURE
- if ("true" in myval):
- myval = 'bilinear'
- if ("false" in myval):
- myval = 'none'
- tmp_list[mykey] = myval
- except Exception:
- raise TableParseError(self.data_table_file,
- iline_count,
- iline)
- # If the fieldname_file is empty (i.e no interpolation just
- # multiplying by a constant), remove fieldname_file,
- # file_name, and interpol_method
- if (tmp_list['fieldname_file'] == ""):
- del tmp_list['fieldname_file']
- del tmp_list['file_name']
- del tmp_list['interpol_method']
- self.data_type['data_table'].append(tmp_list)
-
- def read_and_parse_data_table(self):
- """Open, read, and parse the legacy ascii data_table file"""
- if self.data_table_content != []:
- self.data_table_content = []
- self.read_data_table()
- self.parse_data_table()
-
- def convert_data_table(self):
- """Convert the legacy ascii data_table file to yaml"""
- self.read_and_parse_data_table()
-
- with open(self.yaml_table_file, self.out_file_op) as myfile:
- yaml.dump(self.data_type, myfile, sort_keys=False)
-
-
-def main():
- #: parse user input
- @click.command()
- @click.option('-f',
- '--in_file',
- type=str,
- default="data_table",
- help="Name of the data_table file to convert",
- required=True)
- @click.option('-o',
- '--output',
- type=str,
- default='data_table.yaml',
- help="Ouput file name of the converted YAML \
- (Default: 'diag_table.yaml')",
- required=True)
- @click.option('-F'
- '--force',
- is_flag=True,
- help="Overwrite the output data table yaml file.")
- @click.version_option(version='1.0',
- prog_name='combine_data_table_yaml')
- def data_table_to_yaml(in_file, output, force):
- """
- Converts a legacy ascii data_table to a yaml data_table. \
- Requires pyyaml (https://pyyaml.org/) \
- More details on the data_table yaml format can be found \
- in \
- https://github.com/NOAA-GFDL/FMS/tree/main/data_override")
- """
- try:
- test_class = DataType(data_table_file=in_file,
- yaml_table_file=output,
- force_write=force)
- test_class.convert_data_table()
- except Exception as err:
- raise SystemError(err)
-
-
-if __name__ == "__main__":
- main()
diff --git a/fre/yamltools/data_table/is_valid_data_table_yaml.py b/fre/yamltools/data_table/is_valid_data_table_yaml.py
deleted file mode 100644
index 17d08fbd..00000000
--- a/fre/yamltools/data_table/is_valid_data_table_yaml.py
+++ /dev/null
@@ -1,125 +0,0 @@
-#!/usr/bin/env python3
-"""
-***********************************************************************
-* GNU Lesser General Public License
-*
-* This file is part of the GFDL Flexible Modeling System (FMS) YAML tools.
-*
-* FMS_yaml_tools is free software: you can redistribute it and/or modify it under
-* the terms of the GNU Lesser General Public License as published by
-* the Free Software Foundation, either version 3 of the License, or (at
-* your option) any later version.
-*
-* FMS_yaml_tools is distributed in the hope that it will be useful, but WITHOUT
-* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-* for more details.
-*
-* You should have received a copy of the GNU Lesser General Public
-* License along with FMS. If not, see .
-***********************************************************************
- Determine if a yaml data_table is valid.
- Run `python3 is_valid_data_table_yaml.py -h` for more details
- Author: Uriel Ramirez 05/27/2022
-"""
-
-import yaml
-import click
-
-def check_gridname(grid_name):
- """Check if the input grid_name is valid. Crashes if it not."""
- valid = ["OCN", "LND", "ATM", "ICE"]
- if (grid_name not in valid): raise Exception(grid_name+ ' is not a valid gridname. The only values allowed are "OCN", "LND", "ATM", "ICE"')
-
-def check_fieldname_code(fieldname):
- if (fieldname == ""): raise Exception("Fieldname can't be empty")
-
-def check_filename_and_field(field, interp_method):
- if (field =="" and interp_method != ""): raise Exception('If "fieldname_file" is empty, interp_method must be empty')
-
-def check_interp_method(interp_method):
- """Check if the interp method is valid. Crashes if it not. """
- valid = ["bilinear", "bicubic", "none"]
- if (interp_method not in valid): raise Exception(interp_method + ' is not a valid interp_method. The only values allowed are "bilinear", "bicubic", and "none"')
-
-def check_region_type(region_type):
- """Check if the input region type is valid. Crashes if it is not."""
- valid = ["inside_region", "outside_region"]
- if (region_type not in valid): raise Exception(region_type + 'is not a valid region_type. The only values allowed are "inside_region" and "outside_region"')
-
-def check_if_bounds_present(entry):
- """Check if the region bounds are valid, crashes if they are not """
- if ("lat_start" not in entry): raise Exception('lat_start must be present if region_type is set')
- if ("lat_end" not in entry): raise Exception('lat_end must be present if region_type is set')
- if ("lon_start" not in entry): raise Exception('lon_start must be present if region_type is set')
- if ("lon_end" not in entry): raise Exception('lon_end must be present if region_type is set')
-
-def check_region(my_type, start, end):
- """Check if the region is defined correctly. Crashes if it not. """
- if (start > end): raise Exception(my_type+"_start is greater than "+my_type+"_end")
-
-
-def main():
- @click.command()
- @click.option('-f',
- '--file',
- type=str,
- default="data_table",
- help="Name of the data_table file to convert",
- required=True)
- def is_valid_data_table_yaml(file):
- """
- Determines if a yaml data_table is valid. \
- Requires pyyaml (https://pyyaml.org/) \
- More details on the yaml format can be found in \
- https://github.com/NOAA-GFDL/FMS/tree/main/data_override"
- """
- with open(file, 'r') as fl:
- my_table = yaml.safe_load(fl)
- for key, value in my_table.items():
- for i in range(0, len(value)):
- entry = value[i]
- if "gridname" not in entry:
- raise Exception("gridname is a required key!")
- gridname = entry["gridname"]
- check_gridname(gridname)
-
- if "fieldname_code" not in entry:
- raise Exception("fieldname_code is a required key!")
-
- fieldname_code = entry["fieldname_code"]
- check_fieldname_code(fieldname_code)
-
- if "fieldname_file" in entry:
- fieldname_file = entry["fieldname_file"]
-
- if "file_name" in entry:
- file_name = entry["file_name"]
-
- if "interpol_method" in entry:
- interp_method = entry["interpol_method"]
-
- check_interp_method(interp_method)
- check_filename_and_field(fieldname_file, interp_method)
-
- if "factor" not in entry:
- raise Exception("factor is a required key")
-
- factor = entry["factor"]
-
- if "region_type" in entry:
- region_type = entry["region_type"]
- check_region_type(region_type)
- check_if_bounds_present(entry)
-
- lat_start = entry["lat_start"]
- lat_end = entry["lat_end"]
- check_region("lat", lat_start, lat_end)
-
- lon_start = entry["lon_start"]
- lon_end = entry["lon_end"]
- check_region("lon", lon_start, lon_end)
-
-
-if __name__ == "__main__":
- main()
diff --git a/fre/yamltools/diag_table/__init__.py b/fre/yamltools/diag_table/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/fre/yamltools/diag_table/combine_diag_table_yamls.py b/fre/yamltools/diag_table/combine_diag_table_yamls.py
deleted file mode 100644
index becaf701..00000000
--- a/fre/yamltools/diag_table/combine_diag_table_yamls.py
+++ /dev/null
@@ -1,169 +0,0 @@
-#!/usr/bin/env python3
-# ***********************************************************************
-# * GNU Lesser General Public License
-# *
-# * This file is part of the GFDL Flexible Modeling System (FMS) YAML
-# * tools.
-# *
-# * FMS_yaml_tools is free software: you can redistribute it and/or
-# * modify it under the terms of the GNU Lesser General Public License
-# * as published by the Free Software Foundation, either version 3 of the
-# * License, or (at your option) any later version.
-# *
-# * FMS_yaml_tools is distributed in the hope that it will be useful, but
-# * WITHOUT ANY WARRANTY; without even the implied warranty of
-# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# * General Public License for more details.
-# *
-# * You should have received a copy of the GNU Lesser General Public
-# * License along with FMS. If not, see .
-# ***********************************************************************
-
-from os import path, strerror
-import errno
-import click
-import yaml
-from .. import __version__
-
-""" Combines a series of diag_table.yaml files into one file
- Author: Uriel Ramirez 04/11/2023
-"""
-
-
-def compare_key_value_pairs(entry1, entry2, key, is_optional=False):
- if not is_optional:
- if entry1[key] != entry2[key]:
- raise Exception("The diag_file:" + entry1['file_name'] + " is defined twice " +
- " with different " + key)
- else:
- if key not in entry1 and key not in entry2:
- return
- if key in entry1 and key in entry2:
- if entry1[key] != entry2[key]:
- raise Exception("The diag_file:" + entry1['file_name'] + " is defined twice " +
- " with different " + key)
- if key in entry1 and key not in entry2:
- raise Exception("The diag_file:" + entry1['file_name'] + " is defined twice " +
- " with different " + key)
- if key not in entry1 and key in entry2:
- raise Exception("The diag_file:" + entry1['file_name'] + " is defined twice " +
- " with different " + key)
-
-
-def is_field_duplicate(diag_table, new_entry, file_name):
- for entry in diag_table:
- if entry == new_entry:
- return True
- else:
- if entry['var_name'] != new_entry['var_name']:
- # If the variable name is not the same, then it is a brand new variable
- return False
- elif entry['var_name'] == new_entry['var_name'] and entry['module'] != new_entry['module']:
- # If the variable name is the same but it a different module, then it is a brand new variable
- return False
- else:
- raise Exception("The variable " + entry['var_name'] + " from module " + entry['module'] +
- " in file " + file_name + " is defined twice with different keys")
-
-
-def is_file_duplicate(diag_table, new_entry):
- # Check if a diag_table entry was already defined
- for entry in diag_table:
- if entry == new_entry:
- return True
- else:
- # If the file_name is not the same, then it is a brand new file
- if entry['file_name'] != new_entry['file_name']:
- return False
-
- # Since there are duplicate files, check fhat all the keys are the same:
- compare_key_value_pairs(entry, new_entry, 'freq')
- compare_key_value_pairs(entry, new_entry, 'time_units')
- compare_key_value_pairs(entry, new_entry, 'unlimdim')
-
- compare_key_value_pairs(entry, new_entry, 'write_file', is_optional=True)
- compare_key_value_pairs(entry, new_entry, 'new_file_freq', is_optional=True)
- compare_key_value_pairs(entry, new_entry, 'start_time', is_optional=True)
- compare_key_value_pairs(entry, new_entry, 'file_duration', is_optional=True)
- compare_key_value_pairs(entry, new_entry, 'global_meta', is_optional=True)
- compare_key_value_pairs(entry, new_entry, 'sub_region', is_optional=True)
- compare_key_value_pairs(entry, new_entry, 'is_ocean', is_optional=True)
-
- # Since the file is the same, check if there are any new variables to add to the file:
- for field_entry in new_entry['varlist']:
- if not is_field_duplicate(entry['varlist'], field_entry, entry['file_name']):
- entry['varlist'].append(field_entry)
- return True
-
-
-def combine_yaml(files):
- diag_table = {}
- diag_table['title'] = ""
- diag_table['base_date'] = ""
- diag_table['diag_files'] = []
- for f in files:
- # Check if the file exists
- if not path.exists(f):
- raise FileNotFoundError(errno.ENOENT,
- strerror(errno.ENOENT),
- f)
-
- with open(f) as fl:
- my_table = yaml.safe_load(fl)
-
- if 'base_date' in my_table:
- diag_table['base_date'] = my_table['base_date']
- if 'title' in my_table:
- diag_table['title'] = my_table['title']
-
- if 'diag_files' not in my_table:
- continue
-
- diag_files = my_table['diag_files']
- for entry in diag_files:
- if not is_file_duplicate(diag_table['diag_files'], entry):
- diag_table['diag_files'].append(entry)
- return diag_table
-
-
-def main():
- #: parse user input
- @click.command()
- @click.option('-f',
- '--in_files',
- type=str,
- multiple=True,
- default=["diag_table"],
- help='Space seperated list with the '
- 'Names of the diag_table.yaml files to combine')
- @click.option('-o',
- '--out_file',
- type=str,
- default='diag_table.yaml',
- help="Ouput file name of the converted YAML \
- (Default: 'diag_table.yaml')")
- @click.option('-F',
- '--force',
- is_flag=True,
- help="Overwrite the output diag table yaml file.")
- @click.version_option(version=__version__,
- prog_name='combine_diag_table_yaml')
- def combine_diag_table_yaml(in_files, out_file, force):
- """
- Combines a series of diag_table.yaml files into one file
- Requires pyyaml (https://pyyaml.org/)
- """
- try:
- diag_table = combine_yaml(in_files)
- out_file_op = "x" # Exclusive write
- if force:
- out_file_op = "w"
- with open(out_file, out_file_op) as myfile:
- yaml.dump(diag_table, myfile, default_flow_style=False, sort_keys=False)
-
- except Exception as err:
- raise SystemExit(err)
-
-
-if __name__ == "__main__":
- main()
diff --git a/fre/yamltools/diag_table/diag_table_to_yaml.py b/fre/yamltools/diag_table/diag_table_to_yaml.py
deleted file mode 100755
index 400abee4..00000000
--- a/fre/yamltools/diag_table/diag_table_to_yaml.py
+++ /dev/null
@@ -1,478 +0,0 @@
-#!/usr/bin/env python3
-# ***********************************************************************
-# * GNU Lesser General Public License
-# *
-# * This file is part of the GFDL Flexible Modeling System (FMS) YAML tools.
-# *
-# * FMS_yaml_tools is free software: you can redistribute it and/or modify it under
-# * the terms of the GNU Lesser General Public License as published by
-# * the Free Software Foundation, either version 3 of the License, or (at
-# * your option) any later version.
-# *
-# * FMS_yaml_tools is distributed in the hope that it will be useful, but WITHOUT
-# * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# * for more details.
-# *
-# * You should have received a copy of the GNU Lesser General Public
-# * License along with FMS. If not, see .
-# ***********************************************************************
-
-""" Converts a legacy ascii diag_table to a yaml diag_table.
- Run `python3 diag_table_to_yaml.py -h` for more details
- Author: Uriel Ramirez 05/27/2022
-"""
-
-import copy as cp
-from os import path
-import click
-import yaml
-from .. import __version__#, TableParseError
-
-def main():
- #: parse user input
- @click.command()
- @click.option('-f'
- '--in_file',
- type=str,
- help='Name of the diag_table to convert')
- @click.option('-s',
- '--is_segment',
- is_flag=True,
- help='The diag_table is a segment and a not a full table, \
- so the tile and the base_date are not expected')
- @click.option('-o',
- '--out_file',
- type=str,
- default='diag_table.yaml',
- help="Ouput file name of the converted YAML \
- (Default: 'diag_table.yaml')")
- @click.option('-F',
- '--force',
- is_flag=True,
- help="Overwrite the output data table yaml file.")
- @click.version_option(__version__,
- prog_name='diag_table_to_yaml')
- def diag_table_to_yaml(in_file, is_segment, out_file, force):
- """
- Converts a legacy ascii diag_table to a yaml diag_table
- Requires pyyaml (https://pyyaml.org)
- More details on the diag_table yaml format can be found in
- https://github.com/NOAA-GFDL/FMS/tree/main/diag_table
- """
- #: start
- test_class = DiagTable(diag_table_file=in_file, is_segment=is_segment)
- test_class.read_and_parse_diag_table()
- test_class.construct_yaml(yaml_table_file=out_file, force_write=force)
-
-
-def is_duplicate(current_files, diag_file):
-
- """
- Determine if a diag file has already been defined.
-
- Args:
- current_files (list): List of dictionary containing all the diag files that have been defined
- diag_file (dictionary): Dictionary defining a diag file
-
- Returns:
- logical: If the diag_file has been defined and has the same keys, returns True
- If it has been defined but it does not have the same keys, return an error
- If it has not been defined, return False
- """
- for curr_diag_file in current_files['diag_files']:
- if curr_diag_file['file_name'] != diag_file['file_name']:
- continue
- if curr_diag_file == diag_file:
- return True
- else:
- raise Exception("The diag_table defines " + diag_file['file_name'] + " more than once with different keys")
- return False
-
-
-class DiagTable:
- def __init__(self, diag_table_file='Diag_Table', is_segment=False):
- '''Initialize the diag_table type'''
-
- self.diag_table_file = diag_table_file
- self.is_segment = is_segment
- self.global_section = {}
- self.global_section_keys = ['title', 'base_date']
- self.global_section_fvalues = {'title': str,
- 'base_date': [int, int, int, int, int, int]}
- self.max_global_section = len(self.global_section_keys) - 1 # minus title
-
- self.file_section = []
- self.file_section_keys = ['file_name',
- 'freq_int',
- 'freq_units',
- 'time_units',
- 'unlimdim',
- 'new_file_freq_int',
- 'new_file_freq_units',
- 'start_time',
- 'file_duration_int',
- 'file_duration_units',
- 'filename_time_bounds']
- self.file_section_fvalues = {'file_name': str,
- 'freq_int': int,
- 'freq_units': str,
- 'time_units': str,
- 'unlimdim': str,
- 'new_file_freq_int': int,
- 'new_file_freq_units': str,
- 'start_time': str,
- 'file_duration_int': int,
- 'file_duration_units': str,
- 'filename_time_bounds': str}
- self.max_file_section = len(self.file_section_keys)
-
- self.region_section = []
- self.region_section_keys = ['grid_type',
- 'corner1',
- 'corner2',
- 'corner3',
- 'corner4',
- 'zbounds',
- 'is_only_zbounds',
- 'file_name'
- 'line']
- self.region_section_fvalues = {'grid_type': str,
- 'corner1': [float, float],
- 'corner2': [float, float],
- 'corner3': [float, float],
- 'corner4': [float, float],
- 'zbounds': [float, float],
- 'is_only_zbounds': bool,
- 'file_name': str,
- 'line': str}
- self.max_file_section = len(self.file_section_keys)
- self.field_section = []
- self.field_section_keys = ['module',
- 'var_name',
- 'output_name',
- 'file_name',
- 'reduction',
- 'spatial_ops',
- 'kind',
- 'zbounds']
- self.field_section_fvalues = {'module': str,
- 'var_name': str,
- 'output_name': str,
- 'file_name': str,
- 'reduction': str,
- 'spatial_ops': str,
- 'kind': str,
- 'zbounds': str}
- self.max_field_section = len(self.field_section_keys)
-
- self.diag_table_content = []
-
- #: check if diag_table file exists
- if not path.exists(self.diag_table_file):
- raise Exception('file ' + self.diag_table_file + ' does not exist')
-
- def read_diag_table(self):
- """ Open and read the diag_table"""
- with open(self.diag_table_file, 'r') as myfile:
- self.diag_table_content = myfile.readlines()
-
- def set_sub_region(self, myval, field_dict):
- """
- Loop through the defined sub_regions, determine if the file already has a sub_region defined
- if it does crash. If the sub_region is not already defined add the region to the list
-
- Args:
- myval (string): Defines the subregion as read from the diag_table in the format
- [starting x, ending x, starting y, ending y, starting z, ending z]
- field_dict(dictionary): Defines the field
- """
- tmp_dict2 = {}
- file_name = field_dict['file_name']
- found = False
- is_same = True
- for iregion_dict in self.region_section:
- if iregion_dict['file_name'] == file_name:
- found = True
- if iregion_dict['line'] != myval:
- """
- Here the file has a already a sub_region defined and it is not the same as the current
- subregion.
- """
- is_same = False
- if found and is_same:
- return
-
- tmp_dict2["line"] = myval
- tmp_dict2["file_name"] = file_name
- if "none" in myval:
- tmp_dict2[self.region_section_keys[0]] = myval
- else:
- tmp_dict2[self.region_section_keys[0]] = "latlon"
- stuff = myval.split(' ')
- k = -1
- for j in range(len(stuff)):
- if stuff[j] == "":
- continue # Some lines have extra spaces ("1 10 9 11 -1 -1")
- k = k + 1
-
- # Set any -1 values to -999
- if float(stuff[j]) == -1:
- stuff[j] = "-999"
-
- # Define the 4 corners and the z bounds
- if k == 0:
- corner1 = stuff[j]
- corner2 = stuff[j]
- elif k == 1:
- corner3 = stuff[j]
- corner4 = stuff[j]
- elif k == 2:
- corner1 = corner1 + ' ' + stuff[j]
- corner2 = corner2 + ' ' + stuff[j]
- elif k == 3:
- corner3 = corner3 + ' ' + stuff[j]
- corner4 = corner4 + ' ' + stuff[j]
- elif k == 4:
- zbounds = stuff[j]
- elif k == 5:
- zbounds = zbounds + ' ' + stuff[j]
-
- tmp_dict2["corner1"] = corner1
- tmp_dict2["corner2"] = corner2
- tmp_dict2["corner3"] = corner3
- tmp_dict2["corner4"] = corner4
- tmp_dict2["zbounds"] = zbounds
- tmp_dict2["is_only_zbounds"] = False
- field_dict['zbounds'] = zbounds
-
- if corner1 == "-999 -999" and corner2 == "-999 -999" and corner3 == "-999 -999" and corner4 == "-999 -999":
- tmp_dict2["is_only_zbounds"] = True
- elif not is_same:
- raise Exception("The " + file_name + " has multiple sub_regions defined. Be sure that all the variables"
- "in the file are in the same sub_region! "
- "Region 1:" + myval + "\n"
- "Region 2:" + iregion_dict['line'])
- self.region_section.append(cp.deepcopy(tmp_dict2))
-
- def parse_diag_table(self):
- """ Loop through each line in the diag_table and parse it"""
-
- if self.diag_table_content == []:
- raise Exception('ERROR: The input diag_table is empty!')
-
- iline_count, global_count = 0, 0
-
- if self.is_segment:
- global_count = 2
-
- #: The first two lines should be the title and base_time
- while global_count < 2:
- iline = self.diag_table_content[iline_count]
- iline_count += 1
- # Ignore comments and empty lines
- if iline.strip() != '' and '#' not in iline.strip()[0]:
- #: The second uncommented line is the base date
- if global_count == 1:
- try:
- iline_list, tmp_list = iline.split('#')[0].split(), [] #: not comma separated integers
- mykey = self.global_section_keys[1]
- self.global_section[mykey] = iline.split('#')[0].strip()
- global_count += 1
- except:
- raise Exception(" ERROR with line # " + str(iline_count) + '\n'
- " CHECK: " + str(iline) + '\n'
- " Ensure that the second uncommented line of the diag table defines \n"
- " the base date in the format [year month day hour min sec]")
- #: The first uncommented line is the title
- if global_count == 0:
- try:
- mykey = self.global_section_keys[0]
- myfunct = self.global_section_fvalues[mykey]
- myval = myfunct(iline.strip().strip('"').strip("'"))
- self.global_section[mykey] = myval
- global_count += 1
- except:
- raise Exception(" ERROR with line # " + str(iline_count) + '\n'
- " CHECK: " + str(iline) + '\n'
- " Ensure that the first uncommented line of the diag table defines the title")
-
- #: The rest of the lines are either going to be file or field section
- for iline_in in self.diag_table_content[iline_count:]:
- iline = iline_in.strip().strip(',') # get rid of any leading spaces and the comma that some file lines have in the end #classic
- iline_count += 1
- if iline.strip() != '' and '#' not in iline.strip()[0]: # if not blank line or comment
- iline_list = iline.split('#')[0].split(',') # get rid of any comments in the end of a line
- try:
- #: Fill in the file section
- tmp_dict = {}
- for i in range(len(iline_list)):
- j = i
- # Do not do anything with the "file_format" column
- if i == 3:
- continue
- if i > 3:
- j = i - 1
- mykey = self.file_section_keys[j]
- myfunct = self.file_section_fvalues[mykey]
- myval = myfunct(iline_list[i].strip().strip('"').strip("'"))
-
- # Ignore file_duration if it less than 0
- if i == 9 and myval <= 0:
- continue
-
- # Ignore the file_duration_units if it is an empty string
- if i == 10 and myval == "":
- continue
- tmp_dict[mykey] = myval
- self.file_section.append(cp.deepcopy(tmp_dict))
- except:
- #: Fill in the field section
- try:
- tmp_dict = {}
- for i in range(len(self.field_section_keys)):
- j = i
- buf = iline_list[i]
- # Do nothing with the "time_sampling" section
- if i == 4:
- continue
- if i > 4:
- j = i - 1
- if i == 5:
- # Set the reduction to average or none instead of the other options
- if "true" in buf.lower() or "avg" in buf.lower() or "mean" in buf.lower():
- buf = "average"
- elif "false" in buf.lower():
- buf = "none"
-
- # Set the kind to either "r4" or "r8"
- if i == 7:
- if "2" in buf:
- buf = "r4"
- elif "1" in buf:
- buf = "r8"
- else:
- raise Exception(" ERROR with line # " + str(iline_count) + '\n'
- " CHECK: " + str(iline) + '\n'
- " Ensure that kind is either 1 or 2")
- mykey = self.field_section_keys[j]
- myfunct = self.field_section_fvalues[mykey]
- myval = myfunct(buf.strip().strip('"').strip("'"))
-
- # Do not add the region to the field section. This will be added to the file later
- if i != 6:
- tmp_dict[mykey] = myval
- else:
- self.set_sub_region(myval, tmp_dict)
- self.field_section.append(cp.deepcopy(tmp_dict))
- except:
- raise Exception(" ERROR with line # " + str(iline_count) + '\n'
- " CHECK: " + str(iline) + '\n'
- " Ensure that the line defines a field in the format: \n"
- " 'module_name', 'field_name', 'output_name', 'file_name', 'time_sampling', 'reduction_method',"
- " 'regional_section', 'packing' \n"
- " Or that the line defined a file in the format: \n"
- " 'file_name', 'output_freq', 'output_freq_units', 'file_format', "
- " 'time_axis_units', 'time_axis_name' "
- " 'new_file_freq', 'new_file_freq_units', 'start_time', 'file_duration', 'file_duration_units'")
-
- def construct_yaml(self,
- yaml_table_file='diag_table.yaml',
- force_write=False):
- """ Combine the global, file, field, sub_region sections into 1 """
-
- out_file_op = "x" # Exclusive write
- if force_write:
- out_file_op = "w"
-
- yaml_doc = {}
- #: title
-
- if not self.is_segment:
- mykey = self.global_section_keys[0]
- yaml_doc[mykey] = self.global_section[mykey]
- #: basedate
- mykey = self.global_section_keys[1]
- yaml_doc[mykey] = self.global_section[mykey]
-
- #: diag_files
- yaml_doc['diag_files'] = []
- #: go through each file
- for ifile_dict in self.file_section: #: file_section = [ {}, {}, {} ]
- if 'ocean' in ifile_dict['file_name']:
- ifile_dict['is_ocean'] = True
- ifile_dict['sub_region'] = []
-
- # Combine freq_int and freq_units into 1 key
- ifile_dict['freq'] = str(ifile_dict['freq_int']) + ' ' + ifile_dict['freq_units']
- del ifile_dict['freq_int']
- del ifile_dict['freq_units']
-
- # Combine new_file_freq_int and new_file_freq_units into 1 key
- if "new_file_freq_int" in ifile_dict:
- ifile_dict['new_file_freq'] = str(ifile_dict['new_file_freq_int']) + ' ' + ifile_dict['new_file_freq_units']
- del ifile_dict['new_file_freq_int']
- del ifile_dict['new_file_freq_units']
-
- # Combine file_duration_int and file_duration_units into 1 key
- if "file_duration_int" in ifile_dict:
- ifile_dict['file_duration'] = str(ifile_dict['file_duration_int']) + ' ' + ifile_dict['file_duration_units']
- del ifile_dict['file_duration_int']
- del ifile_dict['file_duration_units']
-
- found = False
- for iregion_dict in self.region_section:
- if iregion_dict['file_name'] == ifile_dict['file_name']:
- tmp_dict = cp.deepcopy(iregion_dict)
- del tmp_dict['file_name']
- del tmp_dict['line']
- if tmp_dict['grid_type'] != "none":
- ifile_dict['sub_region'].append(tmp_dict)
- found = True
- if tmp_dict['is_only_zbounds']:
- found = False
- del tmp_dict['is_only_zbounds']
- del tmp_dict['zbounds']
- if not found:
- del ifile_dict['sub_region']
-
- ifile_dict['varlist'] = []
- found = False
- for ifield_dict in self.field_section: #: field_section = [ {}, {}. {} ]
- if ifield_dict['file_name'] == ifile_dict['file_name']:
- tmp_dict = cp.deepcopy(ifield_dict)
-
- # Ensure that the output_name contains "min"
- # if the reduction method is "min"
- if tmp_dict['reduction'] == "min" and "min" not in tmp_dict['output_name']:
- tmp_dict['output_name'] = tmp_dict['output_name'] + "_min"
-
- # Ensure that the output_name contains "max"
- # if the reduction method is "max"
- if tmp_dict['reduction'] == "max" and "max" not in tmp_dict['output_name']:
- tmp_dict['output_name'] = tmp_dict['output_name'] + "_max"
-
- # If the output_name and the var_name are the same
- # there is no need for output_name
- if tmp_dict['output_name'] == tmp_dict['var_name']:
- del tmp_dict['output_name']
-
- del tmp_dict['file_name']
- ifile_dict['varlist'].append(tmp_dict)
- found = True
- continue
- if not found:
- del ifile_dict['varlist']
- if not is_duplicate(yaml_doc, ifile_dict):
- yaml_doc['diag_files'].append(ifile_dict)
- myfile = open(yaml_table_file, out_file_op)
- yaml.dump(yaml_doc, myfile, sort_keys=False)
-
- def read_and_parse_diag_table(self):
- """ Read and parse the file """
- self.read_diag_table()
- self.parse_diag_table()
-
-
-if __name__ == "__main__":
- main()
diff --git a/fre/yamltools/diag_table/is_valid_diag_table_yaml.py b/fre/yamltools/diag_table/is_valid_diag_table_yaml.py
deleted file mode 100644
index e9ce12a5..00000000
--- a/fre/yamltools/diag_table/is_valid_diag_table_yaml.py
+++ /dev/null
@@ -1,217 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-***********************************************************************
-* GNU Lesser General Public License
-*
-* This file is part of the GFDL Flexible Modeling System (FMS) YAML tools.
-*
-* FMS_yaml_tools is free software: you can redistribute it and/or modify it under
-* the terms of the GNU Lesser General Public License as published by
-* the Free Software Foundation, either version 3 of the License, or (at
-* your option) any later version.
-*
-* FMS_yaml_tools is distributed in the hope that it will be useful, but WITHOUT
-* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-* for more details.
-*
-* You should have received a copy of the GNU Lesser General Public
-* License along with FMS. If not, see .
-***********************************************************************
- Determine if a yaml diag_table is valid.
- Run `python3 is_valid_diag_table_yaml.py -h` for more details
- Author: Uriel Ramirez 05/27/2022
-"""
-
-import sys
-import argparse
-import yaml
-
-parser = argparse.ArgumentParser(prog='is_valid_diag_table_yaml', \
- description="Determine if a yaml diag_table is valid. \
- Requires pyyaml (https://pyyaml.org/) \
- More details on the yaml format can be found in \
- https://github.com/NOAA-GFDL/FMS/tree/main/diag_table")
-parser.add_argument('-f', type=str, help='Name of the diag_table yaml to check' )
-
-in_diag_table = parser.parse_args().f
-
-class UniqueKeyLoader(yaml.SafeLoader):
- """ Special loader to check if duplicate keys are present"""
- def construct_mapping(self, node, deep=False):
- mapping = []
- for key_node, value_node in node.value:
- key = self.construct_object(key_node, deep=deep)
- if key in mapping : sys.exit('ERROR: You have defined the key:' + key + ' multiple times')
- mapping.append(key)
- return super().construct_mapping(node, deep)
-
-def check_time_units(file_name, key_name, time_units) :
- """Check if the input time_units are valid, crashes if they are not """
- valid = ["seconds", "minutes", "hours", "days", "months", "years"]
- if (time_units not in valid) :
- sys.exit('ERROR: ' + time_units + ' is not a valid unit. Check your ' + key_name + ' entry for file:' + file_name)
-
-def check_freq(file_name, freq, freq_units) :
- """Check if the input freq is valid, crashes if they are not """
- if (freq < -1) : sys.exit('ERROR: freq needs to greater than -1. Check your freq entry for file:' + file_name)
- check_time_units(file_name, 'freq_units', freq_units)
-
-def check_required_diag_files_key(diag_file) :
- """Checks if all the required key are present in diag_file block. Crashes if any are missing."""
- if 'file_name' not in diag_file : sys.exit('ERROR: file_name is a required key!')
- if 'freq' not in diag_file : sys.exit('ERROR: freq is a required key! Add it for file:' + diag_file['file_name'])
- if 'freq_units' not in diag_file : sys.exit('ERROR: freq_units is a required key! Add it for file:' + diag_file['file_name'])
- if 'time_units' not in diag_file : sys.exit('ERROR: time_units is a required key! Add it for file:' + diag_file['file_name'])
- if 'unlimdim' not in diag_file : sys.exit('ERROR: unlimdim is a required key! Add it for file:' + diag_file['file_name'])
-
-def check_new_file_freq(diag_file) :
- """Check if the input new_file_freq and new_file_freq_units are valid, crashes if they are not """
- if 'new_file_freq' in diag_file :
- if 'new_file_freq_units' not in diag_file :
- sys.exit('ERROR: new_file_freq is present, but not new_file_freq_units. Check you entry for file:' + diag_file['file_name'])
- if (diag_file['new_file_freq'] < 1) :
- sys.exit('ERROR: new_file_freq needs to be greater than 0. Check your new_file_freq for ' + diag_file['file_name'])
- check_time_units(diag_file['file_name'], 'new_file_freq_units', diag_file['new_file_freq_units'])
- else:
- if 'new_file_freq_units' in diag_file :
- sys.exit('ERROR: new_file_freq_units is present, but not new_file_freq. Check your entry for file:' + diag_file['file_name'])
-
-def check_file_duration(diag_file) :
- """Check if the input file_duration and file_duration_units are valid, crashes if they are not """
- if 'file_duration' in diag_file :
- if 'file_duration_units' not in diag_file :
- sys.exit('ERROR: file_duration is present, but not file_duration_units. Check you entry for file:' + diag_file['file_name'])
- if (diag_file['file_duration'] < 1) :
- sys.exit('ERROR: file_duration_units needs to be greater than 0. Check your file_duration_units for ' + diag_file['file_name'])
- check_time_units(diag_file['file_name'], 'file_duration_units', diag_file['file_duration_units'])
- else:
- if 'file_duration_units' in diag_file :
- sys.exit('ERROR: file_duration_units is present, but not file_duration. Check your entry for file:' + diag_file['file_name'])
-
-def check_start_time(diag_file) :
- """Check if the start_time is valid, crashes if it is not """
- if 'start_time' not in diag_file : return
- if 'file_duration' not in diag_file :
- sys.exit('ERROR: file_duration is needed if start_time is present. Check your entry for file:' + diag_file['file_name'])
- check_date(diag_file['start_time'], 'start_time')
-
-def check_sub_region(diag_file) :
- """Check if the sub_regeion is defined correctly, crashes if it is not """
- if 'sub_region' not in diag_file : return
-
- sub_regions = diag_file['sub_region']
- sub_region = sub_regions[0]
-
- valid = ["latlon", "index"]
- if 'grid_type' not in sub_region :
- sys.exit('ERROR: grid_type is required if defining a sub_region. Add it your file:' + diag_file['file_name'])
-
- if sub_region['grid_type'] not in valid:
- sys.exit('ERROR: the grid_type (' + sub_region['grid_type'] + ') and file:' + diag_file['file_name']+ ' is not valid')
-
- if 'dim1_begin' in sub_region and 'dim1_end' in sub_region :
- if sub_region['dim1_begin'] > sub_region['dim1_end'] :
- sys.exit('ERROR: dim1_begin in your subregion of file:' + diag_file['file_name']+ ' is greater than dim1_end')
-
- if 'dim2_begin' in sub_region and 'dim2_end' in sub_region :
- if sub_region['dim2_begin'] > sub_region['dim2_end'] :
- sys.exit('ERROR: dim2_begin in your subregion of file:' + diag_file['file_name']+ ' is greater than dim2_end')
-
- if 'dim3_begin' in sub_region and 'dim3_end' in sub_region :
- if sub_region['dim3_begin'] > sub_region['dim3_end'] :
- sys.exit('ERROR: dim3_begin in your subregion of file:' + diag_file['file_name']+ ' is greater than dim3_end')
-
- if 'dim4_begin' in sub_region and 'dim4_end' in sub_region :
- if sub_region['dim4_begin'] > sub_region['dim4_end'] :
- sys.exit('ERROR: dim4_begin in your subregion of file:' + diag_file['file_name']+ ' is greater than dim4_end')
-
-def check_diag_file(diag_file) :
- """Check if the diag_file is defined correctly. Crashes if it is not. """
- check_required_diag_files_key(diag_file)
- check_freq(diag_file['file_name'], diag_file['freq'], diag_file['freq_units'])
- check_time_units(diag_file['file_name'], 'time_units', diag_file['time_units'])
- check_new_file_freq(diag_file)
- check_file_duration(diag_file)
- check_start_time(diag_file)
- check_sub_region(diag_file)
-
-def check_required_diag_field_key(diag_field, file_name):
- """Check if all the required keys in the diag_field are present, crashes if any are missing """
- if 'var_name' not in diag_field :
- sys.exit('ERROR: var_name is a required field! Check your var_name in file: ' + file_name)
- if 'module' not in diag_field :
- sys.exit('ERROR: module is a required field! Add it for variable:' + diag_field['var_name'] + ' in file: ' + file_name)
- if 'reduction' not in diag_field :
- sys.exit('ERROR: reduction is a required field! Add it for variable:' + diag_field['var_name'] + ' in file: ' + file_name)
- if 'kind' not in diag_field :
- sys.exit('ERROR: kind is a required field! Add it for variable:' + diag_field['var_name'] + ' in file: ' + file_name)
-
-def check_date(date_str, date_name):
- """Check if the input date is valid, crashes if it is not """
- date_int = date_str.split() # [yr month day hour minute second]
- if (len(date_int) != 6 ) :
- sys.exit('ERROR: The size of ' + date_name + ' (' + date_str + ') should be 6')
- if int(date_int[1]) <= 0 :
- sys.exit('ERROR: The month in the ' + date_name + ' (' + date_str + ') should greater than 0')
- if int(date_int[2]) <= 0 :
- sys.exit('ERROR: The day in the ' + date_name + ' (' + date_str + ') should be greater than 0')
-
-def check_reduction(diag_field, file_name):
- """Check if the reduction is valid, crashes if it is not """
- valid = ["none", "average", "min", "max", "rms"]
- reduction = diag_field['reduction']
-
- if "diurnal" in reduction :
- if int(reduction[7:9]) < 0 :
- sys.exit('ERROR: The number of diurnal samples in ' + reduction + ' for variable:' + diag_field['var_name'] + ' and file:' + file_name + ' is not valid')
- elif "pow" in reduction :
- if int(reduction[3:5]) < 0 :
- sys.exit('ERROR: The power value in ' + reduction + ' for variable:' + diag_field['var_name'] + ' and file:' + file_name + ' is not valid')
- elif reduction not in valid :
- sys.exit('ERROR: The reduction (' + reduction + ') in variable:' + diag_field['var_name'] + ' and file:' + file_name + ' is not valid')
-
-def check_kind(diag_field, file_name) :
- """Check if the variable type is valid. Crashes if it not. """
- valid = ["r4", "r8"]
- if diag_field['kind'] not in valid :
- sys.exit('ERROR: The kind (' + diag_field['kind'] + ') in variable:' + diag_field['var_name'] + ' and file:' + file_name + ' is not valid')
-
-def check_diag_field(diag_field, file_name) :
- """Check if the diag_field is defined correctly, crashes if it is not """
- check_required_diag_field_key(diag_field, file_name)
- check_reduction(diag_field, file_name)
- check_kind(diag_field, file_name)
-
-def check_for_duplicates(my_list, list_name) :
- """Check if there any duplicates in a list. Crashes if they are """
- if len(set(my_list)) != len(my_list):
- sys.exit('ERROR: Found duplicate ' + list_name )
-
-""" Loop thorugh all the files and field and checks if they are defined correctly"""
-file_names = []
-with open(in_diag_table) as fl:
- my_table = yaml.load(fl, Loader=UniqueKeyLoader)
- if 'title' not in my_table : sys.exit('ERROR: title is a required key!')
- if 'base_date' not in my_table : sys.exit('ERROR: base_date is a required key!')
- check_date(my_table['base_date'], 'base_date')
-
- diag_files = my_table['diag_files']
- for i in range(0, len(diag_files)) :
- diag_file = diag_files[i]
- check_diag_file(diag_file)
-
- if 'varlist' not in diag_file: continue
- diag_fields = diag_file['varlist']
- var_names = []
- for j in range(0, len(diag_fields)) :
- diag_field = diag_fields[j]
- check_diag_field(diag_field, diag_file['file_name'])
- if "output_name" in diag_field :
- var_names = var_names + [diag_field['output_name']]
- else :
- var_names = var_names + [diag_field['var_name']]
- check_for_duplicates(var_names, 'var_names in file: ' + diag_file['file_name'])
- file_names = file_names + [diag_file['file_name']]
-check_for_duplicates(file_names, "file_names")
diff --git a/fre/yamltools/field_table/__init__.py b/fre/yamltools/field_table/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/fre/yamltools/field_table/combine_field_table_yamls.py b/fre/yamltools/field_table/combine_field_table_yamls.py
deleted file mode 100644
index c0afb6e6..00000000
--- a/fre/yamltools/field_table/combine_field_table_yamls.py
+++ /dev/null
@@ -1,154 +0,0 @@
-#!/usr/bin/env python3
-# ***********************************************************************
-# * GNU Lesser General Public License
-# *
-# * This file is part of the GFDL Flexible Modeling System (FMS) YAML
-# * tools.
-# *
-# * FMS_yaml_tools is free software: you can redistribute it and/or
-# * modify it under the terms of the GNU Lesser General Public License
-# * as published by the Free Software Foundation, either version 3 of the
-# * License, or (at your option) any later version.
-# *
-# * FMS_yaml_tools is distributed in the hope that it will be useful, but
-# * WITHOUT ANY WARRANTY; without even the implied warranty of
-# * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# * General Public License for more details.
-# *
-# * You should have received a copy of the GNU Lesser General Public
-# * License along with FMS. If not, see .
-# ***********************************************************************
-
-from os import path, strerror
-import errno
-import click
-import yaml
-from .. import __version__
-
-""" Combines a series of field_table.yaml files into one file
- Author: Uriel Ramirez 11/20/2023
-"""
-
-
-def is_duplicate(field_table, new_entry):
- """
- Check if a field_table entry was already defined in a different file
-
- Args:
- field_table: List of dictionaries containing all of the field_table
- entries that have been combined
- new_entry: Dictionary of the field_table entry to check
- """
- is_duplicate = False
- return is_duplicate
-
-def field_type_exists(field_type, curr_entries):
- for entry in curr_entries:
- if field_type == entry['field_type']:
- return True
- return False
-
-def add_new_field(new_entry, curr_entries):
- new_field_type = new_entry['field_type']
- for entry in curr_entries:
- if new_field_type == entry['field_type']:
- if entry == new_entry:
- # If the field_type already exists but it is exactly the same, move on
- continue
- new_modlist = new_entry['modlist']
- for mod in new_modlist:
- if model_type_exists(mod['model_type'], entry):
- add_new_mod(mod, entry)
- else:
- #If the model type does not exist, just append it
- entry['modlist'].append(mod)
-
-def add_new_mod(new_mod, curr_entries):
- model_type = new_mod['model_type']
- for entry in curr_entries['modlist']:
- if model_type == entry['model_type']:
- if new_mod == entry:
- # If the model_type already exists but it is exactly the same, move on
- continue
- new_varlist = new_mod['varlist']
- curr_varlist = entry['varlist']
- for new_var in new_varlist:
- for curr_var in curr_varlist:
- if new_var == curr_var:
- continue
- curr_varlist.append(new_var)
-
-def model_type_exists(model_type, curr_entries):
- for entry in curr_entries['modlist']:
- if model_type == entry['model_type']:
- return True
- return False
-
-def combine_yaml(files):
- """
- Combines a list of yaml files into one
-
- Args:
- files: List of yaml file names to combine
- """
- field_table = {}
- field_table['field_table'] = []
- for f in files:
- print("File:" + f)
- # Check if the file exists
- if not path.exists(f):
- raise FileNotFoundError(errno.ENOENT,
- strerror(errno.ENOENT),
- f)
- with open(f) as fl:
- my_table = yaml.safe_load(fl)
- entries = my_table['field_table']
- for entry in entries:
- if not field_type_exists(entry['field_type'], field_table['field_table']):
- # If the field table does not exist, just add it to the current field table
- field_table['field_table'].append(entry)
- else:
- add_new_field(entry, field_table['field_table'])
- return field_table
-
-def main():
- #: parse user input
- @click.command()
- @click.option('-f',
- '--in_files',
- type=str,
- multiple=True,
- default=["field_table"],
- help='Space seperated list with the '
- 'Names of the field_table.yaml files to combine')
- @click.option('-o',
- '--out_file',
- type=str,
- default='field_table.yaml',
- help="Ouput file name of the converted YAML \
- (Default: 'field_table.yaml')")
- @click.option('-F',
- '--force',
- is_flag=True,
- help="Overwrite the output field table yaml file.")
- @click.version_option(version=__version__,
- prog_name="combine_field_table_yaml")
- def combine_field_table_yaml(in_files, out_file, force):
- """
- Combines a list of field_table.yaml files into one file
- Requires pyyaml (https://pyyaml.org/)
- """
- try:
- field_table = combine_yaml(in_files)
- out_file_op = "x" # Exclusive write
- if force:
- out_file_op = "w"
- with open(out_file, out_file_op) as myfile:
- yaml.dump(field_table, myfile, default_flow_style=False)
-
- except Exception as err:
- raise SystemExit(err)
-
-
-if __name__ == "__main__":
- main()
diff --git a/fre/yamltools/field_table/field_table_to_yaml.py b/fre/yamltools/field_table/field_table_to_yaml.py
deleted file mode 100755
index 0c8be719..00000000
--- a/fre/yamltools/field_table/field_table_to_yaml.py
+++ /dev/null
@@ -1,274 +0,0 @@
-#!/usr/bin/env python3
-"""
-***********************************************************************
-* GNU Lesser General Public License
-*
-* This file is part of the GFDL Flexible Modeling System (FMS) YAML tools.
-*
-* FMS_yaml_tools is free software: you can redistribute it and/or modify it under
-* the terms of the GNU Lesser General Public License as published by
-* the Free Software Foundation, either version 3 of the License, or (at
- * your option) any later version.
-*
-* FMS_yaml_tools is distributed in the hope that it will be useful, but WITHOUT
-* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-* for more details.
-*
-* You should have received a copy of the GNU Lesser General Public
-* License along with FMS. If not, see .
-***********************************************************************
-
-Converts a legacy ascii field_table to a yaml field_table.
- Author: Eric Stofferahn 07/14/2022
-
-"""
-
-import re
-from collections import OrderedDict
-import click
-import yaml
-
-global gverbose
-gverbose = None
-
-def main():
- # Necessary to dump OrderedDict to yaml format
- yaml.add_representer(OrderedDict, lambda dumper, data: dumper.represent_mapping('tag:yaml.org,2002:map', data.items()))
-
- @click.command()
- @click.option('--file',
- '-f',
- type=str,
- help='Name of the field_table file to convert')
- @click.option('--verbose',
- '-v',
- type=bool,
- is_flag=True,
- default=False,
- help='Increase verbosity')
- def convert_field_table_to_yaml(file, verbose):
- gverbose = verbose
- field_table_name = file
-
- if verbose:
- print(field_table_name)
-
- field_yaml = FieldYaml(field_table_name)
- field_yaml.main()
- field_yaml.writeyaml()
-
-def dont_convert_yaml_val(inval):
- # Yaml does some auto-conversions to boolean that we don't want, this will help fix it
- dontconvertus = ["yes", "Yes", "no", "No", "on", "On", "off", "Off"]
-
- if not isinstance(inval, str):
- return yaml.safe_load(inval)
- if inval in dontconvertus:
- return inval
- else:
- return yaml.safe_load(inval)
-
-class Field:
- """ A Field Object, containing the variable attributes, methods, and subparameters """
- def __init__(self, in_field_type, entry_tuple):
- """ Initialize the Field Object with the provided entries, then process as a species or tracer """
- self.field_type = in_field_type
- self.name = entry_tuple[0]
- self.dict = OrderedDict()
- self.num_subparams = 0
- for in_prop in entry_tuple[1]:
- if 'tracer' == self.field_type:
- self.process_tracer(in_prop)
- else:
- self.process_species(in_prop)
-
- def process_species(self, prop):
- """ Process a species field """
- comma_split = prop.split(',')
- if gverbose:
- print(self.name)
- print(self.field_type)
- print(comma_split)
- if len(comma_split) > 1:
- eq_splits = [x.split('=') for x in comma_split]
- if gverbose:
- print('printing eq_splits')
- print(eq_splits)
- for idx, sub_param in enumerate(eq_splits):
- if gverbose:
- print('printing len(sub_param)')
- print(len(sub_param))
- if len(sub_param) < 2:
- eq_splits[0][1] += f',{sub_param[0]}'
- if gverbose:
- print(eq_splits)
- eq_splits = [x for x in eq_splits if len(x) > 1]
- for sub_param in eq_splits:
- if ',' in sub_param[1]:
- val = yaml.safe_load("'" + sub_param[1]+ "'")
- else:
- val = dont_convert_yaml_val(sub_param[1])
- self.dict[sub_param[0].strip()] = val
- else:
- eq_split = comma_split[0].split('=')
- val = dont_convert_yaml_val(eq_split[1])
- self.dict[eq_split[0].strip()] = val
-
- def process_tracer(self, prop):
- """ Process a tracer field """
- if gverbose:
- print(len(prop))
- self.dict[prop[0]] = prop[1]
- if len(prop) > 2:
- self.dict[f'subparams{str(self.num_subparams)}'] = [OrderedDict()]
- self.num_subparams += 1
- if gverbose:
- print(self.name)
- print(self.field_type)
- print(prop[2:])
- for sub_param in prop[2:]:
- eq_split = sub_param.split('=')
- if len(eq_split) < 2:
- self.dict[prop[0]] = 'fm_yaml_null'
- val = dont_convert_yaml_val(eq_split[0])
- if isinstance(val, list):
- val = [dont_convert_yaml_val(b) for b in val]
- self.dict[f'subparams{str(self.num_subparams-1)}'][0][prop[1].strip()] = val
- else:
- val = dont_convert_yaml_val(eq_split[-1])
- if isinstance(val, list):
- val = [dont_convert_yaml_val(b) for b in val]
- self.dict[f'subparams{str(self.num_subparams-1)}'][0][eq_split[0].strip()] = val
-
-def list_items(brief_text, brief_od):
- """ Given text and an OrderedDict, make an OrderedDict and convert to list """
- return list(OrderedDict([(brief_text, brief_od)]).items())
-
-def listify_ordered_dict(in_list, in_list2, in_od):
- """ Given two lists and an OrderedDict, return a list of OrderedDicts. Note this function is recursive. """
- if len(in_list) > 1:
- x = in_list.pop()
- y = in_list2.pop()
- return [OrderedDict(list_items(x, k) + list_items(y, listify_ordered_dict(in_list, in_list2, v))) for k, v in in_od.items()]
- else:
- x = in_list[0]
- y = in_list2[0]
- return [OrderedDict(list_items(x, k) + list_items(y, v)) for k, v in in_od.items()]
-
-def zip_uneven(in_even, in_odd):
- """ Re-splice two uneven lists that have been split apart by a stride of 2 """
- result = [None]*(len(in_even)+len(in_odd))
- result[::2] = in_even
- result[1::2] = in_odd
- return result
-
-def pound_signs_within_quotes(in_lines):
- """ Change pound signs within quotes to the word poundsign so they aren't expunged when eliminating comments. """
- odds = [x.split('"')[1::2] for x in in_lines]
- evens = [x.split('"')[::2] for x in in_lines]
- for idx, line in enumerate(odds):
- odds[idx] = [re.sub('#','poundsign',x) for x in line]
- newfilelines = [zip_uneven(e,o) for e, o in zip(evens,odds)]
- return ''.join(['"'.join(x) for x in newfilelines])
-
-def process_field_file(my_file):
- """ Parse ascii field table into nested lists for further processing """
- with open(my_file, 'r') as fh:
- filelines = fh.readlines()
- # Change literal pound signs to the word poundsign
- whole_file = pound_signs_within_quotes(filelines)
- # Eliminate tabs and quotes
- whole_file = whole_file.replace('"', '').replace('\t', '')
- # Eliminate anything after a comment marker (#)
- whole_file = re.sub("\#"+r'.*'+"\n",'\n',whole_file)
- # Replace the word poundsign with a literal pound sign (#)
- whole_file = re.sub("poundsign","#",whole_file)
- # Eliminate extraneous spaces, but not in value names
- whole_file = re.sub(" *\n *",'\n',whole_file)
- whole_file = re.sub(" *, *",',',whole_file)
- whole_file = re.sub(" */\n",'/\n',whole_file)
- # Eliminate trailing commas (rude)
- whole_file = whole_file.replace(',\n', '\n')
- # Eliminate newline before end of entry
- whole_file = re.sub("\n/",'/',whole_file)
- # Eliminate spaces at very beginning and end
- whole_file = whole_file.strip()
- # Eliminate very last slash
- whole_file = whole_file.strip('/')
- # Split entries based upon the "/" ending character
- into_lines = [x for x in re.split("/\n", whole_file) if x]
- # Eliminate blank lines
- into_lines = [re.sub(r'\n+','\n',x) for x in into_lines]
- into_lines = [x[1:] if '\n' in x[:1] else x for x in into_lines]
- into_lines = [x[:-1] if '\n' in x[-1:] else x for x in into_lines]
- # Split already split entries along newlines to form nested list
- nested_lines = [x.split('\n') for x in into_lines]
- # Split nested lines into "heads" (field_type, model, var_name) and "tails" (the rest)
- heads = [x[0] for x in nested_lines]
- tails = [x[1:] for x in nested_lines]
- return heads, tails
-
-class FieldYaml:
- def __init__(self, field_file):
- self.filename = field_file
- self.out_yaml = OrderedDict()
- self.heads, self.tails = process_field_file(self.filename)
-
- def init_ordered_keys(self):
- """ Get unique combination of field_type and model... in order provided """
- self.ordered_keys = OrderedDict.fromkeys([tuple([y.lower() for y in x.split(',')[:2]]) for x in self.heads])
-
- def initialize_lists(self):
- """ Initialize out_yaml and ordered_keys """
- for k in self.ordered_keys.keys():
- self.ordered_keys[k] = []
- if k[0] not in self.out_yaml.keys():
- self.out_yaml[k[0]] = OrderedDict()
- if k[1] not in self.out_yaml[k[0]].keys():
- self.out_yaml[k[0]][k[1]] = OrderedDict()
-
- def populate_entries(self):
- """ Populate entries as OrderedDicts """
- for h, t in zip(self.heads, self.tails):
- head_list = [y.lower() for y in h.split(',')]
- tail_list = [x.split(',') for x in t]
- if (head_list[0], head_list[1]) in self.ordered_keys.keys():
- if 'tracer' == head_list[0]:
- self.ordered_keys[(head_list[0], head_list[1])].append((head_list[2], tail_list))
- else:
- self.ordered_keys[(head_list[0], head_list[1])].append((head_list[2], t))
-
- def make_objects(self):
- """ Make Tracer and Species objects and assign to out_yaml """
- for k in self.ordered_keys.keys():
- for j in self.ordered_keys[k]:
- my_entry = Field(k[0], j)
- self.out_yaml[k[0]][k[1]][my_entry.name] = my_entry.dict
-
- def convert_yaml(self):
- """ Convert to list-style yaml """
- lists_yaml = listify_ordered_dict(['model_type', 'field_type'], ['varlist', 'modlist'], self.out_yaml)
- for i in range(len(lists_yaml)):
- for j in range(len(lists_yaml[i]['modlist'])):
- lists_yaml[i]['modlist'][j]['varlist'] = [OrderedDict(list(OrderedDict([('variable', k)]).items()) +
- list(v.items())) for k, v in lists_yaml[i]['modlist'][j]['varlist'].items()]
- self.lists_wh_yaml = {"field_table": lists_yaml}
-
- def writeyaml(self):
- """ Write yaml out to file """
- raw_out = yaml.dump(self.lists_wh_yaml, None, default_flow_style=False)
- final_out = re.sub('subparams\d*:','subparams:',raw_out)
- with open(f'{self.filename}.yaml', 'w') as yaml_file:
- yaml_file.write(final_out)
-
- def main(self):
- self.init_ordered_keys()
- self.initialize_lists()
- self.populate_entries()
- self.make_objects()
- self.convert_yaml()
-
-
-if __name__ == '__main__':
- main()