Merge branch 'main' into cmor-snack-prep

.gitignore

@@ -1,3 +1,8 @@
+# fremake generated files that should be ignored
+combined-null_model.yaml
+combined-am5.yaml
+tmp
+makefile_out
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

docs/tools/yamltools.rst

@@ -1,6 +1,13 @@
-.. NEEDS UPDATING #TODO
-   
 ``combine-yamls``
 -----------------
 
-* placehold
+``fre yamltools combine-yamls [options]``
+   - Purpose: Creates a combined yaml file for either compilation or post-processing. 
+              If `--use compile`, the model yaml is combined with the compile and platforms yaml.
+              If `--use pp`, the model yaml is combined with post-processing yamls.
+   - Options:
+        - `-y, --yamlfile [experiment yaml] (required)`
+        - `-e, --experiment [experiment name]`
+        - `-p, --platform [platform] (required)`
+        - `-t, --target [target] (required)`
+        - `--use [compile|pp] (required)`

docs/usage/compile.rst

@@ -82,7 +82,7 @@ Guide
 
 For the container build, parallel checkouts are not supported, so the `-npc` options must be used for the checkout script. In addition the platform must be a container platform.
 
-Users will not be able to create containers unless they have podman access on gaea.
+Gaea users will not be able to create containers unless they have requested and been given podman access.
 
 .. code-block::
 
@@ -103,45 +103,45 @@ Users will not be able to create containers unless they have podman access on ga
 
 Quickstart
 ----------
-The quickstart instructions can be used with the am5-compile examples located in the fre-examples repository: https://github.com/NOAA-GFDL/fre-examples/tree/main/AM5/am5-compile
+The quickstart instructions can be used with the null model example located in the fre-cli repository: https://github.com/NOAA-GFDL/fre-cli/tree/main/fre/make/tests/null_example
 
 1. Bare-metal Build:
 
 .. code-block::
 
   # Create checkout script
-  fre make create-checkout -y am5.yaml -p ncrc5.intel23 -t prod
+  fre make create-checkout -y null_model.yaml -p ncrc5.intel23 -t prod
 
   # Create and run checkout script
-  fre make create-checkout -y am5.yaml -p ncrc5.intel23 -t prod --execute
+  fre make create-checkout -y null_model.yaml -p ncrc5.intel23 -t prod --execute
 
   # Create Makefile
-  fre make create-makefile -y am5.yaml -p ncrc5.intel23 -t prod
+  fre make create-makefile -y null_model.yaml -p ncrc5.intel23 -t prod
 
   # Create the compile script
-  fre make create-compile -y am5.yaml -p ncrc5.intel23 -t prod
+  fre make create-compile -y null_model.yaml -p ncrc5.intel23 -t prod
 
   # Create and run the compile script
-  fre make create-compile -y am5.yaml -p ncrc5.intel23 -t prod --execute
+  fre make create-compile -y null_model.yaml -p ncrc5.intel23 -t prod --execute
 
 2. Bare-metal Build Multi-target:
 
 .. code-block::
 
   # Create checkout script
-  fre make create-checkout -y am5.yaml -p ncrc5.intel23 -t prod -t debug
+  fre make create-checkout -y null_model.yaml -p ncrc5.intel23 -t prod -t debug
 
   # Create and run checkout script
-  fre make create-checkout -y am5.yaml -p ncrc5.intel23 -t prod -t debug --execute
+  fre make create-checkout -y null_model.yaml -p ncrc5.intel23 -t prod -t debug --execute
 
   # Create Makefile
-  fre make create-makefile -y am5.yaml -p ncrc5.intel23 -t prod -t debug
+  fre make create-makefile -y null_model.yaml -p ncrc5.intel23 -t prod -t debug
 
   # Create the compile script
-  fre make create-compile -y am5.yaml -p ncrc5.intel23 -t prod -t debug
+  fre make create-compile -y null_model.yaml -p ncrc5.intel23 -t prod -t debug
 
   # Create and run the compile script
-  fre make create-compile -y am5.yaml -p ncrc5.intel23 -t prod -t debug --execute
+  fre make create-compile -y null_model.yaml -p ncrc5.intel23 -t prod -t debug --execute
 
 3. Container Build:
 
@@ -150,28 +150,34 @@ In order for the container to build successfully, a `-npc`, or `--no-parallel-ch
 .. code-block::
 
   # Create checkout script
-  fre make create-checkout -y am5.yaml -p hpcme.2023 -t prod -npc
+  fre make create-checkout -y null_model.yaml -p hpcme.2023 -t prod -npc
 
   # Create and run checkout script
-  fre make create-checkout -y am5.yaml -p hpcme.2023 -t prod -npc --execute
+  fre make create-checkout -y null_model.yaml -p hpcme.2023 -t prod -npc --execute
 
   # Create Makefile
-  fre make create-makefile -y am5.yaml -p hpcme.2023 -t prod
+  fre make create-makefile -y null_model.yaml -p hpcme.2023 -t prod
 
   # Create Dockerfile
-  fre make create-dockerfile -y am5.yaml -p hpcme.2023 -t prod
+  fre make create-dockerfile -y null_model.yaml -p hpcme.2023 -t prod
 
   # Create and run the Dockerfile
-  fre make create-dockerfile -y am5.yaml -p hpcme.2023 -t prod --execute
+  fre make create-dockerfile -y null_model.yaml -p hpcme.2023 -t prod --execute
 
 4. Run all of fremake:
 
-Currently, run-fremake kicks off the compilation automatically; no ``--execute`` option needed.
+`run-fremake` kicks off the compilation automatically
 
 .. code-block::
 
-  # Bare-metal
-  fre make run-fremake -y am5.yaml -p ncrc5.intel23 -t prod
+  # Bare-metal: create and run checkout script, create makefile, create compile script
+  fre make run-fremake -y null_model.yaml -p ncrc5.intel23 -t prod
 
-  # Container
-  fre make run-fremake -y am5.yaml -p hpcme.2023 -t prod -npc
+  # Bare-metal: create and run checkout script, create makefile, create and run compile script
+  fre make run-fremake -y null_model.yaml -p ncrc5.intel23 -t prod --execute
+
+  # Container: create checkout script, makefile, and dockerfile
+  fre make run-fremake -y null_model.yaml -p hpcme.2023 -t prod -npc
+
+  # Container: create checkout script, makefile, create and run dockerfile to build container
+  fre make run-fremake -y null_model.yaml -p hpcme.2023 -t prod -npc --execute

docs/usage/yaml_framework.rst

@@ -136,16 +136,12 @@ The platform yaml contains user defined information for both bare-metal and cont
         compiler: the compiler you are using
         modulesInit: ["array of commands that are needed to load modules." , "each command must end with a newline character"]
         modules: [array of modules to load including compiler]
-        fc: the name of the fortran compiler
-        cc: the name of the C compiler
         mkTemplate: The location of the mkmf make template
         modelRoot: The root directory of the model (where src, exec, experiments will go)
       - container platform: container platform name
         compiler: compiler you are using
         RUNenv: Commands needed at the beginning of a RUN in dockerfile
         modelRoot: The root directory of the model (where src, exec, experiments will go) INSIDE of the container (/apps)
-        fc: name of fortan compiler
-        cc: name of C compiler
         container: True if this is a container platform
         containerBuild: "podman" - the container build program
         containerRun: "apptainer" - the container run program

environment.yml

@@ -12,6 +12,7 @@ dependencies:
   - jsonschema
   - noaa-gfdl::fre-nctools
   - noaa-gfdl::catalogbuilder
+  - noaa-gfdl::analysis_scripts
   - conda-forge::nccmp
   - conda-forge::cylc-flow>=8.2.0
   - conda-forge::cylc-rose

fre/analysis/freanalysis.py

@@ -0,0 +1,64 @@
+from analysis_scripts import available_plugins
+import click
+
+from .subtools import install_analysis_package, list_plugins, run_analysis, \
+                      uninstall_analysis_package
+
+
+@click.group(help=click.style(" - access fre analysis subcommands", fg=(250, 154, 90)))
+def analysis_cli():
+    """Entry point to fre analysis click commands."""
+    pass
+
+
+@analysis_cli.command()
+@click.option("--url", type=str, required=True, help="URL of the github repository.")
+@click.option("--name", type=str, required=False, help="Subdirectory to pip install.")
+@click.option("--library-directory", type=str, required=False,
+              help="Path to a custom lib directory.")
+def install(url, name, library_directory):
+    """Installs an analysis package."""
+    install_analysis_package(url, name, library_directory)
+
+
+@analysis_cli.command()
+@click.option("--library-directory", type=str, required=False,
+              help="Path to a custom lib directory.")
+def list(library_directory):
+    """List available plugins."""
+    plugins = list_plugins(library_directory)
+    if plugins:
+        print("Installed analysis packages:\n")
+        for plugin in plugins:
+            print(plugin)
+    else:
+        print("No analysis packages found.")
+
+
+@analysis_cli.command()
+@click.option("--name", type=str, required=True, help="Name of the analysis script.")
+@click.option("--catalog", type=str, required=True, help="Path to the data catalog.")
+@click.option("--output-directory", type=str, required=True,
+              help="Path to the output directory.")
+@click.option("--output-yaml", type=str, required=True, help="Path to the output yaml.")
+@click.option("--experiment-yaml", type=str, required=True, help="Path to the experiment yaml.")
+@click.option("--library-directory", type=str, required=False,
+              help="Path to a custom lib directory.")
+def run(name, catalog, output_directory, output_yaml, experiment_yaml,
+        library_directory):
+    """Runs the analysis script and writes the paths to the created figures to a yaml file."""
+    run_analysis(name, catalog, output_directory, output_yaml, experiment_yaml,
+                 library_directory)
+
+
+@analysis_cli.command()
+@click.option("--name", type=str, required=True, help="Name of package to uninstall.")
+@click.option("--library-directory", type=str, required=False,
+              help="Path to a custom lib directory.")
+def uninstall(name, library_directory):
+    """Uninstall an analysis package."""
+    uninstall_analysis_package(name, library_directory)
+
+
+if __name__ == "__main__":
+    analysis_cli()

fre/analysis/subtools.py

@@ -0,0 +1,123 @@
+from pathlib import Path
+from subprocess import run
+from tempfile import TemporaryDirectory
+
+from analysis_scripts import available_plugins, run_plugin, VirtualEnvManager
+from yaml import safe_load
+
+
+def install_analysis_package(url, name=None, library_directory=None):
+    """Installs the analysis package.
+
+    Args:
+        url: URL to the github repository for the analysis package.
+        name: String name of the analysis-script package.
+        library_directory: Directory to install the package in.
+    """
+    # Clean up the url if necessary.
+    if not url.startswith("https://"):
+        url = f"https://{url}"
+    if not url.endswith(".git"):
+        url = f"{url}.git"
+
+    # Get the absolute path of the input library_directory.
+    if library_directory:
+        library_directory = Path(library_directory).resolve()
+
+    if name:
+        # If a name is given, then expect that the analysis script is part or the noaa-gfdl
+        # github repository.
+        with TemporaryDirectory() as tmp:
+            tmp_path = Path(tmp)
+            run(["git", "clone", url, str(tmp_path / "scripts")], check=True)
+
+            if library_directory:
+                # If a library directory is given, install the analysis script in a virtual
+                # environment.
+                env = VirtualEnvManager(library_directory)
+                env.create_env()
+                env.install_package(str(tmp_path / "scripts" / "core" / "analysis_scripts"))
+                env.install_package(str(tmp_path / "scripts" / "core" / "figure_tools"))
+                env.install_package(str(tmp_path / "scripts" / "user-analysis-scripts" / name))
+            else:
+                run(["pip", "install", str(tmp_path / "scripts" / "core" / "figure_tools")],
+                    check=True)
+                run(["pip", "install", str(tmp_path / "scripts" / "user-analysis-scripts" / name)],
+                    check=True)
+    else:
+        if library_directory:
+            env = VirtualEnvManager(library_directory)
+            env.create_env()
+            env.install_package(str(tmp_path / "scripts" / "core" / "analysis_scripts"))
+            env.install_package(f"{url}@main")
+        else:
+            run(["pip", "install", f"{url}@main"])
+
+
+def list_plugins(library_directory=None):
+    """Finds the list of analysis scripts.
+
+    Args:
+        library_directory: Directory where the analysis package is installed.
+
+    Returns:
+        List of string plugin names.
+    """
+    if library_directory:
+        env = VirtualEnvManager(library_directory)
+        return env.list_plugins()
+    else:
+        return available_plugins()
+
+
+def run_analysis(name, catalog, output_directory, output_yaml, experiment_yaml,
+                 library_directory=None):
+    """Runs the analysis script and writes the paths to the created figures to a yaml file.
+
+    Args:
+        name: String name of the analysis script.
+        catalog: Path to the data catalog.
+        output_directory: Path to the output directory.
+        output_yaml:  Path to the output yaml.
+        experiment: Path to the experiment yaml.
+        library_directory: Directory where the analysis package is installed.
+    """
+
+    # Create the directory for the figures.
+    Path(output_directory).mkdir(parents=True, exist_ok=True)
+
+    # Parse the configuration out of the experiment yaml file.
+    with open(experiment_yaml) as file_:
+        config_yaml = safe_load(file_)
+        try:
+            configuration = config_yaml["analysis"][name]["required"]
+        except KeyError:
+            configuration = None
+
+    # Run the analysis.
+    if library_directory:
+        env = VirtualEnvManager(library_directory)
+        figure_paths = env.run_analysis_plugin(name, catalog, output_directory,
+                                               config=configuration)
+    else:
+        figure_paths = run_plugin(name, catalog, output_directory, config=configuration)
+
+    # Write out the figure paths to a file.
+    with open(output_yaml, "w") as output:
+        output.write("figure_paths:\n")
+        for path in figure_paths:
+            output.write(f"  -{Path(path).resolve()}\n")
+
+
+def uninstall_analysis_package(name, library_directory=None):
+    """Uninstalls the analysis package.
+
+    Args:
+        name: String name of the analysis-script package.
+        library_directory: Directory where the package was installed.
+    """
+    if library_directory:
+        env = VirtualEnvManager(library_directory)
+        env.uninstall_package(name)
+    else:
+        run(["pip", "uninstall", name], check=True)