Post SIM-5 changes

.github/workflows/python-test.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11']
 
     env:
       SPATIALITE_LIBRARY_PATH: 'mod_spatialite'

.gitignore

@@ -110,6 +110,9 @@ ENV/
 env.bak/
 venv.bak/
 
+# PyCharm
+.idea
+
 # Spyder project settings
 .spyderproject
 .spyproject

CHANGELOG.rst

@@ -34,15 +34,39 @@ Changed
   vice-versa).
 - junc_image_req_ldst.py got some additional columns to manage the Science Team
   evaluation of images acquired from Image Requests.
+- create_pano.py's make_pano_product() function renamed to make_pano_record().
+- nss_simulator.py - nodata defaults for ideal detector count maps updated to more
+  realistic values for dry terrain.
+- image_statistics.py - Added additional stats in the compute() function, and added
+  a pprint() function to format the info nicely.
 
 Added
 ^^^^^
+- pds.Purpose now provides names and explanations for the PDS-allowable values
+  for "purpose."
 - yamcs_reception_time column added to the image_records.py table.
 - Association table junc_image_pano created which provides a many-to-many
   connection between ImageRecords and PanoRecords and added bidirectional
   relationship entries to each table.
-- pano_records table now has azimuth and elevation angle min/max values to indicate
+- pano_records table now has pan and tilt angle min/max values to indicate
   angular range of panorama coverage.
+- ImageRecord objects will now extract an ImageRequest ID from the provided capture_id
+  if it is larger than the 16 bit range.
+- image_requests.py - "Acquired," "Not Acquired," "Not Planned," and "Not Obtainable"
+  statuses added to enum.  Also added asdict() method.
+- ptu_records.py - Tables to record the pan and tilt of the rover's pan-tilt-unit (PTU).
+- create_browse.py - For making browse products from existing image products.
+- create_mmgis_pano.py - For making pano products for use in MMGIS.
+- create_pano.py - updated to correctly add PanoRecord associations, can now query
+  database for ImageRecords.
+- create_pano_product.py - takes PanoRecords and makes a PDS Pano Product.
+- get_position.py - Gets position and yaw from a REST-based service.
+- create_vis_dbs.py - Now also supports spatialite databases, primarily for testing.
+- create_raw.py - Added components for adding observational intent and data quality
+  to the XML label.
+- labelmaker - A program to help build PDS4 bundle and collection labels.
+- bundle_install - A program to copy just the files related to a PDS4 bundle into a
+  new location.  Fundamentally allowing a "make install" for PDS4 bundles.
 
 Removed
 ^^^^^^^
@@ -157,6 +181,7 @@ Added
 
 Changed
 ^^^^^^^
+- Updated templates and modules for PDS information model 21.
 - Flattened test directory structure.
 - tri2gpkg -v is no longer an alias for --value-names, as it now determines verbosity
   since logging has been added.
@@ -175,6 +200,7 @@ Fixed
 - heatmap's generate_density_heatmap() function now properly returns values of zero
   in the returned out_count numpy array when there are no counts in those grid cells
   instead of the provided nodata value.
+- raw-template.xml can only have one Image_Filter object.
 - tri2gpkg now works correctly if --keep_z is specified
 - tri2gpkg now uses the correct srs if a pre-defined site is selected.
 

environment.yml

@@ -26,3 +26,4 @@ dependencies:
   - shapely
   - sqlalchemy
   - tifftools
+  - yaml

environment_dev.yml

@@ -20,4 +20,5 @@ dependencies:
   - tox
   - types-pillow
   - types-requests
+  - types-pyyaml
   - twine

requirements.txt

@@ -14,3 +14,4 @@ scikit-learn
 shapely
 sqlalchemy
 tifftools
+pyyaml

requirements_dev.txt

@@ -10,5 +10,6 @@ tox
 twine
 types-pillow
 types-requests
+types-pyyaml
 wheel
 watchdog

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.6.1
+current_version = 0.7.0-dev
 commit = True
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+))?
@@ -67,22 +67,30 @@ where = src
 console_scripts = 
 	accrual = vipersci.carto.accrual:main
 	anaglyph = vipersci.vis.anaglyph:main
+	anom_pixel = vipersci.vis.anom_pixel:main
+    bundle_install = vipersci.pds.bundle_install:main
 	colorforge = vipersci.carto.colorforge:main
 	dice_buffer = vipersci.carto.dice_buffer:main
 	dissolve_dice = vipersci.carto.dissolve_dice:main
+	get_position = vipersci.carto.get_position:main
 	image_stats = vipersci.vis.image_statistics:main
+    labelmaker = vipersci.pds.labelmaker.cli:main
 	msolo_simulator = vipersci.carto.msolo_simulator:main
 	nirvss_simulator = vipersci.carto.nirvss_simulator:main
 	nss_modeler = vipersci.carto.nss_modeler:main
 	nss_simulator = vipersci.carto.nss_simulator:main
 	traverse_interpolator = vipersci.carto.traverse_interpolator:main
 	tri2gpkg = vipersci.carto.tri2gpkg:main
 	template_test = vipersci.vis.pds.template_test:main
+    vis_create_browse = vipersci.vis.pds.create_browse:main
 	vis_create_image = vipersci.vis.create_image:main
+    vis_create_mmgis_pano = vipersci.vis.create_mmgis_pano:main
 	vis_create_pano = vipersci.vis.create_pano:main
+    vis_create_pano_product = vipersci.vis.pds.create_pano_product:main
 	vis_create_tif = vipersci.vis.create_tif:main
 	vis_create_raw = vipersci.vis.pds.create_raw:main
 	vis_create_dbs = vipersci.vis.db.create_vis_dbs:main
+    vis_pano_check = vipersci.vis.pano_check:main
 	viseer = vipersci.vis.viseer:main
 
 [bdist_wheel]

src/vipersci/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = """vipersci Developers"""
 __email__ = "[email protected]"
-__version__ = "0.6.1"
+__version__ = "0.7.0-dev"

src/vipersci/carto/get_position.py

@@ -0,0 +1,177 @@
+"""Gather Rover locations from a REST-based service.
+"""
+
+# Copyright 2023, United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Reuse is permitted under the terms of the license.
+# The AUTHORS file and the LICENSE file are at the
+# top level of this library.
+
+import argparse
+import csv
+from datetime import datetime, timezone
+import logging
+from pathlib import Path
+import sys
+
+import pandas as pd
+import requests
+
+from vipersci import util
+
+logger = logging.getLogger(__name__)
+
+
+def arg_parser():
+    parser = argparse.ArgumentParser(
+        description=__doc__, parents=[util.parent_parser()]
+    )
+    parser.add_argument(
+        "-s",
+        "--start",
+        default=datetime(2020, 1, 1, tz=timezone.utc),
+        help="An ISO8601 datetime to start the query at.",
+    )
+    parser.add_argument(
+        "-e",
+        "--end",
+        default=datetime.now(tz=timezone.utc),
+        help="An ISO8601 datetime to end the query at.",
+    )
+    parser.add_argument(
+        "-f",
+        "--frequency",
+        help="A frequency between the start and end times, using pandas 'Offset alias' "
+        "string notation (https://pandas.pydata.org/pandas-docs/stable/user_guide/"
+        "timeseries.html#offset-aliases). Could be 'S' for secondly.",
+    )
+    parser.add_argument(
+        "-u",
+        "--url",
+        required=True,
+        help="URL to which event_times may be posted and for which location and "
+        "orientation will be returned.",
+    )
+    parser.add_argument(
+        "-o", "--output", type=Path, help="Output path for CSV file output."
+    )
+    return parser
+
+
+def main():
+    parser = arg_parser()
+    args = parser.parse_args()
+    util.set_logger(args.verbose)
+
+    t_start = datetime.fromisoformat(args.start)
+    t_end = datetime.fromisoformat(args.end)
+
+    if args.frequency:
+        times = pd.date_range(t_start, t_end, freq=args.frequency).tolist()
+
+        unix_times = []
+        for t in times:
+            unix_times.append(t.timestamp())
+
+        tpp = get_position_and_pose(unix_times, args.url)
+    else:
+        tpp = get_position_and_pose_range(
+            t_start.timestamp(), t_end.timestamp(), args.url
+        )
+
+    if args.output is not None:
+        with open(args.output, "w", newline="") as csvfile:
+            writer = csv.writer(csvfile)
+            writer.writerow(["UTC datetime", "x", "y", "yaw"])
+            for row in tpp:
+                dt = datetime.fromtimestamp(row[0], tz=timezone.utc)
+                writer.writerow(
+                    [
+                        dt.isoformat(),
+                    ]
+                    + list(row[1:])
+                )
+
+
+def get_position_and_pose(times: list, url: str, crs: str = "VIPER:910101"):
+    """
+    Given a list of unix times and a URL that requests can be made against,
+    return a list of two-tuples whose first element is the time and
+    whose second element is a three-tuple of x-location, y-location,
+    and yaw.
+    """
+
+    tpp = list()
+
+    for t in times:
+        logger.info(f"unix timestamp: {t}")
+        position_result = requests.get(
+            url,
+            params={
+                "event_time": t,  # Event time in unix datetime.
+                "margin_seconds": 2,  # Some "margin time" around the event to search.
+                "format": "xyyaw",  # Could also be xyyaw_uncertainty
+                "time_format": "unix_seconds",
+                "source": "ROVER",
+                "limit": 0,
+                "crs_code": crs,
+            },
+            verify=False,
+        )
+        rj = position_result.json()
+        logger.info(rj)
+
+        tpp.append(
+            (rj["event_seconds"], (rj["location"][0], rj["location"][1], rj["yaw"]))
+        )
+
+    return tpp
+
+
+def get_position_and_pose_range(
+    start_time, stop_time, url: str, crs: str = "VIPER:910101"
+):
+    tpp = list()
+
+    track_result = requests.get(
+        url,
+        params={
+            "min_time": start_time,
+            "max_time": stop_time,
+            "all": False,
+            "format": "xyyaw",
+            "crs_code": crs,
+            "time_format": "unix_seconds",
+            "source": "ROVER",
+            "start_end_only": False,
+            "simplify": False,
+            "order": "asc",
+        },
+        verify=False,
+    )
+    rj = track_result.json()
+
+    for time, loc, yaw in zip(rj["event_seconds"], rj["location"], rj["yaw"]):
+        tpp.append((time, loc[0], loc[1], yaw))
+
+    return tpp
+
+
+if __name__ == "__main__":
+    sys.exit(main())

src/vipersci/carto/nss_modeler.py

@@ -93,6 +93,7 @@ def main():
 
 
 def write_tif(path: Path, ending: str, arr: np.typing.ArrayLike, kwds: dict):
-    with rasterio.open(path.with_name(path.name + ending), "w", **kwds) as dst_dataset:
+    p = path.with_name(path.name + ending)
+    with rasterio.open(p, "w", **kwds) as dst_dataset:
         dst_dataset.write(arr, 1)
-    return
+    return p

src/vipersci/carto/nss_simulator.py

@@ -182,8 +182,11 @@
         d2_arr = d2.reshape(bd_data.shape)
 
         kwds = bd_data.profile
-        kwds["nodata"] = 0.0
+        # for Detector 1
+        kwds["nodata"] = 31.0
         write_tif(args.output, "_d1.tif", d1_arr, kwds)
+        # for Detector 2
+        kwds["nodata"] = 60.0
         write_tif(args.output, "_d2.tif", d2_arr, kwds)
         return
 

src/vipersci/pds/__init__.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+"""Provides some generic PDS structures."""
+
+# Copyright 2023, United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Reuse is permitted under the terms of the license.
+# The AUTHORS file and the LICENSE file are at the
+# top level of this library.
+
+import enum
+
+
+class Purpose(enum.Enum):
+    # These definitions are taken from the allowable values for
+    # Product_Observational/Observation_Area/Primary_Result_Summary/purpose
+    # in the PDS4 Data Dictionary.
+    CALIBRATION = "Data collected to determine the relationship between measurement "
+    "values and physical units."
+    CHECKOUT = "Data collected during operational tests."
+    ENGINEERING = "Data collected about support systems and structures, which are "
+    "ancillary to the primary measurements."
+    NAVIGATION = "Data collected to support navigation."
+    OBSERVATION_GEOMETRY = "Data used to compute instrument observation geometry, "
+    "such as SPICE kernels."
+    SCIENCE = "Data collected primarily to answer questions about the targets of "
+    "the investigation."
+    SUPPORTING_OBSERVATION = "A science observation that was acquired to provide "
+    "support for another science observation (e.g., a context image for a very "
+    "high resolution observation, or an image intended to support an observation "
+    "by a spectral imager)."