From d2e1949c6662d9c7ebbc7678e6458488de553cb2 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Mon, 16 Oct 2023 11:58:50 -0700 Subject: [PATCH 01/60] feat(image_requests.py): Added to status enum. --- CHANGELOG.rst | 2 ++ src/vipersci/vis/db/image_requests.py | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e8b24ac..a0a2f1f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -43,6 +43,8 @@ Added relationship entries to each table. - pano_records table now has azimuth and elevation angle min/max values to indicate angular range of panorama coverage. +- image_requests.py - "Acquired," "Not Acquired," and "Not Planned" statuses added to + enum. Removed ^^^^^^^ diff --git a/src/vipersci/vis/db/image_requests.py b/src/vipersci/vis/db/image_requests.py index 7a34caa..9363928 100644 --- a/src/vipersci/vis/db/image_requests.py +++ b/src/vipersci/vis/db/image_requests.py @@ -85,12 +85,14 @@ class Status(enum.Enum): """ This describes the status of an Image Request. """ - WORKING = 1 READY_FOR_VIS = 2 READY_FOR_PLANNING = 3 PLANNED = 4 - IMMEDIATE = 5 + NOT_PLANNED = 5 + IMMEDIATE = 6 + ACQUIRED = 7 + NOT_AQUIRED = 8 class CameraType(enum.Enum): From 7e962e97d6e6e29a1198408e697bd0e1894d782c Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Mon, 16 Oct 2023 12:01:19 -0700 Subject: [PATCH 02/60] feat(ptu_records.py): Added tables for PTU data. --- CHANGELOG.rst | 1 + src/vipersci/vis/db/create_vis_dbs.py | 3 + src/vipersci/vis/db/ptu_records.py | 97 +++++++++++++++++++++++++++ 3 files changed, 101 insertions(+) create mode 100644 src/vipersci/vis/db/ptu_records.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a0a2f1f..6a13748 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -45,6 +45,7 @@ Added angular range of panorama coverage. - image_requests.py - "Acquired," "Not Acquired," and "Not Planned" statuses added to enum. +- ptu_records.py - Tables to record the pan and tilt of the rover's pan-tilt-unit (PTU). Removed ^^^^^^^ diff --git a/src/vipersci/vis/db/create_vis_dbs.py b/src/vipersci/vis/db/create_vis_dbs.py index a658faf..d48b15b 100755 --- a/src/vipersci/vis/db/create_vis_dbs.py +++ b/src/vipersci/vis/db/create_vis_dbs.py @@ -48,6 +48,7 @@ from vipersci.vis.db.ldst import LDST from vipersci.vis.db.light_records import LightRecord from vipersci.vis.db.pano_records import PanoRecord +from vipersci.vis.db.ptu_records import PanRecord, TiltRecord # As new tables are defined, their Classes must be imported above, and # then also added to this tuple: @@ -62,6 +63,8 @@ LDST, LightRecord, PanoRecord, + PanRecord, + TiltRecord, ) logger = logging.getLogger(__name__) diff --git a/src/vipersci/vis/db/ptu_records.py b/src/vipersci/vis/db/ptu_records.py new file mode 100644 index 0000000..08d68d2 --- /dev/null +++ b/src/vipersci/vis/db/ptu_records.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +# coding: utf-8 + +"""Defines the VIS pan and tilt records tables using the SQLAlchemy ORM.""" + +# Copyright 2023, United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +from sqlalchemy import ( + DateTime, + Float, +) +from sqlalchemy.orm import mapped_column, validates + +from vipersci.vis.db import Base +import vipersci.vis.db.validators as vld + + +class PanRecord(Base): + """An object to represent rows in the pan_records table for VIS. Each row + represents a pan angle at a particular time.""" + + # This class is derived from SQLAlchemy's orm.DeclarativeBase + # which means that it has a variety of class properties that are + # then swept up into properties on the instantiated object via + # super().__init__(). + + # The table represents many of these objects, so the __tablename__ is + # plural while the class name is singular. + __tablename__ = "pan_records" + + angle = mapped_column( + Float, + nullable=False, + doc="Value in radians of the pan angle of the rover PTU. Zero is forward (+x) " + "relative to the rover frame.", + ) + datetime = mapped_column( + DateTime(timezone=True), + nullable=False, + primary_key=True, + doc="The datetime at which the pan angle was reported.", + ) + + @validates("datetime") + def validate_datetime_asutc(self, key, value): + return vld.validate_datetime_asutc(key, value) + + +class TiltRecord(Base): + """An object to represent rows in the tilt_records table for VIS. Each row + represents a tilt angle at a particular time.""" + + # This class is derived from SQLAlchemy's orm.DeclarativeBase + # which means that it has a variety of class properties that are + # then swept up into properties on the instantiated object via + # super().__init__(). + + # The table represents many of these objects, so the __tablename__ is + # plural while the class name is singular. + __tablename__ = "tilt_records" + + angle = mapped_column( + Float, + nullable=False, + doc="Value in radians of the tilt angle of the rover PTU. Zero is level " + "relative to the rover frame.", + ) + datetime = mapped_column( + DateTime(timezone=True), + nullable=False, + primary_key=True, + doc="The datetime at which the pan angle was reported.", + ) + + @validates("datetime") + def validate_datetime_asutc(self, key, value): + return vld.validate_datetime_asutc(key, value) From 9bcd17d08e124b460d768e0f36276e4336cb11c1 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Mon, 16 Oct 2023 13:26:41 -0700 Subject: [PATCH 03/60] feat(image_tags.py): Added "missing row(s)" tag. --- src/vipersci/vis/db/image_tags.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/vipersci/vis/db/image_tags.py b/src/vipersci/vis/db/image_tags.py index 53e1677..97125d3 100644 --- a/src/vipersci/vis/db/image_tags.py +++ b/src/vipersci/vis/db/image_tags.py @@ -62,4 +62,6 @@ class ImageTag(Base): "Over Exposed", "Under Exposed", "Compression Artifacts", + "Missing row(s) of pixels", + # Image header damaged in downlink? Not sure how we'd know. ] From 0bf321b4c7fa146d3ee7a6deaf057ad14483a723 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Mon, 16 Oct 2023 20:00:06 -0700 Subject: [PATCH 04/60] feat(pano_records.py & create_pano.py): Various updates to better support the data. --- CHANGELOG.rst | 5 +- src/vipersci/vis/create_pano.py | 73 ++++++++++++++++++++--------- src/vipersci/vis/db/pano_records.py | 38 +++++++-------- tests/test_create_pano.py | 6 +-- tests/test_pano_records.py | 2 +- 5 files changed, 78 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6a13748..5a591c0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -34,6 +34,7 @@ Changed vice-versa). - junc_image_req_ldst.py got some additional columns to manage the Science Team evaluation of images acquired from Image Requests. +- create_pano.py's make_pano_product() function renamed to make_pano_record(). Added ^^^^^ @@ -41,11 +42,13 @@ Added - Association table junc_image_pano created which provides a many-to-many connection between ImageRecords and PanoRecords and added bidirectional relationship entries to each table. -- pano_records table now has azimuth and elevation angle min/max values to indicate +- pano_records table now has pan and tilt angle min/max values to indicate angular range of panorama coverage. - image_requests.py - "Acquired," "Not Acquired," and "Not Planned" statuses added to enum. - ptu_records.py - Tables to record the pan and tilt of the rover's pan-tilt-unit (PTU). +- create_pano.py - updated to correctly add PanoRecord associations, can now query + database for ImageRecords. Removed ^^^^^^^ diff --git a/src/vipersci/vis/create_pano.py b/src/vipersci/vis/create_pano.py index b5f82fe..534ced9 100644 --- a/src/vipersci/vis/create_pano.py +++ b/src/vipersci/vis/create_pano.py @@ -33,18 +33,19 @@ import argparse import logging -from typing import Any, Dict, Iterable, Union, Optional, MutableSequence +from typing import Any, Dict, Union, Optional, MutableSequence from pathlib import Path import numpy as np import numpy.typing as npt from skimage.io import imread, imsave # maybe just imageio here? from skimage.transform import resize -from sqlalchemy import create_engine +from sqlalchemy import create_engine, select from sqlalchemy.orm import Session import vipersci from vipersci.vis.db.image_records import ImageRecord +from vipersci.vis.db.junc_image_pano import JuncImagePano from vipersci.vis.db.pano_records import PanoRecord from vipersci.pds import pid as pds from vipersci.vis.create_image import tif_info, write_json @@ -133,8 +134,8 @@ def main(): def create( - inputs: Iterable[Union[Path, pds.VISID, ImageRecord, str]], - outdir: Path = Path.cwd(), + inputs: MutableSequence[Union[Path, pds.VISID, ImageRecord, str]], + outdir: Optional[Path] = None, session: Optional[Session] = None, json: bool = True, bottom_row: Optional[MutableSequence[Union[Path, str]]] = None, @@ -142,37 +143,52 @@ def create( """ Creates a Panorama Product in *outdir*. Returns None. - At this time, session, xml, and template_path are ignored. + At this time, session is ignored. At this time, *inputs* should be a list of file paths. In the future, it could be a list of product IDs. If a path is provided to *outdir* the created files - will be written there. Defaults to the current working - directory. + will be written there. - If *session* is given, information for the raw product will be - written to the raw_products table. If not, no database activity + If *session* is given, information for the PanoRecord will be + written to the pano_records table. If not, no database activity will occur. - - The *template_path* argument is passed to the write_xml() function, please see - its documentation for details. """ + if outdir is None: + raise ValueError("A Path for outdir must be supplied.") + metadata: Dict[str, Any] = dict( - source_products=[], + source_pids=[], ) source_paths = [] + image_records = [] + + for i, vid in enumerate(inputs): + if isinstance(vid, str): + temp_vid = pds.VISID(vid) + if str(temp_vid) == vid: + vid = temp_vid + else: + continue + + if isinstance(vid, pds.VISID): + if session is not None: + ir = session.scalars( + select(ImageRecord).where(ImageRecord.product_id == str(vid)) + ).first() + if ir is None: + raise ValueError(f"{vid} was not found in the database.") + else: + inputs[i] = ir for i in inputs: if isinstance(i, ImageRecord): - metadata["source_products"].append(i.product_id) + metadata["source_pids"].append(i.product_id) source_paths.append(i.file_path) - elif isinstance(i, pds.VISID): - raise NotImplementedError( - "One day, this will fire up a db connection and get the info needed." - ) + image_records.append(i) elif isinstance(i, (Path, str)): - metadata["source_products"].append([str(pds.VISID(i))]) + metadata["source_pids"].append([str(pds.VISID(i))]) source_paths.append(i) else: raise ValueError( @@ -215,7 +231,7 @@ def create( bot_arr = np.hstack(bottom_list) pano_arr = np.vstack((pano_arr, bot_arr)) - pp = make_pano_product(metadata, pano_arr, outdir) + pp = make_pano_record(metadata, pano_arr, outdir) if json: write_json(pp.asdict(), outdir) @@ -224,12 +240,25 @@ def create( # write_xml(pp.label_dict(), outdir, template_path) if session is not None: - session.add(pp) + if image_records: + to_add = list( + pp, + ) + for ir in image_records: + a = JuncImagePano() + a.image_record = ir + a.pano_record = pp + to_add.append(a) + + session.add_all(to_add) + + else: + session.add(pp) return -def make_pano_product( +def make_pano_record( metadata: dict, image: Union[ImageType, Path, None] = None, outdir: Path = Path.cwd(), diff --git a/src/vipersci/vis/db/pano_records.py b/src/vipersci/vis/db/pano_records.py index 4de194f..2cc9068 100644 --- a/src/vipersci/vis/db/pano_records.py +++ b/src/vipersci/vis/db/pano_records.py @@ -94,33 +94,33 @@ class PanoRecord(Base): nullable=False, doc="The number of lines or rows in the Pano Product image.", ) - rover_az_min = mapped_column( + rover_pan_min = mapped_column( Float, nullable=False, - doc="The minimum or leftmost azimuth of this panorama in degrees, where zero " - "is the rover forward (+x) direction. " - "This azimuth is relative to the rover frame.", + doc="The minimum or leftmost pan angle of this panorama in degrees, where " + "zero is the rover forward (+x) direction. " + "This angle is relative to the rover frame.", ) - rover_az_max = mapped_column( + rover_pan_max = mapped_column( Float, nullable=False, - doc="The maximum or rightmost azimuth of this panorama in degrees, where zero " - "is the rover forward (+x) direction. " - "This azimuth is relative to the rover frame.", + doc="The maximum or rightmost pan angle of this panorama in degrees, where " + "zero is the rover forward (+x) direction. " + "This angle is relative to the rover frame.", ) - rover_el_min = mapped_column( + rover_tilt_min = mapped_column( Float, nullable=False, - doc="The minimum or lower elevation angle of this panorama in degrees, " + doc="The minimum or lower tilt angle of this panorama in degrees, " "where zero is the rover level plane. " - "This elevation angle is relative to the rover frame.", + "This tilt angle is relative to the rover frame.", ) - rover_el_max = mapped_column( + rover_tilt_max = mapped_column( Float, nullable=False, - doc="The maximum or upper elevation angle of this panorama in degrees, " + doc="The maximum or upper tilt angle of this panorama in degrees, " "where zero is the rover level plane. " - "This elevation angle is relative to the rover frame.", + "This tilt angle is relative to the rover frame.", ) _pid = mapped_column( "product_id", String, nullable=False, unique=True, doc="The PDS Product ID." @@ -171,8 +171,8 @@ def __init__(self, **kwargs): f"provided start_time ({kwargs['start_time']}) disagree." ) - elif "image_records" in kwargs: - source_pids = list(map(VISID, kwargs["image_records"])) + elif "source_pids" in kwargs: + source_pids = list(map(VISID, kwargs["source_pids"])) instruments = set([p.instrument for p in source_pids]) inst = "pan" if len(instruments) == 1: @@ -194,11 +194,11 @@ def __init__(self, **kwargs): if v is not None: got[k] = v - if "image_records" in kwargs: - got["image_records"] = kwargs["image_records"] + if "source_pids" in kwargs: + got["source_pids"] = kwargs["source_pids"] raise ValueError( - "Either product_id must be given, or a list of source image_records. " + "Either product_id must be given, or a list of source product IDs. " f"Got: {got}" ) diff --git a/tests/test_create_pano.py b/tests/test_create_pano.py index 8d6935d..39f9f1a 100644 --- a/tests/test_create_pano.py +++ b/tests/test_create_pano.py @@ -29,7 +29,7 @@ def setUp(self) -> None: } def test_no_image(self): - pp = cp.make_pano_product(self.d) + pp = cp.make_pano_record(self.d) self.assertIsInstance(pp, PanoRecord) @patch("vipersci.vis.create_pano.imsave") @@ -47,7 +47,7 @@ def test_no_image(self): ) def test_image(self, mock_tif_info, mock_imsave): image = np.array([[5, 5], [5, 5]], dtype=np.uint16) - pp = cp.make_pano_product(self.d, image, Path("outdir/")) + pp = cp.make_pano_record(self.d, image, Path("outdir/")) self.assertIsInstance(pp, PanoRecord) mock_imsave.assert_called_once() mock_tif_info.assert_called_once() @@ -55,7 +55,7 @@ def test_image(self, mock_tif_info, mock_imsave): mock_imsave.reset_mock() mock_tif_info.reset_mock() - prp = cp.make_pano_product(self.d, Path("dummy.tif"), Path("outdir/")) + prp = cp.make_pano_record(self.d, Path("dummy.tif"), Path("outdir/")) self.assertIsInstance(prp, PanoRecord) mock_imsave.assert_not_called() mock_tif_info.assert_called_once() diff --git a/tests/test_pano_records.py b/tests/test_pano_records.py index 01eb33c..f079b8c 100644 --- a/tests/test_pano_records.py +++ b/tests/test_pano_records.py @@ -46,7 +46,7 @@ def setUp(self): mission_phase="Test", purpose="Engineering", samples=2048, - image_records=self.source_products, + source_pids=self.source_products, start_time=self.startUTC, ) self.extras = dict(foo="bar") From 869c6769ab3d176d75835611d8e77918a1a33ff4 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Mon, 16 Oct 2023 20:01:51 -0700 Subject: [PATCH 05/60] feat(nss_simulator.py): Update to nodata values. --- CHANGELOG.rst | 2 ++ src/vipersci/carto/nss_simulator.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 5a591c0..8788744 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -35,6 +35,8 @@ Changed - junc_image_req_ldst.py got some additional columns to manage the Science Team evaluation of images acquired from Image Requests. - create_pano.py's make_pano_product() function renamed to make_pano_record(). +- nss_simulator.py - nodata defaults for ideal detector count maps updated to more + realistic values for dry terrain. Added ^^^^^ diff --git a/src/vipersci/carto/nss_simulator.py b/src/vipersci/carto/nss_simulator.py index d2295ff..3939e42 100755 --- a/src/vipersci/carto/nss_simulator.py +++ b/src/vipersci/carto/nss_simulator.py @@ -182,8 +182,11 @@ def main(): d2_arr = d2.reshape(bd_data.shape) kwds = bd_data.profile - kwds["nodata"] = 0.0 + # for Detector 1 + kwds["nodata"] = 31.0 write_tif(args.output, "_d1.tif", d1_arr, kwds) + # for Detector 2 + kwds["nodata"] = 60.0 write_tif(args.output, "_d2.tif", d2_arr, kwds) return From b09bd8aad30b0339719a2b5425c4998fca856b1c Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Tue, 24 Oct 2023 11:51:47 -0700 Subject: [PATCH 06/60] feat(image_statistics.py): Added additional stats. --- CHANGELOG.rst | 1 + src/vipersci/vis/image_statistics.py | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8788744..9ac05b8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -37,6 +37,7 @@ Changed - create_pano.py's make_pano_product() function renamed to make_pano_record(). - nss_simulator.py - nodata defaults for ideal detector count maps updated to more realistic values for dry terrain. +- image_statistics.py - Added additional stats in the compute() function. Added ^^^^^ diff --git a/src/vipersci/vis/image_statistics.py b/src/vipersci/vis/image_statistics.py index 8e28fa8..7a0b0b1 100644 --- a/src/vipersci/vis/image_statistics.py +++ b/src/vipersci/vis/image_statistics.py @@ -64,11 +64,19 @@ def main(): return -def compute(image: ImageType) -> dict: +def compute( + image: ImageType, + overexposed_thresh=(4096 * 0.8), + underexposed_thresh=(4096 * 0.2), +) -> dict: d = { "blur": measure.blur_effect(image), "mean": np.mean(image), "std": np.std(image), + "over_exposed": (image > overexposed_thresh).sum(), + "under_exposed": (image < underexposed_thresh).sum(), + # "aggregate_noise": + # number of dead pixels } return d From 092fec707e6f4810f0732d478a31cbf1d51bff60 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Tue, 24 Oct 2023 18:02:14 -0700 Subject: [PATCH 07/60] feat(get_position.py): Added. --- CHANGELOG.rst | 1 + src/vipersci/carto/get_position.py | 176 +++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+) create mode 100644 src/vipersci/carto/get_position.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9ac05b8..24f3567 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -52,6 +52,7 @@ Added - ptu_records.py - Tables to record the pan and tilt of the rover's pan-tilt-unit (PTU). - create_pano.py - updated to correctly add PanoRecord associations, can now query database for ImageRecords. +- get_position.py - Gets position and yaw from a REST-based service. Removed ^^^^^^^ diff --git a/src/vipersci/carto/get_position.py b/src/vipersci/carto/get_position.py new file mode 100644 index 0000000..0be654a --- /dev/null +++ b/src/vipersci/carto/get_position.py @@ -0,0 +1,176 @@ +"""Gather Rover locations from a REST-based service. +""" + +# Copyright 2023, United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +import argparse +import csv +from datetime import datetime, timezone +import logging +from pathlib import Path +import sys + +import pandas as pd +import requests + +from vipersci import util + +logger = logging.getLogger(__name__) + + +def arg_parser(): + parser = argparse.ArgumentParser( + description=__doc__, parents=[util.parent_parser()] + ) + parser.add_argument( + "-s", + "--start", + default=datetime(2020, 1, 1, tz=timezone.utc), + help="An ISO8601 datetime to start the query at." + ) + parser.add_argument( + "-e", + "--end", + default=datetime.now(tz=timezone.utc), + help="An ISO8601 datetime to end the query at." + ) + parser.add_argument( + "-f", + "--frequency", + help="A frequency between the start and end times, using pandas 'Offset alias' " + "string notation " + "(https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#offset-aliases). " + "Could be 'S' for secondly." + ) + parser.add_argument( + "-u", + "--url", + required=True, + help="URL to which event_times may be posted and for which location and " + "orientation will be returned.", + ) + parser.add_argument( + "-o", + "--output", + type=Path, + help="Output path for CSV file output." + ) + return parser + + +def main(): + parser = arg_parser() + args = parser.parse_args() + util.set_logger(args.verbose) + + t_start = datetime.fromisoformat(args.start) + t_end = datetime.fromisoformat(args.end) + + if args.frequency: + times = pd.date_range(t_start, t_end, freq=args.frequency).tolist() + + unix_times = [] + for t in times: + unix_times.append(t.timestamp()) + + tpp = get_position_and_pose(unix_times, args.url) + else: + tpp = get_position_and_pose_range( + t_start.timestamp(), t_end.timestamp(), args.url + ) + + if args.output is not None: + with open(args.output, 'w', newline='') as csvfile: + writer = csv.writer(csvfile) + writer.writerow(["UTC datetime", "x", "y", "yaw"]) + for row in tpp: + dt = datetime.fromtimestamp(row[0], tz=timezone.utc) + writer.writerow([dt.isoformat(), ] + list(row[1:])) + + +def get_position_and_pose( + times: list, url: str, crs: str = "VIPER:910101" +): + """ + Given a list of unix times and a URL that requests can be made against, + return a list of two-tuples whose first element is the time and + whose second element is a three-tuple of x-location, y-location, + and yaw. + """ + + tpp = list() + + for t in times: + logger.info(f"unix timestamp: {t}") + position_result = requests.get( + url, + params={ + "event_time": t, # Event time in unix datetime. + "margin_seconds": 2, # Some "margin time" around the event to search. + "format": "xyyaw", # Could also be xyyaw_uncertainty + "time_format": "unix_seconds", + "source": "ROVER", + "limit": 0, + "crs_code": crs, + }, + verify=False, + ) + rj = position_result.json() + logger.info(rj) + + tpp.append((rj["event_seconds"], (rj["location"][0], rj["location"][1], rj["yaw"]))) + + return tpp + + +def get_position_and_pose_range( + start_time, stop_time, url: str, crs: str = "VIPER:910101" +): + tpp = list() + + track_result = requests.get( + url, + params={ + "min_time": start_time, + "max_time": stop_time, + "all": False, + "format": "xyyaw", + "crs_code": crs, + "time_format": "unix_seconds", + "source": "ROVER", + "start_end_only": False, + "simplify": False, + "order": "asc" + }, + verify=False, + ) + rj = track_result.json() + + for time, loc, yaw in zip(rj["event_seconds"], rj["location"], rj["yaw"]): + tpp.append((time, loc[0], loc[1], yaw)) + + return tpp + + +if __name__ == "__main__": + sys.exit(main()) From 89036917ecc0d30d400c0a9a23a8c0df77c0196d Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Wed, 25 Oct 2023 10:55:29 -0700 Subject: [PATCH 08/60] feat(image_requests.py): Added NOT_OBTAINABLE value to enum. --- CHANGELOG.rst | 4 ++-- src/vipersci/vis/db/image_requests.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 24f3567..e64945e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -47,8 +47,8 @@ Added relationship entries to each table. - pano_records table now has pan and tilt angle min/max values to indicate angular range of panorama coverage. -- image_requests.py - "Acquired," "Not Acquired," and "Not Planned" statuses added to - enum. +- image_requests.py - "Acquired," "Not Acquired," "Not Planned," and "Not Obtainable" + statuses added to enum. - ptu_records.py - Tables to record the pan and tilt of the rover's pan-tilt-unit (PTU). - create_pano.py - updated to correctly add PanoRecord associations, can now query database for ImageRecords. diff --git a/src/vipersci/vis/db/image_requests.py b/src/vipersci/vis/db/image_requests.py index 9363928..08e8b6c 100644 --- a/src/vipersci/vis/db/image_requests.py +++ b/src/vipersci/vis/db/image_requests.py @@ -93,6 +93,7 @@ class Status(enum.Enum): IMMEDIATE = 6 ACQUIRED = 7 NOT_AQUIRED = 8 + NOT_OBTAINABLE = 9 class CameraType(enum.Enum): From a4b239a91c94147a9beda700f073ba8e50cdec42 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Wed, 25 Oct 2023 11:09:09 -0700 Subject: [PATCH 09/60] feat(anom_pixel.py): Added. --- setup.cfg | 2 + src/vipersci/vis/anom_pixel.py | 79 ++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 src/vipersci/vis/anom_pixel.py diff --git a/setup.cfg b/setup.cfg index 8757d74..617bcc6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -67,9 +67,11 @@ where = src console_scripts = accrual = vipersci.carto.accrual:main anaglyph = vipersci.vis.anaglyph:main + anom_pixel = vipersci.vis.anom_pixel:main colorforge = vipersci.carto.colorforge:main dice_buffer = vipersci.carto.dice_buffer:main dissolve_dice = vipersci.carto.dissolve_dice:main + get_position = vipersci.carto.get_position:main image_stats = vipersci.vis.image_statistics:main msolo_simulator = vipersci.carto.msolo_simulator:main nirvss_simulator = vipersci.carto.nirvss_simulator:main diff --git a/src/vipersci/vis/anom_pixel.py b/src/vipersci/vis/anom_pixel.py new file mode 100644 index 0000000..6327692 --- /dev/null +++ b/src/vipersci/vis/anom_pixel.py @@ -0,0 +1,79 @@ +"""Check to see if an image has anomalous pixels which may indicate that those pixels +in the detector may be of concern. +""" + +# Copyright 2023, United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +import argparse +import logging +from pathlib import Path +import sys + +import numpy as np +from skimage.filters import median +from skimage.io import imread + +from vipersci import util + +logger = logging.getLogger(__name__) + + +def arg_parser(): + parser = argparse.ArgumentParser( + description=__doc__, parents=[util.parent_parser()] + ) + parser.add_argument( + "input", type=Path, help="VIS Image." + ) + return parser + + +def main(): + parser = arg_parser() + args = parser.parse_args() + util.set_logger(args.verbose) + + image = imread(args.input) + + print(check(image)) + + return + + +def check(image, tolerance=3): + """ + Returns indices of the elements of *image* which exceed *tolerance* standard + devations of the difference between *image* and a median-filtered version of it. + """ + + blurred = median(image) + difference = image - blurred + threshold = tolerance * np.std(difference) + + anom_pixel_indices = np.nonzero(np.abs(difference) > threshold) + + return anom_pixel_indices + + +if __name__ == "__main__": + sys.exit(main()) From 126104eba9171a9e12a6dc65650a83ea18a769ca Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Wed, 25 Oct 2023 11:13:15 -0700 Subject: [PATCH 10/60] =?UTF-8?q?Bump=20version:=200.6.1=20=E2=86=92=200.7?= =?UTF-8?q?.0-dev?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- setup.cfg | 6 +++--- src/vipersci/__init__.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index 617bcc6..86159c0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.6.1 +current_version = 0.7.0-dev commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+))? @@ -67,11 +67,11 @@ where = src console_scripts = accrual = vipersci.carto.accrual:main anaglyph = vipersci.vis.anaglyph:main - anom_pixel = vipersci.vis.anom_pixel:main + anom_pixel = vipersci.vis.anom_pixel:main colorforge = vipersci.carto.colorforge:main dice_buffer = vipersci.carto.dice_buffer:main dissolve_dice = vipersci.carto.dissolve_dice:main - get_position = vipersci.carto.get_position:main + get_position = vipersci.carto.get_position:main image_stats = vipersci.vis.image_statistics:main msolo_simulator = vipersci.carto.msolo_simulator:main nirvss_simulator = vipersci.carto.nirvss_simulator:main diff --git a/src/vipersci/__init__.py b/src/vipersci/__init__.py index bd2d4b0..e32261c 100644 --- a/src/vipersci/__init__.py +++ b/src/vipersci/__init__.py @@ -2,4 +2,4 @@ __author__ = """vipersci Developers""" __email__ = "rbeyer@seti.org" -__version__ = "0.6.1" +__version__ = "0.7.0-dev" From ef4ba49c272e158b3b12b2ea3f50c04b527e7407 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Wed, 25 Oct 2023 11:28:44 -0700 Subject: [PATCH 11/60] feat(anom_pixel.py): Added command-line tolerance specifier. --- src/vipersci/vis/anom_pixel.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/vipersci/vis/anom_pixel.py b/src/vipersci/vis/anom_pixel.py index 6327692..7371d93 100644 --- a/src/vipersci/vis/anom_pixel.py +++ b/src/vipersci/vis/anom_pixel.py @@ -42,6 +42,14 @@ def arg_parser(): parser = argparse.ArgumentParser( description=__doc__, parents=[util.parent_parser()] ) + parser.add_argument( + "-t", + "--tolerance", + type=float, + default=3, + help="The number of standard deviations of the difference between the image " + "and its median filter to 'trigger' on." + ) parser.add_argument( "input", type=Path, help="VIS Image." ) @@ -55,7 +63,10 @@ def main(): image = imread(args.input) - print(check(image)) + indices = check(image, args.tolerance) + + print(f"There are {len(indices[0])} anomalous pixels.") + print(indices) return @@ -69,6 +80,7 @@ def check(image, tolerance=3): blurred = median(image) difference = image - blurred threshold = tolerance * np.std(difference) + logger.info(f"threshold: {threshold}") anom_pixel_indices = np.nonzero(np.abs(difference) > threshold) From 8816a951cf416215d6cf2a0f0a86d52530d65657 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Thu, 26 Oct 2023 19:49:23 -0700 Subject: [PATCH 12/60] feat(image_statistics.py): Added pprint() function. --- CHANGELOG.rst | 3 ++- src/vipersci/vis/image_statistics.py | 30 ++++++++++++++++++++++++---- src/vipersci/vis/viseer.py | 2 ++ 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e64945e..fd440ae 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -37,7 +37,8 @@ Changed - create_pano.py's make_pano_product() function renamed to make_pano_record(). - nss_simulator.py - nodata defaults for ideal detector count maps updated to more realistic values for dry terrain. -- image_statistics.py - Added additional stats in the compute() function. +- image_statistics.py - Added additional stats in the compute() function, and added + a pprint() function to format the info nicely. Added ^^^^^ diff --git a/src/vipersci/vis/image_statistics.py b/src/vipersci/vis/image_statistics.py index 7a0b0b1..e62bbae 100644 --- a/src/vipersci/vis/image_statistics.py +++ b/src/vipersci/vis/image_statistics.py @@ -25,9 +25,9 @@ import argparse import logging +from textwrap import dedent from typing import Union from pathlib import Path -from pprint import pprint import numpy as np import numpy.typing as npt @@ -40,6 +40,9 @@ ImageType = Union[npt.NDArray[np.uint16], npt.NDArray[np.uint8]] +UNDEREXPOSED_THRESHOLD = 4095 * 0.2 +OVEREXPOSED_THRESHOLD = 4095 * 0.8 + def arg_parser(): parser = argparse.ArgumentParser( @@ -59,15 +62,15 @@ def main(): image = imread(str(args.image)) - pprint(compute(image)) + print(pprint(image)) return def compute( image: ImageType, - overexposed_thresh=(4096 * 0.8), - underexposed_thresh=(4096 * 0.2), + overexposed_thresh=OVEREXPOSED_THRESHOLD, + underexposed_thresh=UNDEREXPOSED_THRESHOLD ) -> dict: d = { "blur": measure.blur_effect(image), @@ -80,3 +83,22 @@ def compute( } return d + + +def pprint( + image: ImageType, + overexposed_thresh=OVEREXPOSED_THRESHOLD, + underexposed_thresh=UNDEREXPOSED_THRESHOLD, +) -> str: + d = compute(image, overexposed_thresh, underexposed_thresh) + + s = dedent( + f"""\ + blur: {d['blur']} (0 for no blur, 1 for maximal blur) + mean: {d['mean']} + std: {d['std']} + over-exposed: {d['over_exposed']} pixels, {100 * d['over_exposed'] / image.size} % + under-exposed: {d['under_exposed']} pixels, {100 * d['under_exposed'] / image.size} %\ + """ + ) + return s diff --git a/src/vipersci/vis/viseer.py b/src/vipersci/vis/viseer.py index b9ce71f..763dd54 100644 --- a/src/vipersci/vis/viseer.py +++ b/src/vipersci/vis/viseer.py @@ -36,6 +36,7 @@ from skimage.exposure import equalize_adapthist from skimage.io import imread +from vipersci.vis import image_statistics from vipersci import util logger = logging.getLogger(__name__) @@ -92,6 +93,7 @@ def main(): imtitle = args.input.name logger.info(describe(image, "image as loaded:")) + logger.info(image_statistics.pprint(image)) if args.clahe: image = equalize_adapthist(image) From 7353113975a6515bb3ff5bcdeaa423584100a0f5 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Fri, 27 Oct 2023 16:42:44 -0700 Subject: [PATCH 13/60] fix(image_records.py): Was not handling a SLoG pid on __init__() correctly. --- src/vipersci/vis/db/image_records.py | 4 ++- tests/test_image_records.py | 40 +++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/src/vipersci/vis/db/image_records.py b/src/vipersci/vis/db/image_records.py index 4df54b1..0288b18 100644 --- a/src/vipersci/vis/db/image_records.py +++ b/src/vipersci/vis/db/image_records.py @@ -475,7 +475,9 @@ def __init__(self, **kwargs): ) else: t = ImageType(self.output_image_mask) - if t.compression_ratio() != vis_compression[pid.compression]: + if ImageType.SLOG_ICER_IMAGE == t and pid.compression == "s": + pass + elif t.compression_ratio() != vis_compression[pid.compression]: raise ValueError( f"The product_id compression code ({pid.compression}) and " f"the compression ratio ({t.compression_ratio()}) based on " diff --git a/tests/test_image_records.py b/tests/test_image_records.py index e83750e..72fc5a1 100644 --- a/tests/test_image_records.py +++ b/tests/test_image_records.py @@ -28,6 +28,16 @@ from vipersci.pds.pid import VISID from vipersci.vis.db import image_records as trp +from vipersci.vis.db.image_requests import ImageRequest # noqa +from vipersci.vis.db.image_stats import ImageStats # noqa +from vipersci.vis.db.image_tags import ImageTag, taglist # noqa +from vipersci.vis.db.junc_image_pano import JuncImagePano # noqa +from vipersci.vis.db.junc_image_record_tags import JuncImageRecordTag # noqa +from vipersci.vis.db.junc_image_req_ldst import JuncImageRequestLDST # noqa +from vipersci.vis.db.ldst import LDST # noqa +from vipersci.vis.db.light_records import LightRecord # noqa +from vipersci.vis.db.pano_records import PanoRecord # noqa +from vipersci.vis.db.ptu_records import PanRecord, TiltRecord # noqa class TestImageType(unittest.TestCase): @@ -104,11 +114,39 @@ def test_init(self): rp = trp.ImageRecord(**self.d) self.assertEqual("220127-000000-ncl-c", str(rp.product_id)) - d = self.d + d = self.d.copy() d.update(self.extras) rpl = trp.ImageRecord(**d) self.assertEqual("220127-000000-ncl-c", str(rpl.product_id)) + d_slog = { + "adcGain": 0, + "autoExposure": 0, + "cameraId": 0, + "captureId": 1, + "exposureTime": 511, + "imageDepth": 2, + "imageHeight": 2048, + "imageWidth": 2048, + "imageId": 0, + "immediateDownloadInfo": 24, + "temperature": 0, + "lobt": 1698350400, + "offset": 0, + "outputImageMask": 16, + "padding": 0, + "pgaGain": 1.0, + "processingInfo": 26, + "product_id": "231026-200000-ncl-s", + "stereo": 1, + "voltageRamp": 0, + "yamcs_generation_time": "2023-10-26T20:00:00Z", + "yamcs_reception_time": "2023-10-26T20:03:00Z", + "yamcs_name": "/ViperGround/Images/ImageData/Navcam_left_slog" + } + ir_slog = trp.ImageRecord(**d_slog) + self.assertEqual("NavCam Left", ir_slog.instrument_name) + # for k in dir(rp): # if k.startswith(("_", "validate_")): # continue From 29b8f19c26b5f8708db83f4720c401d93bb5fd85 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Fri, 27 Oct 2023 16:43:58 -0700 Subject: [PATCH 14/60] fix(create_image.py): Needed a session.commit() to ensure INSERT on the db. --- src/vipersci/vis/create_image.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/vipersci/vis/create_image.py b/src/vipersci/vis/create_image.py index cafac9c..e763b99 100644 --- a/src/vipersci/vis/create_image.py +++ b/src/vipersci/vis/create_image.py @@ -131,6 +131,7 @@ def main(): session, args.json, ) + session.commit() return From 8b4b71578b46008435fac20e54181aa2ef960ce1 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Fri, 27 Oct 2023 16:45:38 -0700 Subject: [PATCH 15/60] feat(create_vis_dbs.py): Now handles spatialite databases. --- CHANGELOG.rst | 1 + src/vipersci/vis/db/create_vis_dbs.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index fd440ae..9607971 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -54,6 +54,7 @@ Added - create_pano.py - updated to correctly add PanoRecord associations, can now query database for ImageRecords. - get_position.py - Gets position and yaw from a REST-based service. +- create_vis_dbs.py - Now also supports spatialite databases, primarily for testing. Removed ^^^^^^^ diff --git a/src/vipersci/vis/db/create_vis_dbs.py b/src/vipersci/vis/db/create_vis_dbs.py index d48b15b..e447686 100755 --- a/src/vipersci/vis/db/create_vis_dbs.py +++ b/src/vipersci/vis/db/create_vis_dbs.py @@ -33,7 +33,9 @@ import csv import logging +from geoalchemy2 import load_spatialite from sqlalchemy import create_engine, insert, inspect, select +from sqlalchemy.event import listen from sqlalchemy.orm import Session from vipersci import util @@ -91,6 +93,9 @@ def main(): util.set_logger(args.verbose) engine = create_engine(args.dburl) + if args.dburl.startswith("sqlite://"): + # This required because we have spatialite tables in the db: + listen(engine, "connect", load_spatialite) # Create tables for t in tables: From 142593a19a003e5972ec14e019e9196d12dd7beb Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Fri, 27 Oct 2023 16:47:46 -0700 Subject: [PATCH 16/60] feat(create_raw.py): Added capability to report data quality and observational intent. --- CHANGELOG.rst | 2 ++ src/vipersci/vis/pds/create_raw.py | 20 +++++++++++++++++++- src/vipersci/vis/pds/data/raw-template.xml | 19 ++++++++++++++++++- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9607971..e04c284 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -55,6 +55,8 @@ Added database for ImageRecords. - get_position.py - Gets position and yaw from a REST-based service. - create_vis_dbs.py - Now also supports spatialite databases, primarily for testing. +- create_raw.py - Added components for adding observational intent and data quality + to the XML label. Removed ^^^^^^^ diff --git a/src/vipersci/vis/pds/create_raw.py b/src/vipersci/vis/pds/create_raw.py index f05650a..16b4a36 100644 --- a/src/vipersci/vis/pds/create_raw.py +++ b/src/vipersci/vis/pds/create_raw.py @@ -121,7 +121,7 @@ def main(): if pid is None or args.input.endswith(".json"): if Path(args.input).exists(): with open(args.input) as f: - ir = ImageRecord(json.load(f)) + ir = ImageRecord(**json.load(f)) else: parser.error(f"The file {args.input} does not exist.") else: @@ -279,6 +279,7 @@ def label_dict(ir: ImageRecord, lights: dict): onoff = {True: "On", False: "Off", None: None} pid = pds.VISID(ir.product_id) d = dict( + data_quality="", lid=f"urn:nasa:pds:viper_vis:raw:{ir.product_id}", mission_lid="urn:nasa:pds:viper", sc_lid=_sclid, @@ -289,6 +290,8 @@ def label_dict(ir: ImageRecord, lights: dict): led_wavelength=453, # nm luminaires={}, compression_class=pid.compression_class(), + minloss=0 if pid.compression_class() == "Lossless" else 12, + observational_intent={}, onboard_compression_ratio=pds.vis_compression[pid.compression], onboard_compression_type="ICER", sample_bits=12, @@ -321,6 +324,21 @@ def label_dict(ir: ImageRecord, lights: dict): d["sample_bits"] = 8 d["sample_bit_mask"] = "2#11111111" + if ir.image_request is not None: + d["observational_intent"]["goal"] = ir.image_request.justification + d["observational_intent"]["task"] = ir.image_request.title + d["observational_intent"]["activity_id"] = f"Image Request {ir.image_request.id}" + d["observational_intent"]["target_id"] = ir.image_request.target_location + + if ir.verified is not None: + if ir.verified: + d["data_quality"] += "Image manually verified." + else: + d["data_quality"] += "Image determined to have errors." + + if ir.verification_notes is not None: + d["data_quality"] += " " + ir.verification_notes + return d diff --git a/src/vipersci/vis/pds/data/raw-template.xml b/src/vipersci/vis/pds/data/raw-template.xml index 338ec51..71918ff 100644 --- a/src/vipersci/vis/pds/data/raw-template.xml +++ b/src/vipersci/vis/pds/data/raw-template.xml @@ -3,6 +3,7 @@ + @@ -95,6 +98,9 @@ image2d imaging_parameters_to_image_object + + ${data_quality} + 1 1 @@ -121,8 +127,11 @@ ${compression_class} - ${onboard_compression_type} ${onboard_compression_ratio} + ${onboard_compression_type} + + ${minloss} + 12 @@ -141,6 +150,14 @@ ${mission_phase} + + + ${observational_intent.goal} + ${observational_intent.task} + ${observational_intent.activity_id} + ${observational_intent.target_id} + + image2d From 7286f16f87c5b23767ff7bfe65cd8cef161274f4 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Fri, 27 Oct 2023 16:53:54 -0700 Subject: [PATCH 17/60] style(various): lint updates. --- src/vipersci/carto/get_position.py | 35 ++++++++++++++------------- src/vipersci/vis/anom_pixel.py | 6 ++--- src/vipersci/vis/db/image_requests.py | 1 + src/vipersci/vis/image_statistics.py | 20 ++++++++------- src/vipersci/vis/pds/create_raw.py | 4 ++- tests/test_image_records.py | 2 +- 6 files changed, 36 insertions(+), 32 deletions(-) diff --git a/src/vipersci/carto/get_position.py b/src/vipersci/carto/get_position.py index 0be654a..91f1d9a 100644 --- a/src/vipersci/carto/get_position.py +++ b/src/vipersci/carto/get_position.py @@ -46,21 +46,20 @@ def arg_parser(): "-s", "--start", default=datetime(2020, 1, 1, tz=timezone.utc), - help="An ISO8601 datetime to start the query at." + help="An ISO8601 datetime to start the query at.", ) parser.add_argument( "-e", "--end", default=datetime.now(tz=timezone.utc), - help="An ISO8601 datetime to end the query at." + help="An ISO8601 datetime to end the query at.", ) parser.add_argument( "-f", "--frequency", help="A frequency between the start and end times, using pandas 'Offset alias' " - "string notation " - "(https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#offset-aliases). " - "Could be 'S' for secondly." + "string notation (https://pandas.pydata.org/pandas-docs/stable/user_guide/" + "timeseries.html#offset-aliases). Could be 'S' for secondly.", ) parser.add_argument( "-u", @@ -70,10 +69,7 @@ def arg_parser(): "orientation will be returned.", ) parser.add_argument( - "-o", - "--output", - type=Path, - help="Output path for CSV file output." + "-o", "--output", type=Path, help="Output path for CSV file output." ) return parser @@ -100,17 +96,20 @@ def main(): ) if args.output is not None: - with open(args.output, 'w', newline='') as csvfile: + with open(args.output, "w", newline="") as csvfile: writer = csv.writer(csvfile) writer.writerow(["UTC datetime", "x", "y", "yaw"]) for row in tpp: dt = datetime.fromtimestamp(row[0], tz=timezone.utc) - writer.writerow([dt.isoformat(), ] + list(row[1:])) + writer.writerow( + [ + dt.isoformat(), + ] + + list(row[1:]) + ) -def get_position_and_pose( - times: list, url: str, crs: str = "VIPER:910101" -): +def get_position_and_pose(times: list, url: str, crs: str = "VIPER:910101"): """ Given a list of unix times and a URL that requests can be made against, return a list of two-tuples whose first element is the time and @@ -138,13 +137,15 @@ def get_position_and_pose( rj = position_result.json() logger.info(rj) - tpp.append((rj["event_seconds"], (rj["location"][0], rj["location"][1], rj["yaw"]))) + tpp.append( + (rj["event_seconds"], (rj["location"][0], rj["location"][1], rj["yaw"])) + ) return tpp def get_position_and_pose_range( - start_time, stop_time, url: str, crs: str = "VIPER:910101" + start_time, stop_time, url: str, crs: str = "VIPER:910101" ): tpp = list() @@ -160,7 +161,7 @@ def get_position_and_pose_range( "source": "ROVER", "start_end_only": False, "simplify": False, - "order": "asc" + "order": "asc", }, verify=False, ) diff --git a/src/vipersci/vis/anom_pixel.py b/src/vipersci/vis/anom_pixel.py index 7371d93..3d19fa9 100644 --- a/src/vipersci/vis/anom_pixel.py +++ b/src/vipersci/vis/anom_pixel.py @@ -48,11 +48,9 @@ def arg_parser(): type=float, default=3, help="The number of standard deviations of the difference between the image " - "and its median filter to 'trigger' on." - ) - parser.add_argument( - "input", type=Path, help="VIS Image." + "and its median filter to 'trigger' on.", ) + parser.add_argument("input", type=Path, help="VIS Image.") return parser diff --git a/src/vipersci/vis/db/image_requests.py b/src/vipersci/vis/db/image_requests.py index 08e8b6c..399282f 100644 --- a/src/vipersci/vis/db/image_requests.py +++ b/src/vipersci/vis/db/image_requests.py @@ -85,6 +85,7 @@ class Status(enum.Enum): """ This describes the status of an Image Request. """ + WORKING = 1 READY_FOR_VIS = 2 READY_FOR_PLANNING = 3 diff --git a/src/vipersci/vis/image_statistics.py b/src/vipersci/vis/image_statistics.py index e62bbae..e1355cd 100644 --- a/src/vipersci/vis/image_statistics.py +++ b/src/vipersci/vis/image_statistics.py @@ -68,9 +68,9 @@ def main(): def compute( - image: ImageType, - overexposed_thresh=OVEREXPOSED_THRESHOLD, - underexposed_thresh=UNDEREXPOSED_THRESHOLD + image: ImageType, + overexposed_thresh=OVEREXPOSED_THRESHOLD, + underexposed_thresh=UNDEREXPOSED_THRESHOLD, ) -> dict: d = { "blur": measure.blur_effect(image), @@ -86,9 +86,9 @@ def compute( def pprint( - image: ImageType, - overexposed_thresh=OVEREXPOSED_THRESHOLD, - underexposed_thresh=UNDEREXPOSED_THRESHOLD, + image: ImageType, + overexposed_thresh=OVEREXPOSED_THRESHOLD, + underexposed_thresh=UNDEREXPOSED_THRESHOLD, ) -> str: d = compute(image, overexposed_thresh, underexposed_thresh) @@ -96,9 +96,11 @@ def pprint( f"""\ blur: {d['blur']} (0 for no blur, 1 for maximal blur) mean: {d['mean']} - std: {d['std']} - over-exposed: {d['over_exposed']} pixels, {100 * d['over_exposed'] / image.size} % - under-exposed: {d['under_exposed']} pixels, {100 * d['under_exposed'] / image.size} %\ + std: {d['std']} + over-exposed: {d['over_exposed']} pixels,\ + {100 * d['over_exposed'] / image.size} % + under-exposed: {d['under_exposed']} pixels,\ + {100 * d['under_exposed'] / image.size} %\ """ ) return s diff --git a/src/vipersci/vis/pds/create_raw.py b/src/vipersci/vis/pds/create_raw.py index 16b4a36..c3d3597 100644 --- a/src/vipersci/vis/pds/create_raw.py +++ b/src/vipersci/vis/pds/create_raw.py @@ -327,7 +327,9 @@ def label_dict(ir: ImageRecord, lights: dict): if ir.image_request is not None: d["observational_intent"]["goal"] = ir.image_request.justification d["observational_intent"]["task"] = ir.image_request.title - d["observational_intent"]["activity_id"] = f"Image Request {ir.image_request.id}" + d["observational_intent"][ + "activity_id" + ] = f"Image Request {ir.image_request.id}" d["observational_intent"]["target_id"] = ir.image_request.target_location if ir.verified is not None: diff --git a/tests/test_image_records.py b/tests/test_image_records.py index 72fc5a1..1e2c2ce 100644 --- a/tests/test_image_records.py +++ b/tests/test_image_records.py @@ -142,7 +142,7 @@ def test_init(self): "voltageRamp": 0, "yamcs_generation_time": "2023-10-26T20:00:00Z", "yamcs_reception_time": "2023-10-26T20:03:00Z", - "yamcs_name": "/ViperGround/Images/ImageData/Navcam_left_slog" + "yamcs_name": "/ViperGround/Images/ImageData/Navcam_left_slog", } ir_slog = trp.ImageRecord(**d_slog) self.assertEqual("NavCam Left", ir_slog.instrument_name) From ba7b247d155652eec1b780d135987f6ffae76731 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Fri, 27 Oct 2023 17:24:32 -0700 Subject: [PATCH 18/60] style(various): mypy updates. --- src/vipersci/vis/create_pano.py | 24 ++++++++++++------------ src/vipersci/vis/db/create_vis_dbs.py | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/vipersci/vis/create_pano.py b/src/vipersci/vis/create_pano.py index 534ced9..8918fd8 100644 --- a/src/vipersci/vis/create_pano.py +++ b/src/vipersci/vis/create_pano.py @@ -33,7 +33,7 @@ import argparse import logging -from typing import Any, Dict, Union, Optional, MutableSequence +from typing import Any, Dict, Union, Optional, MutableSequence, List from pathlib import Path import numpy as np @@ -182,17 +182,17 @@ def create( else: inputs[i] = ir - for i in inputs: - if isinstance(i, ImageRecord): - metadata["source_pids"].append(i.product_id) - source_paths.append(i.file_path) - image_records.append(i) - elif isinstance(i, (Path, str)): - metadata["source_pids"].append([str(pds.VISID(i))]) - source_paths.append(i) + for inp in inputs: + if isinstance(inp, ImageRecord): + metadata["source_pids"].append(inp.product_id) + source_paths.append(inp.file_path) + image_records.append(inp) + elif isinstance(inp, (Path, str)): + metadata["source_pids"].append([str(pds.VISID(inp))]) + source_paths.append(inp) else: raise ValueError( - f"an element in input is not the right type: {i} ({type(i)})" + f"an element in input is not the right type: {inp} ({type(inp)})" ) # At this time, image pointing information is not available, so we assume that @@ -241,9 +241,9 @@ def create( if session is not None: if image_records: - to_add = list( + to_add: List[Union[PanoRecord, JuncImagePano]] = [ pp, - ) + ] for ir in image_records: a = JuncImagePano() a.image_record = ir diff --git a/src/vipersci/vis/db/create_vis_dbs.py b/src/vipersci/vis/db/create_vis_dbs.py index e447686..5ac206d 100755 --- a/src/vipersci/vis/db/create_vis_dbs.py +++ b/src/vipersci/vis/db/create_vis_dbs.py @@ -33,7 +33,7 @@ import csv import logging -from geoalchemy2 import load_spatialite +from geoalchemy2 import load_spatialite # type: ignore from sqlalchemy import create_engine, insert, inspect, select from sqlalchemy.event import listen from sqlalchemy.orm import Session From a6f43c2016ef9d9896ddab73512506da03b193c3 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Wed, 1 Nov 2023 17:54:31 -0700 Subject: [PATCH 19/60] refactor(xml.py): Moved the find_text() convenience function out of image_records.py and into xml.py --- src/vipersci/pds/xml.py | 22 ++++++++++ src/vipersci/vis/db/image_records.py | 65 ++++++++++------------------ 2 files changed, 45 insertions(+), 42 deletions(-) diff --git a/src/vipersci/pds/xml.py b/src/vipersci/pds/xml.py index dedd8c1..41fd7b9 100755 --- a/src/vipersci/pds/xml.py +++ b/src/vipersci/pds/xml.py @@ -38,3 +38,25 @@ ns = {} for k in dd.keys(): ns[k] = f"http://pds.nasa.gov/pds4/{k}/v1" + + +def find_text(root, xpath, unit_check=None, namespace=None): + """Convenience function for returning the text from an element.""" + if namespace is None: + namespace = ns + + element = root.find(xpath, namespace) + if element is not None: + if unit_check is not None: + if element.get("unit") != unit_check: + raise ValueError( + f"The {xpath} element does not have units of " + f"{unit_check}, has {element.get('unit')}" + ) + el_text = element.text + if el_text: + return el_text + else: + raise ValueError(f"The XML {xpath} element contains no information.") + else: + raise ValueError(f"XML text does not have a {xpath} element.") diff --git a/src/vipersci/vis/db/image_records.py b/src/vipersci/vis/db/image_records.py index 0288b18..e0495f8 100644 --- a/src/vipersci/vis/db/image_records.py +++ b/src/vipersci/vis/db/image_records.py @@ -43,7 +43,7 @@ from sqlalchemy.orm import mapped_column, relationship, synonym, validates from vipersci.pds.pid import VISID, vis_instruments, vis_compression -from vipersci.pds.xml import ns +from vipersci.pds.xml import find_text, ns from vipersci.pds.datetime import fromisozformat, isozformat from vipersci.vis.header import pga_gain as header_pga_gain from vipersci.vis.db import Base @@ -646,29 +646,10 @@ def from_xml(cls, text: str): """ d = {} - def _find_text(root, xpath, unit_check=None): - element = root.find(xpath, ns) - if element is not None: - if unit_check is not None: - if element.get("unit") != unit_check: - raise ValueError( - f"The {xpath} element does not have units of " - f"{unit_check}, has {element.get('unit')}" - ) - el_text = element.text - if el_text: - return el_text - else: - raise ValueError( - f"The XML {xpath} element contains no information." - ) - else: - raise ValueError(f"XML text does not have a {xpath} element.") - root = ET.fromstring(text) - lid = _find_text( - root, "./pds:Identification_Area/pds:logical_identifier" - ).split(":") + lid = find_text(root, "./pds:Identification_Area/pds:logical_identifier").split( + ":" + ) if lid[3] != "viper_vis": raise ValueError( @@ -682,19 +663,19 @@ def _find_text(root, xpath, unit_check=None): d["product_id"] = lid[5] d["auto_exposure"] = ( - True if _find_text(root, ".//img:exposure_type") == "Auto" else False + True if find_text(root, ".//img:exposure_type") == "Auto" else False ) d["bad_pixel_table_id"] = int( - _find_text(root, ".//img:bad_pixel_replacement_table_id") + find_text(root, ".//img:bad_pixel_replacement_table_id") ) d["exposure_duration"] = int( - _find_text(root, ".//img:exposure_duration", unit_check="microseconds") + find_text(root, ".//img:exposure_duration", unit_check="microseconds") ) d["file_creation_datetime"] = fromisozformat( - _find_text(root, ".//pds:creation_date_time") + find_text(root, ".//pds:creation_date_time") ) - d["file_path"] = _find_text(root, ".//pds:file_name") + d["file_path"] = find_text(root, ".//pds:file_name") # for k, v in luminaire_names.items(): # light = root.find(f".//img:LED_Illumination_Source[img:name='{k}']", ns) @@ -703,42 +684,42 @@ def _find_text(root, xpath, unit_check=None): # ) osc = root.find(".//pds:Observing_System_Component[pds:type='Instrument']", ns) - d["instrument_name"] = _find_text(osc, "pds:name") + d["instrument_name"] = find_text(osc, "pds:name") d["instrument_temperature"] = float( - _find_text(root, ".//img:temperature_value", unit_check="K") + find_text(root, ".//img:temperature_value", unit_check="K") ) aa = root.find(".//pds:Axis_Array[pds:axis_name='Line']", ns) - d["lines"] = int(_find_text(aa, "./pds:elements")) - d["file_md5_checksum"] = _find_text(root, ".//pds:md5_checksum") - d["mission_phase"] = _find_text(root, ".//msn:mission_phase_name") - d["offset"] = _find_text(root, ".//img:analog_offset") + d["lines"] = int(find_text(aa, "./pds:elements")) + d["file_md5_checksum"] = find_text(root, ".//pds:md5_checksum") + d["mission_phase"] = find_text(root, ".//msn:mission_phase_name") + d["offset"] = find_text(root, ".//img:analog_offset") try: d["onboard_compression_ratio"] = float( - _find_text(root, ".//img:onboard_compression_ratio") + find_text(root, ".//img:onboard_compression_ratio") ) except ValueError: pass - d["purpose"] = _find_text(root, ".//pds:purpose") + d["purpose"] = find_text(root, ".//pds:purpose") aa = root.find(".//pds:Axis_Array[pds:axis_name='Sample']", ns) - d["samples"] = int(_find_text(aa, "./pds:elements")) + d["samples"] = int(find_text(aa, "./pds:elements")) sw = root.find(".//proc:Software", ns) - d["software_name"] = _find_text(sw, "./proc:name") - d["software_version"] = _find_text(sw, "./proc:software_version_id") - d["software_program_name"] = _find_text(sw, "./proc:Software_Program/proc:name") + d["software_name"] = find_text(sw, "./proc:name") + d["software_version"] = find_text(sw, "./proc:software_version_id") + d["software_program_name"] = find_text(sw, "./proc:Software_Program/proc:name") # Start times must be on the whole second, which is why we don't use # fromisozformat() here. d["start_time"] = datetime.strptime( - _find_text(root, ".//pds:start_date_time"), "%Y-%m-%dT%H:%M:%SZ" + find_text(root, ".//pds:start_date_time"), "%Y-%m-%dT%H:%M:%SZ" ).replace(tzinfo=timezone.utc) - d["stop_time"] = fromisozformat(_find_text(root, ".//pds:stop_date_time")) + d["stop_time"] = fromisozformat(find_text(root, ".//pds:stop_date_time")) return cls(**d) From ce8fd0084588cff60dbe5c34fded2f76af1f8bd5 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Wed, 1 Nov 2023 17:55:22 -0700 Subject: [PATCH 20/60] feat(create_raw.py): Changed the "raw" collection name to its proper "data_raw" form. --- src/vipersci/vis/pds/create_raw.py | 2 +- tests/test_create_raw.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/vipersci/vis/pds/create_raw.py b/src/vipersci/vis/pds/create_raw.py index c3d3597..06f3edf 100644 --- a/src/vipersci/vis/pds/create_raw.py +++ b/src/vipersci/vis/pds/create_raw.py @@ -280,7 +280,7 @@ def label_dict(ir: ImageRecord, lights: dict): pid = pds.VISID(ir.product_id) d = dict( data_quality="", - lid=f"urn:nasa:pds:viper_vis:raw:{ir.product_id}", + lid=f"urn:nasa:pds:viper_vis:data_raw:{ir.product_id}", mission_lid="urn:nasa:pds:viper", sc_lid=_sclid, inst_lid=f"{_sclid}.{_inst}", diff --git a/tests/test_create_raw.py b/tests/test_create_raw.py index 84538bf..2331172 100644 --- a/tests/test_create_raw.py +++ b/tests/test_create_raw.py @@ -141,7 +141,9 @@ def test_get_lights(self): def test_label_dict(self): d = cr.label_dict(self.ir, cr.get_lights(self.ir, self.session)) - self.assertEqual(d["lid"], f"urn:nasa:pds:viper_vis:raw:{self.ir.product_id}") + self.assertEqual( + d["lid"], f"urn:nasa:pds:viper_vis:data_raw:{self.ir.product_id}" + ) self.assertEqual(d["exposure_type"], "Manual") self.assertEqual(d["luminaires"][list(luminaire_names.values())[0]], "Off") self.assertEqual( From 80095340491c6defc86bc608bc83f679dfe2366c Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Wed, 8 Nov 2023 18:50:40 -0800 Subject: [PATCH 21/60] feat(labelmaker & bundle_install): Added these to the repo. --- CHANGELOG.rst | 3 + environment.yml | 1 + environment_dev.yml | 1 + requirements.txt | 1 + requirements_dev.txt | 1 + setup.cfg | 2 + src/vipersci/pds/bundle_install.py | 137 ++++++++++ src/vipersci/pds/labelmaker/__init__.py | 276 ++++++++++++++++++++ src/vipersci/pds/labelmaker/bundle.py | 144 +++++++++++ src/vipersci/pds/labelmaker/cli.py | 51 ++++ src/vipersci/pds/labelmaker/collection.py | 162 ++++++++++++ src/vipersci/pds/labelmaker/generic.py | 60 +++++ src/vipersci/pds/labelmaker/inventory.py | 76 ++++++ tests/test_labelmaker.py | 291 ++++++++++++++++++++++ tests/test_labelmaker_bundle.py | 172 +++++++++++++ tests/test_labelmaker_cli.py | 21 ++ tests/test_labelmaker_collection.py | 183 ++++++++++++++ tests/test_labelmaker_generic.py | 53 ++++ tests/test_labelmaker_inventory.py | 51 ++++ 19 files changed, 1686 insertions(+) create mode 100644 src/vipersci/pds/bundle_install.py create mode 100644 src/vipersci/pds/labelmaker/__init__.py create mode 100644 src/vipersci/pds/labelmaker/bundle.py create mode 100644 src/vipersci/pds/labelmaker/cli.py create mode 100644 src/vipersci/pds/labelmaker/collection.py create mode 100644 src/vipersci/pds/labelmaker/generic.py create mode 100644 src/vipersci/pds/labelmaker/inventory.py create mode 100644 tests/test_labelmaker.py create mode 100644 tests/test_labelmaker_bundle.py create mode 100644 tests/test_labelmaker_cli.py create mode 100644 tests/test_labelmaker_collection.py create mode 100644 tests/test_labelmaker_generic.py create mode 100644 tests/test_labelmaker_inventory.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e04c284..328e0a7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -57,6 +57,9 @@ Added - create_vis_dbs.py - Now also supports spatialite databases, primarily for testing. - create_raw.py - Added components for adding observational intent and data quality to the XML label. +- labelmaker - A program to help build PDS4 bundle and collection labels. +- bundle_install - A program to copy just the files related to a PDS4 bundle into a + new location. Fundamentally allowing a "make install" for PDS4 bundles. Removed ^^^^^^^ diff --git a/environment.yml b/environment.yml index e63e3a3..1f95524 100644 --- a/environment.yml +++ b/environment.yml @@ -26,3 +26,4 @@ dependencies: - shapely - sqlalchemy - tifftools + - yaml \ No newline at end of file diff --git a/environment_dev.yml b/environment_dev.yml index db1ce4c..db97a0f 100644 --- a/environment_dev.yml +++ b/environment_dev.yml @@ -20,4 +20,5 @@ dependencies: - tox - types-pillow - types-requests + - types-pyyaml - twine diff --git a/requirements.txt b/requirements.txt index 1b4be2f..6e6ba29 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,3 +14,4 @@ scikit-learn shapely sqlalchemy tifftools +yaml \ No newline at end of file diff --git a/requirements_dev.txt b/requirements_dev.txt index d7a181a..27d83db 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -10,5 +10,6 @@ tox twine types-pillow types-requests +types-pyyaml wheel watchdog diff --git a/setup.cfg b/setup.cfg index 86159c0..b2912f7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -68,11 +68,13 @@ console_scripts = accrual = vipersci.carto.accrual:main anaglyph = vipersci.vis.anaglyph:main anom_pixel = vipersci.vis.anom_pixel:main + bundle_install = vipersci.pds.bundle_install:main colorforge = vipersci.carto.colorforge:main dice_buffer = vipersci.carto.dice_buffer:main dissolve_dice = vipersci.carto.dissolve_dice:main get_position = vipersci.carto.get_position:main image_stats = vipersci.vis.image_statistics:main + labelmaker = vipersci.pds.labelmaker.cli:main msolo_simulator = vipersci.carto.msolo_simulator:main nirvss_simulator = vipersci.carto.nirvss_simulator:main nss_modeler = vipersci.carto.nss_modeler:main diff --git a/src/vipersci/pds/bundle_install.py b/src/vipersci/pds/bundle_install.py new file mode 100644 index 0000000..b49cdd6 --- /dev/null +++ b/src/vipersci/pds/bundle_install.py @@ -0,0 +1,137 @@ +"""Extracts a "publishable" set of bundle and collection files when pointed at +a bundle directory. +""" + +# Copyright 2023, United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +import argparse +import csv +import logging +from pathlib import Path +from shutil import copy2 +import xml.etree.ElementTree as ET + +from vipersci.pds.labelmaker import get_lidvidfile +from vipersci.pds.xml import find_text, ns +from vipersci import util + +logger = logging.getLogger(__name__) + + +def arg_parser(): + parser = argparse.ArgumentParser( + description=__doc__, parents=[util.parent_parser()] + ) + parser.add_argument( + "source_directory", + type=Path, + help="A directory that contains created Bundle files.", + ) + parser.add_argument( + "build_directory", + type=Path, + help="Path to a directory where the bundle files will be installed.", + ) + return parser + + +def main(): + parser = arg_parser() + args = parser.parse_args() + util.set_logger(args.verbose) + + args.build_directory.mkdir(exist_ok=True) + + copy2(args.source_directory / "bundle.xml", args.build_directory) + + bundle = ET.fromstring((args.build_directory / "bundle.xml").read_text()) + readme = bundle.find("./pds:File_Area_Text/pds:File/pds:file_name", ns) + if readme is not None: + copy2(args.source_directory / readme.text, args.build_directory) + + for bme in bundle.findall(".//pds:Bundle_Member_Entry", ns): + col_lidvid = find_text(bme, "pds:lid_reference") + if find_text(bme, "pds:member_status") == "Primary": + if "::" in col_lidvid: + col_lid, col_vid = col_lidvid.split("::") + else: + col_lid = col_lidvid + col_vid = None + + col_name = col_lid.split(":")[-1] + src_col_dir = args.source_directory / col_name + if src_col_dir.exists() is False: + raise FileNotFoundError(f"{src_col_dir} does not exist.") + + bld_col_dir = args.build_directory / col_name + bld_col_dir.mkdir(exist_ok=True) + + col_label_file = f"collection_{col_name}.xml" + copy2(src_col_dir / col_label_file, bld_col_dir) + collection = ET.fromstring((bld_col_dir / col_label_file).read_text()) + this_col_lid = find_text( + collection, "./pds:Identification_Area/pds:logical_identifier" + ) + if col_lid != this_col_lid: + raise ValueError( + f"The collection lid ({this_col_lid}) does not match the " + f"collection lid that the bundle file has ({col_lid})." + ) + if col_vid is not None: + this_col_vid = find_text( + collection, "./pds:Identification_Area/pds:version_id" + ) + if this_col_vid != col_vid: + raise ValueError( + f"The collection vid ({this_col_vid}) for {col_lid} does not " + f"match the collection vid that the bundle file " + f"has ({col_vid})." + ) + inventory = find_text( + collection, "./pds:File_Area_Inventory/pds:File/pds:file_name" + ) + copy2(src_col_dir / inventory, bld_col_dir) + col_lidvids = set() + with open(str(src_col_dir / inventory), newline="") as csvfile: + reader = csv.reader(csvfile) + for row in reader: + if row[0] == "P": + col_lidvids.add(row[1]) + + # At this point we have a list of product lidvids that we need to find + # in this directory or elsewhere. + for p in src_col_dir.rglob("*.xml"): + f = get_lidvidfile(p) + f_lidvid = f["lid"] + "::" + f["vid"] + if f_lidvid in col_lidvids: + dest_path = bld_col_dir / p.relative_to(src_col_dir) + copy2(p, dest_path) + copy2(p.with_name(f["productfile"]), dest_path.parent) + col_lidvids.remove(f_lidvid) + if len(col_lidvids) == 0: + break + + if len(col_lidvids) != 0: + raise ValueError(f"Could not find the following lidvids: {col_lidvids}") + + return diff --git a/src/vipersci/pds/labelmaker/__init__.py b/src/vipersci/pds/labelmaker/__init__.py new file mode 100644 index 0000000..3b1e577 --- /dev/null +++ b/src/vipersci/pds/labelmaker/__init__.py @@ -0,0 +1,276 @@ +"""Helps to build PDS4 Bundle and Collection XML labels and files. +""" + +# Copyright 2023, United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +import csv +import logging +from pathlib import Path +from typing import List, Dict, Optional +import xml.etree.ElementTree as ET + +import pandas as pd + +from vipersci.pds.datetime import fromisozformat, isozformat +from vipersci.pds.xml import find_text, ns +from genshi.template import MarkupTemplate + +logger = logging.getLogger(__name__) + + +def get_common_label_info(element: ET.Element, area="pds:Observation_Area"): + """ + Returns a dict of information harvested from *element* which contains + PDS4 label information. + + The *area* argument contains the XML element which contains elements + like start_date_time. This is likely either 'pds:Observation_Area' (the + default) or 'pds:Context_Area'. + """ + if area is not None: + osc = element.find(".//pds:Observing_System_Component[pds:type='Host']", ns) + + host_name = find_text(osc, "pds:name") + host_lid = find_text(osc, "pds:Internal_Reference/pds:lid_reference") + + instruments = {} + for i in element.findall( + ".//pds:Observing_System_Component[pds:type='Instrument']", ns + ): + instruments[ + find_text(i, "pds:Internal_Reference/pds:lid_reference") + ] = find_text(i, "pds:name") + + purposes = [] + for p in element.findall( + f"./{area}/pds:Primary_Result_Summary/pds:purpose", ns + ): + purposes.append(p.text) + + processing_levels = [] + for p in element.findall( + f"./{area}/pds:Primary_Result_Summary/pds:processing_level", ns + ): + processing_levels.append(p.text) + else: + host_name = None + host_lid = None + instruments = None + purposes = None + processing_levels = None + + lid = find_text(element, "./pds:Identification_Area/pds:logical_identifier") + d = { + "lid": lid, + "vid": find_text(element, "./pds:Identification_Area/pds:version_id"), + "start_date_time": None + if area is None + else fromisozformat( + find_text(element, f"./{area}/pds:Time_Coordinates/pds:start_date_time") + ), + "stop_date_time": None + if area is None + else fromisozformat( + find_text(element, f"./{area}/pds:Time_Coordinates/pds:stop_date_time") + ), + "investigation_name": None + if area is None + else find_text(element, f"./{area}/pds:Investigation_Area/pds:name"), + "investigation_type": None + if area is None + else find_text(element, f"./{area}/pds:Investigation_Area/pds:type"), + "investigation_lid": None + if area is None + else find_text( + element, + f"./{area}/pds:Investigation_Area/pds:Internal_Reference/pds:lid_reference", + ), + "host_name": host_name, + "host_lid": host_lid, + "target_name": None + if area is None + else find_text(element, ".//pds:Target_Identification/pds:name"), + "target_type": None + if area is None + else find_text(element, ".//pds:Target_Identification/pds:type"), + "target_lid": None + if area is None + else find_text( + element, + ".//pds:Target_Identification/pds:Internal_Reference/pds:lid_reference", + ), + "instruments": instruments, + "purposes": purposes, + "processing_levels": processing_levels, + } + + return d + + +def get_lidvid(element: ET.Element): + return { + "lid": find_text(element, "./pds:Identification_Area/pds:logical_identifier"), + "vid": find_text(element, "./pds:Identification_Area/pds:version_id"), + } + + +def get_lidvidfile(path: Path) -> dict: + """ + Returns a dict with three keys: 'lid', 'vid', and 'productfile' whose values + are extracted from the XML file at *path*. + + This is a convenience function to read a PDS4 XML file and extract the following " + values: + ./pds:Identification_Area/pds:logical_identifier, + ./pds:Identification_Area/pds:version_id, and + ./pds:File_Area_Observational/pds:File/pds:file_name . + """ + logger.info(f"Reading {path}") + + root = ET.fromstring(path.read_text()) + + d = get_lidvid(root) + for fxpath in ( + "./pds:File_Area_Observational/pds:File/pds:file_name", + "./pds:Document/pds:Document_Edition/pds:Document_File/pds:file_name", + ): + element = root.find(fxpath, ns) + if element is not None: + el_text = element.text + if el_text: + d["productfile"] = el_text + break + else: + d["productfile"] = None + + return d + + +def gather_info(df, modification_details: List[Dict]): + """ + Returns a dict of information harvested from the provided pandas dataframe, *df*. + + The dataframe is primarily derived from a list of dicts produced by the + get_common_label_info() function. + """ + d = { + "vid": str(vid_max(modification_details, pd.to_numeric(df["vid"]).max())), + "instruments": {}, + "purposes": set(), + "processing_levels": set(), + "start_date_time": isozformat(df["start_date_time"].min()), + "stop_date_time": isozformat(df["stop_date_time"].max()), + } + + for inst_dict in df["instruments"].dropna().tolist(): + d["instruments"].update(inst_dict) + + for p in df["purposes"].dropna().tolist(): + d["purposes"].update(set(p)) + + for pl in df["processing_levels"].dropna().tolist(): + d["processing_levels"].update(set(pl)) + + return d + + +def assert_unique(value, series: pd.Series): + """ + If the *series* has only a single unique value and that value is identical + to *value* then this function silently returns. Otherwise it will raise + a ValueError. + + If the *series* has more than one unique value, this function will raise a + ValueError. If the *series* has a single unique value, but it is not equivalent + to *value*, this function will raise a ValueError. + """ + s = series.dropna() + if s.nunique() == 1: + col_val = s.unique()[0] + if value != col_val: + raise ValueError( + f"The series has a unique value ({col_val}) which does not match the " + f"provided value ({value}) for {series.name}." + ) + else: + raise ValueError( + f"The series has more than one unique value of {series.name}: " + f"{series.unique()} " + ) + return + + +def vid_max(modification_details: List[Dict], max_product_vid: Optional[float] = None): + """ + Returns the maximum VID from the *modification_details* list, and if + *max_product_vid* is not None, then this maximum VID is checked to at least + be greater than *max_product_vid*, otherwise a ValueError is raised. + """ + mod_vids = [] + for detail in modification_details: + mod_vids.append(float(detail["version"])) + max_mod_vid = max(mod_vids) + if max_product_vid is not None: + if max_mod_vid < max_product_vid: + raise ValueError( + f"The largest version in the configuration file is {max_mod_vid} but " + f"the largest version amongst the labels is {max_product_vid} ." + ) + return max_mod_vid + + +def write_inventory(path: Path, labels: List[Dict], member="P"): + """ + Writes a PDS4 collection inventory CSV file to *path* based on the list of + dicts in *labels*. + + Those dictionaries should have a 'lid' and a 'vid' keyword. The value of *member* + indicates whether the label is a Primary (P) member or a Secondary (S) member of + the collection. If the dictionaries in *labels* have a "member" key, that value + will be used instead of *member* for that output row. + """ + with open(path, "w", newline="") as csvfile: + writer = csv.writer(csvfile) + + for label in labels: + lidvid = label["lid"] + "::" + label["vid"] + m = label.get("member", member) + writer.writerow([m, lidvid]) + + return + + +def write_xml(metadata: dict, outpath: Path, template_path: Path): + """ + Convenience function to write an XML file at *outpath* that is the + result of taking the information in *metadata* and populating the XML + template at *template_path* with that information. + + This function uses the genshi markup library: https://genshi.edgewall.org + """ + tmpl = MarkupTemplate(template_path.read_text()) + logger.debug(metadata) + + stream = tmpl.generate(**metadata) + outpath.write_text(stream.render()) + return diff --git a/src/vipersci/pds/labelmaker/bundle.py b/src/vipersci/pds/labelmaker/bundle.py new file mode 100644 index 0000000..0edbee1 --- /dev/null +++ b/src/vipersci/pds/labelmaker/bundle.py @@ -0,0 +1,144 @@ +"""Creates a PDS4 Bundle XML file from the provided Collection XML labels. +""" + +# Copyright 2023, United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +import logging +from pathlib import Path +import xml.etree.ElementTree as ET + +import pandas as pd +import yaml + +from vipersci.pds.labelmaker import ( + assert_unique, + gather_info, + get_common_label_info, + write_xml, +) +from vipersci.pds.xml import find_text, ns +from vipersci import util + +logger = logging.getLogger(__name__) + +# This definition is from Appendix K.2.1.6 (Bundle_Member_Entry) in the +# https://pds.nasa.gov/datastandards/documents/dph/current/PDS4_DataProvidersHandbook_1.20.0.pdf +collection_reference_type = { + "Browse": "bundle_has_browse_collection", + "Calibration": "bundle_has_calibration_collection", + "Context": "bundle_has_context_collection", + "Data": "bundle_has_data_collection", + "Document": "bundle_has_document_collection", + "Geometry": "bundle_has_geometry_collection", + "Miscellaneous": "bundle_has_miscellaneous_collection", + "SPICE Kernel": "bundle_has_spice_kernel_collection", + "Schema": "bundle_has_schema_collection", +} + + +def add_parser(subparsers): + parser = subparsers.add_parser("bundle", help=__doc__) + + parser.add_argument( + "-c", "--config", type=Path, help="YAML file with configuration parameters." + ) + parser.add_argument( + "-t", "--template", type=Path, help="Genshi XML file template. " + ) + parser.add_argument( + "labelfiles", + type=Path, + nargs="+", + help="Path(s) to all Collection XML label files to be included in the output " + "Bundle label file.", + ) + parser.set_defaults(func=main) + return parser + + +def main(args): + util.set_logger(args.verbose) + + d = yaml.safe_load(args.config.read_text()) + + outpath = Path("bundle.xml") + if outpath.exists(): + raise FileExistsError(f"The {outpath} file already exists.") + + labelinfo = [] + for labelpath in args.labelfiles: + logger.info(f"Reading {labelpath}") + labelinfo.append(get_label_info(labelpath)) + + d.update(check_and_derive(d, labelinfo)) + + write_xml(d, outpath, args.template) + logger.info(f"Wrote {outpath}") + + return + + +def check_and_derive(config: dict, labelinfo: list): + df = pd.DataFrame(labelinfo) + + # Check consistency for gathered labels: + for x in ( + "bundle_lid", + "investigation_name", + "investigation_type", + "investigation_lid", + "host_name", + "host_lid", + "target_name", + "target_type", + "target_lid", + ): + assert_unique(config[x], df[x]) + + # Generate values from gathered labels: + d = gather_info(df, config["modification_details"]) + + collections = [] + for label in labelinfo: + collections.append( + { + "lid": label["lid"] + "::" + label["vid"], + "type": collection_reference_type[label["collection_type"]], + } + ) + d["collections"] = collections + + return d + + +def get_label_info(path: Path) -> dict: + root = ET.fromstring(path.read_text()) + + if root.find("pds:Context_Area", ns) is not None: + d = get_common_label_info(root, "pds:Context_Area") + else: + d = get_common_label_info(root, None) + + d["bundle_lid"] = ":".join(d["lid"].split(":")[:-1]) + d["collection_type"] = find_text(root, ".//pds:Collection/pds:collection_type") + return d diff --git a/src/vipersci/pds/labelmaker/cli.py b/src/vipersci/pds/labelmaker/cli.py new file mode 100644 index 0000000..06b7f4a --- /dev/null +++ b/src/vipersci/pds/labelmaker/cli.py @@ -0,0 +1,51 @@ +"""Helps to build PDS4 Bundle and Collection XML labels and files. +""" + +# Copyright 2023, United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +import argparse + +from vipersci.pds.labelmaker.bundle import add_parser as bundle_ap +from vipersci.pds.labelmaker.collection import add_parser as collection_ap +from vipersci.pds.labelmaker.generic import add_parser as generic_ap +from vipersci.pds.labelmaker.inventory import add_parser as inventory_ap +from vipersci import util + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, parents=[util.parent_parser()] + ) + subparsers = parser.add_subparsers( + title="subcommands", + description="valid subcommands", + help="additional help available via '%(prog)s subcommand -h'", + ) + bundle_ap(subparsers) + collection_ap(subparsers) + inventory_ap(subparsers) + generic_ap(subparsers) + args = parser.parse_args() + args.func(args) + + return diff --git a/src/vipersci/pds/labelmaker/collection.py b/src/vipersci/pds/labelmaker/collection.py new file mode 100644 index 0000000..93f9f66 --- /dev/null +++ b/src/vipersci/pds/labelmaker/collection.py @@ -0,0 +1,162 @@ +"""Creates a PDS4 Collection XML file from the provided XML labels. +""" + +# Copyright 2023, United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +from datetime import datetime, timezone +import logging +from pathlib import Path +import xml.etree.ElementTree as ET + +import pandas as pd +import yaml + +from vipersci.pds.datetime import isozformat +from vipersci.pds.labelmaker import ( + assert_unique, + gather_info, + get_common_label_info, + vid_max, + write_inventory, + write_xml, +) +from vipersci.pds.xml import ns +from vipersci import util + +logger = logging.getLogger(__name__) + + +def add_parser(subparsers): + parser = subparsers.add_parser("collection", help=__doc__) + + parser.add_argument( + "-c", "--config", type=Path, help="YAML file with configuration parameters." + ) + parser.add_argument( + "--csv", + type=Path, + help="The collections CSV file. If not given, one will be generated.", + ) + parser.add_argument( + "-t", "--template", type=Path, required=True, help="Genshi XML file template. " + ) + parser.add_argument( + "labelfiles", + type=Path, + nargs="+", + help="Path(s) to all XML label files to be included in the output collection " + "file.", + ) + parser.set_defaults(func=main) + return parser + + +def main(args): + util.set_logger(args.verbose) + + d = yaml.safe_load(args.config.read_text()) + + name = d["collection_lid"].split(":")[-1] + outpath = Path(f"collection_{name}.xml") + if outpath.exists(): + raise FileExistsError(f"The file {outpath} already exists.") + + labelinfo = [] + for labelpath in args.labelfiles: + logger.info(f"Reading {labelpath}") + labelinfo.append(get_label_info(labelpath)) + + d.update(check_and_derive(d, labelinfo)) + + if args.csv is None: + csv_path = Path(f"collection_{name}.csv") + if csv_path.exists(): + raise FileExistsError(f"The file {csv_path} already exists.") + + write_inventory(csv_path, labelinfo) + logger.info(f"Wrote {csv_path}") + d["number_of_records"] = len(labelinfo) + else: + csv_path = args.csv + d["number_of_records"] = len(str.splitlines(args.csv.read_text())) + + d["collection_csv"] = csv_path + d["file_creation_datetime"] = isozformat( + datetime.fromtimestamp(csv_path.stat().st_mtime, timezone.utc) + ) + + write_xml(d, outpath, args.template) + logger.info(f"Wrote {outpath}") + + return + + +def check_and_derive(config: dict, labelinfo: list): + df = pd.DataFrame(labelinfo) + + check = { + "Data": ( + "collection_lid", + "investigation_name", + "investigation_type", + "investigation_lid", + "host_name", + "host_lid", + "target_name", + "target_type", + "target_lid", + ), + "Document": ("collection_lid",), + } + + # Check consistency for gathered labels: + for x in check[config["collection_type"]]: + assert_unique(config[x], df[x]) + + # Generate values from gathered labels: + if config["collection_type"] == "Data": + d = gather_info(df, config["modification_details"]) + elif config["collection_type"] == "Document": + d = { + "vid": str( + vid_max(config["modification_details"], pd.to_numeric(df["vid"]).max()) + ) + } + else: + raise NotImplementedError( + f"Do not have a strategy for {config['collection_type']} collection types." + ) + + return d + + +def get_label_info(path: Path) -> dict: + root = ET.fromstring(path.read_text()) + + if root.find("pds:Observation_Area", ns) is not None: + d = get_common_label_info(root, "pds:Observation_Area") + else: + d = get_common_label_info(root, None) + + d["collection_lid"] = ":".join(d["lid"].split(":")[:-1]) + return d diff --git a/src/vipersci/pds/labelmaker/generic.py b/src/vipersci/pds/labelmaker/generic.py new file mode 100644 index 0000000..b635909 --- /dev/null +++ b/src/vipersci/pds/labelmaker/generic.py @@ -0,0 +1,60 @@ +"""Generically takes a Genshi XML template and a JSON or YAML file to complete +the template. +""" + +# Copyright 2021-2022, United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +import json +import logging +from pathlib import Path +import yaml + +from vipersci.pds.labelmaker import write_xml +from vipersci import util + +logger = logging.getLogger(__name__) + + +def add_parser(subparsers): + parser = subparsers.add_parser("generic", help=__doc__) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("-j", "--json", type=Path, help="Path to .json file to load.") + group.add_argument("-y", "--yaml", type=Path, help="Path to .yml file to load.") + parser.add_argument("template", type=Path, help="Genshi XML file template.") + parser.add_argument("output", type=Path, help="Output XML label.") + parser.set_defaults(func=main) + return parser + + +def main(args): + util.set_logger(args.verbose) + + info = {} + if args.json is not None: + info = json.loads(args.json.read_text()) + elif args.yaml is not None: + info = yaml.safe_load(args.yaml.read_text()) + + write_xml(info, args.output, args.template) + + return diff --git a/src/vipersci/pds/labelmaker/inventory.py b/src/vipersci/pds/labelmaker/inventory.py new file mode 100644 index 0000000..b4afebf --- /dev/null +++ b/src/vipersci/pds/labelmaker/inventory.py @@ -0,0 +1,76 @@ +"""Creates a PDS4 Collection Inventory CSV file from the provided XML labels. +""" + +# Copyright 2023, United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +import logging +from pathlib import Path + +from vipersci.pds.labelmaker import get_lidvidfile, write_inventory +from vipersci import util + +logger = logging.getLogger(__name__) + + +def add_parser(subparsers): + parser = subparsers.add_parser("inventory", help=__doc__) + + parser.add_argument( + "-n", + "--name", + required=True, + help="The name that will be used to form the name of the output file. If " + "'-n foobar' is given, the output file will be collection_foobar.csv ", + ) + parser.add_argument( + "-m", + "--member", + choices=["P", "S"], + default="P", + help="Indicates whether the products are a primary (P) or secondary (S) member " + "of the collection. All products will be marked identically. " + "Default: %(default)s", + ) + parser.add_argument( + "labelfiles", + type=Path, + nargs="+", + help="Path(s) to all XML label files to be included in the output collection " + "file.", + ) + parser.set_defaults(func=main) + return parser + + +def main(args): + util.set_logger(args.verbose) + + outpath = Path(f"collection_{args.name}.csv") + + labels = [] + for path in args.labelfiles: + labels.append(get_lidvidfile(path)) + + write_inventory(outpath, labels, args.member) + + return diff --git a/tests/test_labelmaker.py b/tests/test_labelmaker.py new file mode 100644 index 0000000..939723d --- /dev/null +++ b/tests/test_labelmaker.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python +"""This module has tests for the vis.pds.labelmaker functions.""" + +# Copyright 2023, vipersci developers. +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +from datetime import datetime, timezone +from pathlib import Path +from textwrap import dedent +import unittest +from unittest.mock import call, create_autospec, mock_open, patch +import xml.etree.ElementTree as ET + +from genshi.template import Template +import pandas as pd + +import vipersci.pds.labelmaker as lm + + +class TestFunctions(unittest.TestCase): + def setUp(self): + self.xmltext = dedent( + """\ + + + + + urn:nasa:pds:dummy_lid + 1.0 + + + 2023-10-28 + 0.1 + Bogus testing version + + + + + + 2023-10-26T20:00:00Z + 2023-10-26T20:00:00.000511Z + + + Engineering + Raw + + + MISSIONNAME + Mission + + urn:nasa:pds:missionname + collection_to_investigation + + + + + MISSIONNAME + Host + + urn:nasa:pds:context:instrument_host:spacecraft.missionname + is_instrument_host + + + + NavCam Left + Instrument + + urn:nasa:pds:context:instrument_host:spacecraft.missionname.navcam_left + is_instrument + + + NavCam Right + Instrument + + urn:nasa:pds:context:instrument_host:spacecraft.missionname.navcam_right + is_instrument + + + + + Moon + Satellite + + urn:nasa:pds:context:target:satellite.earth.moon + collection_to_target + + + + + Data + + + + collection_data_raw.csv + 2023-11-02T23:12:59.083415Z + + + 0 + PDS DSV 1 + 4 + Carriage-Return Line-Feed + Comma + + + + """ + ) + + def test_get_lidvidfile(self): + mock_path = create_autospec(Path) + mock_path.read_text.return_value = dedent( + """\ + + + + + urn:nasa:pds:dummy_lid + 1.0 + + + + product.name + + + + """ + ) + + truth = { + "lid": "urn:nasa:pds:dummy_lid", + "vid": "1.0", + "productfile": "product.name", + } + + d = lm.get_lidvidfile(mock_path) + + self.assertEqual(truth, d) + + def test_get_common_label_info(self): + root = ET.fromstring(self.xmltext) + host_lid = "urn:nasa:pds:context:instrument_host:spacecraft.missionname" + truth = { + "lid": "urn:nasa:pds:dummy_lid", + "vid": "1.0", + "start_date_time": datetime(2023, 10, 26, 20, 00, 00, tzinfo=timezone.utc), + "stop_date_time": datetime( + 2023, 10, 26, 20, 00, 00, 511, tzinfo=timezone.utc + ), + "investigation_name": "MISSIONNAME", + "investigation_type": "Mission", + "investigation_lid": "urn:nasa:pds:missionname", + "host_name": "MISSIONNAME", + "host_lid": host_lid, + "target_name": "Moon", + "target_type": "Satellite", + "target_lid": "urn:nasa:pds:context:target:satellite.earth.moon", + "instruments": { + f"{host_lid}.navcam_left": "NavCam Left", + f"{host_lid}.navcam_right": "NavCam Right", + }, + "purposes": [ + "Engineering", + ], + "processing_levels": [ + "Raw", + ], + } + d = lm.get_common_label_info(root, area="pds:Context_Area") + self.assertEqual(truth, d) + + def test_gather_info(self): + df = pd.DataFrame( + data={ + "vid": ["0.1", "0.1", "1.0"], + "instruments": [ + { + "urn:nasa:pds:ncl": "NavCam Left", + "urn:nasa:pds:ncr": "NavCam Right", + }, + { + "urn:nasa:pds:ncl": "NavCam Left", + "urn:nasa:pds:ncr": "NavCam Right", + }, + {"urn:nasa:pds:acl": "AftCam Left"}, + ], + "purposes": [ + [ + "Engineering", + ], + [ + "Science", + ], + [ + "Science", + ], + ], + "processing_levels": [ + [ + "Raw", + ], + [ + "Raw", + ], + [ + "Derived", + ], + ], + "start_date_time": [ + datetime(2023, 10, 1, 20, 00, 00, tzinfo=timezone.utc), + datetime(2023, 10, 15, 20, 00, 00, tzinfo=timezone.utc), + datetime(2023, 11, 1, 20, 00, 00, tzinfo=timezone.utc), + ], + "stop_date_time": [ + datetime(2023, 10, 1, 20, 00, 1, tzinfo=timezone.utc), + datetime(2023, 10, 15, 20, 00, 1, tzinfo=timezone.utc), + datetime(2023, 11, 1, 20, 00, 1, tzinfo=timezone.utc), + ], + } + ) + + truth = { + "vid": "2.0", + "instruments": { + "urn:nasa:pds:ncl": "NavCam Left", + "urn:nasa:pds:ncr": "NavCam Right", + "urn:nasa:pds:acl": "AftCam Left", + }, + "purposes": {"Engineering", "Science"}, + "processing_levels": {"Raw", "Derived"}, + "start_date_time": "2023-10-01T20:00:00Z", + "stop_date_time": "2023-11-01T20:00:01Z", + } + d = lm.gather_info(df, [{"version": "0.1"}, {"version": "2.0"}]) + self.assertEqual(truth, d) + + def test_assert_unique(self): + self.assertIsNone(lm.assert_unique("a", pd.Series(["a", "a"]))) + self.assertRaises(ValueError, lm.assert_unique, "a", pd.Series(["b", "b"])) + self.assertRaises(ValueError, lm.assert_unique, "a", pd.Series(["a", "b"])) + + def test_vid_max(self): + mod_details = [ + {"version": "1.0"}, + {"version": "1.1"}, + {"version": "2.0"}, + ] + self.assertEqual(lm.vid_max(mod_details), 2.0) + self.assertEqual(lm.vid_max(mod_details, 2.0), 2.0) + self.assertRaises(ValueError, lm.vid_max, mod_details, 3.0) + + def test_write_inventory(self): + m = mock_open() + labels = [ + {"lid": "lid1", "vid": "1.0"}, + {"lid": "lid2", "vid": "1.0"}, + ] + with patch("vipersci.pds.labelmaker.open", m): + lm.write_inventory(Path("dummy_path"), labels) + + handle = m() + handle.write.assert_has_calls( + [call("P,lid1::1.0\r\n"), call("P,lid2::1.0\r\n")] + ) + + def test_write_xml(self): + m_path = create_autospec(Path) + m_tmpl_path = create_autospec(Path) + m_tmpl = create_autospec(Template) + with patch( + "vipersci.pds.labelmaker.MarkupTemplate", return_value=m_tmpl + ) as m_markup: + lm.write_xml({"lid": "dummy_lid", "vid": "1.0"}, m_path, m_tmpl_path) + + m_tmpl_path.read_text.assert_called_once() + m_markup.assert_called_once() + m_tmpl.generate.assert_called_once() + m_path.write_text.assert_called_once() diff --git a/tests/test_labelmaker_bundle.py b/tests/test_labelmaker_bundle.py new file mode 100644 index 0000000..90f4f7e --- /dev/null +++ b/tests/test_labelmaker_bundle.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python +"""This module has tests for the vis.pds.labelmaker.bundle functions.""" + +# Copyright 2023, vipersci developers. +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +from argparse import ArgumentParser +from datetime import datetime, timezone +from pathlib import Path +import unittest +from unittest.mock import create_autospec, patch + +import vipersci.pds.labelmaker.bundle as bun +from vipersci import util + + +class TestCollection(unittest.TestCase): + def setUp(self): + self.base = { + "bundle_lid": "urn.pds:bundle", + "investigation_name": "nameo", + "investigation_type": "typeo", + "investigation_lid": "urn:pds:nameo", + "host_name": "hosto", + "host_lid": "urn:pds:hosto", + "target_name": "Moon", + "target_type": "Satellite", + "target_lid": "urn:pds:Moon", + } + + self.label1 = { + "lid": "urn.pds:bundle:collection", + "collection_type": "Data", + "vid": "0.1", + "instruments": {"urn:pds:leftcam": "NavCam Left"}, + "purposes": [ + "Science", + ], + "processing_levels": [ + "Raw", + ], + "start_date_time": datetime(2023, 10, 1, 20, 00, 00, tzinfo=timezone.utc), + "stop_date_time": datetime(2023, 10, 1, 20, 00, 1, tzinfo=timezone.utc), + } + self.label1.update(self.base) + + self.label2 = { + "lid": "urn.pds:bundle:collection", + "collection_type": "Data", + "vid": "2.0", + "instruments": {"urn:pds:rightcam": "NavCam Right"}, + "purposes": [ + "Science", + ], + "processing_levels": [ + "Derived", + ], + "start_date_time": datetime(2023, 11, 1, 20, 00, 00, tzinfo=timezone.utc), + "stop_date_time": datetime(2023, 11, 1, 20, 00, 1, tzinfo=timezone.utc), + } + self.label2.update(self.base) + + def test_add_parser(self): + parser = ArgumentParser() + subparsers = parser.add_subparsers() + bun.add_parser(subparsers) + + d = vars( + parser.parse_args( + [ + "bundle", + "--config", + "dumb.yml", + "-t", + "template.xml", + "file1.xml", + "file2.xml", + ] + ) + ) + self.assertIn("config", d) + self.assertIn("template", d) + self.assertIn("labelfiles", d) + + @patch("vipersci.pds.labelmaker.bundle.ET.fromstring") + @patch( + "vipersci.pds.labelmaker.bundle.get_common_label_info", + return_value={"lid": "urn:pds:dummy:pid"}, + ) + def test_get_label_info(self, m_gcli, m_fromstring): + p = create_autospec(Path) + d = bun.get_label_info(p) + + m_fromstring.assert_called_once() + m_gcli.assert_called_once() + self.assertEqual(d["bundle_lid"], "urn:pds:dummy") + + def test_check_and_derive(self): + config = self.base.copy() + config["modification_details"] = [{"version": "1.0"}, {"version": "2.0"}] + + d = bun.check_and_derive(config, [self.label1, self.label2]) + + self.assertEqual( + { + "collections": [ + { + "lid": "urn.pds:bundle:collection::0.1", + "type": "bundle_has_data_collection", + }, + { + "lid": "urn.pds:bundle:collection::2.0", + "type": "bundle_has_data_collection", + }, + ], + "vid": "2.0", + "instruments": { + "urn:pds:leftcam": "NavCam Left", + "urn:pds:rightcam": "NavCam Right", + }, + "purposes": {"Science"}, + "processing_levels": {"Derived", "Raw"}, + "start_date_time": "2023-10-01T20:00:00Z", + "stop_date_time": "2023-11-01T20:00:01Z", + }, + d, + ) + + def test_main(self): + parser = ArgumentParser(parents=[util.parent_parser()]) + subparsers = parser.add_subparsers() + bun.add_parser(subparsers) + + args = parser.parse_args( + [ + "bundle", + "-c", + "dummy.yml", + "-t", + "template.xml", + "file1.xml", + "file2.xml", + ] + ) + + args.config = create_autospec(Path) + + config = self.base.copy() + config["modification_details"] = [{"version": "1.0"}, {"version": "2.0"}] + + path_mock = create_autospec(Path) + path_mock.exists.return_value = False + + with patch( + "vipersci.pds.labelmaker.bundle.yaml.safe_load", return_value=config + ) as m_yamlload, patch( + "vipersci.pds.labelmaker.bundle.get_label_info", + side_effect=(self.label1, self.label2), + ) as m_get_label, patch( + "vipersci.pds.labelmaker.bundle.write_xml" + ) as m_write_xml, patch( + "vipersci.pds.labelmaker.bundle.Path", return_value=path_mock + ) as m_path: + bun.main(args) + + m_yamlload.assert_called_once() + self.assertEqual(m_get_label.call_count, 2) + m_path.assert_called_once() + m_write_xml.assert_called_once() diff --git a/tests/test_labelmaker_cli.py b/tests/test_labelmaker_cli.py new file mode 100644 index 0000000..8ef070f --- /dev/null +++ b/tests/test_labelmaker_cli.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +"""This module has tests for the vis.pds.labelmaker.inventory functions.""" + +# Copyright 2023, vipersci developers. +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +import unittest +from unittest.mock import patch + +import vipersci.pds.labelmaker.cli as cli + + +class TestMain(unittest.TestCase): + @patch("vipersci.pds.labelmaker.cli.argparse.ArgumentParser", autospec=True) + def test_main(self, m_parser): + cli.main() + + m_parser.assert_called() diff --git a/tests/test_labelmaker_collection.py b/tests/test_labelmaker_collection.py new file mode 100644 index 0000000..ce4ce4f --- /dev/null +++ b/tests/test_labelmaker_collection.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python +"""This module has tests for the vis.pds.labelmaker.collection functions.""" + +# Copyright 2023, vipersci developers. +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +from argparse import ArgumentParser +from datetime import datetime, timezone +from pathlib import Path +import unittest +from unittest.mock import create_autospec, patch + +import vipersci.pds.labelmaker.collection as co +from vipersci import util + + +class TestCollection(unittest.TestCase): + def setUp(self): + self.base = { + "collection_lid": "urn.pds:collection", + "investigation_name": "nameo", + "investigation_type": "typeo", + "investigation_lid": "urn:pds:nameo", + "host_name": "hosto", + "host_lid": "urn:pds:hosto", + "target_name": "Moon", + "target_type": "Satellite", + "target_lid": "urn:pds:Moon", + } + + self.label1 = { + "vid": "0.1", + "instruments": {"urn:pds:leftcam": "NavCam Left"}, + "purposes": [ + "Science", + ], + "processing_levels": [ + "Raw", + ], + "start_date_time": datetime(2023, 10, 1, 20, 00, 00, tzinfo=timezone.utc), + "stop_date_time": datetime(2023, 10, 1, 20, 00, 1, tzinfo=timezone.utc), + } + self.label1.update(self.base) + + self.label2 = { + "vid": "2.0", + "instruments": {"urn:pds:rightcam": "NavCam Right"}, + "purposes": [ + "Science", + ], + "processing_levels": [ + "Derived", + ], + "start_date_time": datetime(2023, 11, 1, 20, 00, 00, tzinfo=timezone.utc), + "stop_date_time": datetime(2023, 11, 1, 20, 00, 1, tzinfo=timezone.utc), + } + self.label2.update(self.base) + + def test_add_parser(self): + parser = ArgumentParser() + subparsers = parser.add_subparsers() + co.add_parser(subparsers) + + d = vars( + parser.parse_args( + [ + "collection", + "--config", + "dumb.yml", + "-t", + "template.xml", + "file1.xml", + "file2.xml", + ] + ) + ) + self.assertIn("config", d) + self.assertIn("template", d) + self.assertIn("labelfiles", d) + + @patch("vipersci.pds.labelmaker.collection.ET.fromstring") + @patch( + "vipersci.pds.labelmaker.collection.get_common_label_info", + return_value={"lid": "urn:pds:dummy:pid"}, + ) + def test_get_label_info(self, m_gcli, m_fromstring): + p = create_autospec(Path) + d = co.get_label_info(p) + + m_fromstring.assert_called_once() + m_gcli.assert_called_once() + self.assertEqual(d["collection_lid"], "urn:pds:dummy") + + def test_check_and_derive(self): + config = self.base.copy() + config["collection_type"] = "Data" + config["modification_details"] = [{"version": "1.0"}, {"version": "2.0"}] + + d = co.check_and_derive(config, [self.label1, self.label2]) + + self.assertEqual( + { + "vid": "2.0", + "instruments": { + "urn:pds:leftcam": "NavCam Left", + "urn:pds:rightcam": "NavCam Right", + }, + "purposes": {"Science"}, + "processing_levels": {"Derived", "Raw"}, + "start_date_time": "2023-10-01T20:00:00Z", + "stop_date_time": "2023-11-01T20:00:01Z", + }, + d, + ) + + def test_main(self): + parser = ArgumentParser(parents=[util.parent_parser()]) + subparsers = parser.add_subparsers() + co.add_parser(subparsers) + + args = parser.parse_args( + [ + "collection", + "-c", + "dummy.yml", + "-t", + "template.xml", + "file1.xml", + "file2.xml", + ] + ) + + args.config = create_autospec(Path) + + config = self.base.copy() + config["collection_type"] = "Data" + config["modification_details"] = [{"version": "1.0"}, {"version": "2.0"}] + + path_mock = create_autospec(Path) + path_mock.exists.return_value = False + + with patch( + "vipersci.pds.labelmaker.collection.yaml.safe_load", return_value=config + ) as m_yamlload, patch( + "vipersci.pds.labelmaker.collection.get_label_info", + side_effect=(self.label1, self.label2), + ) as m_get_label, patch( + "vipersci.pds.labelmaker.collection.write_inventory" + ) as m_write_inventory, patch( + "vipersci.pds.labelmaker.collection.write_xml" + ) as m_write_xml, patch( + "vipersci.pds.labelmaker.collection.Path", return_value=path_mock + ) as m_path: + co.main(args) + + m_yamlload.assert_called_once() + self.assertEqual(m_get_label.call_count, 2) + self.assertEqual(m_path.call_count, 2) + m_write_inventory.assert_called_once() + m_write_xml.assert_called_once() + + # The collection XML label already exists: + path_mock.exists.return_value = True + with patch( + "vipersci.pds.labelmaker.collection.yaml.safe_load", return_value=config + ), patch("vipersci.pds.labelmaker.collection.Path", return_value=path_mock): + self.assertRaises(FileExistsError, co.main, args) + + # The collection XML label doesn't exist, but the collection inventory does. + path_mock2 = create_autospec(Path) + path_mock2.exists.side_effect = (False, True) + with patch( + "vipersci.pds.labelmaker.collection.yaml.safe_load", return_value=config + ), patch( + "vipersci.pds.labelmaker.collection.get_label_info", + side_effect=(self.label1, self.label2), + ), patch( + "vipersci.pds.labelmaker.collection.Path", return_value=path_mock2 + ): + self.assertRaises(FileExistsError, co.main, args) diff --git a/tests/test_labelmaker_generic.py b/tests/test_labelmaker_generic.py new file mode 100644 index 0000000..85b0e07 --- /dev/null +++ b/tests/test_labelmaker_generic.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +"""This module has tests for the vis.pds.labelmaker.generic functions.""" + +# Copyright 2023, vipersci developers. +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +from argparse import ArgumentParser +from pathlib import Path +import unittest +from unittest.mock import create_autospec, patch + +import vipersci.pds.labelmaker.generic as gen +from vipersci import util + + +class TestParser(unittest.TestCase): + def test_add_parser(self): + parser = ArgumentParser() + subparsers = parser.add_subparsers() + gen.add_parser(subparsers) + + d = vars( + parser.parse_args( + ["generic", "--json", "dumb.json", "template.xml", "out.xml"] + ) + ) + self.assertIn("json", d) + self.assertIn("template", d) + self.assertIn("output", d) + + def test_main(self): + parser = ArgumentParser(parents=[util.parent_parser()]) + subparsers = parser.add_subparsers() + gen.add_parser(subparsers) + + args = parser.parse_args( + ["generic", "--json", "dumb.json", "template.xml", "out.xml"] + ) + + args.json = create_autospec(Path) + + with patch( + "vipersci.pds.labelmaker.generic.json.loads", return_value={"dummy": "dict"} + ) as m_jsonloads, patch( + "vipersci.pds.labelmaker.generic.write_xml", + ) as m_write_xml: + gen.main(args) + + m_jsonloads.assert_called_once() + m_write_xml.assert_called_once() diff --git a/tests/test_labelmaker_inventory.py b/tests/test_labelmaker_inventory.py new file mode 100644 index 0000000..ba45093 --- /dev/null +++ b/tests/test_labelmaker_inventory.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +"""This module has tests for the vis.pds.labelmaker.inventory functions.""" + +# Copyright 2023, vipersci developers. +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +from argparse import ArgumentParser +import unittest +from unittest.mock import patch + +import vipersci.pds.labelmaker.inventory as inv +from vipersci import util + + +class TestParser(unittest.TestCase): + def test_add_parser(self): + parser = ArgumentParser() + subparsers = parser.add_subparsers() + inv.add_parser(subparsers) + + d = vars( + parser.parse_args( + ["inventory", "--name", "dumbname", "file1.xml", "file2.xml"] + ) + ) + self.assertIn("name", d) + self.assertIn("member", d) + self.assertIn("labelfiles", d) + + def test_main(self): + parser = ArgumentParser(parents=[util.parent_parser()]) + subparsers = parser.add_subparsers() + inv.add_parser(subparsers) + + args = parser.parse_args( + ["inventory", "--name", "dumbname", "file1.xml", "file2.xml"] + ) + + with patch( + "vipersci.pds.labelmaker.inventory.get_lidvidfile", + return_value={"dummy": "dict"}, + ) as m_get_lidvid, patch( + "vipersci.pds.labelmaker.inventory.write_inventory", + ) as m_write_inv: + inv.main(args) + + self.assertEqual(2, m_get_lidvid.call_count) + m_write_inv.assert_called_once() From 4c14cb11281515451b4fba3420c05dcf44cc4015 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Thu, 9 Nov 2023 15:48:18 -0800 Subject: [PATCH 22/60] feat(image_records.py): Added Purpose Enum and verification_purpose column to allow VIS operator to specify this. --- src/vipersci/vis/db/image_records.py | 30 +++++++++++++++++++++++++--- src/vipersci/vis/pds/create_raw.py | 7 ++++++- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/vipersci/vis/db/image_records.py b/src/vipersci/vis/db/image_records.py index e0495f8..86ee911 100644 --- a/src/vipersci/vis/db/image_records.py +++ b/src/vipersci/vis/db/image_records.py @@ -26,13 +26,14 @@ # top level of this library. from datetime import datetime, timedelta, timezone -from enum import Flag +import enum from warnings import warn import xml.etree.ElementTree as ET from sqlalchemy import ( Boolean, DateTime, + Enum, Float, ForeignKey, Identity, @@ -50,7 +51,7 @@ import vipersci.vis.db.validators as vld -class ImageType(Flag): +class ImageType(enum.Flag): """This Flag class can be used to interpret the outputImageMask but not the immediateDownloadInfo Yamcs parameters, because only a single flag value can be set.""" @@ -75,7 +76,7 @@ def compression_ratio(self): return None -class ProcessingStage(Flag): +class ProcessingStage(enum.Flag): # PROCESS_RESERVED = 1 FLATFIELD = 2 # PROCESS_RESERVED_2 = 4 @@ -83,6 +84,26 @@ class ProcessingStage(Flag): SLOG = 16 +class Purpose(enum.Enum): + # These definitions are taken from the allowable values for + # Product_Observational/Observation_Area/Primary_Result_Summary/purpose + # in the PDS4 Data Dictionary. + CALIBRATION = "Data collected to determine the relationship between measurement " + "values and physical units." + CHECKOUT = "Data collected during operational tests." + ENGINEERING = "Data collected about support systems and structures, which are " + "ancillary to the primary measurements." + NAVIGATION = "Data collected to support navigation." + OBSERVATION_GEOMETRY = "Data used to compute instrument observation geometry, " + "such as SPICE kernels." + SCIENCE = "Data collected primarily to answer questions about the targets of " + "the investigation." + SUPPORTING_OBSERVATION = "A science observation that was acquired to provide " + "support for another science observation (e.g., a context image for a very " + "high resolution observation, or an image intended to support an observation " + "by a spectral imager)." + + class ImageRecord(Base): """An object to represent rows in the image_records table for VIS.""" @@ -292,6 +313,9 @@ class ImageRecord(Base): nullable=True, doc="Any notes about the verification of this image by the VIS Operator.", ) + verification_purpose = mapped_column( + Enum(Purpose), nullable=True, doc="Purpose of Observation, as defined by PDS." + ) verified = mapped_column( Boolean, nullable=True, diff --git a/src/vipersci/vis/pds/create_raw.py b/src/vipersci/vis/pds/create_raw.py index 06f3edf..bf17bb8 100644 --- a/src/vipersci/vis/pds/create_raw.py +++ b/src/vipersci/vis/pds/create_raw.py @@ -145,7 +145,6 @@ def main(): metadata = { "mission_phase": "TEST", "bad_pixel_table_id": 0, - "purpose": "Engineering", } # This allows values in these dicts to override the hard-coded values above. @@ -159,6 +158,12 @@ def main(): "software_program_name": __name__, } ) + if metadata["verification_purpose"] is None: + metadata["purpose"] = "Science" + else: + metadata["purpose"] = ( + metadata["verification_purpose"].value.replace("_", " ").title() + ) if args.input.endswith(".tif"): args.tiff = Path(args.input) From 812c4db79403ee64e8a095df1c6683909ae64cd1 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Wed, 22 Nov 2023 16:36:24 -0800 Subject: [PATCH 23/60] feat(nss_modeler.py): The write_tiff method now returns the path where it wrote the TIFF file. --- src/vipersci/carto/nss_modeler.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/vipersci/carto/nss_modeler.py b/src/vipersci/carto/nss_modeler.py index 59f95ef..af1fad9 100755 --- a/src/vipersci/carto/nss_modeler.py +++ b/src/vipersci/carto/nss_modeler.py @@ -93,6 +93,7 @@ def main(): def write_tif(path: Path, ending: str, arr: np.typing.ArrayLike, kwds: dict): - with rasterio.open(path.with_name(path.name + ending), "w", **kwds) as dst_dataset: + p = path.with_name(path.name + ending) + with rasterio.open(p, "w", **kwds) as dst_dataset: dst_dataset.write(arr, 1) - return + return p From 41801996a6686c87eca858936fe4774d7686752d Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Wed, 22 Nov 2023 17:04:29 -0800 Subject: [PATCH 24/60] refactor(pds.__init__.py and various): Created a top-level Purpose Enum to be used in the module. --- CHANGELOG.rst | 2 ++ src/vipersci/pds/__init__.py | 48 ++++++++++++++++++++++++++++ src/vipersci/vis/db/image_records.py | 21 +----------- src/vipersci/vis/db/pano_records.py | 6 ++-- src/vipersci/vis/db/validators.py | 21 ++++++------ tests/test_validators.py | 2 +- 6 files changed, 65 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 328e0a7..6fcf024 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -42,6 +42,8 @@ Changed Added ^^^^^ +- pds.Purpose now provides names and explanations for the PDS-allowable values + for "purpose." - yamcs_reception_time column added to the image_records.py table. - Association table junc_image_pano created which provides a many-to-many connection between ImageRecords and PanoRecords and added bidirectional diff --git a/src/vipersci/pds/__init__.py b/src/vipersci/pds/__init__.py index e69de29..d48d122 100644 --- a/src/vipersci/pds/__init__.py +++ b/src/vipersci/pds/__init__.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# coding: utf-8 + +"""Provides some generic PDS structures.""" + +# Copyright 2023, United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +import enum + + +class Purpose(enum.Enum): + # These definitions are taken from the allowable values for + # Product_Observational/Observation_Area/Primary_Result_Summary/purpose + # in the PDS4 Data Dictionary. + CALIBRATION = "Data collected to determine the relationship between measurement " + "values and physical units." + CHECKOUT = "Data collected during operational tests." + ENGINEERING = "Data collected about support systems and structures, which are " + "ancillary to the primary measurements." + NAVIGATION = "Data collected to support navigation." + OBSERVATION_GEOMETRY = "Data used to compute instrument observation geometry, " + "such as SPICE kernels." + SCIENCE = "Data collected primarily to answer questions about the targets of " + "the investigation." + SUPPORTING_OBSERVATION = "A science observation that was acquired to provide " + "support for another science observation (e.g., a context image for a very " + "high resolution observation, or an image intended to support an observation " + "by a spectral imager)." diff --git a/src/vipersci/vis/db/image_records.py b/src/vipersci/vis/db/image_records.py index 86ee911..3ff89d7 100644 --- a/src/vipersci/vis/db/image_records.py +++ b/src/vipersci/vis/db/image_records.py @@ -43,6 +43,7 @@ from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import mapped_column, relationship, synonym, validates +from vipersci.pds import Purpose from vipersci.pds.pid import VISID, vis_instruments, vis_compression from vipersci.pds.xml import find_text, ns from vipersci.pds.datetime import fromisozformat, isozformat @@ -84,26 +85,6 @@ class ProcessingStage(enum.Flag): SLOG = 16 -class Purpose(enum.Enum): - # These definitions are taken from the allowable values for - # Product_Observational/Observation_Area/Primary_Result_Summary/purpose - # in the PDS4 Data Dictionary. - CALIBRATION = "Data collected to determine the relationship between measurement " - "values and physical units." - CHECKOUT = "Data collected during operational tests." - ENGINEERING = "Data collected about support systems and structures, which are " - "ancillary to the primary measurements." - NAVIGATION = "Data collected to support navigation." - OBSERVATION_GEOMETRY = "Data used to compute instrument observation geometry, " - "such as SPICE kernels." - SCIENCE = "Data collected primarily to answer questions about the targets of " - "the investigation." - SUPPORTING_OBSERVATION = "A science observation that was acquired to provide " - "support for another science observation (e.g., a context image for a very " - "high resolution observation, or an image intended to support an observation " - "by a spectral imager)." - - class ImageRecord(Base): """An object to represent rows in the image_records table for VIS.""" diff --git a/src/vipersci/vis/db/pano_records.py b/src/vipersci/vis/db/pano_records.py index 2cc9068..2ab31e7 100644 --- a/src/vipersci/vis/db/pano_records.py +++ b/src/vipersci/vis/db/pano_records.py @@ -29,6 +29,7 @@ from sqlalchemy import ( DateTime, + Enum, Float, Identity, Integer, @@ -37,6 +38,7 @@ from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import mapped_column, relationship, validates +from vipersci.pds import Purpose from vipersci.pds.pid import VISID, PanoID from vipersci.pds.datetime import isozformat from vipersci.vis.db import Base @@ -126,8 +128,8 @@ class PanoRecord(Base): "product_id", String, nullable=False, unique=True, doc="The PDS Product ID." ) purpose = mapped_column( - String, - nullable=False, + Enum(Purpose), + nullable=True, doc="This is the value for the PDS " "Observation_Area/Primary_Result_Summary/purpose parameter, it " "has a restricted set of allowable values.", diff --git a/src/vipersci/vis/db/validators.py b/src/vipersci/vis/db/validators.py index 083887a..15b9abf 100644 --- a/src/vipersci/vis/db/validators.py +++ b/src/vipersci/vis/db/validators.py @@ -27,6 +27,7 @@ from datetime import datetime, timedelta, timezone +from vipersci.pds import Purpose from vipersci.pds.datetime import fromisozformat @@ -49,15 +50,11 @@ def validate_datetime_asutc(key, value): def validate_purpose(value: str): - s = { - "Calibration", - "Checkout", - "Engineering", - "Navigation", - "Observation Geometry", - "Science", - "Supporting Observation", - } - if value not in s: - raise ValueError(f"purpose must be one of {s}") - return value + s = set(Purpose.__members__.keys()) + if value in s: + return value + + if value.upper() in s: + return value.upper() + + raise ValueError(f"purpose must be one of {s}") diff --git a/tests/test_validators.py b/tests/test_validators.py index f583ce8..ecc292f 100644 --- a/tests/test_validators.py +++ b/tests/test_validators.py @@ -58,6 +58,6 @@ def test_validate_datetime_asutc(self): ) def test_validate_purpose(self): - self.assertEqual(vld.validate_purpose("Science"), "Science") + self.assertEqual(vld.validate_purpose("Science"), "SCIENCE") self.assertRaises(ValueError, vld.validate_purpose, "not a Purpose") From e6a36805eb7fde60faf3e2576b212d36b2e71628 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Wed, 22 Nov 2023 17:45:55 -0800 Subject: [PATCH 25/60] feat(pano_records.py): Set start time from source products when possible. --- src/vipersci/vis/db/pano_records.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/vipersci/vis/db/pano_records.py b/src/vipersci/vis/db/pano_records.py index 2ab31e7..961f8d1 100644 --- a/src/vipersci/vis/db/pano_records.py +++ b/src/vipersci/vis/db/pano_records.py @@ -182,7 +182,9 @@ def __init__(self, **kwargs): source_pids.sort() st = source_pids[0].datetime() - if self.start_time is not None: + if self.start_time is None: + self.start_time = st + else: st = self.start_time pid = PanoID(st.date(), st.time(), inst) From 29aa87a3a19a9a4e264eef17a66bc10079ac10d5 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Thu, 23 Nov 2023 14:00:16 -0800 Subject: [PATCH 26/60] feat(create_pano.py): Added mechanism to extract times and purposes from the source image_records. Also added dummy pan/tilt info. --- src/vipersci/vis/create_pano.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/src/vipersci/vis/create_pano.py b/src/vipersci/vis/create_pano.py index 8918fd8..aa9082b 100644 --- a/src/vipersci/vis/create_pano.py +++ b/src/vipersci/vis/create_pano.py @@ -32,6 +32,7 @@ # top level of this library. import argparse +from datetime import timezone import logging from typing import Any, Dict, Union, Optional, MutableSequence, List from pathlib import Path @@ -129,6 +130,7 @@ def main(): args.json, args.bottom, ) + session.commit() return @@ -196,7 +198,12 @@ def create( ) # At this time, image pointing information is not available, so we assume that - # the images provided are provided in left-to-right order. + # the images provided are provided in left-to-right order and fake these values: + half_width = (len(inputs) / 2) * 60 + metadata["rover_pan_min"] = -1 * half_width + metadata["rover_pan_max"] = half_width + metadata["rover_tilt_max"] = 15 + metadata["rover_tilt_min"] = -50 if bottom_row is None else -80 image_list = list() for path in source_paths: @@ -233,6 +240,30 @@ def create( pp = make_pano_record(metadata, pano_arr, outdir) + if image_records: + purposes = set() + start_times = [] + stop_times = [] + for ir in image_records: + purposes.add(ir.verification_purpose) + start_times.append( + ir.start_time + if session.get_bind().name != "sqlite" + else ir.start_time.replace(tzinfo=timezone.utc) + ) + stop_times.append( + ir.stop_time + if session.get_bind().name != "sqlite" + else ir.stop_time.replace(tzinfo=timezone.utc) + ) + + purp = purposes.pop() + if purp is not None: + pp.purpose = purp + + pp.start_time = min(start_times) + pp.stop_time = max(stop_times) + if json: write_json(pp.asdict(), outdir) From 24f754cc83115eb5bf2352927be80070cc5348d3 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Fri, 24 Nov 2023 17:44:56 -0800 Subject: [PATCH 27/60] feat(pano_check.py): Now properly ignores stereo pairs if left and right have the same timestamp. Also added entry point. --- setup.cfg | 1 + src/vipersci/vis/pano_check.py | 19 +++++++------------ tests/test_pano_check.py | 6 +++--- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/setup.cfg b/setup.cfg index b2912f7..f5128f3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -87,6 +87,7 @@ console_scripts = vis_create_tif = vipersci.vis.create_tif:main vis_create_raw = vipersci.vis.pds.create_raw:main vis_create_dbs = vipersci.vis.db.create_vis_dbs:main + vis_pano_check = vipersci.vis.pano_check:main viseer = vipersci.vis.viseer:main [bdist_wheel] diff --git a/src/vipersci/vis/pano_check.py b/src/vipersci/vis/pano_check.py index c5735e9..8d64a47 100644 --- a/src/vipersci/vis/pano_check.py +++ b/src/vipersci/vis/pano_check.py @@ -29,7 +29,6 @@ import itertools import logging from pathlib import Path -import sys from typing import Iterable, Union import pandas as pd @@ -100,7 +99,7 @@ def main(): if args.url is not None: gppf = partial(get_position_and_pose_from_mapserver, url=args.url) elif args.csv is not None: - gppf = partial(get_position_and_pose_from_df, path=args.csv) + gppf = partial(get_position_and_pose_from_csv, path=args.csv) else: parser.error( "Neither --url nor --csv were given. Need at least one to be a " @@ -153,7 +152,7 @@ def check( module. These functions may need to be wrapped via functools.partial() so that the function passed here takes only a list of timestamp times. - If the iterable does not contain all ImageRecord objecs or all VISID objects, + If the iterable does not contain all ImageRecord objects or all VISID objects, a TypeError will be raised. """ if get_pos_pose_func is None: @@ -184,9 +183,9 @@ def check( if v.compression == "s": continue ts = v.datetime().timestamp() - - vids_by_time[ts] = v - times.append(ts) + if ts not in vids_by_time: + vids_by_time[ts] = v + times.append(ts) tpp = get_pos_pose_func(times) grouped = groupby_2nd(tpp) @@ -204,7 +203,7 @@ def check( return pano_groups -def get_position_and_pose_from_df( +def get_position_and_pose_from_csv( times: list, path: Path, time_column=0, @@ -215,7 +214,7 @@ def get_position_and_pose_from_df( """ Given a list of timestamp times and a Path to a CSV file with the specified columns, return a list of two-tuples whose first element is the time and whose - second element is an N-tuple of x-location, y-location, and yaw. + second element is three-tuple of x-location, y-location, and yaw. If None is given for any of the x-, y-, or yaw- columns, they are ignored from the CSV read. @@ -297,7 +296,3 @@ def keyfunc(t: tuple): grouped.append((first, second)) return grouped - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/test_pano_check.py b/tests/test_pano_check.py index 16a4d8d..e180856 100644 --- a/tests/test_pano_check.py +++ b/tests/test_pano_check.py @@ -89,7 +89,7 @@ def requests_get_se(url, params): tpp = pc.get_position_and_pose_from_mapserver(self.times, url="foo") self.assertEqual(tpp, self.truth) - def test_get_pp_from_df(self): + def test_get_pp_from_csv(self): my_times = self.times.copy() my_times[0] += 10 @@ -98,7 +98,7 @@ def test_get_pp_from_df(self): ] with patch("vipersci.vis.pano_check.pd.read_csv", return_value=self.df): - tpp = pc.get_position_and_pose_from_df(my_times, Path("dummy.csv")) + tpp = pc.get_position_and_pose_from_csv(my_times, Path("dummy.csv")) self.assertEqual(tpp, my_truth) def test_check(self): @@ -133,7 +133,7 @@ def test_check(self): self.assertRaises(TypeError, pc.check, pid_list, "dummy_function") with patch("vipersci.vis.pano_check.pd.read_csv", return_value=self.df): - gffp = partial(pc.get_position_and_pose_from_df, path="dummy.csv") + gffp = partial(pc.get_position_and_pose_from_csv, path="dummy.csv") p_groups = pc.check(pids, gffp) self.assertEqual(p_groups, truth) From 8cb5d53b1beecf80b16c0bd8148c8a27441b187b Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Fri, 24 Nov 2023 17:45:32 -0800 Subject: [PATCH 28/60] feat(create_pano.py): Can now be given a directory to prepend to inputs when looking for files. --- src/vipersci/vis/create_pano.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/vipersci/vis/create_pano.py b/src/vipersci/vis/create_pano.py index aa9082b..c816daf 100644 --- a/src/vipersci/vis/create_pano.py +++ b/src/vipersci/vis/create_pano.py @@ -99,6 +99,14 @@ def arg_parser(): help="Output directory for label. Defaults to current working directory. " "Output file names are fixed based on product_id, and will be over-written.", ) + parser.add_argument( + "--prefix", + type=Path, + default=Path.cwd(), + help="A directory path that, if given, will be prepended to paths given via " + "inputs or will be prepended to the file_path values returned from a database " + "query." + ) parser.add_argument( "-x", "--xml", action="store_true", help="Create a PDS4 .XML label file." ) @@ -115,6 +123,7 @@ def main(): if args.dburl is None: create( args.inputs, + args.prefix, args.output_dir, None, args.json, @@ -125,6 +134,7 @@ def main(): with Session(engine) as session: create( args.inputs, + args.prefix, args.output_dir, session, args.json, @@ -137,6 +147,7 @@ def main(): def create( inputs: MutableSequence[Union[Path, pds.VISID, ImageRecord, str]], + prefixdir: Optional[Path] = None, outdir: Optional[Path] = None, session: Optional[Session] = None, json: bool = True, @@ -187,11 +198,15 @@ def create( for inp in inputs: if isinstance(inp, ImageRecord): metadata["source_pids"].append(inp.product_id) - source_paths.append(inp.file_path) + source_paths.append( + inp.file_path if prefixdir is None else prefixdir / inp.file_path + ) image_records.append(inp) elif isinstance(inp, (Path, str)): metadata["source_pids"].append([str(pds.VISID(inp))]) - source_paths.append(inp) + source_paths.append( + inp if prefixdir is None else prefixdir / inp + ) else: raise ValueError( f"an element in input is not the right type: {inp} ({type(inp)})" @@ -240,7 +255,8 @@ def create( pp = make_pano_record(metadata, pano_arr, outdir) - if image_records: + if image_records and session is not None: + bound_name = getattr(session.get_bind(), "name", None) purposes = set() start_times = [] stop_times = [] @@ -248,12 +264,12 @@ def create( purposes.add(ir.verification_purpose) start_times.append( ir.start_time - if session.get_bind().name != "sqlite" + if bound_name != "sqlite" else ir.start_time.replace(tzinfo=timezone.utc) ) stop_times.append( ir.stop_time - if session.get_bind().name != "sqlite" + if bound_name != "sqlite" else ir.stop_time.replace(tzinfo=timezone.utc) ) From 7b15d3d10c7f6fc4cd927bcb624ce267ba973bd2 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Fri, 24 Nov 2023 18:00:22 -0800 Subject: [PATCH 29/60] feat(template_test.py): Removed, labelmaker generic does this work now. --- src/vipersci/vis/pds/template_test.py | 65 --------------------------- 1 file changed, 65 deletions(-) delete mode 100644 src/vipersci/vis/pds/template_test.py diff --git a/src/vipersci/vis/pds/template_test.py b/src/vipersci/vis/pds/template_test.py deleted file mode 100644 index 6c75181..0000000 --- a/src/vipersci/vis/pds/template_test.py +++ /dev/null @@ -1,65 +0,0 @@ -""" -Test creation of PDS labels. - -Takes a JSON file and a Genshi XML template, and uses the JSON file to fill -out the template. -""" - -# Copyright 2021-2022, United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# -# Reuse is permitted under the terms of the license. -# The AUTHORS file and the LICENSE file are at the -# top level of this library. - -import argparse -import json -import logging -from pathlib import Path - -# import genshi -# from genshi.template import TemplateLoader -from genshi.template import MarkupTemplate - -from vipersci import util - -logger = logging.getLogger(__name__) - - -def arg_parser(): - parser = argparse.ArgumentParser( - description=__doc__, parents=[util.parent_parser()] - ) - parser.add_argument("-j", "--json", type=Path, help="Path to .json file to load.") - parser.add_argument("input", type=Path, help="Genshi XML file template.") - parser.add_argument("output", type=Path, help="Output XML label.") - return parser - - -def main(): - args = arg_parser().parse_args() - util.set_logger(args.verbose) - - with open(args.json, "r") as f: - info = json.load(f) - - # loader = TemplateLoader() - # tmpl = loader.load(str(args.input)) - tmpl = MarkupTemplate(args.input.read_text()) - stream = tmpl.generate(**info) - args.output.write_text(stream.render()) From 8a47e31860c07ff4218b811f7a94d80c3d8b2095 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Sun, 26 Nov 2023 18:27:36 -0800 Subject: [PATCH 30/60] refactor(pds/__init__.py and create_raw.py): Consolidated some fucntions to the top-level module. --- src/vipersci/vis/pds/__init__.py | 86 ++++++++++++++++++++++++++++++ src/vipersci/vis/pds/create_raw.py | 65 ++++------------------ tests/test_create_raw.py | 4 +- 3 files changed, 97 insertions(+), 58 deletions(-) diff --git a/src/vipersci/vis/pds/__init__.py b/src/vipersci/vis/pds/__init__.py index e69de29..dc53c76 100644 --- a/src/vipersci/vis/pds/__init__.py +++ b/src/vipersci/vis/pds/__init__.py @@ -0,0 +1,86 @@ +"""Various helper functions for VIS PDS operations. +""" + +# Copyright 2023, United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +from datetime import date +from importlib import resources +import logging +from pathlib import Path + +from genshi.template import MarkupTemplate + +logger = logging.getLogger(__name__) + +lids = { + "bundle": "urn:nasa:pds:viper_vis", + "mission": "urn:nasa:pds:viper", + "spacecraft": "urn:nasa:pds:context:instrument_host:spacecraft.viper" +} + + +def version_info(): + # This should reach into a database and do something smart to figure + # out how to populate this, but for now, hardcoding: + d = { + "modification_details": [ + { + "version": 0.1, + "date": date.today().isoformat(), + "description": "Illegal version number for testing", + } + ], + "vid": 0.1, + } + return d + + +def write_xml( + product: dict, + template: str, + outdir: Path = Path.cwd(), +): + """ + Writes a PDS4 XML label in *outdir* based on the contents of + the *product* object, which must be of type Raw_Product. + + The *template_path* can be a path to an appropriate template + XML file, but defaults to the raw-template.xml file provided + with this library. + """ + if Path(template).exists(): + tmpl = MarkupTemplate(Path(template).read_text()) + else: + tmpl = MarkupTemplate( + resources.read_text("vipersci.vis.pds.data", str(template)) + ) + + d = version_info() + d.update(product) + + logger.info(d) + + stream = tmpl.generate(**d) + out_path = (outdir / product["product_id"]).with_suffix(".xml") + out_path.write_text(stream.render()) + return diff --git a/src/vipersci/vis/pds/create_raw.py b/src/vipersci/vis/pds/create_raw.py index bf17bb8..4320c32 100644 --- a/src/vipersci/vis/pds/create_raw.py +++ b/src/vipersci/vis/pds/create_raw.py @@ -38,15 +38,13 @@ # top level of this library. import argparse -from datetime import date, timedelta -from importlib import resources +from datetime import timedelta import json import logging -from typing import Union, Optional +from typing import Union from pathlib import Path from warnings import warn -from genshi.template import MarkupTemplate import numpy as np import numpy.typing as npt from sqlalchemy import and_, create_engine, select @@ -56,6 +54,7 @@ from vipersci.vis.db.image_records import ImageRecord, ProcessingStage from vipersci.vis.db.light_records import LightRecord, luminaire_names from vipersci.vis.create_image import tif_info +from vipersci.vis.pds import lids, write_xml from vipersci.pds import pid as pds from vipersci import util @@ -79,7 +78,7 @@ def arg_parser(): parser.add_argument( "-t", "--template", - type=Path, + default="raw-template.xml", help="Genshi XML file template. Will default to the raw-template.xml " "file distributed with the module. Only relevant when --xml is provided.", ) @@ -183,7 +182,7 @@ def main(): ) metadata.update(t_info) - write_xml(metadata, args.output_dir, args.template) + write_xml(metadata, args.template, args.output_dir) return @@ -280,15 +279,14 @@ def get_lights(ir: ImageRecord, session: Session): def label_dict(ir: ImageRecord, lights: dict): """Returns a dictionary suitable for label generation.""" _inst = ir.instrument_name.lower().replace(" ", "_") - _sclid = "urn:nasa:pds:context:instrument_host:spacecraft.viper" onoff = {True: "On", False: "Off", None: None} pid = pds.VISID(ir.product_id) d = dict( data_quality="", - lid=f"urn:nasa:pds:viper_vis:data_raw:{ir.product_id}", - mission_lid="urn:nasa:pds:viper", - sc_lid=_sclid, - inst_lid=f"{_sclid}.{_inst}", + lid=f"{lids['bundle']}:data_raw:{ir.product_id}", + mission_lid=lids["mission"], + sc_lid=lids["spacecraft"], + inst_lid=f"{lids['spacecraft']}.{_inst}", gain_number=(ir.adc_gain * ir.pga_gain), exposure_type="Auto" if ir.auto_exposure else "Manual", image_filters=list(), @@ -347,48 +345,3 @@ def label_dict(ir: ImageRecord, lights: dict): d["data_quality"] += " " + ir.verification_notes return d - - -def version_info(): - # This should reach into a database and do something smart to figure - # out how to populate this, but for now, hardcoding: - d = { - "modification_details": [ - { - "version": 0.1, - "date": date.today().isoformat(), - "description": "Illegal version number for testing", - } - ], - "vid": 0.1, - } - return d - - -def write_xml( - product: dict, outdir: Path = Path.cwd(), template_path: Optional[Path] = None -): - """ - Writes a PDS4 XML label in *outdir* based on the contents of - the *product* object, which must be of type Raw_Product. - - The *template_path* can be a path to an appropriate template - XML file, but defaults to the raw-template.xml file provided - with this library. - """ - if template_path is None: - tmpl = MarkupTemplate( - resources.read_text("vipersci.vis.pds.data", "raw-template.xml") - ) - else: - tmpl = MarkupTemplate(template_path.read_text()) - - d = version_info() - d.update(product) - - logger.info(d) - - stream = tmpl.generate(**d) - out_path = (outdir / product["product_id"]).with_suffix(".xml") - out_path.write_text(stream.render()) - return diff --git a/tests/test_create_raw.py b/tests/test_create_raw.py index 2331172..445d7b6 100644 --- a/tests/test_create_raw.py +++ b/tests/test_create_raw.py @@ -167,7 +167,7 @@ def test_main(self, m_write_xml, m_create_engine): with patch("vipersci.vis.db.image_records.isozformat", new=isozformat): cr.main() m_write_xml.assert_called_once() - (metadata, outdir, template) = m_write_xml.call_args[0] + (metadata, template, outdir) = m_write_xml.call_args[0] self.assertEqual(metadata["product_id"], self.ir.product_id) self.assertEqual(outdir, Path.cwd()) - self.assertIsNone(template) + self.assertEqual(template, "raw-template.xml") From d74e3c7fb2f7585f285d27a1e2a1faa200f96515 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Sun, 26 Nov 2023 18:28:54 -0800 Subject: [PATCH 31/60] refactor(pano_records.py): label_dict function is really for making PDS XML labels, doesn't belong here. --- src/vipersci/vis/db/pano_records.py | 17 ++--------------- tests/test_pano_records.py | 12 ++++++------ 2 files changed, 8 insertions(+), 21 deletions(-) diff --git a/src/vipersci/vis/db/pano_records.py b/src/vipersci/vis/db/pano_records.py index 961f8d1..7d3ca66 100644 --- a/src/vipersci/vis/db/pano_records.py +++ b/src/vipersci/vis/db/pano_records.py @@ -250,7 +250,8 @@ def asdict(self): else: d[c.name] = getattr(self, c.name) - d.update(self.labelmeta) + if hasattr(self, "labelmeta"): + d.update(self.labelmeta) return d @@ -262,20 +263,6 @@ def from_xml(cls, text: str): """ raise NotImplementedError() - def label_dict(self): - """Returns a dictionary suitable for label generation.""" - _sclid = "urn:nasa:pds:context:instrument_host:spacecraft.viper" - d = dict( - lid=f"urn:nasa:pds:viper_vis:panoramas:{self.product_id}", - mission_lid="urn:nasa:pds:viper", - sc_lid=_sclid, - # inst_lid=f"{_sclid}.{_inst}", - ) - - d.update(self.asdict()) - - return d - def update(self, other): for k, v in other.items(): if k in self.__table__.columns or k in self.__mapper__.synonyms: diff --git a/tests/test_pano_records.py b/tests/test_pano_records.py index f079b8c..0b775e9 100644 --- a/tests/test_pano_records.py +++ b/tests/test_pano_records.py @@ -99,9 +99,9 @@ def test_update(self): p.update(self.extras) self.assertTrue(k in p.labelmeta) - def test_labeldict(self): - din = self.d - din.update(self.extras) - p = tpp.PanoRecord(**din) - d = p.label_dict() - self.assertEqual(d["samples"], p.samples) + # def test_labeldict(self): + # din = self.d + # din.update(self.extras) + # p = tpp.PanoRecord(**din) + # d = p.label_dict() + # self.assertEqual(d["samples"], p.samples) From a5710ace30b7f77b717a17466cb5eca395acf55b Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Sun, 26 Nov 2023 18:30:09 -0800 Subject: [PATCH 32/60] feat(create_pano_product.py): Added. --- CHANGELOG.rst | 1 + setup.cfg | 1 + src/vipersci/vis/pds/create_pano_product.py | 214 ++++++++++++++++++++ 3 files changed, 216 insertions(+) create mode 100644 src/vipersci/vis/pds/create_pano_product.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6fcf024..2e53203 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -55,6 +55,7 @@ Added - ptu_records.py - Tables to record the pan and tilt of the rover's pan-tilt-unit (PTU). - create_pano.py - updated to correctly add PanoRecord associations, can now query database for ImageRecords. +- create_pano_product.py - takes PanoRecords and makes a PDS Pano Product. - get_position.py - Gets position and yaw from a REST-based service. - create_vis_dbs.py - Now also supports spatialite databases, primarily for testing. - create_raw.py - Added components for adding observational intent and data quality diff --git a/setup.cfg b/setup.cfg index f5128f3..a2f0a85 100644 --- a/setup.cfg +++ b/setup.cfg @@ -84,6 +84,7 @@ console_scripts = template_test = vipersci.vis.pds.template_test:main vis_create_image = vipersci.vis.create_image:main vis_create_pano = vipersci.vis.create_pano:main + vis_create_pano_product = vipersci.vis.pds.create_pano_product:main vis_create_tif = vipersci.vis.create_tif:main vis_create_raw = vipersci.vis.pds.create_raw:main vis_create_dbs = vipersci.vis.db.create_vis_dbs:main diff --git a/src/vipersci/vis/pds/create_pano_product.py b/src/vipersci/vis/pds/create_pano_product.py new file mode 100644 index 0000000..e5e2291 --- /dev/null +++ b/src/vipersci/vis/pds/create_pano_product.py @@ -0,0 +1,214 @@ +"""Creates PDS VIS Pano Products. + +This module builds a VIS panorama data products from VIS pano record data. At this +time, a Pano Product is produced from VIS Image Records. That metadata is the basis for +production of the PDS4 XML label file. + +In order to perform the lookup of Image Records and Pano Records requires use +of the database. + +For now, this program still has a variety of hard-coded elements, +that will eventually be extracted from telemetry. + +The command-line version is primarily to aide testing. +""" + +# Copyright 2022-2023, United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +import argparse +from datetime import timezone +import json +import logging +from typing import Union +from pathlib import Path + +import numpy as np +import numpy.typing as npt +from sqlalchemy import create_engine, select +from sqlalchemy.orm import Session + +import vipersci +from vipersci.vis.db.pano_records import PanoRecord +from vipersci.vis.create_image import tif_info +from vipersci.pds import pid as pds +from vipersci import util +from vipersci.vis.pds import lids, write_xml + +logger = logging.getLogger(__name__) + +ImageType = Union[npt.NDArray[np.uint16], npt.NDArray[np.uint8]] + + +def arg_parser(): + parser = argparse.ArgumentParser( + description=__doc__, parents=[util.parent_parser()] + ) + parser.add_argument( + "-d", + "--dburl", + required=True, + help="Database with a pano_products table which will be written to. " + "If not given, no database will be written to. Example: " + "postgresql://postgres:NotTheDefault@localhost/visdb", + ) + parser.add_argument( + "-t", + "--template", + default="pano-template.xml", + help="Genshi XML file template. Will default to the pano-template.xml " + "file distributed with the module.", + ) + parser.add_argument( + "--tiff", + type=Path, + help="Optional pre-existing TIFF file (presumably created by create_image). " + "This file will be inspected and its information added to the output. ", + ) + parser.add_argument( + "-o", + "--output_dir", + type=Path, + default=Path.cwd(), + help="Output directory for label. Defaults to current working directory. " + "Output file names are fixed based on product_id, and will be over-written.", + ) + parser.add_argument( + "input", + help="Product ID (or TIFF file from which a Product ID can be extracted) or a " + "JSON file containing metadata.", + ) + return parser + + +def main(): + parser = arg_parser() + args = parser.parse_args() + util.set_logger(args.verbose) + + engine = create_engine(args.dburl) + with Session(engine) as session: + try: + pid = pds.PanoID(args.input) + except ValueError: + pid = None + + pr = None + if pid is None or args.input.endswith(".json"): + if Path(args.input).exists(): + with open(args.input) as f: + pr = PanoRecord(**json.load(f)) + else: + parser.error(f"The file {args.input} does not exist.") + else: + # We got a valid pid, go look it up in the db. + stmt = select(PanoRecord).where(PanoRecord.product_id == str(pid)) + result = session.scalars(stmt) + rows = result.all() + if len(rows) > 1: + raise ValueError(f"There was more than 1 row returned from {stmt}") + elif len(rows) == 0: + raise ValueError( + f"No records were returned from the database for Product Id {pid}." + ) + + pr = rows[0] + + if pr is None: + raise ValueError(f"Could not extract a PanoRecord from {args.input}") + + # If testing via sqlite, ensure datetimes are UTC aware: + if session.get_bind().name == "sqlite": + pr.file_creation_datetime = pr.file_creation_datetime.replace( + tzinfo=timezone.utc + ) + pr.start_time = pr.start_time.replace(tzinfo=timezone.utc) + pr.stop_time = pr.stop_time.replace(tzinfo=timezone.utc) + + # I'm not sure where these are coming from, let's hard-code them for now: + metadata = { + "mission_phase": "TEST", + } + + # This allows values in these dicts to override the hard-coded values above. + metadata.update(label_dict(pr)) + metadata.update(pr.asdict()) + metadata.update( + { + "software_name": "vipersci", + "software_version": vipersci.__version__, + "software_type": "Python", + "software_program_name": __name__, + } + ) + if metadata["purpose"] is None: + metadata["purpose"] = "Science" + else: + metadata["purpose"] = metadata["purpose"].value.replace("_", " ").title() + + if args.input.endswith(".tif"): + args.tiff = Path(args.input) + + if args.tiff is None: + # Make up some values: + metadata["file_byte_offset"] = 0 + metadata["file_data_type"] = "UnsignedLSB2" + else: + t_info = tif_info(args.tiff) + + for k, v in t_info.items(): + if hasattr(metadata, k) and metadata[k] != v: + raise ValueError( + f"The value of {k} in the metadata ({metadata[k]}) does not match " + f"the value ({v}) in the image ({args.tiff})" + ) + metadata.update(t_info) + + write_xml(metadata, args.template, args.output_dir) + + return + + +def label_dict(pr: PanoRecord): + """Returns a dictionary suitable for label generation.""" + d = dict( + lid=f"{lids['bundle']}:data_derived:{pr.product_id}", + mission_lid=lids["mission"], + sc_lid=lids["spacecraft"], + instruments=[], + source_product_lidvids=[], + source_product_type="data_to_raw_source_product", + ) + + instruments_dict = {} + for ir in pr.image_records: + _inst = ir.instrument_name.lower().replace(" ", "_") + instruments_dict[ir.instrument_name] = { + "name": ir.instrument_name, + "lid": f"{lids['spacecraft']}.{_inst}", + } + d["source_product_lidvids"].append(str(ir.product_id)) # type: ignore + + for inst in instruments_dict.values(): + d["instruments"].append(inst) # type: ignore + + return d From 684ba952def705e420298876c3ec7104c3fd5419 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Sun, 26 Nov 2023 18:47:19 -0800 Subject: [PATCH 33/60] refactor(create_pano.py and pds/__init__.py): Lint cleanup --- src/vipersci/vis/create_pano.py | 6 ++---- src/vipersci/vis/pds/__init__.py | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/vipersci/vis/create_pano.py b/src/vipersci/vis/create_pano.py index c816daf..5bb9cb7 100644 --- a/src/vipersci/vis/create_pano.py +++ b/src/vipersci/vis/create_pano.py @@ -105,7 +105,7 @@ def arg_parser(): default=Path.cwd(), help="A directory path that, if given, will be prepended to paths given via " "inputs or will be prepended to the file_path values returned from a database " - "query." + "query.", ) parser.add_argument( "-x", "--xml", action="store_true", help="Create a PDS4 .XML label file." @@ -204,9 +204,7 @@ def create( image_records.append(inp) elif isinstance(inp, (Path, str)): metadata["source_pids"].append([str(pds.VISID(inp))]) - source_paths.append( - inp if prefixdir is None else prefixdir / inp - ) + source_paths.append(inp if prefixdir is None else prefixdir / inp) else: raise ValueError( f"an element in input is not the right type: {inp} ({type(inp)})" diff --git a/src/vipersci/vis/pds/__init__.py b/src/vipersci/vis/pds/__init__.py index dc53c76..5a8da7a 100644 --- a/src/vipersci/vis/pds/__init__.py +++ b/src/vipersci/vis/pds/__init__.py @@ -35,7 +35,7 @@ lids = { "bundle": "urn:nasa:pds:viper_vis", "mission": "urn:nasa:pds:viper", - "spacecraft": "urn:nasa:pds:context:instrument_host:spacecraft.viper" + "spacecraft": "urn:nasa:pds:context:instrument_host:spacecraft.viper", } From 67b4d922fe4fdfaa6013580c07ce1830a74c8e3a Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Wed, 29 Nov 2023 11:23:53 -0800 Subject: [PATCH 34/60] feat(create_pano.py): Removed vestigal components for when this module also wrote PDS products. --- src/vipersci/vis/create_pano.py | 24 +--- tests/test_create_pano.py | 188 +++++++++++++++++++++++++++++++- 2 files changed, 190 insertions(+), 22 deletions(-) diff --git a/src/vipersci/vis/create_pano.py b/src/vipersci/vis/create_pano.py index 5bb9cb7..e7bd22c 100644 --- a/src/vipersci/vis/create_pano.py +++ b/src/vipersci/vis/create_pano.py @@ -84,13 +84,6 @@ def arg_parser(): dest="json", help="Disables creation of .json output.", ) - parser.add_argument( - "-t", - "--template", - type=Path, - help="Genshi XML file template. Will default to the pano-template.xml " - "file distributed with the module. Only relevant when --xml is provided.", - ) parser.add_argument( "-o", "--output_dir", @@ -108,10 +101,7 @@ def arg_parser(): "query.", ) parser.add_argument( - "-x", "--xml", action="store_true", help="Create a PDS4 .XML label file." - ) - parser.add_argument( - "inputs", nargs="*", help="Either VIS raw product IDs or files." + "inputs", nargs="+", help="Either VIS raw product IDs or files." ) return parser @@ -148,7 +138,7 @@ def main(): def create( inputs: MutableSequence[Union[Path, pds.VISID, ImageRecord, str]], prefixdir: Optional[Path] = None, - outdir: Optional[Path] = None, + outdir: Path = Path.cwd(), session: Optional[Session] = None, json: bool = True, bottom_row: Optional[MutableSequence[Union[Path, str]]] = None, @@ -156,10 +146,7 @@ def create( """ Creates a Panorama Product in *outdir*. Returns None. - At this time, session is ignored. - - At this time, *inputs* should be a list of file paths. In the - future, it could be a list of product IDs. + At this time, *inputs* should be a list of file paths or product IDs. If a path is provided to *outdir* the created files will be written there. @@ -168,8 +155,6 @@ def create( written to the pano_records table. If not, no database activity will occur. """ - if outdir is None: - raise ValueError("A Path for outdir must be supplied.") metadata: Dict[str, Any] = dict( source_pids=[], @@ -281,9 +266,6 @@ def create( if json: write_json(pp.asdict(), outdir) - # if xml: - # write_xml(pp.label_dict(), outdir, template_path) - if session is not None: if image_records: to_add: List[Union[PanoRecord, JuncImagePano]] = [ diff --git a/tests/test_create_pano.py b/tests/test_create_pano.py index 39f9f1a..1fa5159 100644 --- a/tests/test_create_pano.py +++ b/tests/test_create_pano.py @@ -7,17 +7,72 @@ # The AUTHORS file and the LICENSE file are at the # top level of this library. +from argparse import ArgumentParser from datetime import datetime, timezone from pathlib import Path import unittest -from unittest.mock import patch +from unittest.mock import create_autospec, Mock, patch +from geoalchemy2 import load_spatialite import numpy as np +from sqlalchemy import create_engine +from sqlalchemy.event import listen +from sqlalchemy.orm import Session +from vipersci.vis.db import Base +from vipersci.vis.db.image_records import ImageRecord from vipersci.vis.db.pano_records import PanoRecord from vipersci.vis import create_pano as cp +class TestCLI(unittest.TestCase): + def test_arg_parser(self): + p = cp.arg_parser() + self.assertIsInstance(p, ArgumentParser) + # self.assertRaises(SystemExit, p.parse_args) + d = vars( + p.parse_args( + ["--dburl", "db://foo:username@host/db", "product_id_goes_here"] + ) + ) + self.assertIn("dburl", d) + self.assertIn("output_dir", d) + self.assertIn("inputs", d) + + def test_main(self): + pa_ret_val = cp.arg_parser().parse_args( + ["231126-000000-ncl-s.dummy", "231126-000000-ncr-s.dummy"] + ) + with patch("vipersci.vis.create_pano.arg_parser") as parser, patch( + "vipersci.vis.create_pano.create" + ) as m_create: + parser.return_value.parse_args.return_value = pa_ret_val + cp.main() + m_create.assert_called_once() + + pa2_ret_val = cp.arg_parser().parse_args( + [ + "--dburl", + "db://foo:username@host/db", + "231126-000000-ncl-s.dummy", + "231126-000000-ncr-s.dummy", + ] + ) + session_engine_mock = create_autospec(Session) + session_mock = create_autospec(Session) + session_engine_mock.__enter__ = Mock(return_value=session_mock) + with patch("vipersci.vis.create_pano.arg_parser") as parser, patch( + "vipersci.vis.create_pano.create" + ) as m_create, patch("vipersci.vis.create_pano.create_engine"), patch( + "vipersci.vis.create_pano.Session", return_value=session_engine_mock + ): + parser.return_value.parse_args.return_value = pa2_ret_val + cp.main() + m_create.assert_called_once() + session_engine_mock.__enter__.assert_called_once() + session_mock.commit.assert_called_once() + + class TestMakePano(unittest.TestCase): def setUp(self) -> None: self.startUTC = datetime(2022, 1, 27, 0, 0, 0, tzinfo=timezone.utc) @@ -59,3 +114,134 @@ def test_image(self, mock_tif_info, mock_imsave): self.assertIsInstance(prp, PanoRecord) mock_imsave.assert_not_called() mock_tif_info.assert_called_once() + + +class TestCreate(unittest.TestCase): + def test_nodb(self): + self.assertRaises(ValueError, cp.create, [1, 2, 3]) + self.assertRaises( + FileNotFoundError, + cp.create, + ["231126-000000-ncl-s.dummy", "231126-000000-ncr-s.dummy"], + ) + + path_mock = create_autospec(Path) + path_mock.exists.return_value = True + + with patch("vipersci.vis.create_pano.Path", return_value=path_mock), patch( + "vipersci.vis.create_pano.imread" + ) as m_imread, patch("vipersci.vis.create_pano.np.hstack"), patch( + "vipersci.vis.create_pano.make_pano_record" + ) as m_mpr, patch( + "vipersci.vis.create_pano.write_json" + ) as m_write_json, patch( + "vipersci.vis.create_pano.isinstance", + side_effect=[True, True, False, True, False, True], + ): + cp.create(["231126-000000-ncl-s.dummy", "231126-000000-ncr-s.dummy"]) + + self.assertEqual(m_imread.call_count, 2) + self.assertEqual(m_mpr.call_args[0][2], Path.cwd()) + m_write_json.assert_called_once() + + def test_db(self): + ir1 = ImageRecord( + adc_gain=0, + auto_exposure=0, + cameraId=0, + capture_id=1, + exposure_duration=111, + file_byte_offset=256, + file_creation_datetime="2023-08-01T00:51:51.148919Z", + file_data_type="UnsignedLSB2", + file_md5_checksum="b8f1a035e39c223e2b7e236846102c29", + file_path="231126-000000-ncl-s", + imageDepth=2, + image_id=0, + immediateDownloadInfo=16, + instrument_name="NavCam Left", + instrument_temperature=0, + lines=2048, + lobt=1700956800, + offset=0, + outputImageType="SLOG_ICER_IMAGE", + output_image_mask=16, + padding=0, + pga_gain=1.0, + processing_info=26, + product_id="231126-000000-ncl-s", + samples=2048, + software_name="vipersci", + software_program_name="vipersci.vis.create_image", + software_version="0.6.0-dev", + start_time="2023-11-26T00:00:00Z", + stereo=1, + stop_time="2023-11-26T00:00:00.000111Z", + voltage_ramp=0, + yamcs_generation_time="2023-11-26T00:00:00Z", + yamcs_name="/ViperGround/Images/ImageData/Navcam_left_slog", + ) + ir2 = ImageRecord( + adc_gain=0, + auto_exposure=0, + cameraId=1, + capture_id=2, + exposure_duration=111, + file_byte_offset=256, + file_creation_datetime="2023-08-01T00:51:51.148919Z", + file_data_type="UnsignedLSB2", + file_md5_checksum="b8f1a035e39c223e2b7e236846102c29", + file_path="231126-000000-ncr-s", + imageDepth=2, + image_id=0, + immediateDownloadInfo=16, + instrument_name="NavCam Right", + instrument_temperature=0, + lines=2048, + lobt=1700956800, + offset=0, + outputImageType="SLOG_ICER_IMAGE", + output_image_mask=16, + padding=0, + pga_gain=1.0, + processing_info=26, + product_id="231126-000000-ncr-s", + samples=2048, + software_name="vipersci", + software_program_name="vipersci.vis.create_image", + software_version="0.6.0-dev", + start_time="2023-11-26T00:00:00Z", + stereo=1, + stop_time="2023-11-26T00:00:00.000111Z", + voltage_ramp=0, + yamcs_generation_time="2023-11-26T00:00:00Z", + yamcs_name="/ViperGround/Images/ImageData/Navcam_left_slog", + ) + engine = create_engine("sqlite:///:memory:") + listen(engine, "connect", load_spatialite) + session = Session(engine) + Base.metadata.create_all(engine) + session.add(ir1) + session.add(ir2) + session.commit() + + path_mock = create_autospec(Path) + path_mock.exists.return_value = True + + with patch("vipersci.vis.create_pano.Path", return_value=path_mock), patch( + "vipersci.vis.create_pano.imread" + ) as m_imread, patch("vipersci.vis.create_pano.np.hstack"), patch( + "vipersci.vis.create_pano.make_pano_record" + ) as m_mpr, patch( + "vipersci.vis.create_pano.isinstance", + side_effect=[True, True, True, True, True, True], + ): + session.add_all = Mock() + cp.create( + ["231126-000000-ncl-s", "231126-000000-ncr-s"], + session=session, + json=False, + ) + + self.assertEqual(m_imread.call_count, 2) + self.assertEqual(m_mpr.call_args[0][2], Path.cwd()) From c326c263eec60bf2c8f953bb9b33ca5f9535e9ee Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Wed, 29 Nov 2023 17:36:55 -0800 Subject: [PATCH 35/60] test(various): Added a bunch of new tests. --- tests/test_create_pano_product.py | 227 ++++++++++++++++++++++++++++++ tests/test_create_vis_dbs.py | 31 +++- tests/test_vis_pds.py | 79 +++++++++++ tests/test_xml.py | 86 +++++++++++ 4 files changed, 422 insertions(+), 1 deletion(-) create mode 100644 tests/test_create_pano_product.py create mode 100644 tests/test_vis_pds.py create mode 100644 tests/test_xml.py diff --git a/tests/test_create_pano_product.py b/tests/test_create_pano_product.py new file mode 100644 index 0000000..30f4038 --- /dev/null +++ b/tests/test_create_pano_product.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python +"""This module has tests for the vis.pds.create_raw functions.""" + +# Copyright 2022-2023, vipersci developers. +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +from argparse import ArgumentParser +from pathlib import Path +import unittest +from unittest.mock import patch + +from geoalchemy2 import load_spatialite +from sqlalchemy import create_engine +from sqlalchemy.event import listen +from sqlalchemy.orm import Session + +from vipersci.vis.db import Base +from vipersci.vis.db.image_records import ImageRecord +from vipersci.vis.db.junc_image_pano import JuncImagePano +from vipersci.vis.db.pano_records import PanoRecord +from vipersci.vis.pds import create_pano_product as cpp + +from datetime_sqlite import isozformat + + +class TestParser(unittest.TestCase): + def test_arg_parser(self): + p = cpp.arg_parser() + self.assertIsInstance(p, ArgumentParser) + self.assertRaises(SystemExit, p.parse_args) + d = vars( + p.parse_args( + ["--dburl", "db://foo:username@host/db", "product_id_goes_here"] + ) + ) + self.assertIn("dburl", d) + self.assertIn("template", d) + self.assertIn("tiff", d) + self.assertIn("output_dir", d) + self.assertIn("input", d) + + +class TestDatabase(unittest.TestCase): + def setUp(self) -> None: + self.pr = PanoRecord( + file_creation_datetime="2023-11-27T22:19:47.216057Z", + file_md5_checksum="9e9e7de3e8943ca2b355e5a5ffd47858", + file_path="231109-170000-ncl-pan.tif", + lines=2048, + product_id="231109-170000-ncl-pan", + rover_pan_max=180.0, + rover_pan_min=-180.0, + rover_tilt_max=15, + rover_tilt_min=-50, + samples=12288, + software_name="vipersci", + software_program_name="vipersci.vis.create_pano", + software_version="0.7.0-dev", + source_pids=[ + "231109-170000-ncl-c", + "231109-170100-ncl-c", + "231109-170200-ncl-c", + ], + start_time="2023-11-09T17:00:00Z", + stop_time="2023-11-09T17:05:00.000511Z", + ) + ir1 = ImageRecord( + adc_gain=0, + auto_exposure=0, + cameraId=0, + capture_id=1, + exposure_duration=511, + file_byte_offset=256, + file_creation_datetime="2023-11-27T22:18:11.879458Z", + file_data_type="UnsignedLSB2", + file_md5_checksum="cd30229a7803ec35fbf21a3da254ad10", + file_path="231109-170000-ncl-c.tif", + imageDepth=2, + image_id=0, + immediateDownloadInfo=24, + instrument_name="NavCam Left", + instrument_temperature=0, + lines=2048, + lobt=1699549200, + offset=0, + output_image_mask=8, + padding=0, + pga_gain=1.0, + processing_info=10, + product_id="231109-170000-ncl-c", + samples=2048, + software_name="vipersci", + software_program_name="vipersci.vis.create_image", + software_version="0.7.0-dev", + start_time="2023-11-09T17:00:00Z", + stereo=1, + stop_time="2023-11-09T17:02:00.000511Z", + voltage_ramp=0, + yamcs_generation_time="2023-11-09T17:00:00Z", + yamcs_name="/ViperGround/Images/ImageData/Navcam_left_icer", + yamcs_reception_time="2023-11-09T17:05:00Z", + ) + ir2 = ImageRecord( + adc_gain=0, + auto_exposure=0, + cameraId=0, + capture_id=1, + exposure_duration=511, + file_byte_offset=256, + file_creation_datetime="2023-11-27T22:18:12.948617Z", + file_data_type="UnsignedLSB2", + file_md5_checksum="781d510b1f7f7a4048f7a9eea596b9e6", + file_path="231109-170100-ncl-c.tif", + imageDepth=2, + image_id=0, + immediateDownloadInfo=24, + instrument_name="NavCam Left", + instrument_temperature=0, + lines=2048, + lobt=1699549260, + offset=0, + output_image_mask=8, + padding=0, + pga_gain=1.0, + processing_info=10, + product_id="231109-170100-ncl-c", + samples=2048, + software_name="vipersci", + software_program_name="vipersci.vis.create_image", + software_version="0.7.0-dev", + start_time="2023-11-09T17:01:00Z", + stereo=1, + stop_time="2023-11-09T17:01:00.000511Z", + voltage_ramp=0, + yamcs_generation_time="2023-11-09T17:01:00Z", + yamcs_name="/ViperGround/Images/ImageData/Navcam_left_icer", + yamcs_reception_time="2023-11-09T17:06:00Z", + ) + ir3 = ImageRecord( + adc_gain=0, + auto_exposure=0, + cameraId=0, + capture_id=1, + exposure_duration=511, + file_byte_offset=256, + file_creation_datetime="2023-11-27T22:18:14.068725Z", + file_data_type="UnsignedLSB2", + file_md5_checksum="ee7a86d7ed7a436d23dd6b87ae6c3058", + file_path="231109-170200-ncl-c.tif", + imageDepth=2, + image_id=0, + immediateDownloadInfo=24, + instrument_name="NavCam Left", + instrument_temperature=0, + lines=2048, + lobt=1699549320, + offset=0, + output_image_mask=8, + padding=0, + pga_gain=1.0, + processing_info=10, + product_id="231109-170200-ncl-c", + samples=2048, + software_name="vipersci", + software_program_name="vipersci.vis.create_image", + software_version="0.7.0-dev", + start_time="2023-11-09T17:02:00Z", + stereo=1, + stop_time="2023-11-09T17:02:00.000511Z", + voltage_ramp=0, + yamcs_generation_time="2023-11-09T17:02:00Z", + yamcs_name="/ViperGround/Images/ImageData/Navcam_left_icer", + yamcs_reception_time="2023-11-09T17:07:00Z", + ) + self.engine = create_engine("sqlite:///:memory:") + listen(self.engine, "connect", load_spatialite) + self.session = Session(self.engine) + Base.metadata.create_all(self.engine) + to_add = [ + self.pr, + ] + for ir in (ir1, ir2, ir3): + self.session.add(ir) + a = JuncImagePano() + a.image_record = ir + a.pano_record = self.pr + to_add.append(a) + + self.session.add_all(to_add) + self.session.commit() + + def tearDown(self): + Base.metadata.drop_all(self.engine) + + def test_label_dict(self): + d = cpp.label_dict(self.pr) + self.assertEqual( + d["lid"], f"urn:nasa:pds:viper_vis:data_derived:{self.pr.product_id}" + ) + self.assertEqual(d["source_product_type"], "data_to_raw_source_product") + self.assertEqual(d["instruments"][0]["name"], "NavCam Left") + + @patch("vipersci.vis.pds.create_pano_product.create_engine") + @patch("vipersci.vis.pds.create_pano_product.write_xml") + def test_main(self, m_write_xml, m_create_engine): + with patch( + "vipersci.vis.pds.create_pano_product.Session", return_value=self.session + ): + pa_ret_val = cpp.arg_parser().parse_args( + [ + "--dburl", + "db://foo:username@host/db", + self.pr.product_id, + ] + ) + with patch("vipersci.vis.pds.create_pano_product.arg_parser") as parser: + parser.return_value.parse_args.return_value = pa_ret_val + with patch("vipersci.vis.db.image_records.isozformat", new=isozformat): + cpp.main() + m_write_xml.assert_called_once() + (metadata, template, outdir) = m_write_xml.call_args[0] + self.assertEqual(metadata["product_id"], self.pr.product_id) + self.assertEqual(outdir, Path.cwd()) + self.assertEqual(template, "pano-template.xml") diff --git a/tests/test_create_vis_dbs.py b/tests/test_create_vis_dbs.py index 3e8a310..86f4a3f 100644 --- a/tests/test_create_vis_dbs.py +++ b/tests/test_create_vis_dbs.py @@ -12,12 +12,13 @@ from unittest.mock import patch from geoalchemy2 import load_spatialite -from sqlalchemy import create_engine +from sqlalchemy import create_engine, insert from sqlalchemy.event import listen from sqlalchemy.orm import Session from vipersci.vis.db import Base from vipersci.vis.db import create_vis_dbs as cvd +from vipersci.vis.db.image_tags import ImageTag, taglist class TestParser(unittest.TestCase): @@ -46,3 +47,31 @@ def test_main(self): "vipersci.vis.db.create_vis_dbs.create_engine", return_value=self.engine ): cvd.main() + + def test_partial_taglist(self): + Base.metadata.create_all(self.engine) + self.session.execute(insert(ImageTag), {"name": taglist[0]}) + self.session.commit() + + pa_ret_val = cvd.arg_parser().parse_args(["-d", "foo"]) + with patch("vipersci.vis.db.create_vis_dbs.arg_parser") as parser: + parser.return_value.parse_args.return_value = pa_ret_val + with patch( + "vipersci.vis.db.create_vis_dbs.create_engine", return_value=self.engine + ): + self.assertRaises(ValueError, cvd.main) + + def test_full_taglist(self): + Base.metadata.create_all(self.engine) + bad_taglist = taglist.copy() + bad_taglist[0] = "Not a valid tag." + self.session.execute(insert(ImageTag), [{"name": x} for x in bad_taglist]) + self.session.commit() + + pa_ret_val = cvd.arg_parser().parse_args(["-d", "foo"]) + with patch("vipersci.vis.db.create_vis_dbs.arg_parser") as parser: + parser.return_value.parse_args.return_value = pa_ret_val + with patch( + "vipersci.vis.db.create_vis_dbs.create_engine", return_value=self.engine + ): + self.assertRaises(ValueError, cvd.main) diff --git a/tests/test_vis_pds.py b/tests/test_vis_pds.py new file mode 100644 index 0000000..8b4c114 --- /dev/null +++ b/tests/test_vis_pds.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +"""Tests for the `vis/pds` module.""" + +# Copyright 2023, United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +from pathlib import Path +import unittest +from unittest.mock import create_autospec, patch + +from genshi.template import Template + +from vipersci.vis import pds + + +class TestVersion(unittest.TestCase): + def test_version_info(self): + d = pds.version_info() + self.assertEqual(0.1, d["vid"]) + + +class TestXML(unittest.TestCase): + def test_write_xml(self): + product = { + "dummy": "product", + "product_id": "dummy_pid", + } + m_outdir = create_autospec(Path) + m_tmpl = create_autospec(Template) + with patch( + "vipersci.vis.pds.MarkupTemplate", return_value=m_tmpl + ) as m_markup, patch("vipersci.vis.pds.resources.read_text") as m_read_text: + pds.write_xml(product, "raw-template.xml", m_outdir) + + m_read_text.assert_called_once() + m_markup.assert_called_once() + m_tmpl.generate.assert_called_once() + + def test_write_xml_custom_template(self): + product = { + "dummy": "product", + "product_id": "dummy_pid", + } + + path_mock = create_autospec(Path) + path_mock.exists.return_value = True + + m_outdir = create_autospec(Path) + m_tmpl = create_autospec(Template) + + with patch("vipersci.vis.pds.Path", return_value=path_mock), patch( + "vipersci.vis.pds.MarkupTemplate", return_value=m_tmpl + ) as m_markup: + pds.write_xml(product, "this_exists.xml", m_outdir) + + path_mock.read_text.assert_called_once() + m_markup.assert_called_once() + m_tmpl.generate.assert_called_once() diff --git a/tests/test_xml.py b/tests/test_xml.py new file mode 100644 index 0000000..122a3fc --- /dev/null +++ b/tests/test_xml.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +"""Tests for the `pds/xml` module.""" + +# Copyright 2023, United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +from textwrap import dedent +import unittest +import xml.etree.ElementTree as ET + +from vipersci.pds import xml + + +class TestXML(unittest.TestCase): + def setUp(self): + self.xmltext = dedent( + """\ + + + + + + + 2023-11-02T23:12:59.083415Z + + + 0 + PDS DSV 1 + 4 + Carriage-Return Line-Feed + Comma + + + + """ + ) + + def test_find_text(self): + root = ET.fromstring(self.xmltext) + self.assertRaises( + ValueError, + xml.find_text, + root, + ".//pds:File_Area_Inventory/pds:Inventory/pds:offset", + unit_check="bit", + ) + + self.assertRaises( + ValueError, + xml.find_text, + root, + ".//pds:File_Area_Inventory/pds:File/pds:file_name", + ) + + self.assertRaises( + ValueError, + xml.find_text, + root, + ".//pds:File_Area_Inventory/pds:File/pds:md5", + ) From 1d8ec99320f7c43eb23c85845f658d0486700676 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Thu, 30 Nov 2023 14:48:03 -0800 Subject: [PATCH 36/60] fix(bundle_install.py): Subdirectories in source collections will now be properly built in output. --- src/vipersci/pds/bundle_install.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/vipersci/pds/bundle_install.py b/src/vipersci/pds/bundle_install.py index b49cdd6..04c8efe 100644 --- a/src/vipersci/pds/bundle_install.py +++ b/src/vipersci/pds/bundle_install.py @@ -125,6 +125,7 @@ def main(): f_lidvid = f["lid"] + "::" + f["vid"] if f_lidvid in col_lidvids: dest_path = bld_col_dir / p.relative_to(src_col_dir) + dest_path.parent.mkdir(parents=True, exist_ok=True) copy2(p, dest_path) copy2(p.with_name(f["productfile"]), dest_path.parent) col_lidvids.remove(f_lidvid) From 7b2d72f3f3a9e26e7848e9c0d8c98843ef08bcec Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Thu, 30 Nov 2023 17:44:12 -0800 Subject: [PATCH 37/60] chore(.gitignore): Ignore PyCharm .idea directory. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 56e21c3..1ca6b2a 100644 --- a/.gitignore +++ b/.gitignore @@ -110,6 +110,9 @@ ENV/ env.bak/ venv.bak/ +# PyCharm +.idea + # Spyder project settings .spyderproject .spyproject From e0c8b5d5121add5f1e21c208f27131cc77934c8d Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Mon, 4 Dec 2023 16:26:09 -0800 Subject: [PATCH 38/60] feat(image_statistics.py): Added central_clear metric and experimental aggregate_noise function. --- src/vipersci/vis/image_statistics.py | 46 +++++++++++++++++++++++++--- tests/test_image_statistics.py | 2 ++ 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/src/vipersci/vis/image_statistics.py b/src/vipersci/vis/image_statistics.py index e1355cd..792a960 100644 --- a/src/vipersci/vis/image_statistics.py +++ b/src/vipersci/vis/image_statistics.py @@ -31,6 +31,9 @@ import numpy as np import numpy.typing as npt +from rasterio.windows import Window +from scipy.ndimage import generic_filter +from skimage.morphology import disk from skimage.io import imread from skimage import measure @@ -44,6 +47,15 @@ OVEREXPOSED_THRESHOLD = 4095 * 0.8 +class CentralMask: + def __init__(self, image): + self.array = np.zeros_like(image, dtype=int) + quarter_width = int(image.shape[0] / 4) + self.disk = disk(quarter_width) + self.window = Window(quarter_width, quarter_width, *self.disk.shape) + self.array[self.window.toslices()] += self.disk + + def arg_parser(): parser = argparse.ArgumentParser( description=__doc__, parents=[util.parent_parser()] @@ -64,21 +76,42 @@ def main(): print(pprint(image)) + print(f"aggregate_noise: {aggregate_noise(image)} DN") + return +def aggregate_noise(image): + # This seems to yield a high value, and is slooooooow + logger.info("aggregate_noise start") + logger.info("Start std filtering.") + std_filtered = generic_filter(image, np.std, size=(3, 3)) + logger.info("Start mean filtering.") + mean_filtered = generic_filter(image, np.mean, size=(3, 3)) + logger.info("Dividing.") + return np.nansum(np.divide(std_filtered, mean_filtered, where=mean_filtered != 0)) + + def compute( image: ImageType, overexposed_thresh=OVEREXPOSED_THRESHOLD, underexposed_thresh=UNDEREXPOSED_THRESHOLD, ) -> dict: + mask_indices = np.logical_and( + np.logical_and( + image > underexposed_thresh, + image < overexposed_thresh, + ), + CentralMask(image).array, + ) d = { "blur": measure.blur_effect(image), "mean": np.mean(image), "std": np.std(image), "over_exposed": (image > overexposed_thresh).sum(), "under_exposed": (image < underexposed_thresh).sum(), - # "aggregate_noise": + "central_clear": np.count_nonzero(mask_indices), + # "aggregate_noise": aggregate_noise(image) # number of dead pixels } @@ -91,16 +124,21 @@ def pprint( underexposed_thresh=UNDEREXPOSED_THRESHOLD, ) -> str: d = compute(image, overexposed_thresh, underexposed_thresh) + disk_size = CentralMask(image).disk.sum() s = dedent( f"""\ blur: {d['blur']} (0 for no blur, 1 for maximal blur) mean: {d['mean']} std: {d['std']} - over-exposed: {d['over_exposed']} pixels,\ + over-exposed (>{overexposed_thresh}): {d['over_exposed']} pixels,\ {100 * d['over_exposed'] / image.size} % - under-exposed: {d['under_exposed']} pixels,\ - {100 * d['under_exposed'] / image.size} %\ + under-exposed (<{underexposed_thresh}): {d['under_exposed']} pixels,\ + {100 * d['under_exposed'] / image.size} % + central_clear: {d['central_clear']} pixels, \ + {100 * d['central_clear'] / disk_size} % """ + # aggregate_noise: {d['aggregate_noise']} DN + # """ ) return s diff --git a/tests/test_image_statistics.py b/tests/test_image_statistics.py index a510ca2..f5fb49e 100644 --- a/tests/test_image_statistics.py +++ b/tests/test_image_statistics.py @@ -56,7 +56,9 @@ def setUp(self): ) def test_compute(self): + # print(image_statistics.pprint(self.image)) stats = image_statistics.compute(self.image) self.assertEqual(stats["blur"], 1.0) self.assertAlmostEqual(stats["mean"], 2374.36363636) self.assertAlmostEqual(stats["std"], 1310.43637928) + self.assertEqual(stats["central_clear"], 13) From eb0d8f091c964756baeceb205ecf58330ba50130 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Mon, 11 Dec 2023 14:45:26 -0800 Subject: [PATCH 39/60] fix(anom_pixel.py): Previously, the difference math was being done with unsigned ints which wrapped around. --- src/vipersci/vis/anom_pixel.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/vipersci/vis/anom_pixel.py b/src/vipersci/vis/anom_pixel.py index 3d19fa9..e5d8d4a 100644 --- a/src/vipersci/vis/anom_pixel.py +++ b/src/vipersci/vis/anom_pixel.py @@ -64,6 +64,7 @@ def main(): indices = check(image, args.tolerance) print(f"There are {len(indices[0])} anomalous pixels.") + print(f"or {100 * len(indices[0]) / image.size} %") print(indices) return @@ -76,7 +77,7 @@ def check(image, tolerance=3): """ blurred = median(image) - difference = image - blurred + difference = image.astype(int) - blurred.astype(int) threshold = tolerance * np.std(difference) logger.info(f"threshold: {threshold}") From 5fc3967b1bb8a2e04458b0bf1022c5bf478783fe Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Thu, 14 Dec 2023 19:14:26 -0800 Subject: [PATCH 40/60] feat(create_raw.py vis/pds/__init__.py pds/labelmaker/__init__.py): Observing systems are not unique on their lids, but on their Names. --- src/vipersci/pds/labelmaker/__init__.py | 4 ++-- src/vipersci/vis/pds/__init__.py | 1 + src/vipersci/vis/pds/create_raw.py | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/vipersci/pds/labelmaker/__init__.py b/src/vipersci/pds/labelmaker/__init__.py index 3b1e577..37820b3 100644 --- a/src/vipersci/pds/labelmaker/__init__.py +++ b/src/vipersci/pds/labelmaker/__init__.py @@ -58,8 +58,8 @@ def get_common_label_info(element: ET.Element, area="pds:Observation_Area"): ".//pds:Observing_System_Component[pds:type='Instrument']", ns ): instruments[ - find_text(i, "pds:Internal_Reference/pds:lid_reference") - ] = find_text(i, "pds:name") + find_text(i, "pds:name") + ] = find_text(i, "pds:Internal_Reference/pds:lid_reference") purposes = [] for p in element.findall( diff --git a/src/vipersci/vis/pds/__init__.py b/src/vipersci/vis/pds/__init__.py index 5a8da7a..a7ab538 100644 --- a/src/vipersci/vis/pds/__init__.py +++ b/src/vipersci/vis/pds/__init__.py @@ -36,6 +36,7 @@ "bundle": "urn:nasa:pds:viper_vis", "mission": "urn:nasa:pds:viper", "spacecraft": "urn:nasa:pds:context:instrument_host:spacecraft.viper", + "instrument": "urn:nasa:pds:context:instrument:spacecraft.vis", } diff --git a/src/vipersci/vis/pds/create_raw.py b/src/vipersci/vis/pds/create_raw.py index 4320c32..e1d4b28 100644 --- a/src/vipersci/vis/pds/create_raw.py +++ b/src/vipersci/vis/pds/create_raw.py @@ -278,7 +278,7 @@ def get_lights(ir: ImageRecord, session: Session): def label_dict(ir: ImageRecord, lights: dict): """Returns a dictionary suitable for label generation.""" - _inst = ir.instrument_name.lower().replace(" ", "_") + # _inst = ir.instrument_name.lower().replace(" ", "_") onoff = {True: "On", False: "Off", None: None} pid = pds.VISID(ir.product_id) d = dict( @@ -286,7 +286,7 @@ def label_dict(ir: ImageRecord, lights: dict): lid=f"{lids['bundle']}:data_raw:{ir.product_id}", mission_lid=lids["mission"], sc_lid=lids["spacecraft"], - inst_lid=f"{lids['spacecraft']}.{_inst}", + inst_lid=f"{lids['instrument']}", gain_number=(ir.adc_gain * ir.pga_gain), exposure_type="Auto" if ir.auto_exposure else "Manual", image_filters=list(), From 004a366811392c00695f46bd19bf6bae025dd800 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Thu, 28 Dec 2023 17:58:37 -0800 Subject: [PATCH 41/60] feat(create_mmgis_pano.py): Added. --- CHANGELOG.rst | 1 + setup.cfg | 1 + src/vipersci/vis/create_mmgis_pano.py | 202 ++++++++++++++++++++++++++ 3 files changed, 204 insertions(+) create mode 100644 src/vipersci/vis/create_mmgis_pano.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2e53203..e7be031 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -53,6 +53,7 @@ Added - image_requests.py - "Acquired," "Not Acquired," "Not Planned," and "Not Obtainable" statuses added to enum. - ptu_records.py - Tables to record the pan and tilt of the rover's pan-tilt-unit (PTU). +- create_mmgis_pano.py - For making pano products for use in MMGIS. - create_pano.py - updated to correctly add PanoRecord associations, can now query database for ImageRecords. - create_pano_product.py - takes PanoRecords and makes a PDS Pano Product. diff --git a/setup.cfg b/setup.cfg index a2f0a85..70b1893 100644 --- a/setup.cfg +++ b/setup.cfg @@ -83,6 +83,7 @@ console_scripts = tri2gpkg = vipersci.carto.tri2gpkg:main template_test = vipersci.vis.pds.template_test:main vis_create_image = vipersci.vis.create_image:main + vis_create_mmgis_pano = vipersci.vis.create_mmgis_pano:main vis_create_pano = vipersci.vis.create_pano:main vis_create_pano_product = vipersci.vis.pds.create_pano_product:main vis_create_tif = vipersci.vis.create_tif:main diff --git a/src/vipersci/vis/create_mmgis_pano.py b/src/vipersci/vis/create_mmgis_pano.py new file mode 100644 index 0000000..749a001 --- /dev/null +++ b/src/vipersci/vis/create_mmgis_pano.py @@ -0,0 +1,202 @@ +"""Creates a panorama file and JSON stream suitable for use in MMGIS. +""" + +# Copyright 2023, United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +import argparse +import json +import logging +from typing import Union, Optional +from pathlib import Path + +import numpy as np +import numpy.typing as npt +from skimage.io import imread, imsave # maybe just imageio here? +from skimage.exposure import equalize_adapthist, rescale_intensity +from sqlalchemy import create_engine, select +from sqlalchemy.orm import Session + +from vipersci.vis.db.pano_records import PanoRecord +from vipersci.pds import pid as pds +from vipersci import util + +logger = logging.getLogger(__name__) + +ImageType = Union[npt.NDArray[np.uint16], npt.NDArray[np.uint8]] + + +def arg_parser(): + parser = argparse.ArgumentParser( + description=__doc__, parents=[util.parent_parser()] + ) + parser.add_argument( + "-d", + "--dburl", + help="Database with a raw_products table and a panorama_products table which " + "will be read from and written to. If not given, no database will be " + "written to. " + "Example: postgresql://postgres:NotTheDefault@localhost/visdb", + ) + parser.add_argument( + "-m", + "--mapserver", + help="URL that will respond to requests when given an event_time and " + "a crs_code. Alternately, if a float is provided, this will be used as " + "the rover yaw (zero==north) for testing.", + ) + parser.add_argument( + "-o", + "--output_dir", + type=Path, + default=Path.cwd(), + help="Output directory for label. Defaults to current working directory. " + "Output file names are fixed based on product_id, and will be over-written.", + ) + parser.add_argument( + "--prefix", + type=Path, + default=Path.cwd(), + help="A directory path that, if given, will be prepended to paths given via " + "inputs or will be prepended to the file_path values returned from a database " + "query.", + ) + parser.add_argument( + "input", help="Either VIS Pano product IDs or a JSON file." + ) + return parser + + +def main(): + args = arg_parser().parse_args() + util.set_logger(args.verbose) + + create_args = [args.input, args.prefix, args.output_dir, args.mapserver] + + if args.dburl is None: + create_args.append(None) + create(*create_args) + else: + engine = create_engine(args.dburl) + with Session(engine) as session: + create_args.append(session) + create(*create_args) + + session.commit() + + return + + +def create( + info: Union[Path, pds.PanoID, PanoRecord, str], + prefixdir: Optional[Path] = None, + outdir: Path = Path.cwd(), + mapserver: str = None, + session: Optional[Session] = None, +): + """ + Creates an MMGIS Panorama in *outdir*. Returns None. + + At this time, *input* should be a list of file paths or product IDs. + + If a path is provided to *outdir* the created files + will be written there. + + If *session* is given, information for the PanoRecord will be + written to the pano_records table. If not, no database activity + will occur. + """ + + # vid = None + # pano = {} + # source_path = Path() + + if isinstance(info, str): + temp_vid = pds.PanoID(info) + if str(temp_vid) == info: + info = temp_vid + + if isinstance(info, pds.PanoID): + if session is not None: + pr = session.scalars( + select(PanoRecord).where(PanoRecord.product_id == str(info)) + ).first() + if pr is None: + raise ValueError(f"{info} was not found in the database.") + else: + info = pr + else: + raise ValueError(f"Without a database session, can't lookup {info}") + + if isinstance(info, PanoRecord): + vid = pds.PanoID(info.product_id) + source_path = Path(info.file_path) if prefixdir is None else prefixdir / info.file_path + pano = info.asdict() + elif isinstance(info, (Path, str)): + vid = pds.PanoID(info) + with open(info) as f: + pano = json.load(f) + source_path = Path(pano["file_path"]) if prefixdir is None else prefixdir / pano["file_path"] + else: + raise ValueError( + f"an element in input is not the right type: {info} ({type(info)})" + ) + + if mapserver is None: + yaw = 0 + else: + try: + yaw = float(mapserver) + except ValueError: + raise NotImplementedError(f"mapserver queries are not yet implemented.") + + # Convert to PNG + image = equalize_adapthist(imread(str(source_path))) + image8 = rescale_intensity( + image, in_range="image", out_range="uint8" + ).astype("uint8") + + outpath = outdir / source_path.with_suffix(".png").name + + imsave(str(outpath), image8, check_contrast=False) + + d = mmgis_data(pano, yaw) + d["url"] = "not/sure/what/the/path/should/be/to/" + outpath.name + + with open(outpath.stem + "_mmgis.json", "w") as f: + json.dump(d, f, indent=2, sort_keys=True) + + return + + +def mmgis_data(pano_data: dict, yaw=0): + d = { + "azmax": yaw + pano_data["rover_pan_max"], + "azmin": yaw + pano_data["rover_pan_min"], + "columns": pano_data["samples"], + "elmax": pano_data["rover_tilt_max"], + "elmin": pano_data["rover_tilt_min"], + "isPanoramic": True, + "name": pano_data["product_id"], + "rows": pano_data["lines"], + } + return d From 2a65b082459a0caa8357892716aa257e491ff77f Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Thu, 28 Dec 2023 18:00:27 -0800 Subject: [PATCH 42/60] feat(image_requests.py): Added asdict() method. --- CHANGELOG.rst | 2 +- src/vipersci/vis/db/image_requests.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e7be031..704a839 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -51,7 +51,7 @@ Added - pano_records table now has pan and tilt angle min/max values to indicate angular range of panorama coverage. - image_requests.py - "Acquired," "Not Acquired," "Not Planned," and "Not Obtainable" - statuses added to enum. + statuses added to enum. Also added asdict() method. - ptu_records.py - Tables to record the pan and tilt of the rover's pan-tilt-unit (PTU). - create_mmgis_pano.py - For making pano products for use in MMGIS. - create_pano.py - updated to correctly add PanoRecord associations, can now query diff --git a/src/vipersci/vis/db/image_requests.py b/src/vipersci/vis/db/image_requests.py index 399282f..64e8396 100644 --- a/src/vipersci/vis/db/image_requests.py +++ b/src/vipersci/vis/db/image_requests.py @@ -62,6 +62,7 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from datetime import datetime import enum from typing import Sequence, Union @@ -76,6 +77,7 @@ from sqlalchemy.orm import mapped_column, relationship, validates from geoalchemy2 import Geometry # type: ignore +from vipersci.pds.datetime import isozformat from vipersci.pds.pid import vis_instruments from vipersci.vis.db import Base from vipersci.vis.db.light_records import luminaire_names @@ -270,3 +272,17 @@ def validate_listing( ) return ",".join(value) + + def asdict(self): + d = {} + + for c in self.__table__.columns: + if isinstance(getattr(self, c.name), datetime): + d[c.name] = isozformat(getattr(self, c.name)) + else: + d[c.name] = getattr(self, c.name) + + if hasattr(self, "labelmeta"): + d.update(self.labelmeta) + + return d From ff4ce72d5e5c232b4d8520a6dc9dcfeec7feae1e Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Thu, 28 Dec 2023 18:02:34 -0800 Subject: [PATCH 43/60] feat(pano-template.xml and raw-template.xml): Updated to PDS information model 21. --- CHANGELOG.rst | 1 + src/vipersci/vis/pds/data/pano-template.xml | 22 ++++++++--------- src/vipersci/vis/pds/data/raw-template.xml | 26 ++++++++++----------- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 704a839..e3417d6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -178,6 +178,7 @@ Added Changed ^^^^^^^ +- Updated templates and modules for PDS information model 21. - Flattened test directory structure. - tri2gpkg -v is no longer an alias for --value-names, as it now determines verbosity since logging has been added. diff --git a/src/vipersci/vis/pds/data/pano-template.xml b/src/vipersci/vis/pds/data/pano-template.xml index 871001d..4d06f67 100644 --- a/src/vipersci/vis/pds/data/pano-template.xml +++ b/src/vipersci/vis/pds/data/pano-template.xml @@ -1,9 +1,9 @@ - - - - - + + + + + ${lid} ${vid} VIPER Visible Imaging System Panorama - ${product_id} - 1.18.0.0 + 1.21.0.0 Product_Observational diff --git a/src/vipersci/vis/pds/data/raw-template.xml b/src/vipersci/vis/pds/data/raw-template.xml index 71918ff..e500929 100644 --- a/src/vipersci/vis/pds/data/raw-template.xml +++ b/src/vipersci/vis/pds/data/raw-template.xml @@ -1,10 +1,10 @@ - - - - - - + + + + + + ${lid} ${vid} VIPER Visible Imaging System ${instrument_name} image - ${product_id} - 1.18.0.0 + 1.21.0.0 Product_Observational From ad01c7f1d4ebf65fe5c3721d7c3dae183e55306a Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Thu, 28 Dec 2023 18:03:27 -0800 Subject: [PATCH 44/60] fix( raw-template.xml): Apparently you can only have one img:Image_Filter item. --- CHANGELOG.rst | 1 + src/vipersci/vis/pds/data/raw-template.xml | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e3417d6..7a8cce5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -197,6 +197,7 @@ Fixed - heatmap's generate_density_heatmap() function now properly returns values of zero in the returned out_count numpy array when there are no counts in those grid cells instead of the provided nodata value. +- raw-template.xml can only have one Image_Filter object. - tri2gpkg now works correctly if --keep_z is specified - tri2gpkg now uses the correct srs if a pre-defined site is selected. diff --git a/src/vipersci/vis/pds/data/raw-template.xml b/src/vipersci/vis/pds/data/raw-template.xml index e500929..e428854 100644 --- a/src/vipersci/vis/pds/data/raw-template.xml +++ b/src/vipersci/vis/pds/data/raw-template.xml @@ -121,14 +121,14 @@ ${led_wavelength} - - ${venue} - ${algorithm} + + Onboard + ${image_filters} ${compression_class} - ${onboard_compression_ratio} ${onboard_compression_type} + ${onboard_compression_ratio} ${minloss} From 6e8fc52171ab12ed91c753fdfccc342e5751bd8e Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Thu, 28 Dec 2023 18:19:28 -0800 Subject: [PATCH 45/60] fix(create_image.py and create_raw.py): Deal with the fact that an 8-bit image is not an UnsignedLSB1, but an UnsignedByte. --- src/vipersci/vis/create_image.py | 16 +++++++++++++--- src/vipersci/vis/pds/create_raw.py | 21 ++++++++++----------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/src/vipersci/vis/create_image.py b/src/vipersci/vis/create_image.py index e763b99..8ae6574 100644 --- a/src/vipersci/vis/create_image.py +++ b/src/vipersci/vis/create_image.py @@ -194,7 +194,10 @@ def check_bit_depth(pid: pds.VISID, bit_depth: Union[int, str, np.dtype]): if isinstance(bit_depth, int): bd = bit_depth elif isinstance(bit_depth, str): - bd = int(bit_depth[-1]) * 8 + if bit_depth.endswith("Byte"): + bd = 8 + else: + bd = int(bit_depth[-1]) * 8 elif isinstance(bit_depth, np.dtype): if bit_depth == np.uint16: bd = 16 @@ -287,11 +290,18 @@ def tif_info(p: Path) -> dict: end = "MSB" if info["bigEndian"] else "LSB" + # Tag 258 is bits per pixel: + bpp = int(tags[258]['data'][0] / 8) + + if bpp == 1: + dt_end = "Byte" + else: + dt_end = end + str(bpp) + d = { "file_byte_offset": tags[273]["data"][0], # Tag 273 is StripOffsets "file_creation_datetime": dt, - # Tag 258 is bits per pixel: - "file_data_type": f"Unsigned{end}{int(tags[258]['data'][0] / 8)}", + "file_data_type": f"Unsigned{dt_end}", "file_md5_checksum": md5.hexdigest(), "file_path": p.name, "lines": tags[257]["data"][0], # Tag 257 is ImageWidth, diff --git a/src/vipersci/vis/pds/create_raw.py b/src/vipersci/vis/pds/create_raw.py index e1d4b28..211a4e1 100644 --- a/src/vipersci/vis/pds/create_raw.py +++ b/src/vipersci/vis/pds/create_raw.py @@ -167,11 +167,7 @@ def main(): if args.input.endswith(".tif"): args.tiff = Path(args.input) - if args.tiff is None: - # Make up some values: - metadata["file_byte_offset"] = 0 - metadata["file_data_type"] = "UnsignedLSB2" - else: + if args.tiff is not None: t_info = tif_info(args.tiff) for k, v in t_info.items(): @@ -289,7 +285,7 @@ def label_dict(ir: ImageRecord, lights: dict): inst_lid=f"{lids['instrument']}", gain_number=(ir.adc_gain * ir.pga_gain), exposure_type="Auto" if ir.auto_exposure else "Manual", - image_filters=list(), + image_filters="", led_wavelength=453, # nm luminaires={}, compression_class=pid.compression_class(), @@ -314,19 +310,22 @@ def label_dict(ir: ImageRecord, lights: dict): f"processing_info ({ir.processing_info}) is not one " f"of {list(ProcessingStage)}, so assuming a value of {proc_info}" ) + + im_filt = list() if ProcessingStage.FLATFIELD in proc_info: - d["image_filters"].append(("Onboard", "Flat field normalization.")) + im_filt.append("Flat field normalization.") if ProcessingStage.LINEARIZATION in proc_info: - d["image_filters"].append(("Onboard", "Linearization.")) + im_filt.append("Linearization.") if ProcessingStage.SLOG in proc_info: - d["image_filters"].append( - ("Onboard", "Sign of the Laplacian of the Gaussian, SLoG") - ) + im_filt.append("Sign of the Laplacian of the Gaussian, SLoG.") d["sample_bits"] = 8 d["sample_bit_mask"] = "2#11111111" + if len(im_filt ) > 0: + d["image_filters"] = " ".join(im_filt) + if ir.image_request is not None: d["observational_intent"]["goal"] = ir.image_request.justification d["observational_intent"]["task"] = ir.image_request.title From 9ed33a1fba41a1d604e489e501ad1cd9eb128f60 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Thu, 28 Dec 2023 18:20:25 -0800 Subject: [PATCH 46/60] fix(create_pano_product.py): Source lidvids need to be FULL lidvids. Faking the vid for now. --- src/vipersci/vis/pds/create_pano_product.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/vipersci/vis/pds/create_pano_product.py b/src/vipersci/vis/pds/create_pano_product.py index e5e2291..3a155be 100644 --- a/src/vipersci/vis/pds/create_pano_product.py +++ b/src/vipersci/vis/pds/create_pano_product.py @@ -206,7 +206,10 @@ def label_dict(pr: PanoRecord): "name": ir.instrument_name, "lid": f"{lids['spacecraft']}.{_inst}", } - d["source_product_lidvids"].append(str(ir.product_id)) # type: ignore + # todo: need to figure out how to set version id here + d["source_product_lidvids"].append( + f"urn:nasa:pds:viper_vis:data_raw:{ir.product_id}:0.1" + ) # type: ignore for inst in instruments_dict.values(): d["instruments"].append(inst) # type: ignore From 0009fe6ea2253114ccea3ae4036e67490e9756a2 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Thu, 28 Dec 2023 18:30:05 -0800 Subject: [PATCH 47/60] fix(test_create_raw.py and test_labelmaker.py): Updated tests for recent changes. --- tests/test_create_raw.py | 2 +- tests/test_labelmaker.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_create_raw.py b/tests/test_create_raw.py index 445d7b6..06fc3d0 100644 --- a/tests/test_create_raw.py +++ b/tests/test_create_raw.py @@ -147,7 +147,7 @@ def test_label_dict(self): self.assertEqual(d["exposure_type"], "Manual") self.assertEqual(d["luminaires"][list(luminaire_names.values())[0]], "Off") self.assertEqual( - d["image_filters"][0], ("Onboard", "Flat field normalization.") + d["image_filters"], "Flat field normalization. Linearization." ) self.assertEqual(d["sample_bits"], 12) diff --git a/tests/test_labelmaker.py b/tests/test_labelmaker.py index 939723d..62275c4 100644 --- a/tests/test_labelmaker.py +++ b/tests/test_labelmaker.py @@ -169,8 +169,8 @@ def test_get_common_label_info(self): "target_type": "Satellite", "target_lid": "urn:nasa:pds:context:target:satellite.earth.moon", "instruments": { - f"{host_lid}.navcam_left": "NavCam Left", - f"{host_lid}.navcam_right": "NavCam Right", + "NavCam Left": f"{host_lid}.navcam_left", + "NavCam Right": f"{host_lid}.navcam_right", }, "purposes": [ "Engineering", From 2b4b5b367d1c636e3f044e98c967e1d4b5c07434 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Fri, 29 Dec 2023 09:41:42 -0800 Subject: [PATCH 48/60] refactor(create_mmgis_pano.py): lint cleanup --- src/vipersci/vis/create_mmgis_pano.py | 34 +++++++++++++++------------ 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/vipersci/vis/create_mmgis_pano.py b/src/vipersci/vis/create_mmgis_pano.py index 749a001..8bbf941 100644 --- a/src/vipersci/vis/create_mmgis_pano.py +++ b/src/vipersci/vis/create_mmgis_pano.py @@ -61,8 +61,8 @@ def arg_parser(): "-m", "--mapserver", help="URL that will respond to requests when given an event_time and " - "a crs_code. Alternately, if a float is provided, this will be used as " - "the rover yaw (zero==north) for testing.", + "a crs_code. Alternately, if a float is provided, this will be used as " + "the rover yaw (zero==north) for testing.", ) parser.add_argument( "-o", @@ -80,9 +80,7 @@ def arg_parser(): "inputs or will be prepended to the file_path values returned from a database " "query.", ) - parser.add_argument( - "input", help="Either VIS Pano product IDs or a JSON file." - ) + parser.add_argument("input", help="Either VIS Pano product IDs or a JSON file.") return parser @@ -110,7 +108,7 @@ def create( info: Union[Path, pds.PanoID, PanoRecord, str], prefixdir: Optional[Path] = None, outdir: Path = Path.cwd(), - mapserver: str = None, + mapserver: Optional[str] = None, session: Optional[Session] = None, ): """ @@ -148,32 +146,38 @@ def create( raise ValueError(f"Without a database session, can't lookup {info}") if isinstance(info, PanoRecord): - vid = pds.PanoID(info.product_id) - source_path = Path(info.file_path) if prefixdir is None else prefixdir / info.file_path + # vid = pds.PanoID(info.product_id) + source_path = ( + Path(info.file_path) if prefixdir is None else prefixdir / info.file_path + ) pano = info.asdict() elif isinstance(info, (Path, str)): - vid = pds.PanoID(info) + # vid = pds.PanoID(info) with open(info) as f: pano = json.load(f) - source_path = Path(pano["file_path"]) if prefixdir is None else prefixdir / pano["file_path"] + source_path = ( + Path(pano["file_path"]) + if prefixdir is None + else prefixdir / pano["file_path"] + ) else: raise ValueError( f"an element in input is not the right type: {info} ({type(info)})" ) if mapserver is None: - yaw = 0 + yaw = 0.0 else: try: yaw = float(mapserver) except ValueError: - raise NotImplementedError(f"mapserver queries are not yet implemented.") + raise NotImplementedError("mapserver queries are not yet implemented.") # Convert to PNG image = equalize_adapthist(imread(str(source_path))) - image8 = rescale_intensity( - image, in_range="image", out_range="uint8" - ).astype("uint8") + image8 = rescale_intensity(image, in_range="image", out_range="uint8").astype( + "uint8" + ) outpath = outdir / source_path.with_suffix(".png").name From 93ae983c89a21e236b092d2508541d6f55462b34 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Fri, 29 Dec 2023 09:50:31 -0800 Subject: [PATCH 49/60] refactor(various): more lint cleanup --- src/vipersci/pds/labelmaker/__init__.py | 6 +++--- src/vipersci/vis/create_image.py | 2 +- src/vipersci/vis/pds/create_pano_product.py | 4 ++-- src/vipersci/vis/pds/create_raw.py | 2 +- tests/test_create_raw.py | 4 +--- 5 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/vipersci/pds/labelmaker/__init__.py b/src/vipersci/pds/labelmaker/__init__.py index 37820b3..f62615b 100644 --- a/src/vipersci/pds/labelmaker/__init__.py +++ b/src/vipersci/pds/labelmaker/__init__.py @@ -57,9 +57,9 @@ def get_common_label_info(element: ET.Element, area="pds:Observation_Area"): for i in element.findall( ".//pds:Observing_System_Component[pds:type='Instrument']", ns ): - instruments[ - find_text(i, "pds:name") - ] = find_text(i, "pds:Internal_Reference/pds:lid_reference") + instruments[find_text(i, "pds:name")] = find_text( + i, "pds:Internal_Reference/pds:lid_reference" + ) purposes = [] for p in element.findall( diff --git a/src/vipersci/vis/create_image.py b/src/vipersci/vis/create_image.py index 8ae6574..f8b7909 100644 --- a/src/vipersci/vis/create_image.py +++ b/src/vipersci/vis/create_image.py @@ -291,7 +291,7 @@ def tif_info(p: Path) -> dict: end = "MSB" if info["bigEndian"] else "LSB" # Tag 258 is bits per pixel: - bpp = int(tags[258]['data'][0] / 8) + bpp = int(tags[258]["data"][0] / 8) if bpp == 1: dt_end = "Byte" diff --git a/src/vipersci/vis/pds/create_pano_product.py b/src/vipersci/vis/pds/create_pano_product.py index 3a155be..9c5977c 100644 --- a/src/vipersci/vis/pds/create_pano_product.py +++ b/src/vipersci/vis/pds/create_pano_product.py @@ -207,9 +207,9 @@ def label_dict(pr: PanoRecord): "lid": f"{lids['spacecraft']}.{_inst}", } # todo: need to figure out how to set version id here - d["source_product_lidvids"].append( + d["source_product_lidvids"].append( # type: ignore f"urn:nasa:pds:viper_vis:data_raw:{ir.product_id}:0.1" - ) # type: ignore + ) for inst in instruments_dict.values(): d["instruments"].append(inst) # type: ignore diff --git a/src/vipersci/vis/pds/create_raw.py b/src/vipersci/vis/pds/create_raw.py index 211a4e1..0651cfb 100644 --- a/src/vipersci/vis/pds/create_raw.py +++ b/src/vipersci/vis/pds/create_raw.py @@ -323,7 +323,7 @@ def label_dict(ir: ImageRecord, lights: dict): d["sample_bits"] = 8 d["sample_bit_mask"] = "2#11111111" - if len(im_filt ) > 0: + if len(im_filt) > 0: d["image_filters"] = " ".join(im_filt) if ir.image_request is not None: diff --git a/tests/test_create_raw.py b/tests/test_create_raw.py index 06fc3d0..dd9780a 100644 --- a/tests/test_create_raw.py +++ b/tests/test_create_raw.py @@ -146,9 +146,7 @@ def test_label_dict(self): ) self.assertEqual(d["exposure_type"], "Manual") self.assertEqual(d["luminaires"][list(luminaire_names.values())[0]], "Off") - self.assertEqual( - d["image_filters"], "Flat field normalization. Linearization." - ) + self.assertEqual(d["image_filters"], "Flat field normalization. Linearization.") self.assertEqual(d["sample_bits"], 12) @patch("vipersci.vis.pds.create_raw.create_engine") From cf873d1f2950b3814aa674047d119d0da2666a47 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Fri, 29 Dec 2023 19:09:19 -0800 Subject: [PATCH 50/60] refactor(pds/__init__.py and create_pano_product): Switch testing PDS version to 99.99 --- src/vipersci/vis/pds/__init__.py | 8 +++++--- src/vipersci/vis/pds/create_pano_product.py | 2 +- tests/test_vis_pds.py | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/vipersci/vis/pds/__init__.py b/src/vipersci/vis/pds/__init__.py index a7ab538..fdde773 100644 --- a/src/vipersci/vis/pds/__init__.py +++ b/src/vipersci/vis/pds/__init__.py @@ -46,12 +46,14 @@ def version_info(): d = { "modification_details": [ { - "version": 0.1, + "version": 99.99, "date": date.today().isoformat(), - "description": "Illegal version number for testing", + "description": "Dumb information model forces this to be a 'valid' " + "version instead of 0.1 for testing, which should be " + "allowed.", } ], - "vid": 0.1, + "vid": 99.99, } return d diff --git a/src/vipersci/vis/pds/create_pano_product.py b/src/vipersci/vis/pds/create_pano_product.py index 9c5977c..e7d3068 100644 --- a/src/vipersci/vis/pds/create_pano_product.py +++ b/src/vipersci/vis/pds/create_pano_product.py @@ -208,7 +208,7 @@ def label_dict(pr: PanoRecord): } # todo: need to figure out how to set version id here d["source_product_lidvids"].append( # type: ignore - f"urn:nasa:pds:viper_vis:data_raw:{ir.product_id}:0.1" + f"urn:nasa:pds:viper_vis:data_raw:{ir.product_id}::99.99" ) for inst in instruments_dict.values(): diff --git a/tests/test_vis_pds.py b/tests/test_vis_pds.py index 8b4c114..662fc4f 100644 --- a/tests/test_vis_pds.py +++ b/tests/test_vis_pds.py @@ -37,7 +37,7 @@ class TestVersion(unittest.TestCase): def test_version_info(self): d = pds.version_info() - self.assertEqual(0.1, d["vid"]) + self.assertEqual(99.99, d["vid"]) class TestXML(unittest.TestCase): From 879e3d54bd9386cf080b5c7d8ab3f5f39499ba5a Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Tue, 9 Jan 2024 18:47:33 -0800 Subject: [PATCH 51/60] feat(image_records.py): If the captureId is larger than 16 bits, then assume the last 16 bits are a Science Request ID. --- CHANGELOG.rst | 2 ++ src/vipersci/vis/db/image_records.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7a8cce5..0af77ab 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -50,6 +50,8 @@ Added relationship entries to each table. - pano_records table now has pan and tilt angle min/max values to indicate angular range of panorama coverage. +- ImageRecord objects will now extract an ImageRequest ID from the provided capture_id + if it is larger than the 16 bit range. - image_requests.py - "Acquired," "Not Acquired," "Not Planned," and "Not Obtainable" statuses added to enum. Also added asdict() method. - ptu_records.py - Tables to record the pan and tilt of the rover's pan-tilt-unit (PTU). diff --git a/src/vipersci/vis/db/image_records.py b/src/vipersci/vis/db/image_records.py index 3ff89d7..c0da6d0 100644 --- a/src/vipersci/vis/db/image_records.py +++ b/src/vipersci/vis/db/image_records.py @@ -545,6 +545,11 @@ def __init__(self, **kwargs): self._pid = str(pid) + if self.capture_id is not None and self.capture_id > int( + "1111111111111111", base=2 + ): # 65535 + self.image_request_id = int(bin(self.capture_id)[-16:], base=2) + # Is this really a good idea? Not sure. This instance variable plus # label_dict() and update() allow other key/value pairs to be carried around # in this object, which is handy. If these are well enough known, perhaps From 5feccc6a6fdde4c3db84df61545aa4061874df6f Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Tue, 9 Jan 2024 18:51:58 -0800 Subject: [PATCH 52/60] feat(create_raw.py): Changes to enable operation on sqlite databases. --- src/vipersci/vis/pds/create_raw.py | 27 +++++++++++++++++++++++---- tests/test_create_raw.py | 3 ++- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/vipersci/vis/pds/create_raw.py b/src/vipersci/vis/pds/create_raw.py index 0651cfb..3262aae 100644 --- a/src/vipersci/vis/pds/create_raw.py +++ b/src/vipersci/vis/pds/create_raw.py @@ -38,7 +38,7 @@ # top level of this library. import argparse -from datetime import timedelta +from datetime import datetime, timedelta, timezone import json import logging from typing import Union @@ -47,7 +47,9 @@ import numpy as np import numpy.typing as npt +from geoalchemy2 import load_spatialite # type: ignore from sqlalchemy import and_, create_engine, select +from sqlalchemy.event import listen from sqlalchemy.orm import Session import vipersci @@ -56,6 +58,7 @@ from vipersci.vis.create_image import tif_info from vipersci.vis.pds import lids, write_xml from vipersci.pds import pid as pds +from vipersci.pds.datetime import isozformat from vipersci import util logger = logging.getLogger(__name__) @@ -110,6 +113,9 @@ def main(): util.set_logger(args.verbose) engine = create_engine(args.dburl) + if args.dburl.startswith("sqlite://"): + listen(engine, "connect", load_spatialite) + with Session(engine) as session: try: pid = pds.VISID(args.input) @@ -148,6 +154,12 @@ def main(): # This allows values in these dicts to override the hard-coded values above. metadata.update(label_dict(ir, get_lights(ir, session))) + if args.dburl.startswith("sqlite://"): + for c in ir.__table__.columns: + dt = getattr(ir, c.name) + if isinstance(dt, datetime) and dt.tzinfo is None: + setattr(ir, c.name, dt.replace(tzinfo=timezone.utc)) + metadata.update(ir.asdict()) metadata.update( { @@ -169,14 +181,21 @@ def main(): if args.tiff is not None: t_info = tif_info(args.tiff) + else: + t_info = tif_info(Path(ir.file_path)) - for k, v in t_info.items(): - if hasattr(metadata, k) and metadata[k] != v: + for k, v in t_info.items(): + if hasattr(metadata, k): + if metadata[k] != v: raise ValueError( f"The value of {k} in the metadata ({metadata[k]}) does not match " f"the value ({v}) in the image ({args.tiff})" ) - metadata.update(t_info) + else: + if isinstance(v, datetime): + metadata[k] = isozformat(v) + else: + metadata[k] = v write_xml(metadata, args.template, args.output_dir) diff --git a/tests/test_create_raw.py b/tests/test_create_raw.py index dd9780a..7e3e40b 100644 --- a/tests/test_create_raw.py +++ b/tests/test_create_raw.py @@ -151,7 +151,8 @@ def test_label_dict(self): @patch("vipersci.vis.pds.create_raw.create_engine") @patch("vipersci.vis.pds.create_raw.write_xml") - def test_main(self, m_write_xml, m_create_engine): + @patch("vipersci.vis.pds.create_raw.tif_info") + def test_main(self, m_tif_info, m_write_xml, m_create_engine): with patch("vipersci.vis.pds.create_raw.Session", return_value=self.session): pa_ret_val = cr.arg_parser().parse_args( [ From 944dff52d785ec4a744ec53399be874e5efdc954 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Thu, 11 Jan 2024 18:04:48 -0800 Subject: [PATCH 53/60] fix(pds/__init__.py): The instrument context is "viper.vis" not "spacecraft.vis" --- src/vipersci/vis/pds/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vipersci/vis/pds/__init__.py b/src/vipersci/vis/pds/__init__.py index fdde773..c51419a 100644 --- a/src/vipersci/vis/pds/__init__.py +++ b/src/vipersci/vis/pds/__init__.py @@ -36,7 +36,7 @@ "bundle": "urn:nasa:pds:viper_vis", "mission": "urn:nasa:pds:viper", "spacecraft": "urn:nasa:pds:context:instrument_host:spacecraft.viper", - "instrument": "urn:nasa:pds:context:instrument:spacecraft.vis", + "instrument": "urn:nasa:pds:context:instrument:viper.vis", } From 65ab0045cad5c949ea039c8c079c0d5ec93871bc Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Thu, 11 Jan 2024 20:14:12 -0800 Subject: [PATCH 54/60] feat(bundle_install.py): Made this more flexible for lidvid_reference and lid_reference. --- src/vipersci/pds/bundle_install.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/vipersci/pds/bundle_install.py b/src/vipersci/pds/bundle_install.py index 04c8efe..4272b2e 100644 --- a/src/vipersci/pds/bundle_install.py +++ b/src/vipersci/pds/bundle_install.py @@ -70,14 +70,14 @@ def main(): copy2(args.source_directory / readme.text, args.build_directory) for bme in bundle.findall(".//pds:Bundle_Member_Entry", ns): - col_lidvid = find_text(bme, "pds:lid_reference") - if find_text(bme, "pds:member_status") == "Primary": - if "::" in col_lidvid: - col_lid, col_vid = col_lidvid.split("::") - else: - col_lid = col_lidvid - col_vid = None + try: + lidvid_ref = find_text(bme, "pds:lidvid_reference") + col_lid, col_vid = lidvid_ref.split("::") + except ValueError: + col_lid = find_text(bme, "pds:lid_reference") + col_vid = None + if find_text(bme, "pds:member_status") == "Primary": col_name = col_lid.split(":")[-1] src_col_dir = args.source_directory / col_name if src_col_dir.exists() is False: From 70542fbfdd8a0d49e7b167d5cdb16ffe04cd6b4b Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Thu, 11 Jan 2024 20:15:45 -0800 Subject: [PATCH 55/60] feat(create_browse.py and others): Added create_browse --- CHANGELOG.rst | 1 + setup.cfg | 1 + src/vipersci/pds/labelmaker/__init__.py | 1 + src/vipersci/pds/labelmaker/collection.py | 3 +- src/vipersci/vis/pds/create_browse.py | 164 ++++++++++++++++++ src/vipersci/vis/pds/data/browse-template.xml | 43 +++++ 6 files changed, 212 insertions(+), 1 deletion(-) create mode 100644 src/vipersci/vis/pds/create_browse.py create mode 100644 src/vipersci/vis/pds/data/browse-template.xml diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0af77ab..3aa2c1a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -55,6 +55,7 @@ Added - image_requests.py - "Acquired," "Not Acquired," "Not Planned," and "Not Obtainable" statuses added to enum. Also added asdict() method. - ptu_records.py - Tables to record the pan and tilt of the rover's pan-tilt-unit (PTU). +- create_browse.py - For making browse products from existing image products. - create_mmgis_pano.py - For making pano products for use in MMGIS. - create_pano.py - updated to correctly add PanoRecord associations, can now query database for ImageRecords. diff --git a/setup.cfg b/setup.cfg index 70b1893..91404cc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -82,6 +82,7 @@ console_scripts = traverse_interpolator = vipersci.carto.traverse_interpolator:main tri2gpkg = vipersci.carto.tri2gpkg:main template_test = vipersci.vis.pds.template_test:main + vis_create_browse = vipersci.vis.pds.create_browse:main vis_create_image = vipersci.vis.create_image:main vis_create_mmgis_pano = vipersci.vis.create_mmgis_pano:main vis_create_pano = vipersci.vis.create_pano:main diff --git a/src/vipersci/pds/labelmaker/__init__.py b/src/vipersci/pds/labelmaker/__init__.py index f62615b..99cfda6 100644 --- a/src/vipersci/pds/labelmaker/__init__.py +++ b/src/vipersci/pds/labelmaker/__init__.py @@ -153,6 +153,7 @@ def get_lidvidfile(path: Path) -> dict: for fxpath in ( "./pds:File_Area_Observational/pds:File/pds:file_name", "./pds:Document/pds:Document_Edition/pds:Document_File/pds:file_name", + "./pds:File_Area_Browse/pds:File/pds:file_name", ): element = root.find(fxpath, ns) if element is not None: diff --git a/src/vipersci/pds/labelmaker/collection.py b/src/vipersci/pds/labelmaker/collection.py index 93f9f66..4d44685 100644 --- a/src/vipersci/pds/labelmaker/collection.py +++ b/src/vipersci/pds/labelmaker/collection.py @@ -115,6 +115,7 @@ def check_and_derive(config: dict, labelinfo: list): df = pd.DataFrame(labelinfo) check = { + "Browse": ("collection_lid",), "Data": ( "collection_lid", "investigation_name", @@ -136,7 +137,7 @@ def check_and_derive(config: dict, labelinfo: list): # Generate values from gathered labels: if config["collection_type"] == "Data": d = gather_info(df, config["modification_details"]) - elif config["collection_type"] == "Document": + elif config["collection_type"] in ("Document", "Browse"): d = { "vid": str( vid_max(config["modification_details"], pd.to_numeric(df["vid"]).max()) diff --git a/src/vipersci/vis/pds/create_browse.py b/src/vipersci/vis/pds/create_browse.py new file mode 100644 index 0000000..a67bdee --- /dev/null +++ b/src/vipersci/vis/pds/create_browse.py @@ -0,0 +1,164 @@ +"""Creates Browse VIS PDS Products. + +This module builds "browse" VIS data products from VIS Image Products. +""" + +# Copyright 2022-2024, United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# +# Reuse is permitted under the terms of the license. +# The AUTHORS file and the LICENSE file are at the +# top level of this library. + +import argparse +from datetime import datetime, timezone +import hashlib +import logging +from typing import Union +from pathlib import Path +import xml.etree.ElementTree as ET + +import numpy as np +import numpy.typing as npt +from skimage.io import imread, imsave +from skimage.exposure import equalize_adapthist, rescale_intensity +from skimage.transform import resize + +from vipersci.vis.pds import write_xml +from vipersci.pds.labelmaker import get_lidvidfile +from vipersci.pds.datetime import isozformat +from vipersci.pds.xml import find_text, ns +from vipersci import util + +logger = logging.getLogger(__name__) + +ImageType = Union[npt.NDArray[np.uint16], npt.NDArray[np.uint8]] + + +def arg_parser(): + parser = argparse.ArgumentParser( + description=__doc__, parents=[util.parent_parser()] + ) + parser.add_argument( + "-t", + "--template", + default="browse-template.xml", + help="Genshi XML file template. Will default to the browse-template.xml " + "file distributed with the module.", + ) + parser.add_argument( + "-o", + "--output_dir", + type=Path, + default=Path.cwd(), + help="Output directory for label. Defaults to current working directory. " + "Output file names are fixed based on product_id, and will be over-written.", + ) + parser.add_argument( + "input", + type=Path, + help="PDS XML label file of product to create a browse product from.", + ) + return parser + + +def main(): + parser = arg_parser() + args = parser.parse_args() + util.set_logger(args.verbose) + + metadata = get_product_info(args.input) + + if metadata["productfile"] is None: + raise ValueError(f"Could not fine a file_name in {args.input}") + + image_path = args.input.parent / metadata["productfile"] + + try: + image = rescale_intensity( + equalize_adapthist(imread(image_path)), in_range="image", out_range="uint8" + ) + except FileNotFoundError as err: + parser.error(str(err)) + + metadata["source_lidvid"] = metadata["lid"] + "::" + metadata["vid"] + metadata[ + "source_product_type" + ] = f"data_to_{metadata['type'].lower()}_source_product" + + # Adjust LID + lid_tokens = metadata["lid"].split(":") + lid_tokens[4] = "browse" + metadata["product_id"] = lid_tokens[5] + "-browse" + lid_tokens[5] = metadata["product_id"] + metadata["lid"] = ":".join(lid_tokens) + + # Adjust title + title_tokens = metadata["title"].split(" - ") + title_words = title_tokens[0].split() + title_words.insert(-1, "browse") + metadata["title"] = " ".join(title_words) + f" - {metadata['product_id']}" + + # Scale down image to be no larger than 1024 pixels in the maximum direction + max_dim = max(np.shape(image)) + if max_dim > 1024: + scale = max_dim / 1024 + new_shape = tuple(int(x / scale) for x in np.shape(image)) + image = rescale_intensity( + resize(image, new_shape), in_range="image", out_range="uint8" + ) + + metadata["file_path"] = metadata["product_id"] + ".png" + out_im_path = args.output_dir / metadata["file_path"] + imsave(out_im_path, image, check_contrast=False) + + metadata["file_creation_datetime"] = isozformat( + datetime.fromtimestamp(out_im_path.stat().st_mtime, timezone.utc) + ) + md5 = hashlib.md5() + with open(out_im_path, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + md5.update(chunk) + metadata["file_md5_checksum"] = md5.hexdigest() + + write_xml(metadata, args.template, args.output_dir) + + return + + +def get_product_info(xml_path: Path): + d = get_lidvidfile(xml_path) + + label = ET.fromstring(xml_path.read_text()) + d["title"] = find_text(label, "./pds:Identification_Area/pds:title") + d["type"] = find_text( + label, "./pds:Observation_Area/pds:Primary_Result_Summary/pds:processing_level" + ) + d["modification_details"] = get_modification_details(label) + return d + + +def get_modification_details(element_tree: ET.Element): + mod_history = list() + for mod_detail in element_tree.findall(".//pds:Modification_Detail", ns): + d = dict() + for key in ("modification_date", "version_id", "description"): + d[key] = find_text(mod_detail, f"pds:{key}") + mod_history.append(d) + + return mod_history diff --git a/src/vipersci/vis/pds/data/browse-template.xml b/src/vipersci/vis/pds/data/browse-template.xml new file mode 100644 index 0000000..9200c3b --- /dev/null +++ b/src/vipersci/vis/pds/data/browse-template.xml @@ -0,0 +1,43 @@ + + + + + + ${lid} + ${vid} + ${title} + 1.21.0.0 + Product_Browse + + + ${detail.modification_date} + ${detail.version_id} + ${detail.description} + + + + + + ${source_lidvid} + ${source_product_type} + + + + + ${file_path} + ${file_creation_datetime} + + + ${file_md5_checksum} + 0 + PNG + Reduced-size 8-bit version of the source product. + + + From cde0d765120e4a5fd120df869ee05c763f7fbe29 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Thu, 11 Jan 2024 20:56:14 -0800 Subject: [PATCH 56/60] refactor(util.py and others): Created centralized function that takes a PathLike and returns an md5.hexdigest string. --- src/vipersci/util.py | 13 +++++++++++++ src/vipersci/vis/create_image.py | 8 +------- src/vipersci/vis/pds/create_browse.py | 7 +------ tests/test_create_image.py | 4 ++-- tests/test_util.py | 6 ++++++ 5 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/vipersci/util.py b/src/vipersci/util.py index d13bc93..57c4260 100644 --- a/src/vipersci/util.py +++ b/src/vipersci/util.py @@ -24,11 +24,24 @@ # top level of this library. import argparse +import hashlib import logging +import os import vipersci +def md5(path: os.PathLike): + """Returns a string objects of hexadecimal digits representing the md5 hash + of the file at *path*.""" + m = hashlib.md5() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + m.update(chunk) + + return m.hexdigest() + + def parent_parser() -> argparse.ArgumentParser: """Returns a parent parser with common arguments for viss programs.""" parent = argparse.ArgumentParser(add_help=False) diff --git a/src/vipersci/vis/create_image.py b/src/vipersci/vis/create_image.py index f8b7909..3f47040 100644 --- a/src/vipersci/vis/create_image.py +++ b/src/vipersci/vis/create_image.py @@ -34,7 +34,6 @@ import argparse from datetime import datetime, timezone -import hashlib import json import logging from typing import Union, Optional @@ -280,11 +279,6 @@ def tif_info(p: Path) -> dict: """ dt = datetime.fromtimestamp(p.stat().st_mtime, timezone.utc) - md5 = hashlib.md5() - with open(p, "rb") as f: - for chunk in iter(lambda: f.read(4096), b""): - md5.update(chunk) - info = read_tiff(str(p)) tags = info["ifds"][0]["tags"] @@ -302,7 +296,7 @@ def tif_info(p: Path) -> dict: "file_byte_offset": tags[273]["data"][0], # Tag 273 is StripOffsets "file_creation_datetime": dt, "file_data_type": f"Unsigned{dt_end}", - "file_md5_checksum": md5.hexdigest(), + "file_md5_checksum": util.md5(p), "file_path": p.name, "lines": tags[257]["data"][0], # Tag 257 is ImageWidth, "samples": tags[256]["data"][0], # Tag 256 is ImageWidth, diff --git a/src/vipersci/vis/pds/create_browse.py b/src/vipersci/vis/pds/create_browse.py index a67bdee..bdcfe77 100644 --- a/src/vipersci/vis/pds/create_browse.py +++ b/src/vipersci/vis/pds/create_browse.py @@ -27,7 +27,6 @@ import argparse from datetime import datetime, timezone -import hashlib import logging from typing import Union from pathlib import Path @@ -130,11 +129,7 @@ def main(): metadata["file_creation_datetime"] = isozformat( datetime.fromtimestamp(out_im_path.stat().st_mtime, timezone.utc) ) - md5 = hashlib.md5() - with open(out_im_path, "rb") as f: - for chunk in iter(lambda: f.read(4096), b""): - md5.update(chunk) - metadata["file_md5_checksum"] = md5.hexdigest() + metadata["file_md5_checksum"] = util.md5(out_im_path) write_xml(metadata, args.template, args.output_dir) diff --git a/tests/test_create_image.py b/tests/test_create_image.py index be9de01..9d40212 100644 --- a/tests/test_create_image.py +++ b/tests/test_create_image.py @@ -10,7 +10,7 @@ from datetime import datetime, timezone from pathlib import Path import unittest -from unittest.mock import create_autospec, mock_open, patch +from unittest.mock import create_autospec, patch import numpy as np @@ -117,7 +117,7 @@ def test_tif_info(self, mock_datetime): mock_path = create_autospec(Path) mock_path.name = "dummy.tif" - with patch("vipersci.vis.create_image.open", mock_open(read_data=b"test")): + with patch("vipersci.vis.create_image.util.md5", return_value="hex"): with patch("vipersci.vis.create_image.read_tiff", return_value=info): d = ci.tif_info(mock_path) diff --git a/tests/test_util.py b/tests/test_util.py index 7101c75..f218e23 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -28,11 +28,17 @@ import argparse import logging import unittest +from unittest.mock import mock_open, patch import vipersci.util as util class TestUtil(unittest.TestCase): + def test_md5(self): + with patch("vipersci.util.open", mock_open(read_data=b"test")): + h = util.md5("bogus.file") + self.assertEqual("098f6bcd4621d373cade4e832627b4f6", h) + def test_parent_parser(self): self.assertIsInstance(util.parent_parser(), argparse.ArgumentParser) From df1975b9a95e415977ed2d21f55a1ee62aa967a5 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Wed, 24 Jan 2024 19:44:09 -0800 Subject: [PATCH 57/60] fix(create_image.py): Adapted the write_tiff() function to handle bool and int32 arrays, if provided. --- src/vipersci/vis/create_image.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/vipersci/vis/create_image.py b/src/vipersci/vis/create_image.py index 3f47040..3f1f89d 100644 --- a/src/vipersci/vis/create_image.py +++ b/src/vipersci/vis/create_image.py @@ -42,6 +42,7 @@ import numpy as np import numpy.typing as npt from skimage.io import imread, imsave # maybe just imageio here? +from skimage.util import img_as_ubyte, img_as_uint from sqlalchemy import create_engine from sqlalchemy.orm import Session from tifftools import read_tiff @@ -319,6 +320,10 @@ def write_tiff(pid: pds.VISID, image: ImageType, outdir: Path = Path.cwd()) -> P Returns the path where a TIFF with a name based on *pid* and the array *image* was written in *outdir* (defaults to current working directory). """ + if image.dtype == np.bool_: + image = img_as_ubyte(image) + if image.dtype == np.int32: + image = img_as_uint(image) check_bit_depth(pid, image.dtype) From 83d51e6f2335f5c0e115d17d95c159a28a29867f Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Wed, 24 Jan 2024 19:44:36 -0800 Subject: [PATCH 58/60] fix(image_requests.py): Added rover_wait_for column. --- src/vipersci/vis/db/image_requests.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/vipersci/vis/db/image_requests.py b/src/vipersci/vis/db/image_requests.py index 64e8396..ade95c6 100644 --- a/src/vipersci/vis/db/image_requests.py +++ b/src/vipersci/vis/db/image_requests.py @@ -119,6 +119,12 @@ class ImageMode(enum.Enum): CALIBRATION = 5 # Only NavCams: initiates "standard" calibration sequence. +class RoverWaitFor(enum.Enum): + DOWNLINK = 1 + VIS_VERIFICATION = 2 + DOWNLINK_AND_VIS = 3 + + class ImageRequest(Base): """An object to represent rows in the image_requests table for VIS.""" @@ -165,6 +171,9 @@ class ImageRequest(Base): default="any", doc="One-line description of desired rover orientation", ) + rover_wait_for = mapped_column( + Enum(RoverWaitFor), doc="Details of a rover stop request." + ) # TODO: how do we handle attached images? are we responsible for file # management? From 591a938d2e78ad4c6a81d4f3b68475ab68c1c0a3 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Wed, 24 Jan 2024 19:59:27 -0800 Subject: [PATCH 59/60] fix(requirements.txt): Forgot that you import yaml, but pip install pyyaml. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6e6ba29..fbb161e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,4 +14,4 @@ scikit-learn shapely sqlalchemy tifftools -yaml \ No newline at end of file +pyyaml From d63ce636f9f6c87186438bc6965bba1dca16db05 Mon Sep 17 00:00:00 2001 From: Ross Beyer Date: Wed, 24 Jan 2024 20:09:18 -0800 Subject: [PATCH 60/60] test(python-test.yml): Removed the 3.7 tests as Python 3.7 is EoL. --- .github/workflows/python-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index cc9711b..27b7db1 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python-version: ['3.8', '3.9', '3.10', '3.11'] env: SPATIALITE_LIBRARY_PATH: 'mod_spatialite'