diff --git a/notebooks_tsqr/NightLog.ipynb b/notebooks_tsqr/NightLog.ipynb
index 9209941..986ef6c 100644
--- a/notebooks_tsqr/NightLog.ipynb
+++ b/notebooks_tsqr/NightLog.ipynb
@@ -25,7 +25,7 @@
"# day_obs values: TODAY, YESTERDAY, YYYY-MM-DD\n",
"# Report on observing nights that start upto but not included this day.\n",
"#!day_obs = '2024-09-25' # Value to use for local testing (Summit)\n",
- "day_obs = \"2024-09-25\" # TODO Change to 'YESTERDAY' to test with default before push\n",
+ "day_obs = \"2024-09-24\" # TODO Change to 'YESTERDAY' to test with default before push\n",
"\n",
"# Total number of days of data to display (ending on day_obs)\n",
"number_of_days = \"1\" # TODO Change to '1' to test with default before push"
@@ -35,7 +35,11 @@
"cell_type": "code",
"execution_count": null,
"id": "2",
- "metadata": {},
+ "metadata": {
+ "jupyter": {
+ "source_hidden": true
+ }
+ },
"outputs": [],
"source": [
"import datetime as dt\n",
@@ -113,11 +117,7 @@
"cell_type": "code",
"execution_count": null,
"id": "4",
- "metadata": {
- "jupyter": {
- "source_hidden": true
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"# Set default env to \"usdf\" and try before PUSH to repo.\n",
@@ -129,14 +129,23 @@
"# Under Times Square it is ignored.\n",
"server = os.environ.get(\n",
" \"EXTERNAL_INSTANCE_URL\", summit\n",
- ") # TODO try with \"usdf\" before push (else \"summit\")\n",
- "\n",
+ ") # TODO try with \"usdf\" before push (else \"summit\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "5",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Read records from (almost) all sources\n",
"allsrc = AllSources(server_url=server)"
]
},
{
"cell_type": "markdown",
- "id": "5",
+ "id": "6",
"metadata": {},
"source": [
"# Table of Contents\n",
@@ -150,7 +159,7 @@
},
{
"cell_type": "markdown",
- "id": "6",
+ "id": "7",
"metadata": {},
"source": [
"# Overview \n",
@@ -160,7 +169,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "7",
+ "id": "8",
"metadata": {},
"outputs": [],
"source": [
@@ -172,20 +181,27 @@
"- This report will include available data from noon **{min_date}** to noon **{max_date}**.\n",
"- Using ***Prototype* Logging and Reporting** Version: **{lrversion}**\n",
"\"\"\"\n",
- ")\n",
- "\n",
- "sources_md_str = \"\"\n",
- "#! sources_md_str += '\\n- '.join(['',*sad.all_endpoints(server)]) # TODO move\n",
- "sources_md_str += \"\\n- ConsDB\"\n",
- "sources_md_str += \"\\n- EFD\"\n",
- "sources_md_str += \"\\n- (DDV)\"\n",
- "sources_md_str += \"\\n- (Astroplan for Almanac)\"\n",
- "md(f\"### This report uses the following data sources: {sources_md_str}\")"
+ ")"
]
},
{
"cell_type": "markdown",
- "id": "8",
+ "id": "9",
+ "metadata": {},
+ "source": [
+ "### This report uses the following data sources\n",
+ "- NightReport\n",
+ "- Exposurelog\n",
+ "- Narrativelog\n",
+ "- EFD\n",
+ "- ConsDB\n",
+ "- (DDV)\n",
+ "- (Almanac from Astroplan)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "10",
"metadata": {},
"source": [
"\n",
@@ -195,7 +211,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "9",
+ "id": "11",
"metadata": {},
"outputs": [],
"source": [
@@ -209,7 +225,7 @@
},
{
"cell_type": "markdown",
- "id": "10",
+ "id": "12",
"metadata": {},
"source": [
"\n",
@@ -219,7 +235,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "11",
+ "id": "13",
"metadata": {},
"outputs": [],
"source": [
@@ -230,7 +246,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "12",
+ "id": "14",
"metadata": {},
"outputs": [],
"source": [
@@ -244,7 +260,7 @@
},
{
"cell_type": "markdown",
- "id": "13",
+ "id": "15",
"metadata": {},
"source": [
"# Night Report "
@@ -253,7 +269,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "14",
+ "id": "16",
"metadata": {},
"outputs": [],
"source": [
@@ -278,12 +294,13 @@
" md(f\"Used: [API Data]({allsrc.nig_src.source_url})\")\n",
"\n",
"# Display time log\n",
- "nr_rep.time_log_as_markdown()"
+ "nr_rep.time_log_as_markdown()\n",
+ "md(\"-------------\")"
]
},
{
"cell_type": "markdown",
- "id": "15",
+ "id": "17",
"metadata": {},
"source": [
"# Exposure Log"
@@ -292,7 +309,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "16",
+ "id": "18",
"metadata": {},
"outputs": [],
"source": [
@@ -324,12 +341,13 @@
" md(f\"Used: [API Data]({allsrc.exp_src.source_url})\")\n",
"\n",
"# Time Log\n",
- "exposure_rep.time_log_as_markdown()"
+ "exposure_rep.time_log_as_markdown()\n",
+ "md(\"-------------\")"
]
},
{
"cell_type": "markdown",
- "id": "17",
+ "id": "19",
"metadata": {},
"source": [
"# Narrative Log\n"
@@ -338,7 +356,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "18",
+ "id": "20",
"metadata": {},
"outputs": [],
"source": [
@@ -349,12 +367,13 @@
"narrative_rep.overview\n",
"\n",
"# Time Log\n",
- "narrative_rep.time_log_as_markdown()"
+ "narrative_rep.time_log_as_markdown()\n",
+ "md(\"-------------\")"
]
},
{
"cell_type": "markdown",
- "id": "19",
+ "id": "21",
"metadata": {},
"source": [
"# Developer Only Section"
@@ -362,7 +381,7 @@
},
{
"cell_type": "markdown",
- "id": "20",
+ "id": "22",
"metadata": {},
"source": [
"## Where was this run?\n",
@@ -378,7 +397,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "21",
+ "id": "23",
"metadata": {},
"outputs": [],
"source": [
@@ -391,7 +410,7 @@
},
{
"cell_type": "markdown",
- "id": "22",
+ "id": "24",
"metadata": {},
"source": [
"# Finale"
@@ -400,7 +419,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "23",
+ "id": "25",
"metadata": {},
"outputs": [],
"source": [
diff --git a/python/lsst/ts/logging_and_reporting/all_sources.py b/python/lsst/ts/logging_and_reporting/all_sources.py
index 4761961..c320811 100644
--- a/python/lsst/ts/logging_and_reporting/all_sources.py
+++ b/python/lsst/ts/logging_and_reporting/all_sources.py
@@ -119,3 +119,19 @@ async def night_tally_observation_gaps(self, verbose=True):
# https://ts-xml.lsst.io/sal_interfaces/Scheduler.html#slewtime
# edf.get_targets() => "slewTime" # (d,g,h)
return instrument_tally
+
+ def records_per_source(self):
+ sources = [
+ self.nig_src,
+ self.exp_src,
+ self.nar_src,
+ self.efd_src,
+ ]
+
+ # Until efd.get_targets is run, it will report 0 records.
+ # That method is run in: await allsrc.night_tally_observation_gaps()
+ res = {
+ src.service: src.status[src.primary_endpoint]["number_of_records"]
+ for src in sources
+ }
+ return res
diff --git a/python/lsst/ts/logging_and_reporting/efd.py b/python/lsst/ts/logging_and_reporting/efd.py
index 57c77f4..ad22201 100644
--- a/python/lsst/ts/logging_and_reporting/efd.py
+++ b/python/lsst/ts/logging_and_reporting/efd.py
@@ -29,6 +29,11 @@ class Server:
class EfdAdapter(SourceAdapter):
salindex = 2
+ service = "efd"
+ endpoints = [
+ "targets",
+ ]
+ primary_endpoint = "targets"
def __init__(
self,
@@ -62,6 +67,13 @@ def __init__(
)
raise Exception(msg)
+ self.targets = None
+ self.status["targets"] = dict(
+ endpoint_url="NA",
+ number_of_records=0,
+ error=None,
+ )
+
async def get_topics(self):
self.topics = await self.client.get_topics()
return self.topics
@@ -129,6 +141,9 @@ async def query_nights(self, topic, fields, index=301):
# slewTime (and probably others) are EXPECTED times, not ACTUAL.
async def get_targets(self):
+ if self.targets is not None: # is cached
+ return self.targets
+
topic = "lsst.sal.Scheduler.logevent_target"
fields_wanted = [
"blockId",
@@ -143,6 +158,12 @@ async def get_targets(self):
fields_wanted,
index=self.salindex,
)
+ self.targets = targets
+ self.status["targets"] = dict(
+ endpoint_url="NA",
+ number_of_records=len(targets),
+ error=None,
+ )
return targets
async def get_weather(self, days=1):
diff --git a/python/lsst/ts/logging_and_reporting/reports.py b/python/lsst/ts/logging_and_reporting/reports.py
index ab2e129..7b42b57 100644
--- a/python/lsst/ts/logging_and_reporting/reports.py
+++ b/python/lsst/ts/logging_and_reporting/reports.py
@@ -95,13 +95,8 @@ def time_log_as_markdown(
records = adapter.records
service = adapter.service
url = adapter.get_status().get("endpoint_url")
- title = log_title if log_title else ""
if records:
- md(
- f"## Time Log for {self.source_adapter.min_date} "
- f"to {self.source_adapter.max_date}"
- )
- md(f"### {title}")
+ md("## Time Log")
table = self.source_adapter.day_table("date_added")
mdlist(table)
else:
diff --git a/python/lsst/ts/logging_and_reporting/source_adapters.py b/python/lsst/ts/logging_and_reporting/source_adapters.py
index 0b389c2..aa76a1c 100644
--- a/python/lsst/ts/logging_and_reporting/source_adapters.py
+++ b/python/lsst/ts/logging_and_reporting/source_adapters.py
@@ -78,7 +78,7 @@ def __init__(
self,
*,
server_url=None,
- max_dayobs=None, # EXCLUSIVE: default=Today other=YYYY-MM-DD
+ max_dayobs=None, # EXCLUSIVE: default=TODAY other=YYYY-MM-DD
min_dayobs=None, # INCLUSIVE: default=max_dayobs - 1 day
offset=0,
connect_timeout=1.05, # seconds
@@ -101,9 +101,9 @@ def __init__(
self.timeout = (self.c_timeout, self.r_timeout)
self.records = None # else: list of dict
- # Provide the following in subclass
- # self.service = None
- # self.endpoints = None
+
+ # Provide the following in subclasses: self.service, self.endpoints
+
# status[endpoint] = dict(endpoint_url, number_of_records, error)
# e.g. status['messages'] = dict(endpoint_url='.../messages?...', ...)
self.status = dict()
@@ -115,6 +115,7 @@ def __init__(
self.min_date = ut.dayobs2dt(min_dayobs)
else:
self.min_date = self.max_date - timedelta(days=1)
+ assert self.min_date < self.max_date
self.min_dayobs = ut.datetime_to_dayobs(self.min_date)
def get_status(self, endpoint=None):
@@ -138,7 +139,6 @@ def row_str_func(self, datetime_str, rec):
msg = rec["message_text"].strip()
return f"`{datetime_str}`\n```\n{msg}\n```"
- # Break on DAYOBS. Within that, break on DATE, within that only show time.
def day_table(
self,
datetime_field,
@@ -146,6 +146,9 @@ def day_table(
row_str_func=None,
zero_message=False,
):
+ """Break on DAYOBS.
+ Within that, break on DATE, within that only show time."""
+
def obs_night(rec):
if "day_obs" in rec:
return ut.dayobs_str(rec["day_obs"]) # -> # "YYYY-MM-DD"
@@ -277,6 +280,50 @@ def __init__(
if self.min_date:
self.status[self.primary_endpoint] = self.get_records()
+ # Nightreport
+ def day_table(
+ self,
+ datetime_field,
+ dayobs_field=None,
+ row_str_func=None,
+ zero_message=False,
+ ):
+ """Break on TELESCOPE, DATE. Within that only show time."""
+
+ def obs_night(rec):
+ if "day_obs" in rec:
+ return ut.dayobs_str(rec["day_obs"]) # -> # "YYYY-MM-DD"
+ else:
+ dt = datetime.fromisoformat(rec[datetime_field])
+ return ut.datetime_to_dayobs(dt)
+
+ def obs_date(rec):
+ dt = datetime.fromisoformat(rec[datetime_field])
+ return dt.replace(microsecond=0)
+
+ def telescope(rec):
+ return rec["telescope"]
+
+ recs = self.records
+ if len(recs) == 0:
+ if zero_message:
+ print("Nothing to display.")
+ return
+
+ table = list()
+ # Sort by TELESCOPE, then by OBS_DATE.
+ recs = sorted(recs, key=obs_date)
+ recs = sorted(recs, key=telescope)
+ for tele, g0 in itertools.groupby(recs, key=telescope):
+ table.append(f"### Telescope: {tele}")
+ for rec in g0:
+ attrstr = f'{str(obs_date(rec))} {rec.get("user_id")}'
+ table.append(f"{self.row_str_func(attrstr, rec)}")
+ crew_list = rec.get("observers_crew", [])
+ crew_str = ", ".join(crew_list)
+ table.append(f"*Observer Crew: {crew_str}*")
+ return table
+
def row_str_func(self, datetime_str, rec):
msg = rec["summary"].strip()
return f"`{datetime_str}`\n```\n{msg}\n```"
@@ -357,7 +404,7 @@ class NarrativelogAdapter(SourceAdapter):
"time_lost_type",
# 'urls',
# 'user_agent',
- # 'user_id',
+ "user_id",
}
service = "narrativelog"
endpoints = [
@@ -384,6 +431,43 @@ def __init__(
if self.min_date:
self.status[self.primary_endpoint] = self.get_records()
+ # Narrativelog
+ def day_table(
+ self,
+ datetime_field,
+ dayobs_field=None,
+ row_str_func=None,
+ zero_message=False,
+ ):
+ """Break on DATE. Within that show time, author."""
+
+ def obs_night(rec):
+ if "day_obs" in rec:
+ return ut.dayobs_str(rec["day_obs"]) # -> # "YYYY-MM-DD"
+ else:
+ dt = datetime.fromisoformat(rec[datetime_field])
+ return ut.datetime_to_dayobs(dt)
+
+ def obs_date(rec):
+ dt = datetime.fromisoformat(rec[datetime_field])
+ return dt.replace(microsecond=0)
+
+ recs = self.records
+ if len(recs) == 0:
+ if zero_message:
+ print("Nothing to display.")
+ return
+
+ table = list()
+
+ recs = sorted(recs, key=obs_date)
+ recs = sorted(recs, key=obs_night)
+ for tele, g0 in itertools.groupby(recs, key=obs_night):
+ for rec in g0:
+ attrstr = f'{str(obs_date(rec))} {rec.get("user_id")}'
+ table.append(f"{self.row_str_func(attrstr, rec)}")
+ return table
+
def get_records(
self,
site_ids=None,
@@ -561,6 +645,8 @@ def get_instruments(self):
)
return status
+ # RETURNS status: dict[endpoint_url, number_of_records, error]
+ # SIDE-EFFECT: puts records in self.exposures
def get_exposures(self, instrument, registry=1):
qparams = dict(instrument=instrument, registery=registry)
if self.min_dayobs: