diff --git a/notebooks_tsqr/NightLog.ipynb b/notebooks_tsqr/NightLog.ipynb index 566bf28..74805fb 100644 --- a/notebooks_tsqr/NightLog.ipynb +++ b/notebooks_tsqr/NightLog.ipynb @@ -12,7 +12,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "1", "metadata": {}, "outputs": [], @@ -33,7 +33,7 @@ "number_of_days = \"2\" # TODO Change to '1' to test with default before push\n", "\n", "# Experimental Features config\n", - "exp_tally = \"false\" # default: true\n", + "exp_tally = \"false\" # default: true\n", "exp_parsing = \"true\" # default: true\n", "experimental = dict()\n", "experimental[\"exposure_type_tally\"] = exp_tally == \"true\"\n", @@ -42,7 +42,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "2", "metadata": {}, "outputs": [], @@ -75,7 +75,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "3", "metadata": {}, "outputs": [], @@ -100,7 +100,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "4", "metadata": {}, "outputs": [], @@ -119,7 +119,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "5", "metadata": {}, "outputs": [], @@ -154,23 +154,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "7", "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "# Showing data for 2024-09-05 to 2024-09-07 for LSSTComCamSim, LSSTComCam, LATISS" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "instrum_str = \", \".join(list(allsrc.exp_src.instruments.keys()))\n", "md(f\"# Showing data for {min_date.date()} to {max_date.date()} for {instrum_str}\")" @@ -211,131 +198,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "10", "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "### Telescope: AuxTel" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "```\n", - "AuxTel had a rougher day, as the crew was delayed for the handover due to debugging new cRIO issues. We had a light hiccough with the ATAOS system faulting once, but the checkouts ran well after the team deployed another fix to correct data logging on the cRIO. We vented while waiting for the REB temperature to go down. After dinner, we attempted to take calibrations given the so-so sky quality. The ATWhite lamp CSC faulted, and waiting for the cooling cycle to complete before it could be recovered added about 30 minutes to wait time. The ATmonochromter failed next, and we were drifting into science time. We abandoned attempts of calibrations, and got on sky a bit delayed. ATPtg needed a cycle for the telescope timing issues to be sorted, but after that issue, we observed until 2 am without incident. Mostly seeing below 1.5\", nearly photometric conditions. We closed at 01:45 after finishing TMA testing. Darks and biases left running.\n", - "```" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "Telescope Status: Telescope parked, dome and vents closed. All CSCs in STANBY." - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "*Authors: Alysha Shugart, Yijung Kang, Bruno Quint, Karla Pena*" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "### Telescope: Simonyi" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "```\n", - "In the morning, BLOCK-T62 was executed but just to move the TMA and left to trace the record. TMA was firstly moved to 45º. Since the default values for the EUI might be wrong, we used the EUI to set Acc and Vel manually to 0.05º/s(2, 1% performance. At 45deg and Zenith there was data collection with LaserTracker. While this process, there was an issue with a locking pin -X, which fails to move from LOCK to FREE (OBS-585). At the end, TMA was moved to Zenith (90.01 deg) and OSS/PS OFF with engaged locking pins. Note that there were interesting movements detected in M2 tangent links induced by jumping on the Az platform. \n", - "\n", - "Around 21:37pm UTC, we started BLOCK-T4, warm up camera hexapod only without issue. Warm-up M2 hexapod was done while we were waiting for BLOCK-T143 to be ready after dinner. \n", - "\n", - "We checked with Tony J. that ComCam has been already on Playback mode, BLOCK-T143 was executed after some changes updated by Tiago R. Configurations for this block were changed (see the config on OLE/Zephyr test case comment), but we had to stop at Step 2 as there was an issue with IN_PROGRESS ack for filter changes. \n", - "\n", - "M1M3 FA Bump test (BLOCK-T144) was conducted with ignoring FA423 due to its loud periodic noise. Observers went up to the level 3 to check sounds from FAs, pressure (120PSI), and temperature (deltaT ~ 1C). Actuators [133, 138, 220, 236, 240, 320, 330, 335] FAILED the bump test (SAA; ID, Pri Index: 133:32, 138:37, 220:55, 236:70, 240:74, 320:97, 330:107, 335:112, no DAA failed). \n", - "\n", - "We then started M1M3 HP breakaway test (BLOCK-145) until the end of the night. Workout Camera Hexapod (BLOCK-T87) was skipped for tonight and closed the night with BLOCK-T139, Simonyi Hexapods shutdown. MTHexapod1, MTHexapod2, and MTM1M3 were all set to DISABLED.\n", - "```" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "Telescope Status: TMA parked at Zenith. OSS and PS off and locking pins were engaged. MTHexapod1, MTHexapod2, and MTM1M3 were all set to DISABLED. \n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "*Authors: Bruno Quint, Karla Pena, David Sanmartim, Kevin Fanning, Alysha Shugart, Yijung Kang*" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "-------------" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "# Night Report\n", "# Display time log\n", @@ -353,107 +219,10 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "12", "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "**Almanac for the observing night starting 2024-09-05**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
(times in UTC)
Moon Rise2024-09-05 12:12:44.540
Moon Set2024-09-04 23:57:53.331
Moon Illumination4%(% illuminated)
Astronomical Twilight (morning)2024-09-05 09:33:24.426(-18 degrees)
Astronomical Twilight (evening)2024-09-04 23:49:29.974(-18 degrees)
Solar Midnight2024-09-05 04:41:30.776
Nautical Twilight (morning)2024-09-05 10:01:10.597(-12 degrees)
Nautical Twilight (evening)2024-09-04 23:21:41.930(-12 degrees)
Civil Twilight (morning)2024-09-05 10:28:57.809(-6 degrees)
Civil Twilight (evening)2024-09-04 22:53:52.501(-6 degrees)
Sun Rise2024-09-05 10:56:52.310
Sun Set2024-09-04 22:25:55.324
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "# Display various almanac values (for moon, sun)\n", "#!rep.AlmanacReport().day_obs_report(min_day_obs)\n", @@ -479,7 +248,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "15", "metadata": { "jupyter": { @@ -495,138 +264,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "16", "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "## Summary Scalars for the night " - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "(of all available instruments that have at least one exposure)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
LSSTComCamSimLATISS
Total Night (HH:MM:SS)9:43:549:43:54
Total Exposure (HH:MM:SS)0:02:332:45:13
Number of exposures9286
Number of slews4040
Total Detector Read (HH:MM:SS)0:00:200:10:57
Mean Detector Read (HH:MM:SS)0:00:020:00:02
Total Slew (HH:MM:SS)0:41:170:41:17
Mean Slew (HH:MM:SS)0:01:010:01:01
Total Idle (HH:MM:SS)9:00:036:17:23
\n", - "
" - ], - "text/plain": [ - " LSSTComCamSim LATISS\n", - "Total Night (HH:MM:SS) 9:43:54 9:43:54\n", - "Total Exposure (HH:MM:SS) 0:02:33 2:45:13\n", - "Number of exposures 9 286\n", - "Number of slews 40 40\n", - "Total Detector Read (HH:MM:SS) 0:00:20 0:10:57\n", - "Mean Detector Read (HH:MM:SS) 0:00:02 0:00:02\n", - "Total Slew (HH:MM:SS) 0:41:17 0:41:17\n", - "Mean Slew (HH:MM:SS) 0:01:01 0:01:01\n", - "Total Idle (HH:MM:SS) 9:00:03 6:17:23" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "# Exposure Tally\n", "md('')\n", @@ -648,59 +289,10 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "18", "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "- [OBS-582](https://rubinobs.atlassian.net/browse/OBS-582)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- [OBS-584](https://rubinobs.atlassian.net/browse/OBS-584)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- [/rubinobs-lfa-cp/LOVE/OLE/2024/09/05/LOVE_OLE_2024-09-05T14:36:21.262.png](https://s3.cp.lsst.org/rubinobs-lfa-cp/LOVE/OLE/2024/09/05/LOVE_OLE_2024-09-05T14:36:21.262.png)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- [OBS-583](https://rubinobs.atlassian.net/browse/OBS-583)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "if allsrc.urls:\n", " for url in allsrc.urls:\n", @@ -709,23 +301,10 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "19", "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "No tickets found using: [API Data](https://summit-lsp.lsst.codes/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&min_day_obs=20240905&max_day_obs=20240907&limit=999) in `confluence_url`." - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "# Display Jira BLOCKS\n", "front = \"https://rubinobs.atlassian.net/projects/BLOCK?selectedItem=com.atlassian.plugins.atlassian-connect-plugin:com.kanoah.test-manager__main-project-page#!/\"\n", @@ -761,23 +340,10 @@ }, { "cell_type": "code", - "execution_count": 13, - "id": "23", + "execution_count": null, + "id": "22", "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "### Experimental Exposure field Tally DISASBLED" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "# Exposure Report\n", "# Facet counts\n", @@ -793,7 +359,7 @@ }, { "cell_type": "markdown", - "id": "24", + "id": "23", "metadata": {}, "source": [ "| Symbol | Meaning |\n", @@ -805,23 +371,10 @@ }, { "cell_type": "code", - "execution_count": 14, - "id": "25", + "execution_count": null, + "id": "24", "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "-------------" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "# Time Log\n", "allrep.exp_rep.time_log_as_markdown()" @@ -829,7 +382,7 @@ }, { "cell_type": "markdown", - "id": "26", + "id": "25", "metadata": {}, "source": [ "## Narrative Log\n", @@ -838,2907 +391,10 @@ }, { "cell_type": "code", - "execution_count": 15, - "id": "27", + "execution_count": null, + "id": "26", "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "- **2024-09-05 05:44:37.750878**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Around 21:15 local time, shortly after signs that the network to the outside world was disconnected, ATCamera went into a fault state. The CSC reported that the MCM went into a fault state for an \"unspecified reason.\" We stopped tracked and opened the CCS to investigate. Network issues were plaguing the summit and it was a very slow connection to the CCS-console - using shell commands saw a delay of about 1-2 seconds for every keystroke. We managed to identify the error in the ats-ih - according to the logs there was an image header delay.\r\n",
-       "\r\n",
-       "\r\n",
-       "*Unknown if this was due to network outages. \r\n",
-       "\r\n",
-       "\r\n",
-       "We cleared the errors in the ats-ih and mcm, and we recovered the ATCamera. Even without network, we were able to resume scheduler-driven operations. The lengthy recovery time was due to the slow connection to auxtel-fp01 and reading through documentation to diagnose. \r\n",
-       "\r\n",
-       "\r\n",
-       "**A note: LATISS was taking new images, but CCS was not updating, even when connected to Rubin-guest. Is this expected behavior? How might we view images if there is another network outage?
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- Link: [OBS-583](https://rubinobs.atlassian.net/browse/OBS-583)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 06:01:16.348158**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Logging time loss to OBS-584
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 00:01:06.570924**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
After changing the Parked Engineering state, worked
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 00:01:18.684009**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Re-run check_actuators.py
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 00:03:32.477288**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
2024/09/05 00:01:51 TAI\r\n",
-       "Force Actuator ID 240 Following Error 50
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 00:09:50.344086**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Run again  but forgot to killM1M3 actuator test.\r\n",
-       "run_command.py.\r\n",
-       "1) enterEngineering\r\n",
-       "2) killForceActuactorBumptTest\r\n",
-       "\r\n",
-       "\r\n",
-       "and re-run the script.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 00:26:42.103884**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Hernan reports that the LHN has gone down - connecting to Rubin Guest wifi for ops
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 00:50:49.142853**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
SAA (ID, Pri Index): 303:80, 326:103 \r\n",
-       " DAA (ID, Sec Index): failed. \r\n",
-       "bump test done.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 00:54:33.897804**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Block-T4. Hexapod warming up.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 00:57:39.671319**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Executive summary on the bump test: \r\n",
-       "\r\n",
-       "\r\n",
-       "The test went through ignoring FA138, 236, 240, 320, 335. FA 303, 326 failed the test.\r\n",
-       "When there is an error on a given actuator and want to start again the test you need:\r\n",
-       "1. Park in engineering mode first. That was possible with run_command.py (component: MTM1M3, cmd: enterEngineering) to be able to enter on ParkedEngineering mode. \r\n",
-       "2. Kill the failed bump test. That is done using run_command.py (component:MTM1M3, cmd: killForceActuatorBumpTest).\r\n",
-       "3. Relaunch the test (BLOCK-T144) ignoring the desired actuators.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 01:12:51.910240**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
T4 is done without issue. Running Block-T3.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 01:21:45.053232**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Block-T3 done. Starting Block-T89
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 01:32:54.864233**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Measurement of the Laser Tracker went through. \r\n",
-       "Missed 2 measurement(s) from Meas_TMA_CENTRAL_0.35_148.75_0.00_04::TMA_CENTRAL_0.35_148.75_0.00_04.'
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 01:35:24.046506**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Bump test for FA303, 326 from EUI. All passed.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 01:35:50.383761**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Done for Block-T89. Starting Block-T87.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 01:46:37.096069**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "Salindex: 100511. Failed as known network issue at the summit\r\n", - "
RuntimeError: Error on cmd=cmd_start, initial_state=5: msg='Command failed', ackcmd=(ackcmd private_seqNum=1698289930, ack=<SalRetCode.CMD_FAILED: -302>, error=1, result=\"Failed: HTTPSConnectionPool(host='roundtable.lsst.codes', port=443): Max retries exceeded with url: /segwarides/ (Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x7f99c038ed90>, 'Connection to roundtable.lsst.codes timed out. (connect timeout=None)'))\")\r\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 02:16:32.071631**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Resuming scheduler with ATCamera recovered.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 02:59:13.267372**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Network has returned.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 03:02:42.208275**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
network is recovered. Try to enable maintel/scheduler and it worked this time. \r\n",
-       "Resuming Scheduler:1 for config:maintel_fbs_sit_block_t87.yaml
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 03:13:54.780447**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
RuntimeError: Timeout waiting for TCS to report as ready to take data (timeout=30.0).\r\n",
-       "\r\n",
-       "\r\n",
-       "Resuming observations, skipping step\r\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 03:41:46.082133**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
We are experiencing back to back runtime errors: The telescope times out getting into position. Doing some different experiments to recover.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 03:42:55.381132**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Cycled ATCS, but the telescope timed out when trying to correct pointing. Going to shutdown and open up again to force the system to come back in a correct state. LOVE telemetries being unreliable
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 04:07:50.871134**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Cycled ATCS CSC and am attempting to park the telescope again
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 04:22:03.433382**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Running prepare for onsky after telescope went to commanded position.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 04:34:28.000789**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Corrected pointing with success. Resuming scheduler. End of fault loss.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 05:57:27.762005**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Stop scheduler. Block-T87.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 05:58:09.215646**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
BLOCK-T139. Move MTHexapod1 to origin.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 06:01:34.808204**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Move MTHexapod2 to origin.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 06:01:35.897529**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Running shutdown script, closing for the night, end of science time.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 06:03:35.372781**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Put both hexapod on disabled. Put MTPtg in disabled, the same as the initial status.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 20:25:22.980109**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Grabbing messages from BLOCK-T62. \r\n",
-       "Comments:\r\n",
-       "\r\n",
-       "\r\n",
-       "Similarly to other days, we will simply move the TMA.\r\n",
-       "Today we will move from Horizon to Zenith.\r\n",
-       "We will make a stop at 45º so Roberto and Guillem can perform measurements with the LaserTracker.\r\n",
-       "There will be no change in any counterweights.\r\n",
-       "\r\n",
-       "\r\n",
-       "LOVE does not work. So we will perform all the logging here.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 20:25:41.091606**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
BLOCK-T62 - Step 1\r\n",
-       "LOVE does not work. So we cannot pause the MTQueue.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 20:31:47.371588**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
BLOCK-T62 - Step 2\r\n",
-       "The EUI default values on the EUI are 2%.\r\n",
-       "We will set vel/acc/jerk to 1% manually during execution.\r\n",
-       "Current default value acc 0.1 / jerk 0.4 / vel 0.1.\r\n",
-       "We will update this later.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 20:32:02.302123**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
BLOCK-T62 - Step 3\r\n",
-       "No need to check on counterweights today.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 20:32:14.970747**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
BLOCK-T62 - Step 4\r\n",
-       "MTMount already in disabled
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 20:32:28.359818**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
BLOCK-T62 - Step 5\r\n",
-       "Confirmed torques are good yesterday.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 20:32:39.575744**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
BLOCK-T62 - Step 7\r\n",
-       "(Step 6 and Step 7 are swapped).\r\n",
-       "\r\n",
-       "\r\n",
-       "Both pins were already powered on. (Why?)\r\n",
-       "Locking Pin -X went to a fault. This is the one that gave us a problem yesterday, too.\r\n",
-       "This might be concerning.\r\n",
-       "\r\n",
-       "\r\n",
-       "We are having an OMT fault (see fig attached). \r\n",
-       "Freddy visually confirms that the pin is ok.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- Link: [/rubinobs-lfa-cp/LOVE/OLE/2024/09/05/LOVE_OLE_2024-09-05T14:36:21.262.png](https://s3.cp.lsst.org/rubinobs-lfa-cp/LOVE/OLE/2024/09/05/LOVE_OLE_2024-09-05T14:36:21.262.png)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 14:37:24.722652**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Freddy and Ian are debugging. \r\n",
-       "It seems to be a hardware problem. \r\n",
-       "More details to follow.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 14:44:38.380512**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
As part of this debugging, we created OBS-585
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 14:45:27.661561**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
For information, TEST position means 45 mm and FREE is almost 0+.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 14:49:14.394753**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Powering up power supply now
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 14:53:31.606212**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Running HOME on elevation now
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 15:10:17.907398**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
We already asked the TMA to move to 45º.\r\n",
-       "Since the default values for the EUI might be wrong, we used the EUI to set ACC and vel manually to 0.05º/s(2).\r\n",
-       "\r\n",
-       "\r\n",
-       "We are now at Zenith and Guillem will collect data with LaserTracker.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 15:15:01.149666**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Moving the telescope now up to 90.01º
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 20:24:53.229342**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
UTC 15:18:20 - Something happened in Azimuth (which is powered off) and that was also seen by M2. \r\n",
-       "Needs investigation. \r\n",
-       "\r\n",
-       "\r\n",
-       "See the query below:\r\n",
-       "https://summit-lsp.lsst.codes/chronograf/sources/1/dashboards/198?refresh=Paused&tempVars%5BFunction%5D=mean&tempVars%5BDownsample%5D=Default&lower=now%28%29%20-%2015m&zoomedLower=2024-09-05T15%3A17%3A43.890Z&zoomedUpper=2024-09-05T15%3A18%3A50.905Z
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 20:25:05.147620**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
15:23:23 UTC Bruno went up to the az platform and performed three jumps followed by a pause. This can be seen in the tma az telemetry and also on m2.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 20:24:43.604901**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Guillem performed measurements with laser tracker at 90.01º\r\n",
-       "We are now inserting the locking pins. We will put them into test first. \r\n",
-       "\r\n",
-       "\r\n",
-       "We got the same error as in OBS-585.\r\n",
-       "Trying X+ to TEST. \r\n",
-       "\r\n",
-       "\r\n",
-       "We realized we forgot to power up the locking pin.\r\n",
-       "We powered X+ on and moved it successfully. \r\n",
-       "\r\n",
-       "\r\n",
-       "Same with locking pin X-. We are all good.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 20:24:30.317775**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Power off Locking Pin.\r\n",
-       "Power off OSS.\r\n",
-       "Power off Power Supply.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 18:48:38.908840**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "While the daytime checkouts. The Slew_and_take_image_checkout faulted with 
error:\r\n",
-       "\r\n",
-       "\r\n",
-       "Traceback (most recent call last):\r\n",
-       "  File \"/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/ataos/ataos_csc.py\", line 1321, in correction_loop\r\n",
-       "    await corr\r\n",
-       "  File \"/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/ataos/ataos_csc.py\", line 1669, in set_pressure_m1\r\n",
-       "    await self.set_pressure(\r\n",
-       "  File \"/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/ataos/ataos_csc.py\", line 1762, in set_pressure\r\n",
-       "    raise e\r\n",
-       "  File \"/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/ataos/ataos_csc.py\", line 1756, in set_pressure\r\n",
-       "    await cmd_attr.set_start(pressure=pressure, timeout=self.cmd_timeout)\r\n",
-       "  File \"/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 416, in set_start\r\n",
-       "    return await self.start(timeout=timeout, wait_done=wait_done)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File \"/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 487, in start\r\n",
-       "    return await cmd_info.next_ackcmd(timeout=timeout)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File \"/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 191, in next_ackcmd\r\n",
-       "    raise base.AckError(msg=\"Command failed\", ackcmd=ackcmd)\r\n",
-       "lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=1042063791, ack=<SalRetCode.CMD_TIMEOUT: -304>, error=0, result='Timeout in ATPneumatics while waiting for operation to start.')\r\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-06 00:36:18.575477**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "ATmonochromoeter failed while attempting to cnifgure. Sunk around an hour trying to power on the at_calsys but had to give up.\r\n", - "Error in run\r\n", - "Traceback (most recent call last):\r\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py\", line 603, in do_run\r\n", - " await self._run_task\r\n", - " File \"/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/auxtel/calibrations/power_on_atcalsys.py\", line 191, in run\r\n", - " await self.configure_atmonochromator()\r\n", - " File \"/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/auxtel/calibrations/power_on_atcalsys.py\", line 287, in configure_atmonochromator\r\n", - " await self.monochromator.cmd_updateMonochromatorSetup.set_start(\r\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 416, in set_start\r\n", - " return await self.start(timeout=timeout, wait_done=wait_done)\r\n", - " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 487, in start\r\n", - " return await cmd_info.next_ackcmd(timeout=timeout)\r\n", - " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 191, in next_ackcmd\r\n", - " raise base.AckError(msg=\"Command failed\", ackcmd=ackcmd)\r\n", - "
lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=2030279430, ack=<SalRetCode.CMD_TIMEOUT: -304>, error=1, result='Timeout')\r\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- Link: [OBS-582](https://rubinobs.atlassian.net/browse/OBS-582)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 20:31:27.608758**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Proceeding with AT daytime checkout at 16:23 local time
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 20:39:25.532539**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Executive summary AuxTel faults in the afternoon (extracts from Craig Lage):\r\n",
-       "\r\n",
-       "\r\n",
-       "Workings on the ATMCS/ATPneumatics cRIO in the afternoon.\r\n",
-       "In the process a bug made the mount torque data disappeared from the EFD. This was finally tracked down and fixed, and the torque data is flowing again. \r\n",
-       "Totally unrelated, images after seqNum 11 stopped ingesting. Solved on the afternoon.\r\n",
-       "There were several faults when running the daytime checkouts, maybe related with leaving systems in strange states with all of the reboots.\r\n",
-       "The biggest problem remains the ATAOS faulting because of communications with ATPneumatics.\r\n",
-       "\r\n",
-       "\r\n",
-       "On a second attempt for daytime checkouts the slew_and_take_image_checkout passed without issues.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-06 01:29:30.773483**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Lost some sky time due to ATPtg state cycle. Telescope timeouts (non-TCS) were persistent.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- Link: [OBS-584](https://rubinobs.atlassian.net/browse/OBS-584)" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 20:54:55.706808**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
All the Schedulers (starting with AT, then MT, then OCS) at 16:23 local time
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 21:13:33.576699**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "2024/09/05 21:12:22 TAI\r\n", - "Fault! errorCode=8103, errorReport='Correction loop died.'\r\n", - "On ATAOS: \r\n", - "\r\n", - "\r\n", - "\r\n", - "\r\n", - " File \"/opt/lsst/software/stack/miniconda/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 191, in next_ackcmd\r\n", - " raise base.AckError(msg=\"Command failed\", ackcmd=ackcmd)\r\n", - "
lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=1042066578, ack=<SalRetCode.CMD_TIMEOUT: -304>, error=0, result='Timeout in ATPneumatics while waiting for operation to start.')\r\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 21:16:13.963746**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Run vent.py
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 21:38:02.881776**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Block-T4 started. MTHexapod1 was enabled right before.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 22:50:28.389048**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Blcok-T4 done without issue before dinner.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 22:53:42.560749**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Preparing BLOCK-T143. Need to ping Tiago R. and Guillem M. but Tiago Mnetioned that He missed the queue that things were ready and did not rebuilt the MTAOS.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 22:55:47.492969**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Block-T3 run. MThexapod2 enabled.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 23:05:35.562216**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Block-T3 done without issue.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 23:28:09.017654**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
ComCam is already in simulation mode (checked by Tony J.). Tiago finished the update.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 23:31:39.430533**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
BLOCK-T143. Step 1. Run_command.py for CCCamera to play.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 23:38:09.365827**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Cirrus are scattered across the sky. We are going to use this time to attempt a full set of calibrations now that the fiberspectrograph is up.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 23:38:28.108053**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
AtWhite light failing to power on. Going to wait 10 minutes until it is cool to try one more time.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 23:39:04.232661**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Step 1 done even there is no indication on Asummary state CSC. Step 2 - close_loop_comcam.py with g_01 band.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 23:41:32.627163**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "fault on Step 2 with close_loop_comcam.py \r\n", - "2024/09/05 23:39:57 TAI\r\n", - "Error in run\r\n", - "Traceback (most recent call last):\r\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py\", line 603, in do_run\r\n", - " await self._run_task\r\n", - " File \"/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/base_close_loop.py\", line 508, in run\r\n", - " await self.assert_mode_compatibility()\r\n", - " File \"/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/close_loop_comcam.py\", line 77, in assert_mode_compatibility\r\n", - " raise RuntimeError(\"ComCam does not support CWFS mode.\")\r\n", - "
RuntimeError: ComCam does not support CWFS mode.\r\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 23:46:13.165827**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Attempting power_on_calsys again
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 23:51:47.961227**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Change config. from one in Zephyer to reverted version:\r\n",
-       "filter: \"r_03\"\r\n",
-       "mode: FAM\r\n",
-       "exposure_time: 15\r\n",
-       "dz: 1.5\r\n",
-       "max_iter: 1\r\n",
-       "program: \"BLOCK-T143\"\r\n",
-       "reason: \"ComCam_AOS visit 68661\"\r\n",
-       "used_dofs: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]\r\n",
-       "apply_corrections: false\r\n",
-       "ignore: [\r\n",
-       "  \"mtmount\",\r\n",
-       "  \"mtptg\",\r\n",
-       "  \"mtm1m3\",\r\n",
-       "  \"mtm2\",\r\n",
-       "  \"mtrotator\",\r\n",
-       "  \"mtdome\",\r\n",
-       "  \"mtdometrajectory\",\r\n",
-       "]\r\n",
-       "wep_config: \"tasks:\\n generateDonutCatalogWcsTask:\\n config:\\n anyFilterMapsToThis: phot_g_mean\\n connections.refCatalogs: gaia_dr2_20200414\\n donutSelector.magMax: 15.0\\n donutSelector.magMin: 4.0\\n donutSelector.sourceLimit: 5\\n donutSelector.useCustomMagLimit: true\\n\"
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 23:54:54.552138**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Failed with new config too with timeout error due to hexapod. \r\n",
-       "2024/09/05 23:52:18 TAI\r\n",
-       "Error in run\r\n",
-       "\r\n",
-       "\r\n",
-       "  File \"/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/base_close_loop.py\", line 330, in take_intra_extra_focal_images\r\n",
-       "    await self.mtcs.move_camera_hexapod(x=0, y=0, z=z_offset, u=0, v=0)\r\n",
-       "  File \"/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/maintel/mtcs.py\", line 1708, in move_camera_hexapod\r\n",
-       "    await self._handle_in_position(\r\n",
-       "  File \"/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/base_tcs.py\", line 1958, in _handle_in_position\r\n",
-       "    in_position = await in_position_event.next(flush=False, timeout=timeout)\r\n",
-       "                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/read_topic.py\", line 681, in next\r\n",
-       "    return await self._next(timeout=timeout)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/read_topic.py\", line 693, in _next\r\n",
-       "    return await asyncio.wait_for(self._next_task, timeout=timeout)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/tasks.py\", line 502, in wait_for\r\n",
-       "    raise exceptions.TimeoutError() from exc\r\n",
-       "TimeoutError
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-05 23:57:56.588990**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Move MThexapod1 to 0 but MThexapod1 went to fault. Low-level controller went to fault state.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-06 00:05:02.239846**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Moving hexapod again and worked now.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-06 00:04:54.045658**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
re-run close_loop_comcam.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-06 00:06:20.468432**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "024/09/06 00:05:39 TAI\r\n", - "Error in run\r\n", - "Traceback (most recent call last):\r\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py\", line 603, in do_run\r\n", - " await self._run_task\r\n", - " File \"/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/base_close_loop.py\", line 510, in run\r\n", - " await self.arun(True)\r\n", - " File \"/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/base_close_loop.py\", line 443, in arun\r\n", - " await self.operation_model_handlers[self.mode]()\r\n", - " File \"/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/base_close_loop.py\", line 360, in handle_fam_mode\r\n", - " await self.mtcs.rem.mtaos.cmd_runWEP.set_start(\r\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 416, in set_start\r\n", - " return await self.start(timeout=timeout, wait_done=wait_done)\r\n", - " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 487, in start\r\n", - " return await cmd_info.next_ackcmd(timeout=timeout)\r\n", - " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 191, in next_ackcmd\r\n", - " raise base.AckError(msg=\"Command failed\", ackcmd=ackcmd)\r\n", - "
lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=1579091411, ack=<SalRetCode.CMD_FAILED: -302>, error=1, result='Failed: wep_process not set.')\r\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-06 00:08:08.396181**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
ATmonochrometer failed to configure during power_on_atcalsys\r\n",
-       "Already sunk 1 hour into trying to turn on a lamp. Calibrations are being dismissed.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-06 00:10:40.346614**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "While executing the close_loop_comcam.py (BLOCK-T143) MTAOS gives an error (although not in Fault sate):\r\n", - "\r\n", - "\r\n", - "Callback <bound method MTAOS.do_runWEP of <lsst.ts.mtaos.mtaos.MTAOS object at 0x7f4508fa78d0>> failed with data=private_revCode: 73eb1cbb, private_sndStamp: 1725581139.124955, private_rcvStamp: 1725581139.1251993, private_seqNum: 1579091411, private_identity: Script:101082, private_origin: 1895564, visitId: 2024090500002, extraId: 2024090500003, useOCPS: False, config: tasks:\r\n", - " generateDonutCatalogWcsTask:\r\n", - " config:\r\n", - " anyFilterMapsToThis: phot_g_mean\r\n", - " connections.refCatalogs: gaia_dr2_20200414\r\n", - " donutSelector.magMax: 15.0\r\n", - " donutSelector.magMin: 4.0\r\n", - " donutSelector.sourceLimit: 5\r\n", - " donutSelector.useCustomMagLimit: true\r\n", - "Traceback (most recent call last):\r\n", - " File \"/home/saluser/repos/ts_mtaos/python/lsst/ts/mtaos/model.py\", line 664, in handle_wep_process\r\n", - " log_task, config_file = await self._start_wep_process(\r\n", - " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n", - " File \"/home/saluser/repos/ts_mtaos/python/lsst/ts/mtaos/model.py\", line 732, in _start_wep_process\r\n", - " config_file = self._save_wep_configuration(\r\n", - " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n", - " File \"/home/saluser/repos/ts_mtaos/python/lsst/ts/mtaos/model.py\", line 904, in _save_wep_configuration\r\n", - " wep_configuration = self.generate_wep_configuration(\r\n", - " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n", - " File \"/home/saluser/repos/ts_mtaos/python/lsst/ts/mtaos/model.py\", line 878, in generate_wep_configuration\r\n", - " return self.wep_configuration_validation[instrument].validate(wep_configuration)\r\n", - " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/validator.py\", line 192, in validate\r\n", - " self.defaults_validator.validate(result)\r\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/jsonschema/validators.py\", line 451, in validate\r\n", - " raise error\r\n", - "
jsonschema.exceptions.ValidationError: None is not of type 'object'\r\n",
-       "\r\n",
-       "\r\n",
-       "Failed validating 'type' in schema['properties']['tasks']['properties']['generateDonutCatalogWcsTask']:\r\n",
-       "    {'type': 'object',\r\n",
-       "     'additionalProperties': False,\r\n",
-       "     'properties': {'class': {'type': 'string',\r\n",
-       "                              'default': 'lsst.ts.wep.task.generateDonutCatalogWcsTask.GenerateDonutCatalogWcsTask'},\r\n",
-       "                    'config': {'properties': {'filterName': {'type': 'string'}}}}}\r\n",
-       "\r\n",
-       "\r\n",
-       "On instance['tasks']['generateDonutCatalogWcsTask']:\r\n",
-       "    None\r\n",
-       "\r\n",
-       "\r\n",
-       "During handling of the above exception, another exception occurred:\r\n",
-       "\r\n",
-       "\r\n",
-       "Traceback (most recent call last):\r\n",
-       "  File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/controller_command.py\", line 201, in _run_callback\r\n",
-       "    ack = await result  # type: ignore\r\n",
-       "          ^^^^^^^^^^^^\r\n",
-       "  File \"/home/saluser/repos/ts_mtaos/python/lsst/ts/mtaos/mtaos.py\", line 505, in do_runWEP\r\n",
-       "    await self.model.run_wep(\r\n",
-       "  File \"/home/saluser/repos/ts_mtaos/python/lsst/ts/mtaos/utility.py\", line 257, in atimed\r\n",
-       "    result = await func(*args, **kwargs)\r\n",
-       "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File \"/home/saluser/repos/ts_mtaos/python/lsst/ts/mtaos/model.py\", line 525, in run_wep\r\n",
-       "    await self.process_comcam(\r\n",
-       "  File \"/home/saluser/repos/ts_mtaos/python/lsst/ts/mtaos/model.py\", line 615, in process_comcam\r\n",
-       "    async with self.handle_wep_process(\r\n",
-       "  File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/contextlib.py\", line 210, in __aenter__\r\n",
-       "    return await anext(self.gen)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File \"/home/saluser/repos/ts_mtaos/python/lsst/ts/mtaos/model.py\", line 684, in handle_wep_process\r\n",
-       "    await self._finish_wep_process()\r\n",
-       "  File \"/home/saluser/repos/ts_mtaos/python/lsst/ts/mtaos/model.py\", line 837, in _finish_wep_process\r\n",
-       "    raise RuntimeError(\"wep_process not set.\")\r\n",
-       "RuntimeError: wep_process not set.\r\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-06 00:26:56.127468**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Same config but with \r\n",
-       "wep_config: \"tasks:\\n generateDonutCatalogWcsTask:\\n  config:\\n   anyFilterMapsToThis: phot_g_mean\\n   connections.refCatalogs: gaia_dr2_20200414\\n   donutSelector.magMax: 15.0\\n   donutSelector.magMin: 4.0\\n   donutSelector.sourceLimit: 5\\n   donutSelector.useCustomMagLimit: true\\n
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-06 00:29:51.558297**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "Failed at the end of the script:\r\n", - "2024/09/06 00:29:03 TAI\r\n", - "Error in run\r\n", - "Traceback (most recent call last):\r\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py\", line 603, in do_run\r\n", - " await self._run_task\r\n", - " File \"/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/base_close_loop.py\", line 510, in run\r\n", - " await self.arun(True)\r\n", - " File \"/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/base_close_loop.py\", line 443, in arun\r\n", - " await self.operation_model_handlers[self.mode]()\r\n", - " File \"/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/base_close_loop.py\", line 360, in handle_fam_mode\r\n", - " await self.mtcs.rem.mtaos.cmd_runWEP.set_start(\r\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 416, in set_start\r\n", - " return await self.start(timeout=timeout, wait_done=wait_done)\r\n", - " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 487, in start\r\n", - " return await cmd_info.next_ackcmd(timeout=timeout)\r\n", - " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 191, in next_ackcmd\r\n", - " raise base.AckError(msg=\"Command failed\", ackcmd=ackcmd)\r\n", - "
lsst.ts.salobj.base.AckError: msg='Command failed', ackcmd=(ackcmd private_seqNum=1579091411, ack=<SalRetCode.CMD_FAILED: -302>, error=1, result='Failed: wavefrontError write(private_revCode: cdb25a59, private_sndStamp: 1725582543.3115678, private_rcvStamp: 0.0, private_seqNum: 1, private_identity: MTAOS, private_origin: 3159695, sensorId: 0, annularZernikeCoeff: 0.08561753960450617) failed: probably at least one array field is too short')\r\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-06 00:31:46.909487**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Running prepare for onsky.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-06 00:32:12.457994**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
On MTAOS:\r\n",
-       "Callback <bound method MTAOS.do_runWEP of <lsst.ts.mtaos.mtaos.MTAOS object at 0x7f4508fa78d0>> failed with data=private_revCode: 73eb1cbb, private_sndStamp: 1725582417.9892354, private_rcvStamp: 1725582417.9895887, private_seqNum: 1579091411, private_identity: Script:101083, private_origin: 1934283, visitId: 2024090500004, extraId: 2024090500005, useOCPS: False, config: tasks:\r\n",
-       " generateDonutCatalogWcsTask:\r\n",
-       "  config:\r\n",
-       "   anyFilterMapsToThis: phot_g_mean\r\n",
-       "   connections.refCatalogs: gaia_dr2_20200414\r\n",
-       "   donutSelector.magMax: 15.0\r\n",
-       "   donutSelector.magMin: 4.0\r\n",
-       "   donutSelector.sourceLimit: 5\r\n",
-       "   donutSelector.useCustomMagLimit: true
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-06 00:33:13.949549**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Halt BLOCK-T143 here. Need to be checked by Tiago and Guillem.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-06 00:37:31.193400**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
At is on sky after fitzing with calibrations system
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-06 00:50:04.145255**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Tiago found that the MTAOS was running with wrong ts_wep version. It is just swapped.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-06 00:54:40.977624**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Telescope timed out getting into position. LOVE SQ is responding very slowly, so it's a bit difficult to run new steps.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-06 00:57:11.080951**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Timeout error. \r\n",
-       "2024/09/06 00:53:25 TAI\r\n",
-       "Error in run\r\n",
-       "Traceback (most recent call last):\r\n",
-       "  File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/tasks.py\", line 500, in wait_for\r\n",
-       "    return fut.result()\r\n",
-       "           ^^^^^^^^^^^^\r\n",
-       "asyncio.exceptions.CancelledError\r\n",
-       "\r\n",
-       "\r\n",
-       "The above exception was the direct cause of the following exception:\r\n",
-       "\r\n",
-       "\r\n",
-       "Traceback (most recent call last):\r\n",
-       "  File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py\", line 603, in do_run\r\n",
-       "    await self._run_task\r\n",
-       "  File \"/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/base_close_loop.py\", line 510, in run\r\n",
-       "    await self.arun(True)\r\n",
-       "  File \"/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/base_close_loop.py\", line 446, in arun\r\n",
-       "    wavefront_error = await self.mtcs.rem.mtaos.evt_wavefrontError.next(\r\n",
-       "                      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/read_topic.py\", line 681, in next\r\n",
-       "    return await self._next(timeout=timeout)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/read_topic.py\", line 693, in _next\r\n",
-       "    return await asyncio.wait_for(self._next_task, timeout=timeout)\r\n",
-       "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n",
-       "  File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/tasks.py\", line 502, in wait_for\r\n",
-       "    raise exceptions.TimeoutError() from exc\r\n",
-       "TimeoutError
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-06 00:59:39.528550**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Telescope timed out getting into position. LOVE is taking a long time to report errors. Very laggy.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-06 01:03:44.101506**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Re-attemp from the step 1 of BLOCK-T143. \r\n",
-       "- Step 1 passed. \r\n",
-       "- Running Step 2.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-06 01:07:12.586434**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Cycling ATPtg due to telescope timeouts.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-06 01:10:27.434918**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Step 2 passed. Take_image_comcam.py with following setup:\r\n",
-       "exp_times: 15.0\r\n",
-       "note: \"ComCam_AOS visit 68661 2/3 seq 941 - Focus 0.0\"\r\n",
-       "filter: \"r_03\"\r\n",
-       "image_type: \"ACQ\"\r\n",
-       "nimages: 1\r\n",
-       "program: \"BLOCK-T143\"
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-06 01:13:40.548799**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "Failed. \r\n", - "2024/09/06 01:10:46 TAI\r\n", - "Error in run\r\n", - "Traceback (most recent call last):\r\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 189, in next_ackcmd\r\n", - " ackcmd = await self._wait_task\r\n", - " ^^^^^^^^^^^^^^^^^^^^^\r\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 214, in _basic_next_ackcmd\r\n", - " ackcmd = await asyncio.wait_for(\r\n", - " ^^^^^^^^^^^^^^^^^^^^^^^\r\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/tasks.py\", line 502, in wait_for\r\n", - " raise exceptions.TimeoutError() from exc\r\n", - "TimeoutError\r\n", - "\r\n", - "\r\n", - "During handling of the above exception, another exception occurred:\r\n", - "\r\n", - "\r\n", - "Traceback (most recent call last):\r\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py\", line 603, in do_run\r\n", - " await self._run_task\r\n", - " File \"/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/base_take_image.py\", line 243, in run\r\n", - " await asyncio.gather(*setup_tasks)\r\n", - " File \"/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/maintel/comcam.py\", line 206, in setup_instrument\r\n", - " await self.setup_filter(filter=str(kwargs[\"filter\"]))\r\n", - " File \"/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/maintel/comcam.py\", line 226, in setup_filter\r\n", - " await self.rem.cccamera.cmd_setFilter.set_start(\r\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 416, in set_start\r\n", - " return await self.start(timeout=timeout, wait_done=wait_done)\r\n", - " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 487, in start\r\n", - " return await cmd_info.next_ackcmd(timeout=timeout)\r\n", - " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\r\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 205, in next_ackcmd\r\n", - " raise base.AckTimeoutError(\r\n", - "
lsst.ts.salobj.base.AckTimeoutError: msg='Timed out waiting for command acknowledgement', ackcmd=(ackcmd private_seqNum=1520055159, ack=<SalRetCode.CMD_NOACK: -301>, error=0, result='No command acknowledgement seen')\r\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-06 01:19:52.922068**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Issue with timeout. Tiago mentioned that he needs to talk to Tony J about these timeout. Need an IN_PROGRESS ack for filter changes.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-06 01:42:53.756168**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
K.T mentioned that this happened because the header values seem to be changing within a focal plane.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-06 01:44:46.392415**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Re-enabled scheduler.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-06 02:26:33.877824**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Bump test (check_actutator) with the following configuration. BLOCK-T144. \r\n",
-       "program: BLOCK-T144\r\n",
-       "reason: BLOCK-T144\r\n",
-       "ignore_actuators:\r\n",
-       "-423
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-06 03:36:33.267786**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "Bumptest done. 2024/09/06 03:32:29 TAI\r\n", - "Error in run\r\n", - "Traceback (most recent call last):\r\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py\", line 603, in do_run\r\n", - " await self._run_task\r\n", - " File \"/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/base_block_script.py\", line 302, in run\r\n", - " await self.run_block()\r\n", - " File \"/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/maintel/m1m3/check_actuators.py\", line 335, in run_block\r\n", - " raise RuntimeError(error_message)\r\n", - "
RuntimeError: Actuators [133, 138, 220, 236, 240, 320, 330, 335] FAILED the bump test. \r\n",
-       " SAA (ID, Pri Index): 133:32, 138:37, 220:55, 236:70, 240:74, 320:97, 330:107, 335:112 \r\n",
-       " DAA (ID, Sec Index): \r\n",
-       "\r\n",
-       "\r\n",
-       "2024/09/06 03:32:29 TAI\r\n",
-       "Actuators [133, 138, 220, 236, 240, 320, 330, 335] FAILED the bump test. \r\n",
-       " SAA (ID, Pri Index): 133:32, 138:37, 220:55, 236:70, 240:74, 320:97, 330:107, 335:112 \r\n",
-       " DAA (ID, Sec Index): \r\n",
-       "
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-06 03:41:24.795954**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Done for Block-T144. Start BLOCK-T145, hardpoint test.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "- **2024-09-06 05:39:05.272652**" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "\n", - "
Done for BLOCK-T145. Then run BLOCK-T139 to close the night. \r\n",
-       "MTHexapod1, MTHexapod2, and MTM1M3 in disabled now.
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "-------------" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "# Narrative Report\n", "# Time Log\n", @@ -3747,7 +403,7 @@ }, { "cell_type": "markdown", - "id": "28", + "id": "27", "metadata": {}, "source": [ "\n", @@ -3759,7 +415,7 @@ }, { "cell_type": "markdown", - "id": "9bf79dd4-60b7-4bc8-bd33-16ffb6861b21", + "id": "28", "metadata": {}, "source": [ "## Experimental " @@ -3767,7 +423,7 @@ }, { "cell_type": "markdown", - "id": "44e1b0f6-6750-4c4b-8ca6-9e1ddf090d1a", + "id": "29", "metadata": {}, "source": [ "### Parse pasted ERROR" @@ -3775,111 +431,45 @@ }, { "cell_type": "code", - "execution_count": 16, - "id": "7d5c3c53-de57-4f67-b046-54252b89169f", + "execution_count": null, + "id": "30", "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "BEG: This is my text\n", - "that goes on for multiple lines\n", - " and this one started with 3 spaces\n", - " and this one with 4 tabs\n", - "back to first column.:END\n", - "" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "
BEG: This is my text\n",
-       "that goes on for multiple lines\n",
-       "   and this one started with 3 spaces\n",
-       "                and this one with 4 tabs\n",
-       "back to first column.:END\n",
-       "
" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "import lsst.ts.logging_and_reporting.reports as rep\n", - "text = '''BEG: This is my text\n", + "\n", + "text = \"\"\"BEG: This is my text\n", "that goes on for multiple lines\n", " and this one started with 3 spaces\n", " and this one with 4 tabs\n", "back to first column.:END\n", - "'''\n", + "\"\"\"\n", "md(f'{text}')\n", - "md(f'
{text}
')" + "md(\n", + " f'
{text}
'\n", + ")" ] }, { "cell_type": "code", - "execution_count": 17, - "id": "055b4b7f-cb51-4e28-99ed-8c0e4c803aab", + "execution_count": null, + "id": "31", "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "
BEG: This is my text\n",
-       "that goes on for multiple lines\n",
-       "   and this one started with 3 spaces\n",
-       "                and this one with 4 tabs\n",
-       "back to first column.:END\n",
-       "
" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/markdown": [ - "
BEG: This is my text\n",
-       "that goes on for multiple lines\n",
-       "   and this one started with 3 spaces\n",
-       "                and this one with 4 tabs\n",
-       "back to first column.:END\n",
-       "
" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "def htmlcode(text, fgcolor=\"black\", bgcolor=\"white\"):\n", " style = f'style=\"color: {fgcolor}; background-color: {bgcolor};\"'\n", - " elem = 'pre' # 'code'\n", + " elem = \"pre\" # 'code'\n", " return f\"<{elem} {style}>{text}\"\n", "\n", - "md(htmlcode(text, bgcolor='LightGray'))\n", - "md(htmlcode(text, bgcolor='tomato'))" + "\n", + "md(htmlcode(text, bgcolor=\"LightGray\"))\n", + "md(htmlcode(text, bgcolor=\"tomato\"))" ] }, { "cell_type": "markdown", - "id": "edd20807-7942-4020-9e58-99d619ff83a1", + "id": "32", "metadata": {}, "source": [ "### Widgets" @@ -3887,25 +477,10 @@ }, { "cell_type": "code", - "execution_count": 18, - "id": "30", + "execution_count": null, + "id": "33", "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "3f0e0b43f94a4b62a78691ba89e7a7eb", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "interactive(children=(Checkbox(value=False, description='show_records'), Output()), _dom_classes=('widget-inte…" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "from ipywidgets import interact, interactive, fixed, interact_manual\n", "import ipywidgets as widgets\n", @@ -3926,7 +501,7 @@ }, { "cell_type": "markdown", - "id": "32", + "id": "34", "metadata": {}, "source": [ "## Overview \n" @@ -3934,27 +509,21 @@ }, { "cell_type": "code", - "execution_count": 19, - "id": "23484f05-16dd-4827-8865-0b1158cd489a", + "execution_count": null, + "id": "35", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "WARNING: failed to automatically determine site\n" - ] - } - ], + "outputs": [], "source": [ "try:\n", " import lsst.ts.logging_and_reporting.version\n", + "\n", " lrversion = lsst.ts.logging_and_reporting.version.__version__\n", "except:\n", " lrversion = \"LIVE\"\n", "\n", "try:\n", " from lsst.summit.utils import ConsDbClient\n", + "\n", " have_consdb = True\n", "except:\n", " have_consdb = False" @@ -3962,28 +531,10 @@ }, { "cell_type": "code", - "execution_count": 20, - "id": "33", + "execution_count": null, + "id": "36", "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "## Project-Wide Night(s) Report \n", - "- Run on logs and databases from **https://summit-lsp.lsst.codes/**\n", - "- Report **2 observing night(s)** with the last reported night starting on **2024-09-06 12:00:00**.\n", - "- min_dayobs='2024-09-05', max_dayobs='2024-09-07'\n", - "- Using ***Prototype* Logging and Reporting** Version: **0.1.dev145+gf2a5765.d20241014**\n", - "- have_consdb = True\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "# Display overview of Report context\n", "md(\n", @@ -3999,7 +550,7 @@ }, { "cell_type": "markdown", - "id": "34", + "id": "37", "metadata": {}, "source": [ "## This report uses the following data sources\n", @@ -4014,7 +565,7 @@ }, { "cell_type": "markdown", - "id": "35", + "id": "38", "metadata": {}, "source": [ "## DDV " @@ -4022,27 +573,14 @@ }, { "cell_type": "code", - "execution_count": 21, - "id": "36", + "execution_count": null, + "id": "39", "metadata": { "jupyter": { "source_hidden": true } }, - "outputs": [ - { - "data": { - "text/markdown": [ - "Access DDV part of RubinTV: https://summit-lsp.lsst.codes/rubintv-dev/ddv/index.html" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "DDV = (\n", " f\"{server}/rubintv-dev/ddv/index.html\"\n", @@ -4054,7 +592,7 @@ }, { "cell_type": "markdown", - "id": "37", + "id": "40", "metadata": {}, "source": [ "## Where was this run?\n", @@ -4069,7 +607,7 @@ }, { "cell_type": "markdown", - "id": "38", + "id": "41", "metadata": {}, "source": [ "## Section overviews moved here" @@ -4077,132 +615,14 @@ }, { "cell_type": "code", - "execution_count": 22, - "id": "39", + "execution_count": null, + "id": "42", "metadata": { "jupyter": { "source_hidden": true } }, - "outputs": [ - { - "data": { - "text/markdown": [ - "### Overview for Service: `nightreport` [2]" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "None\n" - ] - }, - { - "data": { - "text/markdown": [ - "- Endpoint: https://summit-lsp.lsst.codes/nightreport/reports?is_human=either&is_valid=true&order_by=-day_obs&min_day_obs=20240905&max_day_obs=20240907&limit=999" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "None\n", - "- Got 2 records. \n" - ] - }, - { - "data": { - "text/markdown": [ - "### Overview for Service: `exposurelog` [0]" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "None\n" - ] - }, - { - "data": { - "text/markdown": [ - "- Endpoint: https://summit-lsp.lsst.codes/exposurelog/messages?is_human=either&is_valid=true&order_by=-day_obs&min_day_obs=20240905&max_day_obs=20240907&limit=999" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "None\n", - "- Got 0 records. \n" - ] - }, - { - "data": { - "text/markdown": [ - "### Overview for Service: `narrativelog` [97]" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "None\n" - ] - }, - { - "data": { - "text/markdown": [ - "- Endpoint: https://summit-lsp.lsst.codes/narrativelog/messages?is_human=either&is_valid=true&order_by=-date_added&min_date_added=2024-09-05T00%3A00%3A00&max_date_added=2024-09-07T00%3A00%3A00&limit=999" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "None\n", - "- Got 97 records. \n" - ] - } - ], + "outputs": [], "source": [ "# Night Report Overview\n", "allrep.nig_rep.overview()\n", @@ -4214,8 +634,8 @@ }, { "cell_type": "code", - "execution_count": 23, - "id": "40", + "execution_count": null, + "id": "43", "metadata": {}, "outputs": [], "source": [ @@ -4228,7 +648,7 @@ }, { "cell_type": "markdown", - "id": "41", + "id": "44", "metadata": {}, "source": [ "## Finale" @@ -4236,22 +656,14 @@ }, { "cell_type": "code", - "execution_count": 24, - "id": "42", + "execution_count": null, + "id": "45", "metadata": { "jupyter": { "source_hidden": true } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Finished 2024-10-22 23:35:50.963185\n" - ] - } - ], + "outputs": [], "source": [ "print(f\"Finished {str(dt.datetime.now())}\")" ] diff --git a/python/lsst/ts/logging_and_reporting/parse_message.py b/python/lsst/ts/logging_and_reporting/parse_message.py index f81b69d..112a510 100644 --- a/python/lsst/ts/logging_and_reporting/parse_message.py +++ b/python/lsst/ts/logging_and_reporting/parse_message.py @@ -1,3 +1,4 @@ +import html import re import lsst.ts.logging_and_reporting.reports as rep @@ -28,16 +29,18 @@ def ignore_this(): # TODO remove # BLACK work-around flags = re.DOTALL | re.ASCII | re.MULTILINE | re.IGNORECASE re_err = re.compile(r"(?P\S*error:.*)", flags=flags) +re_tb = re.compile(r"\b(Traceback \(most recent call last\):.*)", flags=flags) def highlight_code(matchobj): - text = matchobj.group("err") - return rep.htmlcode(text, bgcolor="tomato", size="0.875em") + text = html.escape(matchobj.group(1)) + # LightCoral + return rep.htmlcode(text, bgcolor="#FFDDDD", size="0.875em") -def markup_error(records, src_field="message_text"): - """SIDE-EFFECTS: add DEST_FIELD to all records. - The DEST_FIELD will contain modified text from SRC_FIELD (or None). +def markup_errors(records, src_field="message_text"): + """SIDE-EFFECTS: add DEST_FIELD to all records that are marked up. + The DEST_FIELD will contain modified text from SRC_FIELD. """ dest_field = "error_message" @@ -45,9 +48,11 @@ def markup_error(records, src_field="message_text"): orig = r.get(src_field) # If the RE is found, markup the whole string. # TODO remove - # # if re_err.search(orig): - # # r[dest_field] = rep.htmlcode(orig, bgcolor='tomato') + # #! if re_err.search(orig): + # #! r[dest_field] = rep.htmlcode(orig, bgcolor='tomato') - new = re_err.sub(highlight_code, orig) + # #!re.search('|'.join([re_err, re_tb]) + # #!new = re_err.sub(highlight_code, orig) + new = re_tb.sub(highlight_code, orig) if orig != new: r[dest_field] = new diff --git a/python/lsst/ts/logging_and_reporting/reports.py b/python/lsst/ts/logging_and_reporting/reports.py index 3ae525b..83f82b8 100644 --- a/python/lsst/ts/logging_and_reporting/reports.py +++ b/python/lsst/ts/logging_and_reporting/reports.py @@ -65,9 +65,11 @@ def htmlcode(text, fgcolor="black", bgcolor="LightGray", size="1em"): style = "" style += f"color: {fgcolor}; " style += f"background-color: {bgcolor}; " - style += f"font-size: {size}" + style += f"font-size: {size}; " + style += "margin-left: 40px; " - elem = "pre" # code + # code, samp, pre + elem = "pre" return f"<{elem} {style=}>{text}" diff --git a/python/lsst/ts/logging_and_reporting/source_adapters.py b/python/lsst/ts/logging_and_reporting/source_adapters.py index cd52f95..812904d 100644 --- a/python/lsst/ts/logging_and_reporting/source_adapters.py +++ b/python/lsst/ts/logging_and_reporting/source_adapters.py @@ -540,7 +540,7 @@ def get_records( error = str(err) self.keep_fields(recs, self.outfields) - pam.markup_error(recs) + pam.markup_errors(recs) self.records = recs status = dict( endpoint_url=url,