Update observing methods (#1175)

* Update observing methods

* Changes `observatory.observe` to `observatory.take_observation` to have more descriptive name and not interfere with potential state action names.
* `take_observation` will now remove cameras if they are stuck and try to continue observing until no cameras are left.
* Created high-level `pocs.observe_target`, which loops over the exptimes in the observation and sets up processing threads when done with each exposure. This replaces what was in the `observing` state and moves it to a `pocs` method. This allows the mount to stay on a target for the length of the observation without moving between states during each exposures, which helps cadence.

* Remove python 3.10 specific syntax.

src/panoptes/pocs/core.py

@@ -1,13 +1,17 @@
 import os
 from contextlib import suppress
+from multiprocessing import Process
+from typing import Optional
 
 from astropy import units as u
 from panoptes.pocs.base import PanBase
 from panoptes.pocs.observatory import Observatory
+from panoptes.pocs.scheduler.observation.base import Observation
 from panoptes.pocs.state.machine import PanStateMachine
 from panoptes.utils.time import current_time
 from panoptes.utils.utils import get_free_space
 from panoptes.utils.time import CountdownTimer
+from panoptes.pocs.utils import error
 
 
 class POCS(PanStateMachine, PanBase):
@@ -257,6 +261,49 @@ def reset_observing_run(self):
         self.logger.debug("Resetting observing run attempts")
         self._obs_run_retries = self.get_config('pocs.RETRY_ATTEMPTS', default=3)
 
+    def observe_target(self,
+                       observation: Optional[Observation] = None,
+                       park_if_unsafe: bool = True):
+        """Observe something! 🔭🌠
+
+        Note: This is a long-running blocking method.
+
+        This is a high-level method to call the various `observation` methods that
+        allow for observing.
+        """
+        current_observation = observation or self.observatory.current_observation
+        self.say(f"Observing {current_observation}")
+
+        for pic_num in range(current_observation.min_nexp):
+            self.logger.debug(f"Starting observation {pic_num} of {current_observation.min_nexp}")
+            if self.is_safe() is False:
+                self.say(f'Safety warning! Stopping {current_observation}.')
+                if park_if_unsafe:
+                    self.say('Parking the mount!')
+                    self.observatory.mount.park()
+                break
+
+            if not self.observatory.mount.is_tracking:
+                self.say(f'Mount is not tracking, stopping observations.')
+                break
+
+            # Do the observing, once per exptime (usually only one unless a compound observation).
+            for exptime in current_observation.exptimes:
+                self.logger.info(f'Starting {pic_num:03d} of {current_observation.min_nexp:03d} '
+                                 f'with {exptime=}')
+                try:
+                    self.observatory.take_observation(blocking=True)
+                except error.CameraNotFound:
+                    self.logger.error('No cameras available, stopping observation')
+                    break
+
+                # Do processing in background.
+                process_proc = Process(target=self.observatory.process_observation)
+                process_proc.start()
+                self.logger.debug(f'Processing {current_observation} on {process_proc.pid=}')
+
+            pic_num += 1
+
     ################################################################################################
     # Safety Methods
     ################################################################################################

src/panoptes/pocs/observatory.py

@@ -1,5 +1,6 @@
 import os
 from collections import OrderedDict
+from contextlib import suppress
 from datetime import datetime
 from multiprocessing import Process
 from pathlib import Path
@@ -366,7 +367,7 @@ def get_observation(self, *args, **kwargs):
 
         return self.current_observation
 
-    def observe(self, blocking: bool = True):
+    def take_observation(self, blocking: bool = True):
         """Take individual images for the current observation.
 
         This method gets the current observation and takes the next
@@ -377,6 +378,9 @@ def observe(self, blocking: bool = True):
                 exposing before returning, otherwise return immediately.
 
         """
+        if len(self.cameras) == 0:
+            raise error.CameraNotFound("No cameras available, unable to take observation")
+
         # Get observatory metadata
         headers = self.get_standard_headers()
 
@@ -406,8 +410,14 @@ def observe(self, blocking: bool = True):
 
                 timer.sleep(max_sleep=readout_time)
 
+            # If timer expired check cameras and remove if stuck.
             if timer.expired():
-                raise TimeoutError(f'Timer expired waiting for cameras to finish observing')
+                self.logger.warning(f'Timer expired waiting for cameras to finish observing')
+                not_done = [cam_id for cam_id, cam in self.cameras.items() if cam.is_observing]
+                for cam_id in not_done:
+                    self.logger.warning(f'Removing {cam_id} from observatory')
+                    with suppress(KeyError):
+                        del self.cameras[cam_id]
 
     def process_observation(self,
                             compress_fits: Optional[bool] = None,
@@ -424,24 +434,30 @@ def process_observation(self,
             record_observations (bool or None): If observation metadata should be saved.
                 If None (default), checks the `observations.record_observations`
                 config-server key.
-            make_pretty_images (bool or None): If should make a jpg from raw image.
+            make_pretty_images (bool or None): Make a jpg from raw image.
                 If None (default), checks the `observations.make_pretty_images`
                 config-server key.
             plate_solve (bool or None): If images should be plate solved, default None for config.
             upload_image_immediately (bool or None): If images should be uploaded (in a separate
                 process).
         """
         for cam_name in self.cameras.keys():
-            exposure = self.current_observation.exposure_list[cam_name][-1]
-            self.logger.debug(f'Processing observation with {exposure=!r}')
-            metadata = exposure.metadata
             try:
+                exposure = self.current_observation.exposure_list[cam_name][-1]
+            except IndexError:
+                self.logger.warning(f'Unable to get exposure for {cam_name}')
+                continue
+
+            try:
+                self.logger.debug(f'Processing observation with {exposure=!r}')
+                metadata = exposure.metadata
                 image_id = metadata['image_id']
                 seq_id = metadata['sequence_id']
                 file_path = metadata['filepath']
                 exptime = metadata['exptime']
             except KeyError as e:
-                raise error.PanError(f'No information in image metadata, unable to process:  {e!r}')
+                self.logger.warning(f'No information in image metadata, unable to process:  {e!r}')
+                continue
 
             field_name = metadata.get('field_name', '')
 

src/panoptes/pocs/state/states/default/observing.py

@@ -1,5 +1,3 @@
-from multiprocessing import Process
-
 from panoptes.utils import error
 
 
@@ -14,20 +12,12 @@ def on_enter(event_data):
     pocs.next_state = 'parking'
 
     try:
-        # Do the observing, once per exptime (usually only one unless a compound observation).
-        for _ in current_obs.exptimes:
-            pocs.observatory.observe(blocking=True)
-            pocs.say(f"Finished observing! I'll start processing that in the background.")
-
-            # Do processing in background.
-            process_proc = Process(target=pocs.observatory.process_observation)
-            process_proc.start()
-            pocs.logger.debug(f'Processing for {current_obs} started on {process_proc.pid=}')
+        pocs.observe_target()
     except (error.Timeout, error.CameraNotFound):
         pocs.logger.warning("Timeout waiting for images. Something wrong with cameras, parking.")
     except Exception as e:
         pocs.logger.warning(f"Problem with imaging: {e!r}")
         pocs.say("Hmm, I'm not sure what happened with that exposure.")
     else:
-        pocs.logger.debug('Finished with observing, going to analyze')
         pocs.next_state = 'analyzing'
+        pocs.logger.debug('Finished with observing, going to {pocs.next_state}')

tests/test_observatory.py

@@ -315,7 +315,7 @@ def test_observe(observatory):
     assert len(observatory.scheduler.observed_list) == 1
 
     assert observatory.current_observation.current_exp_num == 0
-    observatory.observe()
+    observatory.take_observation()
     assert observatory.current_observation.current_exp_num == 1