diff --git a/src/panoptes/pocs/core.py b/src/panoptes/pocs/core.py index 8826a6e11..76b3dca95 100644 --- a/src/panoptes/pocs/core.py +++ b/src/panoptes/pocs/core.py @@ -1,13 +1,17 @@ import os from contextlib import suppress +from multiprocessing import Process +from typing import Optional from astropy import units as u from panoptes.pocs.base import PanBase from panoptes.pocs.observatory import Observatory +from panoptes.pocs.scheduler.observation.base import Observation from panoptes.pocs.state.machine import PanStateMachine from panoptes.utils.time import current_time from panoptes.utils.utils import get_free_space from panoptes.utils.time import CountdownTimer +from panoptes.pocs.utils import error class POCS(PanStateMachine, PanBase): @@ -257,6 +261,49 @@ def reset_observing_run(self): self.logger.debug("Resetting observing run attempts") self._obs_run_retries = self.get_config('pocs.RETRY_ATTEMPTS', default=3) + def observe_target(self, + observation: Optional[Observation] = None, + park_if_unsafe: bool = True): + """Observe something! 🔭🌠 + + Note: This is a long-running blocking method. + + This is a high-level method to call the various `observation` methods that + allow for observing. + """ + current_observation = observation or self.observatory.current_observation + self.say(f"Observing {current_observation}") + + for pic_num in range(current_observation.min_nexp): + self.logger.debug(f"Starting observation {pic_num} of {current_observation.min_nexp}") + if self.is_safe() is False: + self.say(f'Safety warning! Stopping {current_observation}.') + if park_if_unsafe: + self.say('Parking the mount!') + self.observatory.mount.park() + break + + if not self.observatory.mount.is_tracking: + self.say(f'Mount is not tracking, stopping observations.') + break + + # Do the observing, once per exptime (usually only one unless a compound observation). + for exptime in current_observation.exptimes: + self.logger.info(f'Starting {pic_num:03d} of {current_observation.min_nexp:03d} ' + f'with {exptime=}') + try: + self.observatory.take_observation(blocking=True) + except error.CameraNotFound: + self.logger.error('No cameras available, stopping observation') + break + + # Do processing in background. + process_proc = Process(target=self.observatory.process_observation) + process_proc.start() + self.logger.debug(f'Processing {current_observation} on {process_proc.pid=}') + + pic_num += 1 + ################################################################################################ # Safety Methods ################################################################################################ diff --git a/src/panoptes/pocs/observatory.py b/src/panoptes/pocs/observatory.py index 24ed1d202..e63ea620d 100644 --- a/src/panoptes/pocs/observatory.py +++ b/src/panoptes/pocs/observatory.py @@ -1,5 +1,6 @@ import os from collections import OrderedDict +from contextlib import suppress from datetime import datetime from multiprocessing import Process from pathlib import Path @@ -366,7 +367,7 @@ def get_observation(self, *args, **kwargs): return self.current_observation - def observe(self, blocking: bool = True): + def take_observation(self, blocking: bool = True): """Take individual images for the current observation. This method gets the current observation and takes the next @@ -377,6 +378,9 @@ def observe(self, blocking: bool = True): exposing before returning, otherwise return immediately. """ + if len(self.cameras) == 0: + raise error.CameraNotFound("No cameras available, unable to take observation") + # Get observatory metadata headers = self.get_standard_headers() @@ -406,8 +410,14 @@ def observe(self, blocking: bool = True): timer.sleep(max_sleep=readout_time) + # If timer expired check cameras and remove if stuck. if timer.expired(): - raise TimeoutError(f'Timer expired waiting for cameras to finish observing') + self.logger.warning(f'Timer expired waiting for cameras to finish observing') + not_done = [cam_id for cam_id, cam in self.cameras.items() if cam.is_observing] + for cam_id in not_done: + self.logger.warning(f'Removing {cam_id} from observatory') + with suppress(KeyError): + del self.cameras[cam_id] def process_observation(self, compress_fits: Optional[bool] = None, @@ -424,7 +434,7 @@ def process_observation(self, record_observations (bool or None): If observation metadata should be saved. If None (default), checks the `observations.record_observations` config-server key. - make_pretty_images (bool or None): If should make a jpg from raw image. + make_pretty_images (bool or None): Make a jpg from raw image. If None (default), checks the `observations.make_pretty_images` config-server key. plate_solve (bool or None): If images should be plate solved, default None for config. @@ -432,16 +442,22 @@ def process_observation(self, process). """ for cam_name in self.cameras.keys(): - exposure = self.current_observation.exposure_list[cam_name][-1] - self.logger.debug(f'Processing observation with {exposure=!r}') - metadata = exposure.metadata try: + exposure = self.current_observation.exposure_list[cam_name][-1] + except IndexError: + self.logger.warning(f'Unable to get exposure for {cam_name}') + continue + + try: + self.logger.debug(f'Processing observation with {exposure=!r}') + metadata = exposure.metadata image_id = metadata['image_id'] seq_id = metadata['sequence_id'] file_path = metadata['filepath'] exptime = metadata['exptime'] except KeyError as e: - raise error.PanError(f'No information in image metadata, unable to process: {e!r}') + self.logger.warning(f'No information in image metadata, unable to process: {e!r}') + continue field_name = metadata.get('field_name', '') diff --git a/src/panoptes/pocs/state/states/default/observing.py b/src/panoptes/pocs/state/states/default/observing.py index 9cb19cdf8..e731431f0 100644 --- a/src/panoptes/pocs/state/states/default/observing.py +++ b/src/panoptes/pocs/state/states/default/observing.py @@ -1,5 +1,3 @@ -from multiprocessing import Process - from panoptes.utils import error @@ -14,20 +12,12 @@ def on_enter(event_data): pocs.next_state = 'parking' try: - # Do the observing, once per exptime (usually only one unless a compound observation). - for _ in current_obs.exptimes: - pocs.observatory.observe(blocking=True) - pocs.say(f"Finished observing! I'll start processing that in the background.") - - # Do processing in background. - process_proc = Process(target=pocs.observatory.process_observation) - process_proc.start() - pocs.logger.debug(f'Processing for {current_obs} started on {process_proc.pid=}') + pocs.observe_target() except (error.Timeout, error.CameraNotFound): pocs.logger.warning("Timeout waiting for images. Something wrong with cameras, parking.") except Exception as e: pocs.logger.warning(f"Problem with imaging: {e!r}") pocs.say("Hmm, I'm not sure what happened with that exposure.") else: - pocs.logger.debug('Finished with observing, going to analyze') pocs.next_state = 'analyzing' + pocs.logger.debug('Finished with observing, going to {pocs.next_state}') diff --git a/tests/test_observatory.py b/tests/test_observatory.py index 637cda8f7..73bfb7e0d 100644 --- a/tests/test_observatory.py +++ b/tests/test_observatory.py @@ -315,7 +315,7 @@ def test_observe(observatory): assert len(observatory.scheduler.observed_list) == 1 assert observatory.current_observation.current_exp_num == 0 - observatory.observe() + observatory.take_observation() assert observatory.current_observation.current_exp_num == 1