diff --git a/README.md b/README.md index 9358c234..29b934e7 100644 --- a/README.md +++ b/README.md @@ -271,6 +271,7 @@ To make deployment with docker easier, most of the important configuration optio - FLATHUNTER_GOOGLE_CLOUD_PROJECT_ID - the Google Cloud Project ID, for Google Cloud deployments - FLATHUNTER_VERBOSE_LOG - set to any value to enable verbose logging - FLATHUNTER_LOOP_PERIOD_SECONDS - a number in seconds for the crawling interval + - FLATHUNTER_LOOP_REFRESH_CONFIG - set to any value to enable live editing config file (refresh on each loop) - FLATHUNTER_MESSAGE_FORMAT - a format string for the notification messages, where `#CR#` will be replaced by newline - FLATHUNTER_NOTIFIERS - a comma-separated list of notifiers to enable (e.g. `telegram,mattermost,slack`) - FLATHUNTER_TELEGRAM_BOT_TOKEN - the token for the Telegram notifier diff --git a/config.yaml.dist b/config.yaml.dist index 655b90f3..5da66256 100644 --- a/config.yaml.dist +++ b/config.yaml.dist @@ -10,6 +10,7 @@ loop: active: yes sleeping_time: 600 + refresh_config: no # Location of the Database to store already seen offerings # Defaults to the current directory diff --git a/flathunt.py b/flathunt.py index f5e9d112..06683810 100644 --- a/flathunt.py +++ b/flathunt.py @@ -22,6 +22,44 @@ __status__ = "Production" +def check_config(config): + logger.info("Checking config for errors") + # check config + notifiers = config.notifiers() + if 'mattermost' in notifiers \ + and not config.mattermost_webhook_url(): + logger.error("No Mattermost webhook configured. Starting like this would be pointless...") + return + if 'telegram' in notifiers: + if not config.telegram_bot_token(): + logger.error( + "No Telegram bot token configured. Starting like this would be pointless..." + ) + return + if len(config.telegram_receiver_ids()) == 0: + logger.warning("No Telegram receivers configured - nobody will get notifications.") + if 'apprise' in notifiers \ + and not config.get('apprise', {}): + logger.error("No apprise url configured. Starting like this would be pointless...") + return + if 'slack' in notifiers \ + and not config.slack_webhook_url(): + logger.error("No Slack webhook url configured. Starting like this would be pointless...") + return + + if len(config.target_urls()) == 0: + logger.error("No URLs configured. Starting like this would be pointless...") + return + + return True + + +def get_heartbeat_instructions(args, config): + # get heartbeat instructions + heartbeat_interval = args.heartbeat + heartbeat = Heartbeat(config, heartbeat_interval) + return heartbeat + def launch_flat_hunt(config, heartbeat: Heartbeat): """Starts the crawler / notification loop""" id_watch = IdMaintainer(f'{config.database_location()}/processed_ids.db') @@ -41,6 +79,36 @@ def launch_flat_hunt(config, heartbeat: Heartbeat): counter += 1 counter = heartbeat.send_heartbeat(counter) time.sleep(config.loop_period_seconds()) + + if config.loop_refresh_config(): + args = parse() + config_handle = args.config + if config_handle is not None: + new_config = Config(filename=config_handle.name, silent=True) + + if config.last_modified_time is None or new_config.last_modified_time is None: + logger.warning("Could not compare last modification time of config file." + "Keeping the old configuration") + elif config.last_modified_time < new_config.last_modified_time: + if not check_config(new_config): + logger.warning("Config changed but new config had errors. Keeping old config") + else: + config = new_config + # setup logging + configure_logging(config) + + # initialize search plugins for config + config.init_searchers() + + id_watch = IdMaintainer(f'{new_config.database_location()}/processed_ids.db') + + time_from = dtime.fromisoformat(new_config.loop_pause_from()) + time_till = dtime.fromisoformat(new_config.loop_pause_till()) + + wait_during_period(time_from, time_till) + + hunter = Hunter(new_config, id_watch) + hunter.hunt_flats() @@ -60,31 +128,7 @@ def main(): # initialize search plugins for config config.init_searchers() - # check config - notifiers = config.notifiers() - if 'mattermost' in notifiers \ - and not config.mattermost_webhook_url(): - logger.error("No Mattermost webhook configured. Starting like this would be pointless...") - return - if 'telegram' in notifiers: - if not config.telegram_bot_token(): - logger.error( - "No Telegram bot token configured. Starting like this would be pointless..." - ) - return - if len(config.telegram_receiver_ids()) == 0: - logger.warning("No Telegram receivers configured - nobody will get notifications.") - if 'apprise' in notifiers \ - and not config.get('apprise', {}): - logger.error("No apprise url configured. Starting like this would be pointless...") - return - if 'slack' in notifiers \ - and not config.slack_webhook_url(): - logger.error("No Slack webhook url configured. Starting like this would be pointless...") - return - - if len(config.target_urls()) == 0: - logger.error("No URLs configured. Starting like this would be pointless...") + if not check_config(config): return # get heartbeat instructions diff --git a/flathunter/config.py b/flathunter/config.py index 9c49ce4c..ded1d595 100644 --- a/flathunter/config.py +++ b/flathunter/config.py @@ -46,6 +46,8 @@ class Env: FLATHUNTER_VERBOSE_LOG = _read_env("FLATHUNTER_VERBOSE_LOG") FLATHUNTER_LOOP_PERIOD_SECONDS = _read_env( "FLATHUNTER_LOOP_PERIOD_SECONDS") + FLATHUNTER_LOOP_REFRESH_CONFIG = _read_env( + "FLATHUNTER_LOOP_REFRESH_CONFIG") FLATHUNTER_LOOP_PAUSE_FROM = _read_env("FLATHUNTER_LOOP_PAUSE_FROM") FLATHUNTER_LOOP_PAUSE_TILL = _read_env("FLATHUNTER_LOOP_PAUSE_TILL") FLATHUNTER_MESSAGE_FORMAT = _read_env("FLATHUNTER_MESSAGE_FORMAT") @@ -106,6 +108,12 @@ def __init__(self, config=None): self.config = config self.__searchers__ = [] self.check_deprecated() + if filename := self.config.get("filename"): + self.last_modified_time: Optional[float] = self.get_last_modified_time(filename) + else: + self.last_modified_time: Optional[float] = None + + def __iter__(self): """Emulate dictionary""" @@ -129,6 +137,14 @@ def init_searchers(self): VrmImmo(self) ] + def get_last_modified_time(self, filename: str) -> Optional[float]: + """Gets the time the config file was last modified at the time of initialization""" + try: + return os.path.getmtime(filename) + except OSError as e: + logger.error(e) + return None + def check_deprecated(self): """Notifies user of deprecated config items""" captcha_config = self.config.get("captcha") @@ -209,6 +225,10 @@ def loop_is_active(self): """Return true if flathunter should be crawling in a loop""" return self._read_yaml_path('loop.active', False) + def loop_refresh_config(self): + """Return true if flathunter should refresh the config for every loop""" + return self._read_yaml_path('loop.refresh_config', False) + def loop_period_seconds(self): """Number of seconds to wait between crawls when looping""" return self._read_yaml_path('loop.sleeping_time', 60 * 10) @@ -404,16 +424,18 @@ class Config(CaptchaEnvironmentConfig): # pylint: disable=too-many-public-metho environment variable overrides """ - def __init__(self, filename=None): + def __init__(self, filename=None, silent=False): if filename is None and Env.FLATHUNTER_TARGET_URLS is None: raise ConfigException( "Config file loaction must be specified, or FLATHUNTER_TARGET_URLS must be set") if filename is not None: - logger.info("Using config path %s", filename) + if not silent: + logger.info("Using config path %s", filename) if not os.path.exists(filename): raise ConfigException("No config file found at location %s") with open(filename, encoding="utf-8") as file: config = yaml.safe_load(file) + config["filename"] = filename else: config = {} super().__init__(config) @@ -444,6 +466,11 @@ def loop_period_seconds(self): return int(Env.FLATHUNTER_LOOP_PERIOD_SECONDS) return super().loop_period_seconds() + def loop_refresh_config(self): + if Env.FLATHUNTER_LOOP_REFRESH_CONFIG is not None: + return True + return super().loop_refresh_config() + def loop_pause_from(self): if Env.FLATHUNTER_LOOP_PAUSE_FROM is not None: return str(Env.FLATHUNTER_LOOP_PAUSE_FROM)