From 8564853221317e3dc6cb6b099cc149ea634b7c6c Mon Sep 17 00:00:00 2001 From: James Hewitt Date: Wed, 29 May 2024 23:25:33 +0100 Subject: [PATCH] Allow reporters to be specified multiple times This allows different jobs to be selected for reporting in different ways, for example, allowing different changes to be emailed to different addresses. Closes #790 Signed-off-by: James Hewitt --- CHANGELOG.md | 1 + lib/urlwatch/command.py | 5 +++- lib/urlwatch/handler.py | 5 ++-- lib/urlwatch/jobs.py | 4 +-- lib/urlwatch/main.py | 10 +++++++- lib/urlwatch/reporters.py | 53 ++++++++++++++++++++++----------------- 6 files changed, 49 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index faa9703a..ad7d7390 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format mostly follows [Keep a Changelog](http://keepachangelog.com/en/1.0.0/ - New option `ignore_incomplete_reads` (Requested in #725 by wschoot, contributed in #787 by wfrisch) - New option `wait_for` in browser jobs (Requested in #763 by yuis-ice, contributed in #810 by jamstah) - Added tags to jobs and the ability to select them at the command line (#789 by jamstah) +- Allow reporters to be specified multiple times (#822 by jamstah) ### Changed diff --git a/lib/urlwatch/command.py b/lib/urlwatch/command.py index 01f09a97..e9ea3198 100644 --- a/lib/urlwatch/command.py +++ b/lib/urlwatch/command.py @@ -360,7 +360,10 @@ def set_error(job_state, message): 'Same Old, Same Old\n')) report.error(set_error(build_job('Error Reporting', 'http://example.com/error', '', ''), 'Oh Noes!')) - report.finish_one(name) + reported = report.finish_one(name) + + if not reported: + raise ValueError(f'Reporter not enabled: {name}') sys.exit(0) diff --git a/lib/urlwatch/handler.py b/lib/urlwatch/handler.py index f58323f3..1e1ea0fe 100644 --- a/lib/urlwatch/handler.py +++ b/lib/urlwatch/handler.py @@ -57,6 +57,7 @@ def __init__(self, cache_storage, job): self.timestamp = None self.current_timestamp = None self.exception = None + self.reported_count = 0 self.traceback = None self.tries = 0 self.etag = None @@ -214,10 +215,10 @@ def finish(self): end = datetime.datetime.now() duration = (end - self.start) - ReporterBase.submit_all(self, self.job_states, duration) + return ReporterBase.submit_all(self, self.job_states, duration) def finish_one(self, name): end = datetime.datetime.now() duration = (end - self.start) - ReporterBase.submit_one(name, self, self.job_states, duration) + return ReporterBase.submit_one(name, self, self.job_states, duration) diff --git a/lib/urlwatch/jobs.py b/lib/urlwatch/jobs.py index 1262443c..d6748666 100644 --- a/lib/urlwatch/jobs.py +++ b/lib/urlwatch/jobs.py @@ -199,8 +199,8 @@ class Job(JobBase): __required__ = () __optional__ = ('name', 'filter', 'max_tries', 'diff_tool', 'compared_versions', 'diff_filter', 'enabled', 'treat_new_as_changed', 'user_visible_url', 'tags') - def matching_tags(self, tags: Set[str]) -> Set[str]: - return self.tags & tags + def matching_tags(self, tags: Iterable[str]) -> Set[str]: + return self.tags.intersection(tags) # determine if hyperlink "a" tag is used in HtmlReporter def location_is_url(self): diff --git a/lib/urlwatch/main.py b/lib/urlwatch/main.py index 2c27c921..4f3b5d5f 100644 --- a/lib/urlwatch/main.py +++ b/lib/urlwatch/main.py @@ -109,5 +109,13 @@ def run_jobs(self): run_jobs(self) def close(self): - self.report.finish() + reported = self.report.finish() + + if not reported: + logger.warning('No reporters enabled.') + + for job_state in self.report.job_states: + if not job_state.reported_count: + logger.warning(f'Job {job_state.job.pretty_name()} was not reported on') + self.cache_storage.close() diff --git a/lib/urlwatch/reporters.py b/lib/urlwatch/reporters.py index 2fc67a33..b8c5d2cb 100644 --- a/lib/urlwatch/reporters.py +++ b/lib/urlwatch/reporters.py @@ -28,6 +28,7 @@ import asyncio +from collections.abc import Mapping import difflib import re import email.utils @@ -79,14 +80,17 @@ WDIFF_ADDED_RE = r'[{][+].*?[+][}]' WDIFF_REMOVED_RE = r'[\[][-].*?[-][]]' +def filter_by_tags(job_states, tags): + return job_states if not tags else [job_state for job_state in job_states if job_state.job.matching_tags(tags)] class ReporterBase(object, metaclass=TrackSubClasses): __subclasses__ = {} - def __init__(self, report, config, job_states, duration): + def __init__(self, report, config, job_states, job_count_total, duration): self.report = report self.config = config self.job_states = job_states + self.job_count_total = job_count_total self.duration = duration def get_signature(self): @@ -96,8 +100,10 @@ def get_signature(self): copyright=urlwatch.__copyright__), 'Website: {url}'.format(url=urlwatch.__url__), 'Support urlwatch development: https://github.com/sponsors/thp', - 'watched {count} URLs in {duration} seconds'.format(count=len(self.job_states), - duration=self.duration.seconds), + 'reported {count} of {total} watched URLs in {duration} seconds'.format( + count=len(self.job_states), + total=self.job_count_total, + duration=self.duration.seconds), ) def convert(self, othercls): @@ -123,35 +129,36 @@ def reporter_documentation(cls): @classmethod def submit_one(cls, name, report, job_states, duration): + any_enabled = False subclass = cls.__subclasses__[name] - cfg = report.config['report'].get(name, {'enabled': False}) - if cfg['enabled']: - base_config = subclass.get_base_config(report) - if base_config.get('separate', False): - for job_state in job_states: - subclass(report, cfg, [job_state], duration).submit() - else: - subclass(report, cfg, job_states, duration).submit() - else: - raise ValueError('Reporter not enabled: {name}'.format(name=name)) + cfgs = report.config['report'].get(name, {'enabled': False}) + if isinstance(cfgs, Mapping): + cfgs = [cfgs] - @classmethod - def submit_all(cls, report, job_states, duration): - any_enabled = False - for name, subclass in cls.__subclasses__.items(): - cfg = report.config['report'].get(name, {}) + for cfg in cfgs: if cfg.get('enabled', False): any_enabled = True logger.info('Submitting with %s (%r)', name, subclass) base_config = subclass.get_base_config(report) + matching_job_states = filter_by_tags(job_states, cfg.get("tags", [])) if base_config.get('separate', False): - for job_state in job_states: - subclass(report, cfg, [job_state], duration).submit() + for job_state in matching_job_states: + subclass(report, cfg, [job_state], len(job_states), duration).submit() + job_state.reported_count = job_state.reported_count + 1 else: - subclass(report, cfg, job_states, duration).submit() + subclass(report, cfg, matching_job_states, len(job_states), duration).submit() + for job_state in matching_job_states: + job_state.reported_count = job_state.reported_count + 1 + + return any_enabled + + @classmethod + def submit_all(cls, report, job_states, duration): + any_enabled = False + for name in cls.__subclasses__.keys(): + any_enabled = any_enabled | ReporterBase.submit_one(name, report, job_states, duration) - if not any_enabled: - logger.warning('No reporters enabled.') + return any_enabled def submit(self): raise NotImplementedError()