Allow reporters to be specified multiple times

This allows different jobs to be selected for reporting in different
ways, for example, allowing different changes to be emailed to different
addresses.

Closes #790

Signed-off-by: James Hewitt <[email protected]>

CHANGELOG.md

@@ -12,6 +12,7 @@ The format mostly follows [Keep a Changelog](http://keepachangelog.com/en/1.0.0/
 - New option `ignore_incomplete_reads` (Requested in #725 by wschoot, contributed in #787 by wfrisch)
 - New option `wait_for` in browser jobs (Requested in #763 by yuis-ice, contributed in #810 by jamstah)
 - Added tags to jobs and the ability to select them at the command line (#789 by jamstah)
+- Allow reporters to be specified multiple times (#822 by jamstah)
 
 ### Changed
 

lib/urlwatch/command.py

@@ -360,7 +360,10 @@ def set_error(job_state, message):
                                    'Same Old, Same Old\n'))
         report.error(set_error(build_job('Error Reporting', 'http://example.com/error', '', ''), 'Oh Noes!'))
 
-        report.finish_one(name)
+        reported = report.finish_one(name)
+
+        if not reported:
+            raise ValueError(f'Reporter not enabled: {name}')
 
         sys.exit(0)
 

lib/urlwatch/handler.py

@@ -57,6 +57,7 @@ def __init__(self, cache_storage, job):
         self.timestamp = None
         self.current_timestamp = None
         self.exception = None
+        self.reported_count = 0
         self.traceback = None
         self.tries = 0
         self.etag = None
@@ -214,10 +215,10 @@ def finish(self):
         end = datetime.datetime.now()
         duration = (end - self.start)
 
-        ReporterBase.submit_all(self, self.job_states, duration)
+        return ReporterBase.submit_all(self, self.job_states, duration)
 
     def finish_one(self, name):
         end = datetime.datetime.now()
         duration = (end - self.start)
 
-        ReporterBase.submit_one(name, self, self.job_states, duration)
+        return ReporterBase.submit_one(name, self, self.job_states, duration)

lib/urlwatch/jobs.py

@@ -199,8 +199,8 @@ class Job(JobBase):
     __required__ = ()
     __optional__ = ('name', 'filter', 'max_tries', 'diff_tool', 'compared_versions', 'diff_filter', 'enabled', 'treat_new_as_changed', 'user_visible_url', 'tags')
 
-    def matching_tags(self, tags: Set[str]) -> Set[str]:
-        return self.tags & tags
+    def matching_tags(self, tags: Iterable[str]) -> Set[str]:
+        return self.tags.intersection(tags)
 
     # determine if hyperlink "a" tag is used in HtmlReporter
     def location_is_url(self):

lib/urlwatch/main.py

@@ -109,5 +109,13 @@ def run_jobs(self):
         run_jobs(self)
 
     def close(self):
-        self.report.finish()
+        reported = self.report.finish()
+
+        if not reported:
+            logger.warning('No reporters enabled.')
+
+        for job_state in self.report.job_states:
+            if not job_state.reported_count:
+                logger.warning(f'Job {job_state.job.pretty_name()} was not reported on')
+
         self.cache_storage.close()

lib/urlwatch/reporters.py

@@ -28,6 +28,7 @@
 
 
 import asyncio
+from collections.abc import Mapping
 import difflib
 import re
 import email.utils
@@ -79,14 +80,17 @@
 WDIFF_ADDED_RE = r'[{][+].*?[+][}]'
 WDIFF_REMOVED_RE = r'[\[][-].*?[-][]]'
 
+def filter_by_tags(job_states, tags):
+    return job_states if not tags else [job_state for job_state in job_states if job_state.job.matching_tags(tags)]
 
 class ReporterBase(object, metaclass=TrackSubClasses):
     __subclasses__ = {}
 
-    def __init__(self, report, config, job_states, duration):
+    def __init__(self, report, config, job_states, job_count_total, duration):
         self.report = report
         self.config = config
         self.job_states = job_states
+        self.job_count_total = job_count_total
         self.duration = duration
 
     def get_signature(self):
@@ -96,8 +100,10 @@ def get_signature(self):
                                                       copyright=urlwatch.__copyright__),
             'Website: {url}'.format(url=urlwatch.__url__),
             'Support urlwatch development: https://github.com/sponsors/thp',
-            'watched {count} URLs in {duration} seconds'.format(count=len(self.job_states),
-                                                                duration=self.duration.seconds),
+            'reported {count} of {total} watched URLs in {duration} seconds'.format(
+                count=len(self.job_states),
+                total=self.job_count_total,
+                duration=self.duration.seconds),
         )
 
     def convert(self, othercls):
@@ -123,35 +129,36 @@ def reporter_documentation(cls):
 
     @classmethod
     def submit_one(cls, name, report, job_states, duration):
+        any_enabled = False
         subclass = cls.__subclasses__[name]
-        cfg = report.config['report'].get(name, {'enabled': False})
-        if cfg['enabled']:
-            base_config = subclass.get_base_config(report)
-            if base_config.get('separate', False):
-                for job_state in job_states:
-                    subclass(report, cfg, [job_state], duration).submit()
-            else:
-                subclass(report, cfg, job_states, duration).submit()
-        else:
-            raise ValueError('Reporter not enabled: {name}'.format(name=name))
+        cfgs = report.config['report'].get(name, {'enabled': False})
+        if isinstance(cfgs, Mapping):
+                cfgs = [cfgs]
 
-    @classmethod
-    def submit_all(cls, report, job_states, duration):
-        any_enabled = False
-        for name, subclass in cls.__subclasses__.items():
-            cfg = report.config['report'].get(name, {})
+        for cfg in cfgs:
             if cfg.get('enabled', False):
                 any_enabled = True
                 logger.info('Submitting with %s (%r)', name, subclass)
                 base_config = subclass.get_base_config(report)
+                matching_job_states = filter_by_tags(job_states, cfg.get("tags", []))
                 if base_config.get('separate', False):
-                    for job_state in job_states:
-                        subclass(report, cfg, [job_state], duration).submit()
+                    for job_state in matching_job_states:
+                        subclass(report, cfg, [job_state], len(job_states), duration).submit()
+                        job_state.reported_count = job_state.reported_count + 1
                 else:
-                    subclass(report, cfg, job_states, duration).submit()
+                    subclass(report, cfg, matching_job_states, len(job_states), duration).submit()
+                    for job_state in matching_job_states:
+                        job_state.reported_count = job_state.reported_count + 1
+
+        return any_enabled
+
+    @classmethod
+    def submit_all(cls, report, job_states, duration):
+        any_enabled = False
+        for name in cls.__subclasses__.keys():
+            any_enabled = any_enabled | ReporterBase.submit_one(name, report, job_states, duration)
 
-        if not any_enabled:
-            logger.warning('No reporters enabled.')
+        return any_enabled
 
     def submit(self):
         raise NotImplementedError()