From 432ee1236df3e17f42897f39aa2bcf1ec9d98c90 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 15 Jan 2024 23:34:53 +0100 Subject: [PATCH 01/12] WIP --- changedetectionio/content_fetcher.py | 1 + changedetectionio/flask_app.py | 18 +++++++- changedetectionio/forms.py | 4 +- changedetectionio/plugins/__init__.py | 6 +++ changedetectionio/plugins/default.py | 44 +++++++++++++++++++ changedetectionio/plugins/hookspecs.py | 20 +++++++++ changedetectionio/processors/__init__.py | 22 +++++++++- .../processors/text_json_diff.py | 2 +- changedetectionio/templates/edit.html | 29 ++++++------ changedetectionio/update_worker.py | 7 +++ 10 files changed, 131 insertions(+), 22 deletions(-) create mode 100644 changedetectionio/plugins/__init__.py create mode 100644 changedetectionio/plugins/default.py create mode 100644 changedetectionio/plugins/hookspecs.py diff --git a/changedetectionio/content_fetcher.py b/changedetectionio/content_fetcher.py index 4c8a382c8bb..a7e82314def 100644 --- a/changedetectionio/content_fetcher.py +++ b/changedetectionio/content_fetcher.py @@ -101,6 +101,7 @@ class Fetcher(): error = None fetcher_description = "No description" headers = {} + is_plaintext = None instock_data = None instock_data_js = "" status_code = None diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index f077e688562..469320ff4aa 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -49,6 +49,19 @@ update_q = queue.PriorityQueue() notification_q = queue.Queue() +from .plugins import hookspecs +from .plugins import default as default_plugin + + + +def get_plugin_manager(): + import pluggy + pm = pluggy.PluginManager("eggsample") + pm.add_hookspecs(hookspecs) + pm.load_setuptools_entrypoints("eggsample") + pm.register(default_plugin) + return pm + app = Flask(__name__, static_url_path="", static_folder="static", @@ -95,7 +108,6 @@ def init_app_secret(datastore_path): return secret - @app.template_global() def get_darkmode_state(): css_dark_mode = request.cookies.get('css_dark_mode', 'false') @@ -626,7 +638,6 @@ def edit_page(uuid): form.fetch_backend.choices.append(p) form.fetch_backend.choices.append(("system", 'System settings default')) - # form.browser_steps[0] can be assumed that we 'goto url' first if datastore.proxy_list is None: @@ -727,6 +738,8 @@ def edit_page(uuid): if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'): is_html_webdriver = True + processor_config = next((p[2] for p in processors.available_processors() if p[0] == watch.get('processor')), None) + # Only works reliably with Playwright visualselector_enabled = os.getenv('PLAYWRIGHT_DRIVER_URL', False) and is_html_webdriver output = render_template("edit.html", @@ -741,6 +754,7 @@ def edit_page(uuid): is_html_webdriver=is_html_webdriver, jq_support=jq_support, playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False), + processor_config=processor_config, settings_application=datastore.data['settings']['application'], using_global_webdriver_wait=default['webdriver_delay'] is None, uuid=uuid, diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 9f72a748cd0..7653ad2483a 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -410,7 +410,7 @@ class quickWatchForm(Form): url = fields.URLField('URL', validators=[validateURL()]) tags = StringTagUUID('Group tag', [validators.Optional()]) watch_submit_button = SubmitField('Watch', render_kw={"class": "pure-button pure-button-primary"}) - processor = RadioField(u'Processor', choices=processors.available_processors(), default="text_json_diff") + processor = RadioField(u'Processor', choices=[t[:2] for t in processors.available_processors()], default="text_json_diff") edit_and_watch_submit_button = SubmitField('Edit > Watch', render_kw={"class": "pure-button pure-button-primary"}) @@ -427,7 +427,7 @@ class commonSettingsForm(Form): message="Should contain one or more seconds")]) class importForm(Form): from . import processors - processor = RadioField(u'Processor', choices=processors.available_processors(), default="text_json_diff") + processor = RadioField(u'Processor', choices=[t[:2] for t in processors.available_processors()], default="text_json_diff") urls = TextAreaField('URLs') xlsx_file = FileField('Upload .xlsx file', validators=[FileAllowed(['xlsx'], 'Must be .xlsx file!')]) file_mapping = SelectField('File mapping', [validators.DataRequired()], choices={('wachete', 'Wachete mapping'), ('custom','Custom mapping')}) diff --git a/changedetectionio/plugins/__init__.py b/changedetectionio/plugins/__init__.py new file mode 100644 index 00000000000..7cbc3f9c9dd --- /dev/null +++ b/changedetectionio/plugins/__init__.py @@ -0,0 +1,6 @@ +import pluggy + +hookimpl = pluggy.HookimplMarker("eggsample") +"""Marker to be imported and used in plugins (and for own implementations)""" + +x=1 \ No newline at end of file diff --git a/changedetectionio/plugins/default.py b/changedetectionio/plugins/default.py new file mode 100644 index 00000000000..55289b9da2a --- /dev/null +++ b/changedetectionio/plugins/default.py @@ -0,0 +1,44 @@ +from ..plugins import hookimpl +import changedetectionio.processors.text_json_diff as text_json_diff +from changedetectionio import content_fetcher +import whois + +# would be changedetectionio.plugins in other apps + +class text_json_filtering_whois(text_json_diff.perform_site_check): + + def __init__(self, *args, datastore, watch_uuid, **kwargs): + super().__init__(*args, datastore=datastore, watch_uuid=watch_uuid, **kwargs) + + def call_browser(self): + + # the whois data + self.fetcher = content_fetcher.Fetcher() + self.fetcher.is_plaintext = True + + from urllib.parse import urlparse + parsed = urlparse(self.watch.link) + w = whois.whois(parsed.hostname) + self.fetcher.content= w.text + + +@hookimpl +def extra_processor(): + from changedetectionio.processors import default_processor_config + processor_config = dict(default_processor_config) + # Which UI elements are not used + processor_config['needs_request_fetch_method'] = False + processor_config['needs_browsersteps'] = False + processor_config['needs_visualselector'] = False + return ('plugin_processor_whois', "Whois domain information fetch", processor_config) + +@hookimpl +def processor_call(processor_name, datastore, watch_uuid): + if processor_name == 'plugin_processor_whois': + x = text_json_filtering_whois(datastore=datastore, watch_uuid=watch_uuid) + return x + return None + +@hookimpl +def eggsample_prep_condiments(condiments): + condiments["mint sauce"] = 1 diff --git a/changedetectionio/plugins/hookspecs.py b/changedetectionio/plugins/hookspecs.py new file mode 100644 index 00000000000..c46756001e6 --- /dev/null +++ b/changedetectionio/plugins/hookspecs.py @@ -0,0 +1,20 @@ +import pluggy +from changedetectionio.store import ChangeDetectionStore + +hookspec = pluggy.HookspecMarker("eggsample") + + +@hookspec +def extra_processor(): + """Defines a new fetch method + + :return: a tuples, (machine_name, description) + """ + +@hookspec(firstresult=True) +def processor_call(processor_name: str, datastore: ChangeDetectionStore, watch_uuid: str): + """ + Call processors with processor name + :param processor_name: as defined in extra_processors + :return: data? + """ \ No newline at end of file diff --git a/changedetectionio/processors/__init__.py b/changedetectionio/processors/__init__.py index 7aa8994a5d6..25c77ae8ce5 100644 --- a/changedetectionio/processors/__init__.py +++ b/changedetectionio/processors/__init__.py @@ -6,6 +6,15 @@ from copy import deepcopy from distutils.util import strtobool +# Which UI elements in settings the processor requires +# For example, restock monitor isnt compatible with visualselector and filters +default_processor_config = { + 'needs_request_fetch_method': True, + 'needs_browsersteps': True, + 'needs_visualselector': True, + 'needs_filters': True, +} + class difference_detection_processor(): browser_steps = None @@ -131,6 +140,15 @@ def run_changedetection(self, uuid, skip_when_checksum_same=True): def available_processors(): from . import restock_diff, text_json_diff - x=[('text_json_diff', text_json_diff.name), ('restock_diff', restock_diff.name)] - # @todo Make this smarter with introspection of sorts. + from ..flask_app import get_plugin_manager + pm = get_plugin_manager() + x = [('text_json_diff', text_json_diff.name, dict(default_processor_config)), + ('restock_diff', restock_diff.name, dict(default_processor_config)) + ] + + plugin_choices = pm.hook.extra_processor() + if plugin_choices: + for p in plugin_choices: + x.append(p) + return x diff --git a/changedetectionio/processors/text_json_diff.py b/changedetectionio/processors/text_json_diff.py index b503c5bec23..64949b59dea 100644 --- a/changedetectionio/processors/text_json_diff.py +++ b/changedetectionio/processors/text_json_diff.py @@ -155,7 +155,7 @@ def run_changedetection(self, uuid, skip_when_checksum_same=True): html_content = self.fetcher.content # If not JSON, and if it's not text/plain.. - if 'text/plain' in self.fetcher.get_all_headers().get('content-type', '').lower(): + if 'text/plain' in self.fetcher.get_all_headers().get('content-type', '').lower() or self.fetcher.is_plaintext: # Don't run get_text or xpath/css filters on plaintext stripped_text_from_html = html_content else: diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index d43ed6665f1..cd685d0eae5 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -39,12 +39,15 @@ - + +

Text filtering

From 42c6f8fc374b16339c0beea04904b29587e29a25 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Thu, 18 Jan 2024 23:19:00 +0100 Subject: [PATCH 10/12] some plugin config --- changedetectionio/flask_app.py | 9 ++++++--- changedetectionio/model/App.py | 1 + changedetectionio/templates/settings.html | 7 +++++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index ea13892862c..a5e35a3ed26 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -835,11 +835,14 @@ def settings_page(): flash("An error occurred, please see below.", "error") output = render_template("settings.html", - form=form, - hide_remove_pass=os.getenv("SALTED_PASS", False), api_key=datastore.data['settings']['application'].get('api_access_token'), emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False), - settings_application=datastore.data['settings']['application']) + form=form, + hide_remove_pass=os.getenv("SALTED_PASS", False), + settings_application=datastore.data['settings']['application'], + plugins=[] + + ) return output diff --git a/changedetectionio/model/App.py b/changedetectionio/model/App.py index 1202d5db198..36ecc8be0a6 100644 --- a/changedetectionio/model/App.py +++ b/changedetectionio/model/App.py @@ -38,6 +38,7 @@ class model(dict): 'notification_format': default_notification_format, 'notification_title': default_notification_title, 'notification_urls': [], # Apprise URL list + 'plugins': [], # list of dict, keyed by plugin name, with dict of the config and enabled true/false 'pager_size': 50, 'password': False, 'render_anchor_tag_content': False, diff --git a/changedetectionio/templates/settings.html b/changedetectionio/templates/settings.html index 508f49b2494..3be95e6d641 100644 --- a/changedetectionio/templates/settings.html +++ b/changedetectionio/templates/settings.html @@ -22,6 +22,7 @@
  • Global Filters
  • API
  • CAPTCHA & Proxies
    • +
  • Plugins
    • @@ -243,6 +244,12 @@ {{ render_field(form.requests.form.extra_browsers) }}
    +
    + available plugin on/off stuff here + + how to let each one expose config? +
    +
    {{ render_button(form.save_button) }} From 1aa0070ae206b2cfa04c30f68e121b55472e9e16 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 23 Jan 2024 17:00:02 +0100 Subject: [PATCH 11/12] remove example hooks --- changedetectionio/flask_app.py | 4 ++-- changedetectionio/plugins/__init__.py | 2 +- changedetectionio/plugins/hookspecs.py | 2 +- changedetectionio/plugins/whois.py | 3 --- 4 files changed, 4 insertions(+), 7 deletions(-) diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index a5e35a3ed26..6f3bed092d5 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -54,9 +54,9 @@ def get_plugin_manager(): from changedetectionio.plugins import hookspecs from changedetectionio.plugins import whois as whois_plugin - pm = pluggy.PluginManager("eggsample") + pm = pluggy.PluginManager("changedetectionio_plugin") pm.add_hookspecs(hookspecs) - pm.load_setuptools_entrypoints("eggsample") + pm.load_setuptools_entrypoints("changedetectionio_plugin") pm.register(whois_plugin) return pm diff --git a/changedetectionio/plugins/__init__.py b/changedetectionio/plugins/__init__.py index 7cbc3f9c9dd..98ade07d82d 100644 --- a/changedetectionio/plugins/__init__.py +++ b/changedetectionio/plugins/__init__.py @@ -1,6 +1,6 @@ import pluggy -hookimpl = pluggy.HookimplMarker("eggsample") +hookimpl = pluggy.HookimplMarker("changedetectionio_plugin") """Marker to be imported and used in plugins (and for own implementations)""" x=1 \ No newline at end of file diff --git a/changedetectionio/plugins/hookspecs.py b/changedetectionio/plugins/hookspecs.py index c46756001e6..c6d4905b664 100644 --- a/changedetectionio/plugins/hookspecs.py +++ b/changedetectionio/plugins/hookspecs.py @@ -1,7 +1,7 @@ import pluggy from changedetectionio.store import ChangeDetectionStore -hookspec = pluggy.HookspecMarker("eggsample") +hookspec = pluggy.HookspecMarker("changedetectionio_plugin") @hookspec diff --git a/changedetectionio/plugins/whois.py b/changedetectionio/plugins/whois.py index 568bab125f2..3203f24c29d 100644 --- a/changedetectionio/plugins/whois.py +++ b/changedetectionio/plugins/whois.py @@ -44,6 +44,3 @@ def processor_call(processor_name, datastore, watch_uuid): return x return None -@hookimpl -def eggsample_prep_condiments(condiments): - condiments["mint sauce"] = 1 From 65bc76f11b0e25c744d9bc92e14a347a49b3434b Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 23 Jan 2024 17:03:02 +0100 Subject: [PATCH 12/12] add comments --- changedetectionio/plugins/whois.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/changedetectionio/plugins/whois.py b/changedetectionio/plugins/whois.py index 3203f24c29d..96ad890467e 100644 --- a/changedetectionio/plugins/whois.py +++ b/changedetectionio/plugins/whois.py @@ -2,6 +2,8 @@ Whois information lookup - Fetches using whois - Extends the 'text_json_diff' so that text filters can still be used with whois information + +@todo publish to pypi and github as a separate plugin """ from ..plugins import hookimpl @@ -26,9 +28,12 @@ def call_browser(self): w = whois.whois(parsed.hostname) self.fetcher.content= w.text - @hookimpl def extra_processor(): + """ + Advertise a new processor + :return: + """ from changedetectionio.processors import default_processor_config processor_config = dict(default_processor_config) # Which UI elements are not used @@ -37,9 +42,11 @@ def extra_processor(): processor_config['needs_visualselector'] = False return ('plugin_processor_whois', "Whois domain information fetch", processor_config) +# @todo When a watch chooses this extra_process processor, the watch should ONLY use this one. +# (one watch can only have one extra_processor) @hookimpl def processor_call(processor_name, datastore, watch_uuid): - if processor_name == 'plugin_processor_whois': + if processor_name == 'plugin_processor_whois': # could be removed, see above note x = text_json_filtering_whois(datastore=datastore, watch_uuid=watch_uuid) return x return None