diff --git a/notifier/config/remote.py b/notifier/config/remote.py index d2eb278e..30c757a7 100644 --- a/notifier/config/remote.py +++ b/notifier/config/remote.py @@ -3,14 +3,13 @@ from typing import Any, Dict, List, Tuple, cast import boto3 -import requests import tomlkit from tomlkit.exceptions import TOMLKitError from notifier.database.drivers.base import BaseDatabaseDriver from notifier.database.utils import try_cache from notifier.types import LocalConfig, SupportedWikiConfig -from notifier.wikiconnection import Connection +from notifier.wikidot import Wikidot logger = logging.getLogger(__name__) @@ -25,22 +24,22 @@ def get_global_config( local_config: LocalConfig, database: BaseDatabaseDriver, - connection: Connection, + wikidot: Wikidot, ) -> None: """Retrieve remote global config for wikis.""" try_cache( - get=lambda: fetch_supported_wikis(local_config, connection), + get=lambda: fetch_supported_wikis(local_config, wikidot), store=database.store_supported_wikis, do_not_store=[], ) def fetch_supported_wikis( - local_config: LocalConfig, connection: Connection + local_config: LocalConfig, wikidot: Wikidot ) -> List[SupportedWikiConfig]: """Fetch the list of supported wikis from the configuration wiki.""" configs = [] - for config_soup in connection.listpages( + for config_soup in wikidot.listpages( local_config["config_wiki"], category=local_config["wiki_config_category"], module_body=wiki_config_listpages_body, diff --git a/notifier/config/user.py b/notifier/config/user.py index 5adc962d..c6075907 100644 --- a/notifier/config/user.py +++ b/notifier/config/user.py @@ -15,7 +15,7 @@ Subscription, SubscriptionCardinality, ) -from notifier.wikiconnection import Connection +from notifier.wikidot import Wikidot logger = logging.getLogger(__name__) @@ -40,22 +40,22 @@ def get_user_config( local_config: LocalConfig, database: BaseDatabaseDriver, - connection: Connection, + wikidot: Wikidot, ) -> None: """Retrieve remote user config.""" try_cache( - get=lambda: find_valid_user_configs(local_config, connection), + get=lambda: find_valid_user_configs(local_config, wikidot), store=database.store_user_configs, do_not_store=[], ) def find_valid_user_configs( - local_config: LocalConfig, connection: Connection + local_config: LocalConfig, wikidot: Wikidot ) -> List[RawUserConfig]: """Fetches user configs and returns those that are valid.""" configs: List[RawUserConfig] = [] - for slug, config in fetch_user_configs(local_config, connection): + for slug, config in fetch_user_configs(local_config, wikidot): if not user_config_is_valid(slug, config): # Only accept configs for the user who created the page logger.warning( @@ -78,7 +78,7 @@ def user_config_is_valid(slug: str, config: RawUserConfig) -> bool: def fetch_user_configs( - local_config: LocalConfig, connection: Connection + local_config: LocalConfig, wikidot: Wikidot ) -> List[Tuple[str, RawUserConfig]]: """Fetches a list of user configurations from the configuration wiki. @@ -86,7 +86,7 @@ def fetch_user_configs( cached in the database. """ configs: List[Tuple[str, RawUserConfig]] = [] - for config_soup in connection.listpages( + for config_soup in wikidot.listpages( local_config["config_wiki"], category=local_config["user_config_category"], module_body=user_config_listpages_body, diff --git a/notifier/deletions.py b/notifier/deletions.py index 23c082f6..912094c3 100644 --- a/notifier/deletions.py +++ b/notifier/deletions.py @@ -21,7 +21,7 @@ from notifier.database.drivers.base import BaseDatabaseDriver from notifier import timing from notifier.types import LocalConfig, PostMeta -from notifier.wikiconnection import Connection, ThreadNotExists +from notifier.wikidot import Wikidot, ThreadNotExists logger = logging.getLogger(__name__) @@ -32,7 +32,7 @@ def clear_deleted_posts( - database: BaseDatabaseDriver, connection: Connection + database: BaseDatabaseDriver, wikidot: Wikidot ) -> None: """Check for posts that have been deleted on the remote and delete them here too. @@ -47,13 +47,13 @@ def clear_deleted_posts( now_hour_ts = int(datetime.timestamp(now_hour)) posts = database.get_posts_to_check_for_deletion(now_hour_ts) - delete_posts(posts, database, connection) + delete_posts(posts, database, wikidot) def delete_posts( posts: List[PostMeta], database: BaseDatabaseDriver, - connection: Connection, + wikidot: Wikidot, ) -> None: """Sync post deletion states. @@ -77,7 +77,7 @@ def delete_posts( try: # Throws ThreadNotExists if the thread doesn't exist - thread_meta, thread_posts = connection.thread( + thread_meta, thread_posts = wikidot.thread( post["wiki_id"], post["thread_id"], post["post_id"] ) @@ -143,14 +143,14 @@ def delete_posts( def rename_invalid_user_config_pages( - local_config: LocalConfig, connection: Connection + local_config: LocalConfig, wikidot: Wikidot ) -> None: """Prepares invalid user config pages for deletion.""" logger.info("Finding invalid user configs to prepare for deletion") # Get all user configs and filter out any that are valid invalid_configs = [ (slug, config) - for slug, config in fetch_user_configs(local_config, connection) + for slug, config in fetch_user_configs(local_config, wikidot) if not user_config_is_valid(slug, config) ] logger.debug( @@ -158,7 +158,7 @@ def rename_invalid_user_config_pages( ) for slug, config in invalid_configs: try: - connection.rename_page( + wikidot.rename_page( local_config["config_wiki"], slug, f"deleted:{uuid4()}" ) except Exception as error: @@ -171,11 +171,11 @@ def rename_invalid_user_config_pages( def delete_prepared_invalid_user_pages( - local_config: LocalConfig, connection: Connection + local_config: LocalConfig, wikidot: Wikidot ) -> None: """Deletes prepared invalid user config pages.""" logger.info("Finding pages marked for deletion") - pages_to_delete = connection.listpages( + pages_to_delete = wikidot.listpages( local_config["config_wiki"], category="deleted", module_body="%%fullname%%", @@ -183,7 +183,7 @@ def delete_prepared_invalid_user_pages( for page in pages_to_delete: slug = page.get_text() try: - connection.delete_page(local_config["config_wiki"], slug) + wikidot.delete_page(local_config["config_wiki"], slug) except Exception as error: logger.error( "Couldn't delete page %s", diff --git a/notifier/newposts.py b/notifier/newposts.py index cd81706e..4f26f553 100644 --- a/notifier/newposts.py +++ b/notifier/newposts.py @@ -8,7 +8,7 @@ from notifier.config.user import parse_thread_url from notifier.database.drivers.base import BaseDatabaseDriver from notifier.types import RawPost, RawThreadMeta -from notifier.wikiconnection import Connection +from notifier.wikidot import Wikidot logger = logging.getLogger(__name__) @@ -19,7 +19,7 @@ def get_new_posts( database: BaseDatabaseDriver, - connection: Connection, + wikidot: Wikidot, limit_wikis: Optional[List[str]] = None, ) -> None: """For each configured wiki, retrieve and store new posts. @@ -36,7 +36,7 @@ def get_new_posts( continue logger.info("Getting new posts %s", {"for wiki_id": wiki["id"]}) try: - fetch_posts_with_context(wiki["id"], database, connection) + fetch_posts_with_context(wiki["id"], database, wikidot) except Exception as error: logger.error( "Failed getting new posts %s", @@ -47,7 +47,7 @@ def get_new_posts( def fetch_posts_with_context( - wiki_id: str, database: BaseDatabaseDriver, connection: Connection + wiki_id: str, database: BaseDatabaseDriver, wikidot: Wikidot ) -> None: """Look up new posts for a wiki and then attach their context. Stores the posts in the cache. @@ -110,7 +110,7 @@ def fetch_posts_with_context( "post_id": post_id, }, ) - thread_meta, thread_page_posts = connection.thread( + thread_meta, thread_page_posts = wikidot.thread( wiki_id, thread_id, post_id ) post = next( @@ -165,7 +165,7 @@ def fetch_posts_with_context( "Downloading first thread page %s", {"wiki_id": wiki_id, "thread_id": thread_id}, ) - thread_first_post = connection.thread(wiki_id, thread_id)[1][0] + thread_first_post = wikidot.thread(wiki_id, thread_id)[1][0] database.store_context_thread( { "thread_id": thread_id, diff --git a/notifier/notify.py b/notifier/notify.py index 9e806ce9..7a60fe06 100644 --- a/notifier/notify.py +++ b/notifier/notify.py @@ -25,7 +25,7 @@ EmailAddresses, LocalConfig, ) -from notifier.wikiconnection import Connection, NotLoggedIn, RestrictedInbox +from notifier.wikidot import Wikidot, NotLoggedIn, RestrictedInbox logger = logging.getLogger(__name__) @@ -120,21 +120,19 @@ def notify( logger.warning("No active channels; aborting") return - connection = Connection( - database.get_supported_wikis(), dry_run=dry_run - ) + wikidot = Wikidot(database.get_supported_wikis(), dry_run=dry_run) activation_log_dump.update({"config_start_timestamp": timestamp()}) if dry_run: logger.info("Dry run: skipping remote config acquisition") else: logger.info("Getting remote config...") - get_global_config(config, database, connection) + get_global_config(config, database, wikidot) logger.info("Getting user config...") - get_user_config(config, database, connection) + get_user_config(config, database, wikidot) # Refresh the connection to add any newly-configured wikis - connection = Connection(database.get_supported_wikis()) + wikidot = Wikidot(database.get_supported_wikis()) activation_log_dump.update({"config_end_timestamp": timestamp()}) activation_log_dump.update({"getpost_start_timestamp": timestamp()}) @@ -142,7 +140,7 @@ def notify( logger.info("Dry run: skipping new post acquisition") else: logger.info("Getting new posts...") - get_new_posts(database, connection, limit_wikis) + get_new_posts(database, wikidot, limit_wikis) # The timestamp immediately after downloading posts will be used as the # upper bound of posts to notify users about activation_log_dump.update({"getpost_end_timestamp": timestamp()}) @@ -150,9 +148,7 @@ def notify( if dry_run: logger.info("Dry run: skipping Wikidot login") else: - connection.login( - config["wikidot_username"], auth["wikidot_password"] - ) + wikidot.login(config["wikidot_username"], auth["wikidot_password"]) activation_log_dump.update({"notify_start_timestamp": timestamp()}) logger.info("Notifying...") @@ -164,7 +160,7 @@ def notify( config=config, auth=auth, database=database, - connection=connection, + wikidot=wikidot, force_initial_search_timestamp=force_initial_search_timestamp, dry_run=dry_run, ) @@ -182,11 +178,11 @@ def notify( database.delete_non_notifiable_posts() logger.info("Checking for deleted posts") - clear_deleted_posts(database, connection) + clear_deleted_posts(database, wikidot) logger.info("Purging invalid user config pages...") - delete_prepared_invalid_user_pages(config, connection) - rename_invalid_user_config_pages(config, connection) + delete_prepared_invalid_user_pages(config, wikidot) + rename_invalid_user_config_pages(config, wikidot) def notify_active_channels( @@ -196,7 +192,7 @@ def notify_active_channels( config: LocalConfig, auth: AuthConfig, database: BaseDatabaseDriver, - connection: Connection, + wikidot: Wikidot, force_initial_search_timestamp: Optional[int] = None, dry_run: bool = False, ) -> None: @@ -212,7 +208,7 @@ def notify_active_channels( force_initial_search_timestamp=force_initial_search_timestamp, config=config, database=database, - connection=connection, + wikidot=wikidot, digester=digester, emailer=emailer, dry_run=dry_run, @@ -226,7 +222,7 @@ def notify_channel( force_initial_search_timestamp: Optional[int] = None, config: LocalConfig, database: BaseDatabaseDriver, - connection: Connection, + wikidot: Wikidot, digester: Digester, emailer: Emailer, dry_run: bool = False, @@ -283,7 +279,7 @@ def notify_channel( force_initial_search_timestamp=force_initial_search_timestamp, config=config, database=database, - connection=connection, + wikidot=wikidot, digester=digester, emailer=emailer, addresses=addresses, @@ -337,7 +333,7 @@ def notify_user( force_initial_search_timestamp: Optional[int] = None, config: LocalConfig, database: BaseDatabaseDriver, - connection: Connection, + wikidot: Wikidot, digester: Digester, emailer: Emailer, addresses: EmailAddresses, @@ -416,7 +412,7 @@ def update_tags_for_user( if old_tags == new_tags: return new_tags_string = " ".join(map(str, new_tags)) - connection.set_tags( + wikidot.set_tags( config["config_wiki"], f"{config['user_config_category']}:{str(user['user_id'])}", new_tags_string, @@ -457,7 +453,7 @@ def remove_error_tags_from_user() -> None: {"to user": user["username"], "via": "pm", "channel": channel}, ) try: - connection.send_message(user["user_id"], subject, body) + wikidot.send_message(user["user_id"], subject, body) except RestrictedInbox: # If the inbox is restricted, inform the user logger.debug( @@ -479,7 +475,7 @@ def remove_error_tags_from_user() -> None: # Only get the contacts when there is actually a user who # needs to be emailed logger.info("Retrieving email contacts") - addresses.update(connection.get_contacts()) + addresses.update(wikidot.get_contacts()) logger.debug( "Retrieved email contacts %s", {"address_count": len(addresses)}, @@ -528,7 +524,7 @@ def remove_error_tags_from_user() -> None: # If the delivery was successful, remove any error tags if user["tags"] != "": - connection.set_tags( + wikidot.set_tags( config["config_wiki"], ":".join([config["user_config_category"], user["user_id"]]), "", diff --git a/notifier/wikiconnection.py b/notifier/wikidot.py similarity index 99% rename from notifier/wikiconnection.py rename to notifier/wikidot.py index c6d74ba3..18c5e5fd 100644 --- a/notifier/wikiconnection.py +++ b/notifier/wikidot.py @@ -63,7 +63,7 @@ class OngoingConnectionError(Exception): by trying it multiple times.""" -class Connection: +class Wikidot: """Connection to Wikidot facilitating communications with it.""" PAGINATION_DELAY_S = 1.0