diff --git a/notifier/notify.py b/notifier/notify.py index 71533a6..9e806ce 100644 --- a/notifier/notify.py +++ b/notifier/notify.py @@ -1,7 +1,8 @@ +from contextlib import contextmanager import logging import re from smtplib import SMTPAuthenticationError -from typing import FrozenSet, Iterable, List, Optional, Set, Tuple +from typing import FrozenSet, Iterable, Iterator, List, Optional, Set, Tuple from notifier.config.remote import get_global_config from notifier.config.user import get_user_config @@ -15,7 +16,7 @@ from notifier.dumps import LogDumpCacher, record_activation_log from notifier.emailer import Emailer from notifier.newposts import get_new_posts -from notifier.timing import channel_is_now, channel_will_be_next, timestamp +from notifier.timing import channel_is_now, timestamp from notifier.types import ( ActivationLogDump, AuthConfig, @@ -70,6 +71,28 @@ def pick_channels_to_notify( return channels +@contextmanager +def activation_log_dump_context( + config: LocalConfig, database: BaseDatabaseDriver, dry_run: bool +) -> Iterator[LogDumpCacher]: + """Creates a log dump context that ends the long if the wrapped process fails.""" + activation_log_dump = LogDumpCacher[ActivationLogDump]( + {"start_timestamp": timestamp()}, + database.store_activation_log_dump, + dry_run, + ) + try: + yield activation_log_dump + + finally: + # Even if the run failed, record the end timestamp and upload if possible + activation_log_dump.update({"end_timestamp": timestamp()}) + + if not dry_run: + logger.info("Uploading log dumps...") + record_activation_log(config, database) + + def notify( *, config: LocalConfig, @@ -87,87 +110,83 @@ def notify( getting data for new posts) and then triggers the relevant notification schedules. """ - activation_log_dump = LogDumpCacher[ActivationLogDump]( - {"start_timestamp": timestamp()}, - database.store_activation_log_dump, - dry_run, - ) - - # If there are no active channels, which shouldn't happen, there is - # nothing to do - if len(active_channels) == 0: - logger.warning("No active channels; aborting") - return - - connection = Connection(database.get_supported_wikis(), dry_run=dry_run) - - activation_log_dump.update({"config_start_timestamp": timestamp()}) - if dry_run: - logger.info("Dry run: skipping remote config acquisition") - else: - logger.info("Getting remote config...") - get_global_config(config, database, connection) - logger.info("Getting user config...") - get_user_config(config, database, connection) - - # Refresh the connection to add any newly-configured wikis - connection = Connection(database.get_supported_wikis()) - activation_log_dump.update({"config_end_timestamp": timestamp()}) - activation_log_dump.update({"getpost_start_timestamp": timestamp()}) - if dry_run: - logger.info("Dry run: skipping new post acquisition") - else: - logger.info("Getting new posts...") - get_new_posts(database, connection, limit_wikis) - # The timestamp immediately after downloading posts will be used as the - # upper bound of posts to notify users about - activation_log_dump.update({"getpost_end_timestamp": timestamp()}) + with activation_log_dump_context( + config, database, dry_run + ) as activation_log_dump: + # If there are no active channels, which shouldn't happen, there is + # nothing to do + if len(active_channels) == 0: + logger.warning("No active channels; aborting") + return - if dry_run: - logger.info("Dry run: skipping Wikidot login") - else: - connection.login(config["wikidot_username"], auth["wikidot_password"]) - - activation_log_dump.update({"notify_start_timestamp": timestamp()}) - logger.info("Notifying...") - notify_active_channels( - active_channels, - current_timestamp=activation_log_dump.data.get( - "getpost_end_timestamp", timestamp() - ), - config=config, - auth=auth, - database=database, - connection=connection, - force_initial_search_timestamp=force_initial_search_timestamp, - dry_run=dry_run, - ) - activation_log_dump.update({"notify_end_timestamp": timestamp()}) + connection = Connection( + database.get_supported_wikis(), dry_run=dry_run + ) - # Notifications have been sent, so perform time-insensitive maintenance + activation_log_dump.update({"config_start_timestamp": timestamp()}) + if dry_run: + logger.info("Dry run: skipping remote config acquisition") + else: + logger.info("Getting remote config...") + get_global_config(config, database, connection) + logger.info("Getting user config...") + get_user_config(config, database, connection) + + # Refresh the connection to add any newly-configured wikis + connection = Connection(database.get_supported_wikis()) + activation_log_dump.update({"config_end_timestamp": timestamp()}) + + activation_log_dump.update({"getpost_start_timestamp": timestamp()}) + if dry_run: + logger.info("Dry run: skipping new post acquisition") + else: + logger.info("Getting new posts...") + get_new_posts(database, connection, limit_wikis) + # The timestamp immediately after downloading posts will be used as the + # upper bound of posts to notify users about + activation_log_dump.update({"getpost_end_timestamp": timestamp()}) + + if dry_run: + logger.info("Dry run: skipping Wikidot login") + else: + connection.login( + config["wikidot_username"], auth["wikidot_password"] + ) - if dry_run: - logger.info("Dry run: skipping cleanup") - return + activation_log_dump.update({"notify_start_timestamp": timestamp()}) + logger.info("Notifying...") + notify_active_channels( + active_channels, + current_timestamp=activation_log_dump.data.get( + "getpost_end_timestamp", timestamp() + ), + config=config, + auth=auth, + database=database, + connection=connection, + force_initial_search_timestamp=force_initial_search_timestamp, + dry_run=dry_run, + ) + activation_log_dump.update({"notify_end_timestamp": timestamp()}) - logger.info("Cleaning up...") + # Notifications have been sent, so perform time-insensitive maintenance - logger.info("Removing non-notifiable posts...") - database.delete_non_notifiable_posts() + if dry_run: + logger.info("Dry run: skipping cleanup") + return - logger.info("Checking for deleted posts") - clear_deleted_posts(database, connection) + logger.info("Cleaning up...") - logger.info("Purging invalid user config pages...") - delete_prepared_invalid_user_pages(config, connection) - rename_invalid_user_config_pages(config, connection) + logger.info("Removing non-notifiable posts...") + database.delete_non_notifiable_posts() - activation_log_dump.update({"end_timestamp": timestamp()}) + logger.info("Checking for deleted posts") + clear_deleted_posts(database, connection) - assert not dry_run - logger.info("Uploading log dumps...") - record_activation_log(config, database) + logger.info("Purging invalid user config pages...") + delete_prepared_invalid_user_pages(config, connection) + rename_invalid_user_config_pages(config, connection) def notify_active_channels( diff --git a/poetry.lock b/poetry.lock index 6addf49..00cda25 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "apscheduler" @@ -1219,44 +1219,50 @@ files = [ [[package]] name = "mypy" -version = "0.910" +version = "1.10.0" description = "Optional static typing for Python" optional = false -python-versions = ">=3.5" +python-versions = ">=3.8" files = [ - {file = "mypy-0.910-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:a155d80ea6cee511a3694b108c4494a39f42de11ee4e61e72bc424c490e46457"}, - {file = "mypy-0.910-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:b94e4b785e304a04ea0828759172a15add27088520dc7e49ceade7834275bedb"}, - {file = "mypy-0.910-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:088cd9c7904b4ad80bec811053272986611b84221835e079be5bcad029e79dd9"}, - {file = "mypy-0.910-cp35-cp35m-win_amd64.whl", hash = "sha256:adaeee09bfde366d2c13fe6093a7df5df83c9a2ba98638c7d76b010694db760e"}, - {file = "mypy-0.910-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:ecd2c3fe726758037234c93df7e98deb257fd15c24c9180dacf1ef829da5f921"}, - {file = "mypy-0.910-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:d9dd839eb0dc1bbe866a288ba3c1afc33a202015d2ad83b31e875b5905a079b6"}, - {file = "mypy-0.910-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:3e382b29f8e0ccf19a2df2b29a167591245df90c0b5a2542249873b5c1d78212"}, - {file = "mypy-0.910-cp36-cp36m-win_amd64.whl", hash = "sha256:53fd2eb27a8ee2892614370896956af2ff61254c275aaee4c230ae771cadd885"}, - {file = "mypy-0.910-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b6fb13123aeef4a3abbcfd7e71773ff3ff1526a7d3dc538f3929a49b42be03f0"}, - {file = "mypy-0.910-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e4dab234478e3bd3ce83bac4193b2ecd9cf94e720ddd95ce69840273bf44f6de"}, - {file = "mypy-0.910-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:7df1ead20c81371ccd6091fa3e2878559b5c4d4caadaf1a484cf88d93ca06703"}, - {file = "mypy-0.910-cp37-cp37m-win_amd64.whl", hash = "sha256:0aadfb2d3935988ec3815952e44058a3100499f5be5b28c34ac9d79f002a4a9a"}, - {file = "mypy-0.910-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ec4e0cd079db280b6bdabdc807047ff3e199f334050db5cbb91ba3e959a67504"}, - {file = "mypy-0.910-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:119bed3832d961f3a880787bf621634ba042cb8dc850a7429f643508eeac97b9"}, - {file = "mypy-0.910-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:866c41f28cee548475f146aa4d39a51cf3b6a84246969f3759cb3e9c742fc072"}, - {file = "mypy-0.910-cp38-cp38-win_amd64.whl", hash = "sha256:ceb6e0a6e27fb364fb3853389607cf7eb3a126ad335790fa1e14ed02fba50811"}, - {file = "mypy-0.910-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1a85e280d4d217150ce8cb1a6dddffd14e753a4e0c3cf90baabb32cefa41b59e"}, - {file = "mypy-0.910-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:42c266ced41b65ed40a282c575705325fa7991af370036d3f134518336636f5b"}, - {file = "mypy-0.910-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:3c4b8ca36877fc75339253721f69603a9c7fdb5d4d5a95a1a1b899d8b86a4de2"}, - {file = "mypy-0.910-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:c0df2d30ed496a08de5daed2a9ea807d07c21ae0ab23acf541ab88c24b26ab97"}, - {file = "mypy-0.910-cp39-cp39-win_amd64.whl", hash = "sha256:c6c2602dffb74867498f86e6129fd52a2770c48b7cd3ece77ada4fa38f94eba8"}, - {file = "mypy-0.910-py3-none-any.whl", hash = "sha256:ef565033fa5a958e62796867b1df10c40263ea9ded87164d67572834e57a174d"}, - {file = "mypy-0.910.tar.gz", hash = "sha256:704098302473cb31a218f1775a873b376b30b4c18229421e9e9dc8916fd16150"}, + {file = "mypy-1.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:da1cbf08fb3b851ab3b9523a884c232774008267b1f83371ace57f412fe308c2"}, + {file = "mypy-1.10.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:12b6bfc1b1a66095ab413160a6e520e1dc076a28f3e22f7fb25ba3b000b4ef99"}, + {file = "mypy-1.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e36fb078cce9904c7989b9693e41cb9711e0600139ce3970c6ef814b6ebc2b2"}, + {file = "mypy-1.10.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2b0695d605ddcd3eb2f736cd8b4e388288c21e7de85001e9f85df9187f2b50f9"}, + {file = "mypy-1.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:cd777b780312ddb135bceb9bc8722a73ec95e042f911cc279e2ec3c667076051"}, + {file = "mypy-1.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3be66771aa5c97602f382230165b856c231d1277c511c9a8dd058be4784472e1"}, + {file = "mypy-1.10.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8b2cbaca148d0754a54d44121b5825ae71868c7592a53b7292eeb0f3fdae95ee"}, + {file = "mypy-1.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ec404a7cbe9fc0e92cb0e67f55ce0c025014e26d33e54d9e506a0f2d07fe5de"}, + {file = "mypy-1.10.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e22e1527dc3d4aa94311d246b59e47f6455b8729f4968765ac1eacf9a4760bc7"}, + {file = "mypy-1.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:a87dbfa85971e8d59c9cc1fcf534efe664d8949e4c0b6b44e8ca548e746a8d53"}, + {file = "mypy-1.10.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a781f6ad4bab20eef8b65174a57e5203f4be627b46291f4589879bf4e257b97b"}, + {file = "mypy-1.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b808e12113505b97d9023b0b5e0c0705a90571c6feefc6f215c1df9381256e30"}, + {file = "mypy-1.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f55583b12156c399dce2df7d16f8a5095291354f1e839c252ec6c0611e86e2e"}, + {file = "mypy-1.10.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4cf18f9d0efa1b16478c4c129eabec36148032575391095f73cae2e722fcf9d5"}, + {file = "mypy-1.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:bc6ac273b23c6b82da3bb25f4136c4fd42665f17f2cd850771cb600bdd2ebeda"}, + {file = "mypy-1.10.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9fd50226364cd2737351c79807775136b0abe084433b55b2e29181a4c3c878c0"}, + {file = "mypy-1.10.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f90cff89eea89273727d8783fef5d4a934be2fdca11b47def50cf5d311aff727"}, + {file = "mypy-1.10.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fcfc70599efde5c67862a07a1aaf50e55bce629ace26bb19dc17cece5dd31ca4"}, + {file = "mypy-1.10.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:075cbf81f3e134eadaf247de187bd604748171d6b79736fa9b6c9685b4083061"}, + {file = "mypy-1.10.0-cp38-cp38-win_amd64.whl", hash = "sha256:3f298531bca95ff615b6e9f2fc0333aae27fa48052903a0ac90215021cdcfa4f"}, + {file = "mypy-1.10.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fa7ef5244615a2523b56c034becde4e9e3f9b034854c93639adb667ec9ec2976"}, + {file = "mypy-1.10.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3236a4c8f535a0631f85f5fcdffba71c7feeef76a6002fcba7c1a8e57c8be1ec"}, + {file = "mypy-1.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a2b5cdbb5dd35aa08ea9114436e0d79aceb2f38e32c21684dcf8e24e1e92821"}, + {file = "mypy-1.10.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:92f93b21c0fe73dc00abf91022234c79d793318b8a96faac147cd579c1671746"}, + {file = "mypy-1.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:28d0e038361b45f099cc086d9dd99c15ff14d0188f44ac883010e172ce86c38a"}, + {file = "mypy-1.10.0-py3-none-any.whl", hash = "sha256:f8c083976eb530019175aabadb60921e73b4f45736760826aa1689dda8208aee"}, + {file = "mypy-1.10.0.tar.gz", hash = "sha256:3d087fcbec056c4ee34974da493a826ce316947485cef3901f511848e687c131"}, ] [package.dependencies] -mypy-extensions = ">=0.4.3,<0.5.0" -toml = "*" -typing-extensions = ">=3.7.4" +mypy-extensions = ">=1.0.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = ">=4.1.0" [package.extras] dmypy = ["psutil (>=4.0)"] -python2 = ["typed-ast (>=1.4.0,<1.5.0)"] +install-types = ["pip"] +mypyc = ["setuptools (>=50)"] +reports = ["lxml"] [[package]] name = "mypy-boto3-cloudformation" @@ -1358,12 +1364,13 @@ typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.9\""} [[package]] name = "mypy-extensions" -version = "0.4.4" -description = "Experimental type system extensions for programs checked with the mypy typechecker." +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." optional = false -python-versions = ">=2.7" +python-versions = ">=3.5" files = [ - {file = "mypy_extensions-0.4.4.tar.gz", hash = "sha256:c8b707883a96efe9b4bb3aaf0dcc07e7e217d7d8368eec4db4049ee9e142f4fd"}, + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] [[package]] @@ -1989,4 +1996,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "410c24fb9ba9065f2bcab1b9eed081447794482c611b45433bdc1d3f3d7ddff7" +content-hash = "0a753b65e600bba704e423dd237c2cc45eae1d890565a80e32a48bb8982b8976" diff --git a/pyproject.toml b/pyproject.toml index cf59f1e..5f2d547 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ black = "^22.3.0" pytest = "^6.2.4" isort = "^5.9.2" pylint = "^2.9.6" -mypy = "^0.910" +mypy = "^1.10.0" boto3-stubs = {extras = ["essential"], version = "^1.28.2"} types-beautifulsoup4 = "^4.12.0.5" types-requests = "^2.31.0.1" diff --git a/stats-frontend/assets/sample-notifications/last-activation-early-finish.json b/stats-frontend/assets/sample-notifications/last-activation-early-finish.json new file mode 100644 index 0000000..d184af2 --- /dev/null +++ b/stats-frontend/assets/sample-notifications/last-activation-early-finish.json @@ -0,0 +1,15 @@ +{ + "activations": [ + { + "start_timestamp": 1691179249, + "config_start_timestamp": 1691179249, + "config_end_timestamp": null, + "getpost_start_timestamp": null, + "getpost_end_timestamp": null, + "notify_start_timestamp": null, + "notify_end_timestamp": null, + "end_timestamp": 1691179281 + } + ], + "channels": [] +} \ No newline at end of file