Merge pull request #100 from croque-scp/reopt-post-download

Track last seen post timestamp per wiki

notifier/database/drivers/base.py

@@ -111,6 +111,12 @@ def store_supported_wikis(self, wikis: List[SupportedWikiConfig]) -> None:
         """Stores a set of supported wikis in the database, overwriting any
         that are already present."""
 
+    @abstractmethod
+    def store_latest_post_timestamp(
+        self, wiki_id: str, timestamp: int
+    ) -> None:
+        """Stores the latest seen post timestamp for the given wiki."""
+
     @abstractmethod
     def store_post(self, post: NotifiablePost) -> None:
         """Store a post."""

notifier/database/drivers/mysql.py

@@ -365,6 +365,14 @@ def store_supported_wikis(self, wikis: List[SupportedWikiConfig]) -> None:
                 )
             # Wikis that were removed from the service since the last run are still available as context
 
+    def store_latest_post_timestamp(
+        self, wiki_id: str, timestamp: int
+    ) -> None:
+        self.execute_named(
+            "store_latest_post_timestamp",
+            {"wiki_id": wiki_id, "timestamp": timestamp},
+        )
+
     def store_post(self, post: NotifiablePost) -> None:
         self.execute_named(
             "store_post",

notifier/database/migrations/010-track-latest-timestamp-per-wiki.down.sql

@@ -0,0 +1,4 @@
+ALTER TABLE
+  context_wiki
+DROP COLUMN
+  new_posts_checked_timestamp;

notifier/database/migrations/010-track-latest-timestamp-per-wiki.up.sql

@@ -0,0 +1,9 @@
+ALTER TABLE
+  context_wiki
+ADD COLUMN
+  new_posts_checked_timestamp INT UNSIGNED NOT NULL;
+
+UPDATE
+  context_wiki
+SET
+  new_posts_checked_timestamp = 0;

notifier/database/queries/delete_non_notifiable_posts.sql

@@ -15,8 +15,8 @@ WHERE
   notifiable_post.posted_timestamp < ((
     SELECT latest_timestamp FROM (
       SELECT
-        MAX(posted_timestamp) AS latest_timestamp
-      FROM notifiable_post
+        MAX(new_posts_checked_timestamp) AS latest_timestamp
+      FROM context_wiki
     ) AS t
   ) - (60 * 60 * 24 * 365))
 

notifier/database/queries/get_latest_post_timestamp.sql

@@ -1,6 +1,6 @@
 SELECT
-  MAX(posted_timestamp) as posted_timestamp
+  new_posts_checked_timestamp AS posted_timestamp
 FROM
-  notifiable_post
+  context_wiki
 WHERE
-  notifiable_post.context_wiki_id = %(wiki_id)s
+  context_wiki.wiki_id = %(wiki_id)s

notifier/database/queries/store_context_wiki.sql

@@ -4,14 +4,16 @@ INSERT INTO
     wiki_id,
     wiki_name,
     wiki_service_configured,
-    wiki_uses_https
+    wiki_uses_https,
+    new_posts_checked_timestamp
   )
 VALUES
   (
     %(wiki_id)s,
     %(wiki_name)s,
     %(wiki_service_configured)s,
-    %(wiki_uses_https)s
+    %(wiki_uses_https)s,
+    0
   )
 ON DUPLICATE KEY UPDATE
   wiki_name = %(wiki_name)s,

notifier/database/queries/store_latest_post_timestamp.sql

@@ -0,0 +1,6 @@
+UPDATE
+  context_wiki
+SET
+  new_posts_checked_timestamp = %(timestamp)s
+WHERE
+  context_wiki.wiki_id = %(wiki_id)s

notifier/newposts.py

@@ -75,6 +75,12 @@ def fetch_posts_with_context(
         },
     )
 
+    # If there's at least one post, store the new highest timestamp for this wiki
+    if len(new_posts) > 0:
+        database.store_latest_post_timestamp(
+            wiki_id, max(post["posted_timestamp"] for post in new_posts)
+        )
+
     # Cache the latest downloaded thread to prevent multiple identical downloads when multiple posts share a thread
     thread_meta: RawThreadMeta = None  # type:ignore
     thread_page_posts: List[RawPost] = []