Cleaned up mailman daily jobs. Ready to rock on Heroku again.

Author: teajaymars

getactivity_mailman.py

@@ -3,7 +3,7 @@
 import argparse
 from lxml import html
 import requests
-from lib import uil
+from lib import util
 from lib.backend import Session
 from lib.backend.model import ActivityInMailman
 from datetime import datetime
@@ -13,27 +13,27 @@ def get_activity(verbose=False):
     for l in lists:
         if verbose: print 'Processing activity for %s...' % l['name']
         latest = Session.query(ActivityInMailman)\
-                .filter(ActivityInMailman.list_name==l['name']
+                .filter(ActivityInMailman.list_name==l['name'])\
                 .order_by(ActivityInMailman.message_id.desc())\
                 .first()
         # Walk through message history from the web front-end
         archive_url = l['link'].replace('mailman/listinfo','pipermail')
-        limit = 100
+        limit = 1000
         latest_id = latest.message_id if latest else -1
         for msg in _yield_messages(archive_url,latest_id, verbose=verbose):
             if verbose: print '  -> got msg #%d (%s: "%s")' % (msg['id'],msg['email'],msg['subject'])
             Session.add( ActivityInMailman(
-                listname  = l['name'],
+                list_name  = l['name'],
                 message_id = msg['id'], 
                 subject = msg['subject'],
                 author = msg['author'],
                 email = msg['email'],
                 link = msg['link'],
                 timestamp = msg['date'] ) )
             limit -= 1
-            if limit==0: 
-                if verbose: print '  -> Reached activity limit (100)'
-                break;
+            #if limit==0: 
+            #if verbose: print '  -> Reached activity limit (100)'
+            #break;
         Session.commit()
 
 def _yield_messages(url, latest_id, verbose=False):

lib/backend/model.py

@@ -151,7 +151,7 @@ def toJson(self):
 
 class ActivityInMailman(Base):
     __tablename__='activity_mailman'
-    listname  = Column(String, primary_key=True)
+    list_name  = Column(String, primary_key=True)
     message_id = Column(Integer, primary_key=True)
     subject = Column(String)
     author = Column(String)

lib/util.py

@@ -1,3 +1,5 @@
+from lxml import html
+import requests
 
 def list_mailman_lists(verbose=False):
     """Scrape the server for a catalogue of all mailman lists."""

snapshot_mailman.py

@@ -15,13 +15,13 @@ def snapshot_mailman(verbose=False):
     for l in lists:
         if verbose: print 'Processing snapshots for %s...' % l['name']
         latest = Session.query(SnapshotOfMailman)\
-                .filter(SnapshotOfMailman.name==l['name'])\
+                .filter(SnapshotOfMailman.list_name==l['name'])\
                 .order_by(SnapshotOfMailman.timestamp.desc())\
                 .first()
         # By default, gather 30 days of snapshots
-        since = today - timedelta(days=30)
+        since = today - timedelta(days=180)
         if latest:
-            if latest.timestamp>=until:
+            if latest.timestamp>=today:
                 if verbose: print ' -> most recent snapshots have already been processed.'
                 continue
             since = latest.timestamp + timedelta(days=1)
@@ -32,7 +32,7 @@ def snapshot_mailman(verbose=False):
         while since<today:
             posts_today = Session.query(ActivityInMailman)\
                             .filter(ActivityInMailman.list_name==l['name'])\
-                            .filter(ActivityInMailman.timestamp.between(date,date+day))\
+                            .filter(ActivityInMailman.timestamp.between(since,since+timedelta(days=1)))\
                             .count()
             sn = SnapshotOfMailman(\
                     list_name=l['name'],\