models: change tracking of domains

Author: lnielsen

invenio_accounts/api.py

@@ -16,8 +16,10 @@ def __init__(self):
         """Constructor."""
         self.updated_users = set()
         self.updated_roles = set()
+        self.updated_domains = set()
         self.deleted_users = set()
         self.deleted_roles = set()
+        self.deleted_domains = set()
 
 
 class DBUsersChangeHistory:
@@ -43,6 +45,12 @@ def add_updated_role(self, session_id, role_id):
         session = self._get_session(session_id)
         session.updated_roles.add(role_id)
 
+    def add_updated_domain(self, session_id, domain_id):
+        """Adds a user to the updated domains list."""
+        assert domain_id is not None
+        session = self._get_session(session_id)
+        session.updated_domains.add(domain_id)
+
     def add_deleted_user(self, session_id, user_id):
         """Adds a user to the deleted users list."""
         assert user_id is not None
@@ -55,6 +63,12 @@ def add_deleted_role(self, session_id, role_id):
         session = self._get_session(session_id)
         session.deleted_roles.add(role_id)
 
+    def add_deleted_domain(self, session_id, domain_id):
+        """Adds a role to the deleted domain list."""
+        assert domain_id is not None
+        session = self._get_session(session_id)
+        session.deleted_domains.add(domain_id)
+
     def clear_dirty_sets(self, session):
         """Removes session object."""
         sid = id(session)

invenio_accounts/datastore.py

@@ -12,6 +12,7 @@
 
 from flask import current_app
 from flask_security import SQLAlchemyUserDatastore, user_confirmed
+from sqlalchemy.orm import joinedload
 
 from .models import Domain, Role, User
 from .proxies import current_db_change_history
@@ -27,19 +28,17 @@ def verify_user(self, user):
         now = datetime.utcnow()
         user.blocked_at = None
         user.verified_at = now
+        user.active = True
         if user.confirmed_at is None:
             user.confirmed_at = now
-        if not user.active:
-            user.active = True
         return True
 
     def block_user(self, user):
         """Verify a user."""
         now = datetime.utcnow()
         user.blocked_at = now
         user.verified_at = None
-        if user.active:
-            user.active = False
+        user.active = False
         delete_user_sessions(user)
         return True
 
@@ -113,7 +112,11 @@ def find_role_by_id(self, role_id):
 
     def find_domain(self, domain):
         """Find a domain."""
-        return Domain.query.filter_by(domain=domain).one_or_none()
+        return (
+            Domain.query.filter_by(domain=domain)
+            .options(joinedload(Domain.category_name))
+            .one_or_none()
+        )
 
     def create_domain(self, domain, **kwargs):
         """Create a new domain."""

invenio_accounts/domains.py

@@ -20,6 +20,7 @@ def on_user_confirmed(app, user):
     domain = datastore.find_domain(user.domain)
     if domain is None:
         domain = datastore.create_domain(user.domain)
+        datastore.mark_changed(id(datastore.db.session), model=domain)
 
     # Verify user if domain is verified.
     if domain.status == DomainStatus.verified:

invenio_accounts/models.py

@@ -498,9 +498,6 @@ class DomainOrg(db.Model):
     parent = db.relationship("DomainOrg", remote_side=[id])
     """Relationship to parent."""
 
-    domains = db.relationship("Domain", back_populates="org")
-    """Relationship to domains for this organisation."""
-
     @classmethod
     def create(cls, pid, name, json=None, parent=None):
         """Create a domain organisation."""
@@ -564,12 +561,15 @@ class Domain(db.Model, Timestamp):
     org_id = db.Column(db.Integer(), db.ForeignKey(DomainOrg.id), nullable=True)
     """Organisation associated with domain."""
 
-    org = db.relationship("DomainOrg", back_populates="domains")
+    org = db.relationship("DomainOrg", backref="domains")
 
     # spammer, mail-provider, organisation, company
     category = db.Column(db.Integer(), db.ForeignKey(DomainCategory.id), nullable=True)
     """Category of domain."""
 
+    category_name = db.relationship("DomainCategory", backref="domains")
+    """Relationship to category"""
+
     num_users = db.Column(db.Integer(), default=0, nullable=False)
     """Computed property to store number of users in domain."""
 

invenio_accounts/tasks.py

@@ -122,27 +122,33 @@ def update_domain_status():
     # If statistics are updated regularly, the number of updates is relatively
     # low and hence fit in memory. We read all data first, to avoid starting
     # to modify the same table we're reading from.
-    domain_updates = []
-    for row in stmt.all():
-        domain_updates.append(row)
-
-    # Update the database
-    i = 0
-    for row in domain_updates:
-        domain, users, active, inactive, confirmed, verified, blocked = row
-        db.session.query(Domain).filter(Domain.domain == domain).update(
-            {
-                "num_users": users,
-                "num_active": active,
-                "num_inactive": inactive,
-                "num_confirmed": confirmed,
-                "num_verified": verified,
-                "num_blocked": blocked,
-            }
-        )
-        i += 1
-        # Commit batches of 500 updates
-        if i % 500 == 0:
-            db.session.commit()
-    if i % 500 != 0:
-        db.session.commit()
+    domain_updates = list(stmt.all())
+
+    # Commit batches of 500 updates
+    batch_size = 500
+    now = datetime.utcnow()
+
+    # Process updates in batches
+    for i in range(0, len(domain_updates), batch_size):
+        with db.session.begin_nested():  # Use nested transactions for safety
+            for (
+                domain,
+                users,
+                active,
+                inactive,
+                confirmed,
+                verified,
+                blocked,
+            ) in domain_updates[i : i + batch_size]:
+                db.session.query(Domain).filter(Domain.domain == domain).update(
+                    {
+                        "num_users": users,
+                        "num_active": active,
+                        "num_inactive": inactive,
+                        "num_confirmed": confirmed,
+                        "num_verified": verified,
+                        "num_blocked": blocked,
+                        "updated": now,
+                    }
+                )
+        db.session.commit()  # Commit after each batch