Merge pull request #852 from ZeitOnline/WCM-285

WCM-285: new columns channel and subchannels

core/docs/changelog/WCM-285.change

@@ -0,0 +1 @@
+WCM-285: add new columns channel and subchannels

core/src/zeit/connector/converter.py

@@ -0,0 +1,82 @@
+from sqlalchemy.dialects.postgresql import JSONB
+import grokcore.component as grok
+import sqlalchemy
+import zope.interface
+
+import zeit.connector.interfaces
+
+
+@grok.implementer(zeit.connector.interfaces.IConverter)
+@grok.adapter(sqlalchemy.Column)
+def converter_from_column_type(column):
+    return zeit.connector.interfaces.IConverter(column.type)
+
+
+@grok.implementer(zeit.connector.interfaces.IConverter)
+class DefaultConverter(grok.Adapter):
+    grok.context(zope.interface.Interface)
+
+    def serialize(self, value):
+        return value
+
+    def deserialize(self, value):
+        return value
+
+
+class BoolConverter(DefaultConverter):
+    grok.context(sqlalchemy.Boolean)
+
+    def serialize(self, value):
+        return zeit.cms.content.dav.BoolProperty._toProperty(value)
+
+    def deserialize(self, value):
+        return zeit.cms.content.dav.BoolProperty._fromProperty(value)
+
+
+class IntConverter(DefaultConverter):
+    grok.context(sqlalchemy.Integer)
+
+    def serialize(self, value):
+        return str(value)
+
+    def deserialize(self, value):
+        return int(value)
+
+
+class DatetimeConverter(DefaultConverter):
+    grok.context(sqlalchemy.TIMESTAMP)
+
+    def serialize(self, value):
+        return zeit.cms.content.dav.DatetimeProperty._toProperty(value)
+
+    def deserialize(self, value):
+        return zeit.cms.content.dav.DatetimeProperty._fromProperty(value)
+
+
+class ChannelsConverter(DefaultConverter):
+    grok.context(JSONB)
+    grok.name('channels')
+
+    def serialize(self, value):
+        if not value:
+            return ''
+        elements = []
+        for channel, subchannels in value.items():
+            if subchannels:
+                elements.append(f"{channel} {' '.join(subchannels)}")
+            else:
+                elements.append(channel)
+        return ';'.join(elements)
+
+    def deserialize(self, value):
+        channels = {}
+        if value:
+            elements = [i.split() for i in value.split(';') if i.strip()]
+            for element in elements:
+                channel = element[0]
+                subchannels = element[1:] if len(element) > 1 else []
+                if channel in channels:
+                    channels[channel].extend(subchannels)
+                else:
+                    channels[channel] = subchannels
+        return channels

core/src/zeit/connector/filesystem.py

@@ -7,15 +7,11 @@
 import os.path
 
 import gocept.cache.property
-import grokcore.component as grok
 import lxml.etree
-import sqlalchemy
 import zope.app.file.image
 import zope.interface
 
-from zeit.cms.content.sources import FEATURE_TOGGLES
 from zeit.connector.interfaces import ID_NAMESPACE, CannonicalId
-from zeit.connector.models import ContentWithMetadataColumns as Content
 import zeit.cms.config
 import zeit.cms.content.dav
 import zeit.connector.interfaces
@@ -255,24 +251,13 @@ def _get_properties(self, id):
             return properties
 
         properties.update(parse_properties(xml))
-        self._convert_sql_types(properties)
 
         if zeit.connector.interfaces.RESOURCE_TYPE_PROPERTY not in properties:
             properties[zeit.connector.interfaces.RESOURCE_TYPE_PROPERTY] = self._guess_type(id)
 
         self.property_cache[id] = properties
         return properties
 
-    def _convert_sql_types(self, properties):
-        if not FEATURE_TOGGLES.find('read_metadata_columns'):
-            return
-        for key, value in properties.items():
-            column = Content.column_by_name(*key)
-            if column is None:
-                continue
-            converter = IConverter(column)
-            properties[key] = converter.deserialize(value)
-
     def _guess_type(self, id):
         path = self._path(id)
         if os.path.isdir(path):
@@ -318,58 +303,3 @@ def parse_properties(xml):
         value += '</tag:rankedTags>'
         properties[('keywords', 'http://namespaces.zeit.de/CMS/tagging')] = value
     return properties
-
-
-class IConverter(zope.interface.Interface):
-    def serialize(value):
-        pass
-
-    def deserialize(value):
-        pass
-
-
-@grok.implementer(IConverter)
-@grok.adapter(sqlalchemy.Column)
-def converter_from_column_type(column):
-    return IConverter(column.type)
-
-
-@grok.implementer(IConverter)
-class DefaultConverter(grok.Adapter):
-    grok.context(zope.interface.Interface)
-
-    def serialize(self, value):
-        return value
-
-    def deserialize(self, value):
-        return value
-
-
-class BoolConverter(DefaultConverter):
-    grok.context(sqlalchemy.Boolean)
-
-    def serialize(self, value):
-        return zeit.cms.content.dav.BoolProperty._toProperty(value)
-
-    def deserialize(self, value):
-        return zeit.cms.content.dav.BoolProperty._fromProperty(value)
-
-
-class IntConverter(DefaultConverter):
-    grok.context(sqlalchemy.Integer)
-
-    def serialize(self, value):
-        return str(value)
-
-    def deserialize(self, value):
-        return int(value)
-
-
-class DatetimeConverter(DefaultConverter):
-    grok.context(sqlalchemy.TIMESTAMP)
-
-    def serialize(self, value):
-        return zeit.cms.content.dav.DatetimeProperty._toProperty(value)
-
-    def deserialize(self, value):
-        return zeit.cms.content.dav.DatetimeProperty._fromProperty(value)

core/src/zeit/connector/interfaces.py

@@ -354,6 +354,16 @@ class IResourceInvalidatedEvent(zope.interface.Interface):
     id = zope.interface.Attribute('Unique id of resource')
 
 
+class IConverter(zope.interface.Interface):
+    """Converts webdav values to and from the postgresql database."""
+
+    def serialize(value):
+        pass
+
+    def deserialize(value):
+        pass
+
+
 @zope.interface.implementer(IResourceInvalidatedEvent)
 class ResourceInvalidatedEvent:
     def __init__(self, id):

core/src/zeit/connector/migrations/postdeploy/20240912_1058-6cc99f5afdc5_add_channels_columns_.py

@@ -0,0 +1,34 @@
+"""add channels columns index
+
+Revision ID: 6cc99f5afdc5
+Revises: cf24009572b7
+Create Date: 2024-09-12 10:58:00.181930
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision: str = '6cc99f5afdc5'
+down_revision: Union[str, None] = 'cf24009572b7'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    with op.get_context().autocommit_block():
+        op.create_index(
+            'ix_properties_channels',
+            'properties',
+            ['channels'],
+            unique=False,
+            postgresql_using='gin',
+            postgresql_concurrently=True,
+            if_not_exists=True,
+        )
+
+
+def downgrade() -> None:
+    op.drop_index('ix_properties_channels', table_name='properties', postgresql_using='gin')

core/src/zeit/connector/migrations/predeploy/20240912_1056-9aba9394d011_add_channels_columns.py

@@ -0,0 +1,27 @@
+"""add channels columns
+
+Revision ID: 9aba9394d011
+Revises: 5f2720a9a131
+Create Date: 2024-09-12 10:56:52.266201
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+from sqlalchemy.dialects.postgresql import JSONB
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = '9aba9394d011'
+down_revision: Union[str, None] = '5f2720a9a131'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.add_column('properties', sa.Column('channels', JSONB(astext_type=sa.Text()), nullable=True))
+
+
+def downgrade() -> None:
+    op.drop_column('properties', 'channels')

core/src/zeit/connector/mock.py

@@ -14,8 +14,6 @@
 import sqlalchemy
 import zope.event
 
-from zeit.cms.content.sources import FEATURE_TOGGLES
-from zeit.connector.filesystem import DefaultConverter
 from zeit.connector.interfaces import (
     ID_NAMESPACE,
     UUID_PROPERTY,
@@ -370,7 +368,6 @@ def _get_properties(self, id):
             properties = super()._get_properties(id)
         else:
             properties = properties.copy()
-            self._convert_sql_types(properties)
         return properties
 
     def _set_properties(self, id, properties):
@@ -386,14 +383,6 @@ def _set_properties(self, id, properties):
                 stored_properties.pop((name, namespace), None)
                 continue
 
-            if FEATURE_TOGGLES.find('write_metadata_columns'):
-                column = Content.column_by_name(name, namespace)
-                converter = zeit.connector.filesystem.IConverter(column)
-                value = converter.serialize(value)
-            else:
-                converter = DefaultConverter(None)
-            if isinstance(converter, DefaultConverter) and not isinstance(value, str):
-                raise ValueError('Expected str, got %s: %r' % (type(value), value))
             stored_properties[(name, namespace)] = value
         self._properties[id] = stored_properties
 

core/src/zeit/connector/models.py

@@ -16,6 +16,7 @@
 from sqlalchemy.orm import declared_attr, mapped_column, relationship
 import pytz
 import sqlalchemy
+import zope.component
 
 from zeit.cms.content.sources import FEATURE_TOGGLES
 from zeit.connector.interfaces import INTERNAL_PROPERTY, DeleteProperty, LockStatus
@@ -31,10 +32,23 @@ class Base(sqlalchemy.orm.DeclarativeBase):
 
 
 class CommonMetadata:
+    @staticmethod
+    def table_args(tablename):
+        return (Index(f'ix_{tablename}_channels', 'channels', postgresql_using='gin'),)
+
+    # converter, use name to lookup IConverter instead of type
+    channels = mapped_column(
+        JSONB,
+        nullable=True,
+        info={'namespace': 'document', 'name': 'channels', 'converter': 'channels'},
+    )
+
+
+class DevelopmentCommonMetadata:
     access = mapped_column(Unicode, index=True, info={'namespace': 'document', 'name': 'access'})
 
 
-class ZeitWeb:
+class DevelopmentZeitWeb:
     overscrolling_enabled = mapped_column(
         Boolean, info={'namespace': 'document', 'name': 'overscrolling'}
     )
@@ -120,6 +134,17 @@ def lock_status(self):
 
     NS = 'http://namespaces.zeit.de/CMS/'
 
+    @staticmethod
+    def converter(column):
+        if 'converter' in column.info:
+            return zope.component.queryAdapter(
+                column.type,
+                zeit.connector.interfaces.IConverter,
+                column.info['converter'],
+            )
+        else:
+            return zeit.connector.interfaces.IConverter(column)
+
     def to_webdav(self):
         if self.unsorted is None:
             return {}
@@ -137,7 +162,9 @@ def to_webdav(self):
         if FEATURE_TOGGLES.find('read_metadata_columns'):
             for column in self._columns_with_name():
                 namespace, name = column.info['namespace'], column.info['name']
-                props[(name, self.NS + namespace)] = getattr(self, column.name)
+                value = getattr(self, column.name)
+                converter = self.converter(column)
+                props[(name, self.NS + namespace)] = converter.serialize(value)
 
         if self.lock:
             props[('lock_principal', INTERNAL_PROPERTY)] = self.lock.principal
@@ -165,7 +192,8 @@ def from_webdav(self, props):
                 namespace, name = column.info['namespace'], column.info['name']
                 value = props.get((name, self.NS + namespace), self)
                 if value is not self:
-                    setattr(self, column.name, value)
+                    converter = self.converter(column)
+                    setattr(self, column.name, converter.deserialize(value))
 
         unsorted = collections.defaultdict(dict)
         for (k, ns), v in props.items():
@@ -218,19 +246,26 @@ def status(self):
             return LockStatus.FOREIGN
 
 
-class Content(Base, ContentBase):
+class Content(Base, ContentBase, CommonMetadata):
     lock_class = 'Lock'
 
+    @declared_attr.directive
+    def __table_args__(cls):
+        """every new inheritance level needs to re-apply the table_args"""
+        return super().__table_args__ + CommonMetadata.table_args(cls.__tablename__)
+
 
 class Lock(Base, LockBase):
     content_class = 'Content'
 
 
 class DevelopmentBase(sqlalchemy.orm.DeclarativeBase):
-    pass
+    """Experimental development features, not ready for any deployment or migration!"""
 
 
-class ContentWithMetadataColumns(DevelopmentBase, ContentBase, CommonMetadata, ZeitWeb):
+class ContentWithMetadataColumns(
+    DevelopmentBase, ContentBase, CommonMetadata, DevelopmentCommonMetadata, DevelopmentZeitWeb
+):
     lock_class = 'LockWithMetadataColumns'