Merge pull request #626 from ZeitOnline/ZO-4627

ZO-4627: Use lxml.etree for content objects instead of lxml.objectify

core/docs/changelog/ZO-4627.change

@@ -0,0 +1 @@
+ZO-4627: Replace lxml.objectify with plain lxml.etree usage

core/pyproject.toml

@@ -14,7 +14,6 @@ dependencies = [
     "filetype",
     "gocept.cache >= 2.1",
     "gocept.form[formlib]>=0.7.5",  # XXX Should be [ui], but is entrenched
-    "gocept.lxml>=0.2.1",
     "gocept.runner>0.5.3",
     "google-cloud-storage>=2.1.0.dev0",
     "grokcore.component",
@@ -208,7 +207,6 @@ deploy = [
 
 zon = [
     "gocept.form==0.8.0+py3",
-    "gocept.lxml==0.3.0+lxml5",  # https://github.com/ZeitOnline/gocept.lxml/tree/py3
     "zope.app.locking==3.5.0+py3.1",
     "zope.xmlpickle==4.0.0+py3k1",  # https://github.com/ZeitOnline/zope.xmlpickle/tree/py3
     # We created our own py310 wheels on devpi.zeit.de for these:

core/src/zeit/campus/tests/test_article.py

@@ -24,4 +24,4 @@ def test_topic_should_generate_proper_xml(self):
             )
             == 1
         )
-        assert tplink.xml.xpath('//head/topic/label')[0] == 'Moep'
+        assert tplink.xml.xpath('//head/topic/label')[0].text == 'Moep'

core/src/zeit/cms/application.zcml

@@ -16,7 +16,6 @@
     <include package="zope.securitypolicy" />
     <include package="zope.traversing" />
 
-    <include package="gocept.lxml" />
     <include package="zc.sourcefactory" />
 
     <include package="zope.app.appsetup" />

core/src/zeit/cms/checkout/tests/test_webhook.py

@@ -1,7 +1,7 @@
 from unittest import mock
 
 import celery.exceptions
-import lxml.objectify
+import lxml.etree
 import plone.testing
 import requests.exceptions
 
@@ -33,7 +33,7 @@ def setUp(self):
         )
         self.patch = mock.patch(
             'zeit.cms.checkout.webhook.HookSource._get_tree',
-            side_effect=lambda: lxml.objectify.fromstring(self.config),
+            side_effect=lambda: lxml.etree.fromstring(self.config),
         )
         self.patch.start()
         source = zeit.cms.checkout.webhook.HOOKS.factory

core/src/zeit/cms/configure.zcml

@@ -55,11 +55,13 @@
 
   <!-- lxml security settings -->
 
-  <class class="lxml.objectify.ObjectifiedElement">
+  <class class="lxml.etree._Element">
+    <implements interface=".interfaces.IXMLElement" />
+
     <require permission="zope.View"
       attributes="
       __getitem__
-      countchildren
+      getchildren
       get
       find
       findall
@@ -80,16 +82,19 @@
       " />
   </class>
 
+  <!-- BBB -->
+  <class class="lxml.objectify.ObjectifiedElement">
+    <implements interface=".interfaces.IXMLElement" />
+    <require like_class="lxml.etree._Element" />
+  </class>
   <class class="lxml.objectify.StringElement">
-    <require like_class="lxml.objectify.ObjectifiedElement" />
+    <require like_class="lxml.etree._Element" />
   </class>
-
   <class class="lxml.objectify.IntElement">
-    <require like_class="lxml.objectify.ObjectifiedElement" />
+    <require like_class="lxml.etree._Element" />
   </class>
-
   <class class="lxml.objectify.NoneElement">
-    <require like_class="lxml.objectify.ObjectifiedElement" />
+    <require like_class="lxml.etree._Element" />
   </class>
 
   <class class="lxml.etree.ElementChildIterator">

core/src/zeit/cms/content/adapter.txt

@@ -9,9 +9,9 @@ The xml source adapter adapts IXMLRepresentation to IXMLSource. Let's make sure
 processing instructions which are not inside the document tree are preserverd
 Create a dummy object:
 
->>> import lxml.objectify
+>>> import lxml.etree
 >>> class XML:
-...     xml = lxml.objectify.fromstring('<a/><?foo?>')
+...     xml = lxml.etree.fromstring('<a/><?foo?>')
 
 
 Call the adapter factory -- the PI is still there. We also get the XML

core/src/zeit/cms/content/browser/widget.txt

@@ -19,11 +19,11 @@ Create a schema:
 Create a content object:
 
 
->>> import lxml.objectify
+>>> import lxml.etree
 >>> import zeit.cms.content.property
 >>> @zope.interface.implementer(IContent)
 ... class Content:
-...     xml = lxml.objectify.XML('<art/><?foo?>')
+...     xml = lxml.etree.fromstring('<art/><?foo?>')
 ...     snippet = zeit.cms.content.property.Structure('.title')
 >>> content = Content()
 
@@ -54,8 +54,8 @@ Editing sub-nodes
 The widget also supports editing subnodes. That is that the data being edited
 is not a full tree but a node in a tree.
 
->>> content.whole_tree = lxml.objectify.XML('<a><b/><editme><c/></editme></a>')
->>> content.xml = content.whole_tree.editme
+>>> content.whole_tree = lxml.etree.fromstring('<a><b/><editme><c/></editme></a>')
+>>> content.xml = content.whole_tree.find('editme')
 >>> widget.setRenderedValue(content.xml)
 >>> widget._getFormValue()
 '<editme>\r\n    <c/>\r\n</editme>\r\n'

core/src/zeit/cms/content/field.py

@@ -1,5 +1,4 @@
 import lxml.etree
-import lxml.objectify
 import zope.interface
 import zope.location.location
 import zope.proxy
@@ -8,7 +7,7 @@
 import zope.security.checker
 import zope.security.proxy
 
-from zeit.cms.content.util import objectify_soup_fromstring
+from zeit.cms.content.util import etree_soup_fromstring
 
 
 DEFAULT_MARKER = object()
@@ -25,9 +24,9 @@ class _XMLBase(zope.schema.Field):
     def __init__(self, *args, **kw):
         tidy_input = kw.pop('tidy_input', False)
         if tidy_input:
-            self.parse = objectify_soup_fromstring
+            self.parse = etree_soup_fromstring
         else:
-            self.parse = lxml.objectify.fromstring
+            self.parse = lxml.etree.fromstring
         super().__init__(*args, **kw)
 
     def fromUnicode(self, text):

core/src/zeit/cms/content/field.txt

@@ -5,6 +5,7 @@ Fields
 XML Tree
 ========
 
+>>> from lxml.builder import E
 >>> from zeit.cms.content.field import XMLTree
 >>> import zeit.cms.testing
 >>> field = XMLTree()
@@ -18,8 +19,8 @@ XML Tree
 True
 
 >>> content2 = Content()
->>> content.xml['child'] = 'child'
->>> content2.xml = content.xml['child']
+>>> content.xml.append(E.child('child'))
+>>> content2.xml = content.xml.find('child')
 >>> tree2 = field.fromUnicode('<child>MyNewValue</child>')
 >>> field.set(content2, tree2)
 >>> print(zeit.cms.testing.xmltotext(content.xml))
@@ -31,15 +32,15 @@ True
 Replacing node when there are siblings present
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
->>> import lxml.objectify
->>> root = lxml.objectify.E.root()
->>> root.append(lxml.objectify.E.child())
->>> root.append(lxml.objectify.E.child())
+>>> import lxml.builder
+>>> root = lxml.builder.E.root()
+>>> root.append(E.child())
+>>> root.append(E.child())
 >>> content = Content()
 >>> field = XMLTree()
 >>> field.__name__ = 'xml'
->>> field.set(content, root.child)
->>> field.set(content, lxml.objectify.E.new())
+>>> field.set(content, root.find('child'))
+>>> field.set(content, E.new())
 >>> print(zeit.cms.testing.xmltotext(root))
 <root...>
   <new/>
@@ -55,4 +56,4 @@ Tidying broken input
 >>> tree = field.fromUnicode(
 ...     '<a href="http://www.youtube.com/v/oIr8-f2OWhs&hl=en_US&fs=1&">')
 >>> print(zeit.cms.testing.xmltotext(tree))
-<a href="http://www.youtube.com/v/oIr8-f2OWhs&amp;hl=en_US&amp;fs=1&amp;"/>
+<a href="http://www.youtube.com/v/oIr8-f2OWhs&amp;hl=en_US&amp;fs=1&amp;"/>

core/src/zeit/cms/content/interfaces.py

@@ -374,7 +374,7 @@ class IXMLReference(zope.interface.Interface):
     might be references inside the <head> that always use a <reference> tag.
     (NOTE: These are just examples, not actual zeit.cms policy!)
 
-    Adapting to IXMLReference yields an lxml.objectify tree::
+    Adapting to IXMLReference yields an lxml.etree::
 
         node = zope.component.getAdapter(
            content, zeit.cms.content.interfaces.IXMLReference, name='image')
@@ -389,7 +389,7 @@ class IXMLReferenceUpdater(zope.interface.Interface):
     def update(xml_node, suppress_errors=False):
         """Update xml_node with data from the content object.
 
-        xml_node: lxml.objectify'ed element
+        xml_node: lxml.etree.Element
         """
 
 

core/src/zeit/cms/content/lxmlpickle.py

@@ -8,26 +8,30 @@
 log = logging.getLogger(__name__)
 
 
-def treeFactory(state):
-    """Un-Pickle factory."""
+def deserialize(state):
     try:
-        return lxml.objectify.fromstring(state)
+        return lxml.etree.fromstring(state)
     except Exception as e:
         log.error('Error during unpickling', exc_info=True)
-        return lxml.objectify.fromstring(
-            '<error><!-- XML-FEHLER: %s\n\n%s\n\n--></error>' % (e, state)
-        )
+        return lxml.etree.fromstring('<error><!-- XML-FEHLER: %s\n\n%s\n\n--></error>' % (e, state))
 
 
-copyreg.constructor(treeFactory)
+copyreg.constructor(deserialize)
 
 
-def reduceObjectifiedElement(object):
-    """Reduce function for lxml.objectify trees.
+def serialize(obj):
+    """Reduce function for lxml trees.
     See http://docs.python.org/lib/pickle-protocol.html for details.
     """
-    state = lxml.etree.tostring(object.getroottree())
-    return (treeFactory, (state,))
+    state = lxml.etree.tostring(obj.getroottree())
+    return (
+        deserialize,
+        (state,),
+    )
 
 
-copyreg.pickle(lxml.objectify.ObjectifiedElement, reduceObjectifiedElement, treeFactory)
+copyreg.pickle(lxml.etree._Element, serialize)
+
+# BBB
+treeFactory = deserialize
+copyreg.pickle(lxml.objectify.ObjectifiedElement, serialize)

core/src/zeit/cms/content/lxmlpickle.txt

@@ -5,12 +5,12 @@ lxml pickle support
 Verify the lxml pickle support.
 
 >>> import pickle
->>> import lxml.objectify
+>>> import lxml.etree
 >>> import zeit.cms.content.lxmlpickle
->>> xml = lxml.objectify.fromstring('<foo><b>zoot</b></foo><?bar?>')
+>>> xml = lxml.etree.fromstring('<foo><b>zoot</b></foo><?bar?>')
 >>> p = pickle.dumps(xml)
 >>> restored_xml = pickle.loads(p)
->>> print(zeit.cms.testing.xmltotext(restored_xml.getroottree()))
+>>> print(lxml.etree.tostring(restored_xml.getroottree(), pretty_print=True, encoding=str))
 <foo>
   <b>zoot</b>
 </foo>