Merge branch 'master' into stable/0.7.x

ChangeLog

@@ -1,3 +1,18 @@
+Version 0.7.5
+https://github.com/edgewall/genshi/releases/tag/0.7.5
+(Nov 18 2020, from branches/stable/0.7.x)
+
+ * Fix handling of slices containing function call, variable name and attribute
+   lookup AST nodes in Python 3.9 in template scripts (template expressions
+   already correctly handled these cases). Thank you to Roger Leigh for
+   finding this issue and contributing the fix for it.
+ * C speedup module now available for Python >= 3.3. Support was added for
+   PEP 393 (flexible string representation). Thank you to Inada Naoki for
+   contributing this major enhancement.
+ * Remove the custom 2to3 fixers (no longer used since the removal of 2to3
+   in 0.7.4).
+
+
 Version 0.7.4
 https://github.com/edgewall/genshi/releases/tag/0.7.4
 (Nov 3 2020, from branches/stable/0.7.x)

MANIFEST.in

@@ -1,12 +1,9 @@
 include ChangeLog
 include COPYING
 recursive-include genshi *.py *.c *.txt *.html
-recursive-include scripts *.py
-recursive-include fixes *.py
 recursive-include examples *
 recursive-include doc *.html *.css *.txt *.png *.gif *.py *.ini COPYING
 recursive-exclude doc/logo.lineform *
 exclude doc/2000ft.graffle
 global-exclude *.pyc
-include fixes/*.*
 recursive-include genshi/template/tests/templates *.html *.txt

genshi/_speedups.c

@@ -56,6 +56,8 @@ PyDoc_STRVAR(Markup__doc__,
 "Marks a string as being safe for inclusion in HTML/XML output without\n\
 needing to be escaped.");
 
+#if PY_VERSION_HEX < 0x03030000
+
 static PyObject *
 escape(PyObject *text, int quotes)
 {
@@ -169,6 +171,132 @@ escape(PyObject *text, int quotes)
     return ret;
 }
 
+#else /* if PY_VERSION_HEX < 0x03030000 */
+
+static PyObject *
+escape(PyObject *text, int quotes)
+{
+    PyObject *args, *ret;
+
+    if (PyObject_TypeCheck(text, &MarkupType)) {
+        Py_INCREF(text);
+        return text;
+    }
+    if (PyObject_HasAttrString(text, "__html__")) {
+        ret = PyObject_CallMethod(text, "__html__", NULL);
+        args = PyTuple_New(1);
+        if (args == NULL) {
+            Py_DECREF(ret);
+            return NULL;
+        }
+        PyTuple_SET_ITEM(args, 0, ret);
+        ret = MarkupType.tp_new(&MarkupType, args, NULL);
+        Py_DECREF(args);
+        return ret;
+    }
+
+    PyObject *in = PyObject_Str(text);
+    if (in == NULL) {
+        return NULL;
+    }
+    Py_ssize_t utf8_len;
+    const char *utf8 = PyUnicode_AsUTF8AndSize(in, &utf8_len);
+    if (utf8 == NULL) {
+        Py_DECREF(in);
+        return NULL;
+    }
+
+    /* First we need to figure out how long the escaped string will be */
+    Py_ssize_t len = 0, inn = 0;
+    for (Py_ssize_t i = 0; i < utf8_len; i++) {
+        switch (utf8[i]) {
+            case '&': len += 5; inn++;                                 break;
+            case '"': len += quotes ? 5 : 1; inn += quotes ? 1 : 0;    break;
+            case '<':
+            case '>': len += 4; inn++;                                 break;
+            default:  len++;
+        }
+    }
+
+    /* Do we need to escape anything at all? */
+    if (!inn) {
+        args = PyTuple_New(1);
+        if (args == NULL) {
+            Py_DECREF((PyObject *) in);
+            return NULL;
+        }
+        PyTuple_SET_ITEM(args, 0, in);
+        ret = MarkupType.tp_new(&MarkupType, args, NULL);
+        Py_DECREF(args);
+        return ret;
+    }
+
+    char *buffer = (char*)PyMem_Malloc(len);
+    if (buffer == NULL) {
+        Py_DECREF(in);
+        return NULL;
+    }
+
+    Py_ssize_t outn = 0;
+    const char *inp = utf8;
+    char *outp = buffer;
+    while (utf8_len > inp - utf8) {
+        if (outn == inn) {
+            /* copy rest of string if we have already replaced everything */
+            memcpy(outp, inp, utf8_len - (inp - utf8));
+            break;
+        }
+        switch (*inp) {
+            case '&':
+                memcpy(outp, "&amp;", 5);
+                outp += 5;
+                outn++;
+                break;
+            case '"':
+                if (quotes) {
+                    memcpy(outp, "&#34;", 5);
+                    outp += 5;
+                    outn++;
+                } else {
+                    *outp++ = *inp;
+                }
+                break;
+            case '<':
+                memcpy(outp, "&lt;", 4);
+                outp += 4;
+                outn++;
+                break;
+            case '>':
+                memcpy(outp, "&gt;", 4);
+                outp += 4;
+                outn++;
+                break;
+            default:
+                *outp++ = *inp;
+        }
+        inp++;
+    }
+
+    Py_DECREF(in);
+
+    PyObject *out = PyUnicode_FromStringAndSize(buffer, len);
+    PyMem_Free(buffer);
+    if (out == NULL) {
+        return NULL;
+    }
+    args = PyTuple_New(1);
+    if (args == NULL) {
+        Py_DECREF(out);
+        return NULL;
+    }
+    PyTuple_SET_ITEM(args, 0, out);
+    ret = MarkupType.tp_new(&MarkupType, args, NULL);
+    Py_DECREF(args);
+    return ret;
+}
+
+#endif /* if PY_VERSION_HEX < 0x03030000 */
+
 PyDoc_STRVAR(escape__doc__,
 "Create a Markup instance from a string and escape special characters\n\
 it may contain (<, >, & and \").\n\

genshi/filters/i18n.py

@@ -346,7 +346,7 @@ def __init__(self, value, template=None, namespaces=None, lineno=-1,
     def attach(cls, template, stream, value, namespaces, pos):
         if type(value) is dict:
             numeral = value.get('numeral', '').strip()
-            assert numeral is not '', "at least pass the numeral param"
+            assert numeral != '', "at least pass the numeral param"
             params = [v.strip() for v in value.get('params', '').split(',')]
             value = '%s; ' % numeral + ', '.join(params)
         return super(ChooseDirective, cls).attach(template, stream, value,

genshi/template/astutil.py

@@ -734,17 +734,13 @@ def _process_slice(node):
                     self.visit(node.step)
             elif isinstance(node, _ast.Index):
                 self.visit(node.value)
-            elif isinstance(node, _ast_Constant):
-                self.visit_Constant(node)
-            elif isinstance(node, _ast.UnaryOp):
-                self.visit_UnaryOp(node)
             elif isinstance(node, _ast.ExtSlice):
                 self.visit(node.dims[0])
                 for dim in node.dims[1:]:
                     self._write(', ')
                     self.visit(dim)
             else:
-                raise NotImplementedError('Slice type not implemented')
+                self.visit(node)
         _process_slice(node.slice)
         self._write(']')
 

genshi/template/tests/eval.py

@@ -360,6 +360,31 @@ def test_slice_negative_end(self):
         res = expr.evaluate({'numbers': list(range(5))})
         self.assertEqual([0, 1, 2, 3], res)
 
+    def test_slice_constant(self):
+        expr = Expression("numbers[1]")
+        res = expr.evaluate({"numbers": list(range(5))})
+        self.assertEqual(res, 1)
+
+    def test_slice_call(self):
+        def f():
+            return 2
+        expr = Expression("numbers[f()]")
+        res = expr.evaluate({"numbers": list(range(5)), "f": f})
+        self.assertEqual(res, 2)
+
+    def test_slice_name(self):
+        expr = Expression("numbers[v]")
+        res = expr.evaluate({"numbers": list(range(5)), "v": 2})
+        self.assertEqual(res, 2)
+
+    def test_slice_attribute(self):
+        class ValueHolder:
+            def __init__(self):
+                self.value = 3
+        expr = Expression("numbers[obj.value]")
+        res = expr.evaluate({"numbers": list(range(5)), "obj": ValueHolder()})
+        self.assertEqual(res, 3)
+
     def test_access_undefined(self):
         expr = Expression("nothing", filename='index.html', lineno=50,
                           lookup='lenient')
@@ -941,6 +966,71 @@ def test_with_statement_with_multiple_items(self):
             finally:
                 os.remove(path)
 
+    def test_slice(self):
+        suite = Suite("x = numbers[0:2]")
+        data = {"numbers": [0, 1, 2, 3]}
+        suite.execute(data)
+        self.assertEqual([0, 1], data["x"])
+
+    def test_slice_with_vars(self):
+        suite = Suite("x = numbers[start:end]")
+        data = {"numbers": [0, 1, 2, 3], "start": 0, "end": 2}
+        suite.execute(data)
+        self.assertEqual([0, 1], data["x"])
+
+    def test_slice_copy(self):
+        suite = Suite("x = numbers[:]")
+        data = {"numbers": [0, 1, 2, 3]}
+        suite.execute(data)
+        self.assertEqual([0, 1, 2, 3], data["x"])
+
+    def test_slice_stride(self):
+        suite = Suite("x = numbers[::stride]")
+        data = {"numbers": [0, 1, 2, 3, 4], "stride": 2}
+        suite.execute(data)
+        self.assertEqual([0, 2, 4], data["x"])
+
+    def test_slice_negative_start(self):
+        suite = Suite("x = numbers[-1:]")
+        data = {"numbers": [0, 1, 2, 3, 4], "stride": 2}
+        suite.execute(data)
+        self.assertEqual([4], data["x"])
+
+    def test_slice_negative_end(self):
+        suite = Suite("x = numbers[:-1]")
+        data = {"numbers": [0, 1, 2, 3, 4], "stride": 2}
+        suite.execute(data)
+        self.assertEqual([0, 1, 2, 3], data["x"])
+
+    def test_slice_constant(self):
+        suite = Suite("x = numbers[1]")
+        data = {"numbers": [0, 1, 2, 3, 4]}
+        suite.execute(data)
+        self.assertEqual(1, data["x"])
+
+    def test_slice_call(self):
+        def f():
+            return 2
+        suite = Suite("x = numbers[f()]")
+        data = {"numbers": [0, 1, 2, 3, 4], "f": f}
+        suite.execute(data)
+        self.assertEqual(2, data["x"])
+
+    def test_slice_name(self):
+        suite = Suite("x = numbers[v]")
+        data = {"numbers": [0, 1, 2, 3, 4], "v": 2}
+        suite.execute(data)
+        self.assertEqual(2, data["x"])
+
+    def test_slice_attribute(self):
+        class ValueHolder:
+            def __init__(self):
+                self.value = 3
+        suite = Suite("x = numbers[obj.value]")
+        data = {"numbers": [0, 1, 2, 3, 4], "obj": ValueHolder()}
+        suite.execute(data)
+        self.assertEqual(3, data["x"])
+
 
 def suite():
     suite = unittest.TestSuite()

setup.py

@@ -71,7 +71,7 @@ def _unavailable(self, exc):
     # - CPython >= 3.3 (the new Unicode C API is not supported yet)
     speedups = Feature(
         "optional C speed-enhancements",
-        standard = not is_pypy and sys.version_info < (3, 3),
+        standard = not is_pypy,
         ext_modules = [
             Extension('genshi._speedups', ['genshi/_speedups.c']),
         ],