pypy support: fix a cpyext incompatibility in chunkers/reader, adjust test expectations, see #1755

src/borg/chunkers/reader.pyx

@@ -8,6 +8,8 @@ import stat
 import time
 from collections import namedtuple
 
+from libc.string cimport memcpy
+
 from ..platform import safe_fadvise
 from ..constants import CH_DATA, CH_ALLOC, CH_HOLE, zeros
 
@@ -16,9 +18,14 @@ from ..constants import CH_DATA, CH_ALLOC, CH_HOLE, zeros
 # because the FS also needs to support this.
 has_seek_hole = hasattr(os, 'SEEK_DATA') and hasattr(os, 'SEEK_HOLE')
 
-# os.readv is POSIX; on platforms without it (win32) we fall back to os.read + copy.
+# os.readv is POSIX; on platforms without it (win32, pypy) we fall back to os.read + copy.
 has_readv = hasattr(os, 'readv')
 
+# Upper bound for a single os.read in that fallback: os.read allocates a buffer of the
+# requested size (pypy also zeroes it), no matter how few bytes it then returns. Asking for
+# the whole free scan buffer would thus cost MBs of allocation per small file, see #1755.
+READ_FALLBACK_SIZE = 256 * 1024
+
 _Chunk = namedtuple('_Chunk', 'meta data')
 _Chunk.__doc__ = """\
     Chunk namedtuple
@@ -59,6 +66,18 @@ def release_chunk_data(data):
         data.release()
 
 
+cdef _copy_into(target, Py_ssize_t t_offset, source, Py_ssize_t s_offset, Py_ssize_t count):
+    """memcpy count bytes from source[s_offset:] into target[t_offset:].
+
+    We do not use memoryview slice assignment here: pypy's cpyext memoryview does not
+    support it, and one memcpy is cheaper than creating the slice objects anyway.
+    """
+    cdef unsigned char[::1] dst = target
+    cdef const unsigned char[::1] src = source
+    if count:
+        memcpy(&dst[t_offset], &src[s_offset], count)
+
+
 def dread(offset, size, fd=None, fh=-1):
     use_fh = fh >= 0
     if use_fh:
@@ -391,9 +410,11 @@ class FileReader:
                 if has_readv:
                     got = os.readv(self.fh, [tv[pos:size]])
                 else:
-                    data = os.read(self.fh, size - pos)
+                    # os.read allocates a buffer of the requested size, so ask for the
+                    # block size rather than for the whole free scan buffer, see #1755.
+                    data = os.read(self.fh, min(size - pos, READ_FALLBACK_SIZE))
                     got = len(data)
-                    tv[pos:pos + got] = data
+                    _copy_into(tv, pos, data, 0, got)
                 if got > 0:
                     safe_fadvise(self.fh, self.direct_offset, got, "DONTNEED")
             else:
@@ -403,7 +424,7 @@ class FileReader:
                     # file-like object without readinto: fall back to read + copy
                     data = self.fd.read(size - pos)
                     got = len(data)
-                    tv[pos:pos + got] = data
+                    _copy_into(tv, pos, data, 0, got)
             if not got:
                 break  # EOF
             pos += got
@@ -470,12 +491,11 @@ class FileReader:
 
                 if allocation == CH_DATA:
                     assert data is not None
-                    # one memcpy: block -> target (the source slice is a view, not a copy)
-                    with memoryview(data) as dv:
-                        tv[bytes_read:bytes_read + to_read] = dv[self.offset:self.offset + to_read]
+                    # one memcpy: block -> target
+                    _copy_into(tv, bytes_read, data, self.offset, to_read)
                 else:
                     # holes / all-zero blocks: write zeros (target may contain stale data)
-                    tv[bytes_read:bytes_read + to_read] = zeros[:to_read]
+                    _copy_into(tv, bytes_read, zeros, 0, to_read)
 
                 bytes_read += to_read
 

src/borg/platformflags.py

@@ -20,3 +20,6 @@
 
 # MSYS2 (on Windows)
 is_msystem = is_win32 and "MSYSTEM" in os.environ
+
+# Python implementation
+is_pypy = sys.implementation.name == "pypy"

src/borg/testsuite/archiver/lock_cmds_test.py

@@ -28,13 +28,14 @@ def test_with_lock(tmp_path):
     print("sys.path: %r" % sys.path)
     print("PYTHONPATH: %s" % env.get("PYTHONPATH", ""))
     print("PATH: %s" % env.get("PATH", ""))
-    command0 = "python3", "-m", "borg", "repo-create", "--encryption=none-sha256"
+    python = sys.executable or "python3"
+    command0 = python, "-m", "borg", "repo-create", "--encryption=none-sha256"
     # Timings must be adjusted so that command1 keeps running while command2 tries to get the lock,
     # so that lock acquisition for command2 fails as the test expects it.
     lock_wait = 2
-    command1 = ("python3", "-c", 'import sys; print("first command - acquires the lock", flush=True); sys.stdin.read()')
-    command2 = "python3", "-c", 'print("second command - should never get executed")'
-    borgwl = "python3", "-m", "borg", "with-lock", f"--lock-wait={lock_wait}"
+    command1 = (python, "-c", 'import sys; print("first command - acquires the lock", flush=True); sys.stdin.read()')
+    command2 = python, "-c", 'print("second command - should never get executed")'
+    borgwl = python, "-m", "borg", "with-lock", f"--lock-wait={lock_wait}"
     popen_options = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env)
     subprocess.run(command0, env=env, check=True, text=True, capture_output=True)
     assert repo_path.exists()
@@ -50,7 +51,8 @@ def test_with_lock(tmp_path):
             assert "Failed to create/acquire the lock" in err_out
             assert p2.returncode == 73  # LockTimeout: could not acquire the lock, p1 already has it
         out, err_out = p1.communicate(input="")  # Unblock command1 and read output
-        assert not err_out
+        # ignore the pure-python msgpack warning borg emits on pypy
+        assert not [line for line in err_out.splitlines() if "pure-python msgpack" not in line]
         assert p1.returncode == 0
 
 

src/borg/testsuite/helpers/msgpack_test.py

@@ -3,6 +3,7 @@
 
 from ...helpers.msgpack import is_slow_msgpack
 from ...platform import is_cygwin
+from ...platformflags import is_pypy
 
 
 def expected_py_mp_slow_combination():
@@ -13,6 +14,9 @@ def expected_py_mp_slow_combination():
     # msgpack is slow on Cygwin
     if is_cygwin:
         return True
+    # pypy only has the pure-python msgpack (which pypy's jit hopefully makes fast enough)
+    if is_pypy:
+        return True
     # msgpack < 1.0.6 did not have Python 3.12 wheels
     if sys.version_info[:2] == (3, 12) and msgpack.version < (1, 0, 6):
         return True

src/borg/testsuite/item_test.py

@@ -4,6 +4,7 @@
 from ..item import Item, chunks_contents_equal
 from ..helpers import StableDict
 from ..helpers.msgpack import Timestamp
+from ..platformflags import is_pypy
 
 
 def test_item_empty():
@@ -131,6 +132,7 @@ def test_item_dict_property():
     assert item.as_dict() == {"xattrs": {"foo": "bar", "bar": "baz"}}
 
 
+@pytest.mark.xfail(is_pypy, reason="setting undeclared attributes on cdef class instances is not blocked on pypy")
 def test_unknown_property():
     # We do not want the user to be able to set unknown attributes —
     # they will not appear in the .as_dict() result dictionary.