diff --git a/src/borg/chunkers/reader.pyx b/src/borg/chunkers/reader.pyx index dccfbc9129..78d4f19186 100644 --- a/src/borg/chunkers/reader.pyx +++ b/src/borg/chunkers/reader.pyx @@ -8,6 +8,8 @@ import stat import time from collections import namedtuple +from libc.string cimport memcpy + from ..platform import safe_fadvise from ..constants import CH_DATA, CH_ALLOC, CH_HOLE, zeros @@ -16,9 +18,14 @@ from ..constants import CH_DATA, CH_ALLOC, CH_HOLE, zeros # because the FS also needs to support this. has_seek_hole = hasattr(os, 'SEEK_DATA') and hasattr(os, 'SEEK_HOLE') -# os.readv is POSIX; on platforms without it (win32) we fall back to os.read + copy. +# os.readv is POSIX; on platforms without it (win32, pypy) we fall back to os.read + copy. has_readv = hasattr(os, 'readv') +# Upper bound for a single os.read in that fallback: os.read allocates a buffer of the +# requested size (pypy also zeroes it), no matter how few bytes it then returns. Asking for +# the whole free scan buffer would thus cost MBs of allocation per small file, see #1755. +READ_FALLBACK_SIZE = 256 * 1024 + _Chunk = namedtuple('_Chunk', 'meta data') _Chunk.__doc__ = """\ Chunk namedtuple @@ -59,6 +66,18 @@ def release_chunk_data(data): data.release() +cdef _copy_into(target, Py_ssize_t t_offset, source, Py_ssize_t s_offset, Py_ssize_t count): + """memcpy count bytes from source[s_offset:] into target[t_offset:]. + + We do not use memoryview slice assignment here: pypy's cpyext memoryview does not + support it, and one memcpy is cheaper than creating the slice objects anyway. + """ + cdef unsigned char[::1] dst = target + cdef const unsigned char[::1] src = source + if count: + memcpy(&dst[t_offset], &src[s_offset], count) + + def dread(offset, size, fd=None, fh=-1): use_fh = fh >= 0 if use_fh: @@ -391,9 +410,11 @@ class FileReader: if has_readv: got = os.readv(self.fh, [tv[pos:size]]) else: - data = os.read(self.fh, size - pos) + # os.read allocates a buffer of the requested size, so ask for the + # block size rather than for the whole free scan buffer, see #1755. + data = os.read(self.fh, min(size - pos, READ_FALLBACK_SIZE)) got = len(data) - tv[pos:pos + got] = data + _copy_into(tv, pos, data, 0, got) if got > 0: safe_fadvise(self.fh, self.direct_offset, got, "DONTNEED") else: @@ -403,7 +424,7 @@ class FileReader: # file-like object without readinto: fall back to read + copy data = self.fd.read(size - pos) got = len(data) - tv[pos:pos + got] = data + _copy_into(tv, pos, data, 0, got) if not got: break # EOF pos += got @@ -470,12 +491,11 @@ class FileReader: if allocation == CH_DATA: assert data is not None - # one memcpy: block -> target (the source slice is a view, not a copy) - with memoryview(data) as dv: - tv[bytes_read:bytes_read + to_read] = dv[self.offset:self.offset + to_read] + # one memcpy: block -> target + _copy_into(tv, bytes_read, data, self.offset, to_read) else: # holes / all-zero blocks: write zeros (target may contain stale data) - tv[bytes_read:bytes_read + to_read] = zeros[:to_read] + _copy_into(tv, bytes_read, zeros, 0, to_read) bytes_read += to_read diff --git a/src/borg/platformflags.py b/src/borg/platformflags.py index c6f98270b7..92cc83f7b2 100644 --- a/src/borg/platformflags.py +++ b/src/borg/platformflags.py @@ -20,3 +20,6 @@ # MSYS2 (on Windows) is_msystem = is_win32 and "MSYSTEM" in os.environ + +# Python implementation +is_pypy = sys.implementation.name == "pypy" diff --git a/src/borg/testsuite/archiver/lock_cmds_test.py b/src/borg/testsuite/archiver/lock_cmds_test.py index 68a3e1f03d..935b9cc8ea 100644 --- a/src/borg/testsuite/archiver/lock_cmds_test.py +++ b/src/borg/testsuite/archiver/lock_cmds_test.py @@ -28,13 +28,14 @@ def test_with_lock(tmp_path): print("sys.path: %r" % sys.path) print("PYTHONPATH: %s" % env.get("PYTHONPATH", "")) print("PATH: %s" % env.get("PATH", "")) - command0 = "python3", "-m", "borg", "repo-create", "--encryption=none-sha256" + python = sys.executable or "python3" + command0 = python, "-m", "borg", "repo-create", "--encryption=none-sha256" # Timings must be adjusted so that command1 keeps running while command2 tries to get the lock, # so that lock acquisition for command2 fails as the test expects it. lock_wait = 2 - command1 = ("python3", "-c", 'import sys; print("first command - acquires the lock", flush=True); sys.stdin.read()') - command2 = "python3", "-c", 'print("second command - should never get executed")' - borgwl = "python3", "-m", "borg", "with-lock", f"--lock-wait={lock_wait}" + command1 = (python, "-c", 'import sys; print("first command - acquires the lock", flush=True); sys.stdin.read()') + command2 = python, "-c", 'print("second command - should never get executed")' + borgwl = python, "-m", "borg", "with-lock", f"--lock-wait={lock_wait}" popen_options = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env) subprocess.run(command0, env=env, check=True, text=True, capture_output=True) assert repo_path.exists() @@ -50,7 +51,8 @@ def test_with_lock(tmp_path): assert "Failed to create/acquire the lock" in err_out assert p2.returncode == 73 # LockTimeout: could not acquire the lock, p1 already has it out, err_out = p1.communicate(input="") # Unblock command1 and read output - assert not err_out + # ignore the pure-python msgpack warning borg emits on pypy + assert not [line for line in err_out.splitlines() if "pure-python msgpack" not in line] assert p1.returncode == 0 diff --git a/src/borg/testsuite/helpers/msgpack_test.py b/src/borg/testsuite/helpers/msgpack_test.py index 41960da784..37a64895d5 100644 --- a/src/borg/testsuite/helpers/msgpack_test.py +++ b/src/borg/testsuite/helpers/msgpack_test.py @@ -3,6 +3,7 @@ from ...helpers.msgpack import is_slow_msgpack from ...platform import is_cygwin +from ...platformflags import is_pypy def expected_py_mp_slow_combination(): @@ -13,6 +14,9 @@ def expected_py_mp_slow_combination(): # msgpack is slow on Cygwin if is_cygwin: return True + # pypy only has the pure-python msgpack (which pypy's jit hopefully makes fast enough) + if is_pypy: + return True # msgpack < 1.0.6 did not have Python 3.12 wheels if sys.version_info[:2] == (3, 12) and msgpack.version < (1, 0, 6): return True diff --git a/src/borg/testsuite/item_test.py b/src/borg/testsuite/item_test.py index c795430fb3..e220eb9b27 100644 --- a/src/borg/testsuite/item_test.py +++ b/src/borg/testsuite/item_test.py @@ -4,6 +4,7 @@ from ..item import Item, chunks_contents_equal from ..helpers import StableDict from ..helpers.msgpack import Timestamp +from ..platformflags import is_pypy def test_item_empty(): @@ -131,6 +132,7 @@ def test_item_dict_property(): assert item.as_dict() == {"xattrs": {"foo": "bar", "bar": "baz"}} +@pytest.mark.xfail(is_pypy, reason="setting undeclared attributes on cdef class instances is not blocked on pypy") def test_unknown_property(): # We do not want the user to be able to set unknown attributes — # they will not appear in the .as_dict() result dictionary.