diff --git a/docs/man/borg-create.1 b/docs/man/borg-create.1 index 7729fbbfad..26f9aa43c3 100644 --- a/docs/man/borg-create.1 +++ b/docs/man/borg-create.1 @@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "borg-create" "1" "2026-08-15" "" "borg backup tool" +.TH "borg-create" "1" "2026-08-17" "" "borg backup tool" .SH Name borg-create \- Creates a new archive. .SH SYNOPSIS @@ -320,6 +320,15 @@ open and read block and char device files as well as FIFOs as if they were regul .TP .BI \-\-read\-special\-timeout \ SECONDS when reading from FIFOs or character devices (see \-\-read\-special): skip the file with an error if no data arrives for more than SECONDS (this includes waiting for a FIFO\(aqs writer to connect). Give 0 to wait forever. default: 1800 seconds. +.TP +.BI \-\-map \ MAPFILE +give a map file describing the content ranges of the (single) input file, so borg does not need to read all of it. See the \fIInput maps\fP section below. +.TP +.BI \-\-reuse\-from \ ARCHIVE +reuse the chunks of this reference archive for the input map\(aqs \fBsame\fP ranges (requires \-\-map). See the \fIInput maps\fP section below. +.TP +.BI \-\-reuse\-path \ PATH +archive\-internal path of the reference item in the \-\-reuse\-from archive (only needed if that archive contains more than one file item). .UNINDENT .SS Archive options .INDENT 0.0 @@ -581,6 +590,53 @@ creation a bit. .sp By default, the content read from stdin is stored in a file called \(aqstdin\(aq. Use \fB\-\-stdin\-name\fP to change the name. +.SS Input maps +.sp +Usually, borg reads the complete input to determine its contents. If you already +know the contents of parts of the input from an external source of truth, you can +give that information via \fB\-\-map MAPFILE\fP and borg will not read the known parts. +The primary use case is backing up snapshots of large block devices (e.g. LVM thin +volumes), where the storage layer knows which ranges are in use. +.sp +\fB\-\-map\fP requires giving exactly one input path, which must be a regular file or +(with \fB\-\-read\-special\fP) a block device. +.sp +The map file must describe the whole input: one range per line, in the form +\fBSTART LENGTH STATE\fP (byte values, decimal or 0x\-prefixed hexadecimal). The +ranges must be sorted, non\-overlapping and contiguous, starting at offset 0 and +covering the exact input size. \fB#\fP starts a comment, empty lines are ignored. +STATE is one of: +.INDENT 0.0 +.IP \(bu 2 +\fBdata\fP: the range\(aqs contents are read and backed up. +.IP \(bu 2 +\fBzero\fP: the range is known to read as all\-zero bytes. borg stores a hole +(all\-zero range) of that size without reading the range. +.IP \(bu 2 +\fBsame\fP: the range is known to be identical to the same range of the input +backed up in the \fB\-\-reuse\-from REFARCHIVE\fP reference archive (usually: the +previous backup of an earlier snapshot of the same device). borg reuses the +reference archive\(aqs chunks for such ranges without reading them. This state +requires \fB\-\-reuse\-from\fP\&. +.UNINDENT +.sp +The reference archive must contain exactly one file item; if it contains more, +select the reference item with \fB\-\-reuse\-path PATH\fP (its archive\-internal path). +Reference chunks that only partially overlap \fBsame\fP ranges are re\-read from +the input, so any chunker gives correct results \- but a fixed block size chunker +(e.g. \fB\-\-chunker\-params fixed,4194304\fP, same parameters as used for the +reference archive) avoids re\-reading at the edges of changed ranges and gives +stable chunk boundaries across backups. +.sp +\fBThe map is trusted\fP: if it is wrong (e.g. a range marked \fBzero\fP actually +contains data, or a range marked \fBsame\fP actually changed), the archive will +not match the input and borg cannot detect that. Independently verify the +source producing the maps, and consider doing a periodic full read backup +(without \fB\-\-map\fP). +.sp +For LVM thin volume snapshots, maps can be generated from \fBthin_dump\fP / +\fBthin_delta\fP XML with the \fBscripts/lvm\-thin\-map.py\fP converter from the +borg sources; its docstring shows the complete workflow. .SS Feeding all file paths from externally .sp Usually, you give a starting path (recursion root) to borg and then borg diff --git a/docs/usage/create.rst.inc b/docs/usage/create.rst.inc index 268b6387f1..5380031c6f 100644 --- a/docs/usage/create.rst.inc +++ b/docs/usage/create.rst.inc @@ -99,6 +99,12 @@ borg create +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``--read-special-timeout SECONDS`` | when reading from FIFOs or character devices (see --read-special): skip the file with an error if no data arrives for more than SECONDS (this includes waiting for a FIFO's writer to connect). Give 0 to wait forever. default: 1800 seconds. | +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``--map MAPFILE`` | give a map file describing the content ranges of the (single) input file, so borg does not need to read all of it. See the *Input maps* section below. | + +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``--reuse-from ARCHIVE`` | reuse the chunks of this reference archive for the input map's ``same`` ranges (requires --map). See the *Input maps* section below. | + +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``--reuse-path PATH`` | archive-internal path of the reference item in the --reuse-from archive (only needed if that archive contains more than one file item). | + +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | **Archive options** | +-------------------------------------------------------+---------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``--comment COMMENT`` | add a comment text to the archive | @@ -173,6 +179,9 @@ borg create --files-changed MODE specify how to detect if a file has changed during backup (ctime, mtime, disabled). default: ctime (on Windows: mtime, because ctime is file creation time there). --read-special open and read block and char device files as well as FIFOs as if they were regular files. Also follows symlinks pointing to these kinds of files. --read-special-timeout SECONDS when reading from FIFOs or character devices (see --read-special): skip the file with an error if no data arrives for more than SECONDS (this includes waiting for a FIFO's writer to connect). Give 0 to wait forever. default: 1800 seconds. + --map MAPFILE give a map file describing the content ranges of the (single) input file, so borg does not need to read all of it. See the *Input maps* section below. + --reuse-from ARCHIVE reuse the chunks of this reference archive for the input map's ``same`` ranges (requires --map). See the *Input maps* section below. + --reuse-path PATH archive-internal path of the reference item in the --reuse-from archive (only needed if that archive contains more than one file item). Archive options @@ -445,6 +454,51 @@ creation a bit. By default, the content read from stdin is stored in a file called 'stdin'. Use ``--stdin-name`` to change the name. +Input maps +++++++++++ + +Usually, borg reads the complete input to determine its contents. If you already +know the contents of parts of the input from an external source of truth, you can +give that information via ``--map MAPFILE`` and borg will not read the known parts. +The primary use case is backing up snapshots of large block devices (e.g. LVM thin +volumes), where the storage layer knows which ranges are in use. + +``--map`` requires giving exactly one input path, which must be a regular file or +(with ``--read-special``) a block device. + +The map file must describe the whole input: one range per line, in the form +``START LENGTH STATE`` (byte values, decimal or 0x-prefixed hexadecimal). The +ranges must be sorted, non-overlapping and contiguous, starting at offset 0 and +covering the exact input size. ``#`` starts a comment, empty lines are ignored. +STATE is one of: + +- ``data``: the range's contents are read and backed up. +- ``zero``: the range is known to read as all-zero bytes. borg stores a hole + (all-zero range) of that size without reading the range. +- ``same``: the range is known to be identical to the same range of the input + backed up in the ``--reuse-from REFARCHIVE`` reference archive (usually: the + previous backup of an earlier snapshot of the same device). borg reuses the + reference archive's chunks for such ranges without reading them. This state + requires ``--reuse-from``. + +The reference archive must contain exactly one file item; if it contains more, +select the reference item with ``--reuse-path PATH`` (its archive-internal path). +Reference chunks that only partially overlap ``same`` ranges are re-read from +the input, so any chunker gives correct results - but a fixed block size chunker +(e.g. ``--chunker-params fixed,4194304``, same parameters as used for the +reference archive) avoids re-reading at the edges of changed ranges and gives +stable chunk boundaries across backups. + +**The map is trusted**: if it is wrong (e.g. a range marked ``zero`` actually +contains data, or a range marked ``same`` actually changed), the archive will +not match the input and borg cannot detect that. Independently verify the +source producing the maps, and consider doing a periodic full read backup +(without ``--map``). + +For LVM thin volume snapshots, maps can be generated from ``thin_dump`` / +``thin_delta`` XML with the ``scripts/lvm-thin-map.py`` converter from the +borg sources; its docstring shows the complete workflow. + Feeding all file paths from externally ++++++++++++++++++++++++++++++++++++++ diff --git a/docs/usage/notes.rst b/docs/usage/notes.rst index d6f846ff3c..622b5888a2 100644 --- a/docs/usage/notes.rst +++ b/docs/usage/notes.rst @@ -180,6 +180,26 @@ Now, let's see how to restore some LVs from such a backup. $ borg extract --stdout arch dev/vg0/root-snapshot > /dev/vg0/root $ borg extract --stdout arch dev/vg0/home-snapshot > /dev/vg0/home +Efficient backups of LVM thin volume snapshots +++++++++++++++++++++++++++++++++++++++++++++++ + +Backing up a block device as shown above reads the whole device every time. +For snapshots of LVM *thin* volumes, the thin pool's metadata already knows +which ranges are allocated and which ranges changed between two snapshots, +so most of the reading can be skipped: see the ``--map`` and ``--reuse-from`` +options of ``borg create`` (section *Input maps* in ``borg create --help``) +and the ``scripts/lvm-thin-map.py`` converter in the borg sources, which +turns ``thin_dump`` / ``thin_delta`` XML into borg input maps. The script's +docstring shows the complete workflow: an initial full backup using the +allocation map (unallocated ranges are stored as holes without reading them), +then incremental backups that only read the ranges that changed since the +previous snapshot, while reusing the previous archive's chunks for everything +else. Use ``--chunker-params fixed,4194304`` (or similar) for such backups. + +Note that borg trusts these maps - it cannot detect a wrong or stale map, so +keep the snapshot discipline described in the script's docstring and consider +doing a periodic full read backup (without ``--map``). + .. _separate_compaction: diff --git a/scripts/lvm-thin-map.py b/scripts/lvm-thin-map.py new file mode 100755 index 0000000000..b97d83dd89 --- /dev/null +++ b/scripts/lvm-thin-map.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +""" +Generate a borg input map (borg create --map) from LVM thin pool metadata, see #4363. + +This converts XML produced by the thin-provisioning-tools (thin_dump / thin_delta, +version >= 0.7.4 required - older thin_delta versions had bugs) into the map format +expected by ``borg create --map``, so borg only reads the parts of a thin LV snapshot +that actually contain (changed) data. + +full mode - initial (or periodic full-read) backup of a thin LV snapshot: + + lvcreate -s -n snap1 vg/lv # snapshot to back up (keep it for delta mode!) + lvchange -ay -Ky vg/snap1 + dmsetup message vg-pool-tpool 0 reserve_metadata_snap + thin_dump -m --dev-id $(lvs --noheadings -o thin_id vg/snap1) /dev/mapper/vg-pool_tmeta \ + | lvm-thin-map.py full --device /dev/vg/snap1 > snap1.map + dmsetup message vg-pool-tpool 0 release_metadata_snap + borg create --read-special --chunker-params fixed,4194304 \ + --map snap1.map lv-backup /dev/vg/snap1 + + Unallocated ranges read as zeros, so borg stores them as holes without reading them. + +delta mode - incremental backup against the previous snapshot's archive: + + lvcreate -s -n snap2 vg/lv + lvchange -ay -Ky vg/snap2 + dmsetup message vg-pool-tpool 0 reserve_metadata_snap + thin_delta -m --snap1 $(lvs --noheadings -o thin_id vg/snap1) \ + --snap2 $(lvs --noheadings -o thin_id vg/snap2) /dev/mapper/vg-pool_tmeta \ + | lvm-thin-map.py delta --device /dev/vg/snap2 > snap2.map + dmsetup message vg-pool-tpool 0 release_metadata_snap + borg create --read-special --chunker-params fixed,4194304 \ + --map snap2.map --reuse-from lv-backup lv-backup /dev/vg/snap2 + lvremove vg/snap1 # snap2 is the reference for the next delta + + Ranges that are identical in both snapshots become "same" ranges: borg reuses the + chunks of the --reuse-from reference archive for them, without reading the device. + +Notes: + +- The reference snapshot (--snap1) must be the snapshot that was backed up into the + --reuse-from archive - THIS IS NOT CHECKED and cannot be. A wrong pairing silently + produces an archive that does not match the device. +- Remember to release_metadata_snap; a leftover metadata snapshot blocks future reserves. +- The pool's chunk size (data_block_size) defines the map granularity. borg's fixed + chunker block size does not need to match it; ranges are given in exact bytes. +""" + +import argparse +import os +import sys +import xml.etree.ElementTree as ET + +DELTA_STATES = { + "same": "same", # unchanged between the two snapshots -> reuse reference chunks + "different": "data", # changed -> read + "right_only": "data", # newly allocated -> read + "left_only": "zero", # deallocated (discarded) -> reads as zeros now +} +SECTOR = 512 + + +def die(msg): + print(f"lvm-thin-map: error: {msg}", file=sys.stderr) + sys.exit(2) + + +def device_size(args): + if args.size is not None: + return args.size + fd = os.open(args.device, os.O_RDONLY) + try: + return os.lseek(fd, 0, os.SEEK_END) + finally: + os.close(fd) + + +def parse_ranges(xml_file, mode, meta): + """Yield (start_block, length_blocks, state) from thin_dump / thin_delta XML; fill *meta* in place.""" + inside_device = False + context = ET.iterparse(xml_file, events=("start", "end")) + for event, elem in context: + if event == "start": + if elem.tag == "superblock": + meta["data_block_size"] = int(elem.get("data_block_size")) * SECTOR + elif elem.tag == "diff": + meta["left"], meta["right"] = elem.get("left"), elem.get("right") + elif elem.tag == "device": + if "dev_id" in meta: + die("XML contains more than one , re-run thin_dump with --dev-id") + meta["dev_id"] = elem.get("dev_id") + inside_device = True + elif elem.tag == "range": + die("nested elements found - run thin_delta without --verbose") + continue + # end events + if elem.tag == "device": + inside_device = False + elif mode == "delta" and elem.tag in DELTA_STATES: + yield int(elem.get("begin")), int(elem.get("length")), DELTA_STATES[elem.tag] + elif mode == "full" and elem.tag in ("single_mapping", "range_mapping"): + if not inside_device: + die(f"<{elem.tag}> outside of a element - unsupported thin_dump output") + if elem.tag == "single_mapping": + yield int(elem.get("origin_block")), 1, "data" + else: + yield int(elem.get("origin_begin")), int(elem.get("length")), "data" + elem.clear() + + +def main(): + parser = argparse.ArgumentParser( + description="convert thin_dump/thin_delta XML into a borg input map (borg create --map)" + ) + parser.add_argument("mode", choices=("full", "delta"), help="full: thin_dump XML, delta: thin_delta XML") + parser.add_argument("xml", nargs="?", default="-", help="XML input file (default: stdin)") + size_group = parser.add_mutually_exclusive_group(required=True) + size_group.add_argument("--device", help="get the map's total size from this block device") + size_group.add_argument("--size", type=int, help="give the map's total size in bytes") + args = parser.parse_intermixed_args() + + size = device_size(args) + xml_file = sys.stdin.buffer if args.xml == "-" else args.xml + + out = [] # coalesced [start, end, state] ranges, in bytes + offset = 0 # next expected byte offset + meta = {} + + def emit(start, end, state): + nonlocal offset + if start < offset: + die(f"XML ranges overlap or are not sorted (at byte offset {start})") + if start > offset: + emit_range(offset, start, "zero") # gap: unallocated in all snapshots, reads as zeros + emit_range(start, end, state) + offset = end + + def emit_range(start, end, state): + if out and out[-1][2] == state and out[-1][1] == start: + out[-1][1] = end + else: + out.append([start, end, state]) + + for begin, length, state in parse_ranges(xml_file, args.mode, meta): + block_size = meta["data_block_size"] + emit(begin * block_size, (begin + length) * block_size, state) + + if "data_block_size" not in meta: + die("no found in the XML input") + if offset > size: + die(f"XML mappings end at {offset}, beyond the given size {size} - wrong device or size?") + if offset < size: + emit_range(offset, size, "zero") + + print(f"# borg input map generated by lvm-thin-map.py ({args.mode} mode)") + print(f"# size: {size} bytes, thin pool chunk size: {meta['data_block_size']} bytes") + if args.mode == "delta": + print(f"# thin_delta left (reference) dev_id: {meta.get('left')}, right dev_id: {meta.get('right')}") + else: + print(f"# thin_dump dev_id: {meta.get('dev_id')}") + for start, end, state in out: + print(f"{start} {end - start} {state}") + + +if __name__ == "__main__": + main() diff --git a/src/borg/archive.py b/src/borg/archive.py index 23dc784323..80063ee178 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -43,6 +43,7 @@ from .helpers import safe_ns from .helpers import ellipsis_truncate, ProgressIndicatorPercent, log_multi, get_progress_dt from .helpers import os_open, flags_normal, flags_dir, O_, SpecialFileReader +from .helpers import MAP_DATA, MAP_ZERO, MAP_SAME, input_map_check_size from .helpers import os_stat from .helpers import msgpack from .helpers.lrucache import LRUCache @@ -1299,7 +1300,7 @@ def __init__(self, *, key, cache, add_item, rechunkify): self.add_item = add_item self.rechunkify = rechunkify - def process_file_chunks(self, item, cache, stats, show_progress, chunk_iter, chunk_processor=None): + def process_file_chunks(self, item, cache, stats, show_progress, chunk_iter, chunk_processor=None, append=False): if not chunk_processor: def chunk_processor(chunk): @@ -1312,7 +1313,8 @@ def chunk_processor(chunk): release_chunk_data(data) return chunk_entry - item.chunks = [] + if not append: + item.chunks = [] # a --reuse-from caller calls this repeatedly, appending to existing chunks. for chunk in chunk_iter: chunk_entry = chunk_processor(chunk) item.chunks.append(chunk_entry) @@ -1332,6 +1334,71 @@ def maybe_exclude_by_attr(item): raise BackupItemExcluded +def build_reuse_plan(input_map, ref_chunks, size, seen_chunk): + """ + Build a processing plan for --map with --reuse-from, see #4363. + + A reference chunk is reused iff its whole extent lies within "same" map ranges and it + still exists in the repo (seen_chunk). Everything else is read from the input (with + "zero" ranges stored as holes without reading). Reading whole reference chunks that + intersect changed ranges keeps the result correct for any chunker; with the fixed + chunker, reference chunk boundaries and read windows align exactly. + + Returns a list of parts covering [0, size) in order: + ("reuse", [ChunkListEntry, ...]) or ("read", [(start, length, is_data), ...]). + """ + # merge adjacent "same" ranges, so a reference chunk spanning two of them is still reusable. + same = [] + for start, length, state in input_map: + if state != MAP_SAME: + continue + if same and same[-1][1] == start: + same[-1][1] = start + length + else: + same.append([start, start + length]) + + def read_part(a, b): + # intersect extent [a, b) with the map's ranges: "zero" ranges become holes (not read), + # everything else (data, or same parts of non-reusable reference chunks) is read. + fmap = [] + for start, length, state in input_map: + sub_start, sub_end = max(start, a), min(start + length, b) + if sub_start < sub_end: + fmap.append((sub_start, sub_end - sub_start, state != MAP_ZERO)) + return "read", fmap + + parts = [] + read_start = None # start of the current not-yet-flushed read extent + offset = 0 + si = 0 # index into same[], both same[] and the chunks are sorted by offset + for entry in ref_chunks: + start, end = offset, offset + entry.size + offset = end + if start >= size: + break + while si < len(same) and same[si][1] <= start: + si += 1 + reusable = ( + end <= size and si < len(same) and same[si][0] <= start and end <= same[si][1] and seen_chunk(entry.id) + ) + if reusable: + if read_start is not None: + parts.append(read_part(read_start, start)) + read_start = None + if parts and parts[-1][0] == "reuse": + parts[-1][1].append(entry) + else: + parts.append(("reuse", [entry])) + elif read_start is None: + read_start = start + if read_start is not None: + # covers non-reusable chunks at the end and any input tail beyond the reference chunks. + parts.append(read_part(read_start, size)) + elif offset < size: + parts.append(read_part(offset, size)) + return parts + + class FilesystemObjectProcessors: # When ported to threading, then this doesn't need chunker, cache, key any more. # process_file becomes a callback passed to __init__. @@ -1351,6 +1418,8 @@ def __init__( file_status_printer=None, files_changed="mtime" if is_win32 else "ctime", read_special_timeout=None, + input_map=None, + reuse_chunks=None, ): self.metadata_collector = metadata_collector self.cache = cache @@ -1361,6 +1430,8 @@ def __init__( self.print_file_status = file_status_printer or (lambda *args: None) self.files_changed = files_changed self.read_special_timeout = read_special_timeout + self.input_map = input_map # --map: content range info for the single input file, see #4363 + self.reuse_chunks = reuse_chunks # --reuse-from: the reference archive item's chunk list, see #4363 self.hlm = HardLinkManager(id_type=tuple, info_type=(list, type(None))) # (dev, ino) -> chunks or None self.stats = Statistics(output_json=log_json) # threading: done by cache (including progress) @@ -1552,7 +1623,11 @@ def process_file(self, *, path, parent_fd, name, st, cache, flags=flags_normal, chunk_entry = cache.reuse_chunk(chunk_id, chunk_size, self.stats) item.chunks.append(chunk_entry) else: # normal case, no "2nd+" hard link - if not is_special_file: + if self.input_map is not None: + # --map: the given map replaces the files cache as content/change information, see #4363. + hashed_path = path_hash = None + known, chunks = False, None + elif not is_special_file: hashed_path = safe_encode(item.path) # path as in archive item! started_hashing = time.monotonic() path_hash = self.key.id_hash(hashed_path) @@ -1593,16 +1668,54 @@ def process_file(self, *, path, parent_fd, name, st, cache, flags=flags_normal, # and still commit the archive -- referencing chunks that were never durably # stored. An unwrapped repository OSError is critical and aborts create before # archive.save() runs (see the BackupOSError docstring). - if read_special_timeout is not None: + if self.input_map is not None: + # --map: read only the "data" ranges, store "zero" ranges as holes without + # reading them, see #4363. Non-seekable inputs (fifo/chr) are rejected earlier. + with backup_io("seek"): + input_size = st.st_size if stat.S_ISREG(st.st_mode) else os.lseek(fd, 0, os.SEEK_END) + os.lseek(fd, 0, os.SEEK_SET) + input_map_check_size(self.input_map, input_size) + if self.reuse_chunks is not None: + # --reuse-from: reuse the reference archive's chunks for "same" ranges + # without reading them. Each "read" part gets its own chunkify call, so + # chunks never span the gap left by reused parts. + plan = build_reuse_plan(self.input_map, self.reuse_chunks, input_size, cache.seen_chunk) + item.chunks = [] + for kind, payload in plan: + if kind == "reuse": + for entry in payload: + item.chunks.append(cache.reuse_chunk(entry.id, entry.size, self.stats)) + if self.show_progress: + self.stats.show_progress(item=item) + else: + chunk_iter = self.chunker.chunkify(None, fd, fmap=payload, st=st) + self.process_file_chunks( + item, + cache, + self.stats, + self.show_progress, + backup_io_iter(chunk_iter), + append=True, + ) + else: + fmap = [(start, length, state == MAP_DATA) for start, length, state in self.input_map] + chunk_iter = self.chunker.chunkify(None, fd, fmap=fmap, st=st) + self.process_file_chunks( + item, cache, self.stats, self.show_progress, backup_io_iter(chunk_iter) + ) + elif read_special_timeout is not None: # all reads go through the timeout-enforcing wrapper (fh stays unused). chunk_iter = self.chunker.chunkify(SpecialFileReader(fd, read_special_timeout), st=st) + self.process_file_chunks( + item, cache, self.stats, self.show_progress, backup_io_iter(chunk_iter) + ) else: # passing st saves FileReader a stat call; regular files take the # direct read path, special files (--read-special) the buffered one. chunk_iter = self.chunker.chunkify(None, fd, st=st) - self.process_file_chunks( - item, cache, self.stats, self.show_progress, backup_io_iter(chunk_iter) - ) + self.process_file_chunks( + item, cache, self.stats, self.show_progress, backup_io_iter(chunk_iter) + ) self.stats.chunking_time = self.chunker.chunking_time end_reading = time.time_ns() with backup_io("fstat2"): diff --git a/src/borg/archiver/create_cmd.py b/src/borg/archiver/create_cmd.py index 5dbc2ec455..6181177d45 100644 --- a/src/borg/archiver/create_cmd.py +++ b/src/borg/archiver/create_cmd.py @@ -18,6 +18,7 @@ from ..helpers import comment_validator, ChunkerParams, FilesystemPathSpec, CompressionSpec from ..helpers import archivename_validator, FilesCacheMode, files_cache_mode_no_ctime from ..helpers import octal_int, nonnegative_seconds +from ..helpers import read_input_map from ..helpers import eval_escapes from ..helpers import timestamp, archive_ts_now from ..helpers import get_cache_dir, os_stat, get_strip_prefix, slashify @@ -71,6 +72,48 @@ def do_create(self, args, repository, manifest): read_special_timeout = READ_SPECIAL_TIMEOUT_DEFAULT if read_special_timeout == 0: read_special_timeout = None # wait forever + if args.reuse_from is not None and args.input_map is None: + raise CommandError("--reuse-from requires --map.") + if args.reuse_path is not None and args.reuse_from is None: + raise CommandError("--reuse-path requires --reuse-from.") + input_map = None + if args.input_map is not None: + # --map only makes sense for a single, seekable input file, see #4363. + if args.paths_from_stdin or args.paths_from_command or args.paths_from_shell_command: + raise CommandError("--map cannot be used with --paths-from-*.") + if args.content_from_command: + raise CommandError("--map cannot be used with --content-from-command.") + if len(args.paths) != 1: + raise CommandError("--map requires exactly one input path.") + if args.paths[0] == "-": + raise CommandError("--map cannot be used with stdin input.") + try: + st_map = os.stat(args.paths[0], follow_symlinks=True) + except OSError as e: + raise CommandError(f"--map input: {args.paths[0]}: {e}") + if stat.S_ISBLK(st_map.st_mode): + if not args.read_special: + raise CommandError("--map with a block device requires --read-special.") + elif not stat.S_ISREG(st_map.st_mode): + raise CommandError("--map input must be a regular file or a block device.") + input_map = read_input_map(args.input_map, allow_same=args.reuse_from is not None) + reuse_chunks = None + if args.reuse_from is not None: + ref_info = manifest.archives.get_one([args.reuse_from]) + ref_archive = Archive(manifest, ref_info.id) + ref_items = [item for item in ref_archive.iter_items() if "chunks" in item] + if args.reuse_path is not None: + ref_items = [item for item in ref_items if item.path == args.reuse_path] + if not ref_items: + raise CommandError( + f"--reuse-from: no file item with path {args.reuse_path!r} in reference archive." + ) + if len(ref_items) != 1: + raise CommandError( + f"--reuse-from: reference archive has {len(ref_items)} file items, " + f"use --reuse-path to select the reference item." + ) + reuse_chunks = ref_items[0].chunks if is_win32: # st_ctime is the file *creation* time on Windows, not the "metadata change time", # so a ctime based files cache mode would not detect content changes of a file that @@ -315,6 +358,8 @@ def create_inner(archive, cache, fso): file_status_printer=self.print_file_status, files_changed=args.files_changed, read_special_timeout=read_special_timeout, + input_map=input_map, + reuse_chunks=reuse_chunks, ) create_inner(archive, cache, fso) args.stats |= args.json @@ -955,6 +1000,51 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser): By default, the content read from stdin is stored in a file called 'stdin'. Use ``--stdin-name`` to change the name. + Input maps + ++++++++++ + + Usually, borg reads the complete input to determine its contents. If you already + know the contents of parts of the input from an external source of truth, you can + give that information via ``--map MAPFILE`` and borg will not read the known parts. + The primary use case is backing up snapshots of large block devices (e.g. LVM thin + volumes), where the storage layer knows which ranges are in use. + + ``--map`` requires giving exactly one input path, which must be a regular file or + (with ``--read-special``) a block device. + + The map file must describe the whole input: one range per line, in the form + ``START LENGTH STATE`` (byte values, decimal or 0x-prefixed hexadecimal). The + ranges must be sorted, non-overlapping and contiguous, starting at offset 0 and + covering the exact input size. ``#`` starts a comment, empty lines are ignored. + STATE is one of: + + - ``data``: the range's contents are read and backed up. + - ``zero``: the range is known to read as all-zero bytes. borg stores a hole + (all-zero range) of that size without reading the range. + - ``same``: the range is known to be identical to the same range of the input + backed up in the ``--reuse-from REFARCHIVE`` reference archive (usually: the + previous backup of an earlier snapshot of the same device). borg reuses the + reference archive's chunks for such ranges without reading them. This state + requires ``--reuse-from``. + + The reference archive must contain exactly one file item; if it contains more, + select the reference item with ``--reuse-path PATH`` (its archive-internal path). + Reference chunks that only partially overlap ``same`` ranges are re-read from + the input, so any chunker gives correct results - but a fixed block size chunker + (e.g. ``--chunker-params fixed,4194304``, same parameters as used for the + reference archive) avoids re-reading at the edges of changed ranges and gives + stable chunk boundaries across backups. + + **The map is trusted**: if it is wrong (e.g. a range marked ``zero`` actually + contains data, or a range marked ``same`` actually changed), the archive will + not match the input and borg cannot detect that. Independently verify the + source producing the maps, and consider doing a periodic full read backup + (without ``--map``). + + For LVM thin volume snapshots, maps can be generated from ``thin_dump`` / + ``thin_delta`` XML with the ``scripts/lvm-thin-map.py`` converter from the + borg sources; its docstring shows the complete workflow. + Feeding all file paths from externally ++++++++++++++++++++++++++++++++++++++ @@ -1146,6 +1236,30 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser): "for a FIFO's writer to connect). Give 0 to wait forever. default: %d seconds." % READ_SPECIAL_TIMEOUT_DEFAULT, ) + fs_group.add_argument( + "--map", + metavar="MAPFILE", + dest="input_map", + action=Highlander, + help="give a map file describing the content ranges of the (single) input file, " + "so borg does not need to read all of it. See the *Input maps* section below.", + ) + fs_group.add_argument( + "--reuse-from", + metavar="ARCHIVE", + dest="reuse_from", + action=Highlander, + help="reuse the chunks of this reference archive for the input map's ``same`` " + "ranges (requires --map). See the *Input maps* section below.", + ) + fs_group.add_argument( + "--reuse-path", + metavar="PATH", + dest="reuse_path", + action=Highlander, + help="archive-internal path of the reference item in the --reuse-from archive " + "(only needed if that archive contains more than one file item).", + ) archive_group = subparser.add_argument_group("Archive options") archive_group.add_argument( diff --git a/src/borg/helpers/__init__.py b/src/borg/helpers/__init__.py index eb2a976098..2f7420eca1 100644 --- a/src/borg/helpers/__init__.py +++ b/src/borg/helpers/__init__.py @@ -24,6 +24,7 @@ from .fs import dir_is_tagged, dir_is_cachedir, remove_dotdot_prefixes, make_path_safe, scandir_inorder from .fs import secure_erase, safe_unlink, dash_open, os_open, os_stat, get_strip_prefix, umount, slashify from .fs import SpecialFileReader +from .fs import MAP_DATA, MAP_ZERO, MAP_SAME, read_input_map, input_map_check_size from .fs import O_, flags_dir, flags_dir_follow, flags_special_follow, flags_special from .fs import flags_base, flags_normal, flags_normal_follow, flags_noatime from .fs import HardLinkManager diff --git a/src/borg/helpers/fs.py b/src/borg/helpers/fs.py index 653371a1b5..e7a3d44ebf 100644 --- a/src/borg/helpers/fs.py +++ b/src/borg/helpers/fs.py @@ -640,6 +640,59 @@ def seek(self, pos, whence=os.SEEK_SET): raise OSError(errno.ESPIPE, os.strerror(errno.ESPIPE)) +# input map range states (borg create --map), see #4363. +MAP_DATA = "data" # range contains data, read and store it +MAP_ZERO = "zero" # range is known to read as all-zero, store a hole without reading it +MAP_SAME = "same" # range is known unchanged vs. a reference archive, reuse its chunks (--reuse-from) + + +def read_input_map(path, *, allow_same=False): + """ + Read and validate an input map file (borg create --map), see #4363. + + Format: one range per line: "START LENGTH STATE" (decimal or 0x-prefixed hex), + STATE being data, zero or same. '#' starts a comment, empty lines are ignored. + The ranges must be sorted, non-overlapping and contiguous, starting at offset 0. + + Returns a list of (start, length, state) tuples. + """ + states = {MAP_DATA, MAP_ZERO} | ({MAP_SAME} if allow_same else set()) + input_map = [] + expected_start = 0 + with open(path) as f: + for lineno, line in enumerate(f, start=1): + line = line.split("#", 1)[0].strip() + if not line: + continue + err = f"{path}:{lineno}: invalid input map line" + try: + start_s, length_s, state = line.split() + start, length = int(start_s, 0), int(length_s, 0) + except ValueError: + raise Error(f"{err}: expected 'START LENGTH STATE', got: {line!r}") from None + if state not in states: + raise Error(f"{err}: invalid state {state!r} (expected: {', '.join(sorted(states))})") + if length <= 0: + raise Error(f"{err}: LENGTH must be positive") + if start != expected_start: + raise Error( + f"{err}: ranges must be sorted, contiguous and start at offset 0 " + f"(expected START {expected_start}, got {start})" + ) + input_map.append((start, length, state)) + expected_start = start + length + if not input_map: + raise Error(f"{path}: empty input map") + return input_map + + +def input_map_check_size(input_map, size): + """Check that *input_map* covers [0, size) exactly - a mismatch means the map does not belong to this input.""" + covered = input_map[-1][0] + input_map[-1][1] + if covered != size: + raise Error(f"input map covers {covered} bytes, but the input has {size} bytes") + + def umount(mountpoint): from . import set_ec diff --git a/src/borg/testsuite/archive_test.py b/src/borg/testsuite/archive_test.py index cb2aee520e..d1c5e53b96 100644 --- a/src/borg/testsuite/archive_test.py +++ b/src/borg/testsuite/archive_test.py @@ -579,3 +579,74 @@ def test_archives_get_by_id_missing_returns_none(): manifest = Mock() archives = Archives(repo, manifest) assert archives.get_by_id(b"\x01" * 32) is None + + +class TestBuildReusePlan: + """Tests for the --map/--reuse-from processing plan, see #4363.""" + + @staticmethod + def plan(input_map, chunk_sizes, size, missing=()): + from ..archive import build_reuse_plan + + ref_chunks = [ChunkListEntry(i.to_bytes(4, "big"), chunk_size) for i, chunk_size in enumerate(chunk_sizes)] + return ref_chunks, build_reuse_plan(input_map, ref_chunks, size, lambda id: id not in missing) + + def test_all_same(self): + ref, plan = self.plan([(0, 8192, "same")], [4096, 4096], 8192) + assert plan == [("reuse", ref)] + + def test_all_data_and_zero(self): + _, plan = self.plan([(0, 4096, "data"), (4096, 4096, "zero")], [4096, 4096], 8192) + assert plan == [("read", [(0, 4096, True), (4096, 4096, False)])] + + def test_mixed(self): + # chunks 0 and 2 reusable, chunk 1 changed. + input_map = [(0, 4096, "same"), (4096, 4096, "data"), (8192, 4096, "same")] + ref, plan = self.plan(input_map, [4096, 4096, 4096], 12288) + assert plan == [("reuse", ref[0:1]), ("read", [(4096, 4096, True)]), ("reuse", ref[2:3])] + + def test_partial_overlap_reads_whole_chunk(self): + # a reference chunk that only partially lies in a "same" range is re-read completely, + # including its "same" part. + input_map = [(0, 6000, "same"), (6000, 2192, "data")] + ref, plan = self.plan(input_map, [4096, 4096], 8192) + assert plan == [("reuse", ref[0:1]), ("read", [(4096, 1904, True), (6000, 2192, True)])] + + def test_adjacent_same_ranges_merge(self): + # a chunk spanning two adjacent "same" map ranges is still reusable. + input_map = [(0, 2048, "same"), (2048, 6144, "same")] + ref, plan = self.plan(input_map, [4096, 4096], 8192) + assert plan == [("reuse", ref)] + + def test_zero_inside_read_part(self): + # a "zero" range makes its chunk non-reusable, but is stored as a hole, not read. + input_map = [(0, 4096, "same"), (4096, 4096, "zero")] + ref, plan = self.plan(input_map, [4096, 4096], 8192) + assert plan == [("reuse", ref[0:1]), ("read", [(4096, 4096, False)])] + + def test_grown_input(self): + # the tail beyond the reference chunks must be read. + input_map = [(0, 8192, "same"), (8192, 4096, "data")] + ref, plan = self.plan(input_map, [4096, 4096], 12288) + assert plan == [("reuse", ref), ("read", [(8192, 4096, True)])] + + def test_shrunk_input(self): + # reference chunks beyond the new input size are dropped. + input_map = [(0, 4096, "same")] + ref, plan = self.plan(input_map, [4096, 4096, 4096], 4096) + assert plan == [("reuse", ref[0:1])] + + def test_shrunk_input_straddling_chunk(self): + # a reference chunk straddling the new end of the input cannot be reused. + input_map = [(0, 6000, "same")] + ref, plan = self.plan(input_map, [4096, 4096], 6000) + assert plan == [("reuse", ref[0:1]), ("read", [(4096, 1904, True)])] + + def test_missing_chunk_is_read(self): + input_map = [(0, 8192, "same")] + ref, plan = self.plan(input_map, [4096, 4096], 8192, missing={ref_id(1)}) + assert plan == [("reuse", ref[0:1]), ("read", [(4096, 4096, True)])] + + +def ref_id(i): + return i.to_bytes(4, "big") diff --git a/src/borg/testsuite/archiver/create_cmd_test.py b/src/borg/testsuite/archiver/create_cmd_test.py index c8d8077f22..f668b11bb8 100644 --- a/src/borg/testsuite/archiver/create_cmd_test.py +++ b/src/borg/testsuite/archiver/create_cmd_test.py @@ -17,7 +17,7 @@ from ...platformflags import is_msystem from ...repository import Repository from ...helpers import CommandError, BackupPermissionError, BackupTimeoutError, BackupBrokenSymlinkError -from ...helpers import BackupWarning +from ...helpers import BackupWarning, Error from .. import has_lchflags, has_mknod from .. import changedir from .. import ( @@ -1552,3 +1552,273 @@ def test_exclude_nodump_dir_with_file(archivers, request): list_output = cmd(archiver, "list", "test", "--short") assert "input/nd\n" not in list_output assert "input/nd/file_in_ndir\n" not in list_output + + +def test_create_map(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + block = 4096 + data_a, data_b, data_c = os.urandom(block), os.urandom(block), os.urandom(block) + # data_b is real data, but the map claims that range reads as zero: + # borg must not read "zero" ranges, so the archive must contain zeros there. + create_regular_file(archiver.input_path, "file", contents=data_a + data_b + data_c + b"\0" * block) + map_path = os.fspath(archiver.tmpdir / "input.map") + with open(map_path, "w") as f: + f.write("# test input map\n") + f.write(f"0 {block} data\n") + f.write(f"0x1000 {block} zero\n") + f.write(f"{2 * block} {block} data\n") + f.write(f"{3 * block} {block} zero\n") + expected = data_a + b"\0" * block + data_c + b"\0" * block + for name, chunker_args in [("test-fixed", ("--chunker-params", "fixed,4096")), ("test-default", ())]: + cmd(archiver, "create", *chunker_args, "--map", map_path, name, "input/file") + with changedir("output"): + cmd(archiver, "extract", name) + with open(os.path.join("output", "input", "file"), "rb") as f: + assert f.read() == expected + shutil.rmtree("output") + os.mkdir("output") + + +def test_create_map_errors(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + create_regular_file(archiver.input_path, "file", size=2 * 4096) + create_regular_file(archiver.input_path, "file2", size=100) + map_path = os.fspath(archiver.tmpdir / "input.map") + with open(map_path, "w") as f: + f.write("0 4096 data\n") + + def expect_error(exc_class, *args): + if archiver.FORK_DEFAULT: + cmd(archiver, *args, exit_code=exc_class().exit_code) + else: + with pytest.raises(exc_class): + cmd(archiver, *args) + + # --map requires exactly one input path + expect_error(CommandError, "create", "--map", map_path, "test", "input/file", "input/file2") + # --map input must not be a directory + expect_error(CommandError, "create", "--map", map_path, "test", "input") + # --map cannot be used with stdin input + expect_error(CommandError, "create", "--map", map_path, "test", "-") + # the map covers 4096 bytes, but the input file has 8192 bytes + expect_error(Error, "create", "--map", map_path, "test", "input/file") + + +def test_create_map_reuse_from_fixed(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + block = 4096 + blocks = [os.urandom(block) for _ in range(4)] + fname = os.path.join(archiver.input_path, "file") + create_regular_file(archiver.input_path, "file", contents=b"".join(blocks)) + map_path = os.fspath(archiver.tmpdir / "input.map") + with open(map_path, "w") as f: + f.write(f"0 {4 * block} data\n") + cmd(archiver, "create", "--chunker-params", "fixed,4096", "--map", map_path, "ref", "input/file") + # modify block 1 (declared "data" below) and block 2 (declared "same"!): the new archive + # must contain the new block 1, but the OLD block 2 - proving that borg reused the + # reference archive's chunks instead of reading the "same" ranges. + new_block1, sneaky_block2 = os.urandom(block), os.urandom(block) + with open(fname, "r+b") as f: + f.seek(block) + f.write(new_block1) + f.write(sneaky_block2) + with open(map_path, "w") as f: + f.write(f"0 {block} same\n") + f.write(f"{block} {block} data\n") + f.write(f"{2 * block} {block} same\n") + f.write(f"{3 * block} {block} zero\n") + cmd( + archiver, + "create", + "--chunker-params", + "fixed,4096", + "--map", + map_path, + "--reuse-from", + "ref", + "test", + "input/file", + ) + with changedir("output"): + cmd(archiver, "extract", "test") + with open(os.path.join("output", "input", "file"), "rb") as f: + assert f.read() == blocks[0] + new_block1 + blocks[2] + b"\0" * block + + +def test_create_map_reuse_from_cdc(archivers, request): + # the reuse plan re-reads reference chunks that only partially overlap changed ranges, + # so it must also be correct with content-defined chunking (default chunker). + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + size = 1 << 20 + content = bytearray(os.urandom(size)) + fname = os.path.join(archiver.input_path, "file") + create_regular_file(archiver.input_path, "file", contents=bytes(content)) + map_path = os.fspath(archiver.tmpdir / "input.map") + with open(map_path, "w") as f: + f.write(f"0 {size} data\n") + cmd(archiver, "create", "ref", "input/file") + # change an unaligned range in the middle, keep everything else really unchanged. + start, length = 400000, 5000 + content[start : start + length] = os.urandom(length) + with open(fname, "wb") as f: + f.write(content) + with open(map_path, "w") as f: + f.write(f"0 {start} same\n") + f.write(f"{start} {length} data\n") + f.write(f"{start + length} {size - start - length} same\n") + cmd(archiver, "create", "--map", map_path, "--reuse-from", "ref", "test", "input/file") + with changedir("output"): + cmd(archiver, "extract", "test") + with open(os.path.join("output", "input", "file"), "rb") as f: + assert f.read() == bytes(content) + + +def test_create_map_reuse_from_resize(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + block = 4096 + blocks = [os.urandom(block) for _ in range(3)] + fname = os.path.join(archiver.input_path, "file") + create_regular_file(archiver.input_path, "file", contents=blocks[0] + blocks[1]) + map_path = os.fspath(archiver.tmpdir / "input.map") + with open(map_path, "w") as f: + f.write(f"0 {2 * block} data\n") + cmd(archiver, "create", "--chunker-params", "fixed,4096", "ref", "input/file") + # input grew: the tail beyond the reference chunks must be read. + with open(fname, "ab") as f: + f.write(blocks[2]) + with open(map_path, "w") as f: + f.write(f"0 {2 * block} same\n") + f.write(f"{2 * block} {block} data\n") + cmd( + archiver, + "create", + "--chunker-params", + "fixed,4096", + "--map", + map_path, + "--reuse-from", + "ref", + "grown", + "input/file", + ) + # input shrank: reference chunks beyond the new size must be dropped. + with open(fname, "r+b") as f: + f.truncate(block) + with open(map_path, "w") as f: + f.write(f"0 {block} same\n") + cmd( + archiver, + "create", + "--chunker-params", + "fixed,4096", + "--map", + map_path, + "--reuse-from", + "ref", + "shrunk", + "input/file", + ) + with changedir("output"): + cmd(archiver, "extract", "grown") + with open(os.path.join("output", "input", "file"), "rb") as f: + assert f.read() == blocks[0] + blocks[1] + blocks[2] + shutil.rmtree("output") + os.mkdir("output") + with changedir("output"): + cmd(archiver, "extract", "shrunk") + with open(os.path.join("output", "input", "file"), "rb") as f: + assert f.read() == blocks[0] + + +def test_create_map_reuse_from_missing_chunk(archivers, request, monkeypatch): + # when a reference chunk is not in the repo (any more), borg must fall back to reading. + archiver = request.getfixturevalue(archivers) + if archiver.FORK_DEFAULT: + pytest.skip("needs in-process monkeypatching") + cmd(archiver, "repo-create", RK_ENCRYPTION) + block = 4096 + content = os.urandom(2 * block) + create_regular_file(archiver.input_path, "file", contents=content) + map_path = os.fspath(archiver.tmpdir / "input.map") + with open(map_path, "w") as f: + f.write(f"0 {2 * block} data\n") + cmd(archiver, "create", "--chunker-params", "fixed,4096", "ref", "input/file") + with open(map_path, "w") as f: + f.write(f"0 {2 * block} same\n") + from ...cache import AdHocWithFilesCache + + monkeypatch.setattr(AdHocWithFilesCache, "seen_chunk", lambda self, id, size=None: False) + cmd( + archiver, + "create", + "--chunker-params", + "fixed,4096", + "--map", + map_path, + "--reuse-from", + "ref", + "test", + "input/file", + ) + with changedir("output"): + cmd(archiver, "extract", "test") + with open(os.path.join("output", "input", "file"), "rb") as f: + assert f.read() == content + + +def test_create_map_reuse_from_errors(archivers, request): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + create_regular_file(archiver.input_path, "file", size=4096) + create_regular_file(archiver.input_path, "file2", size=4096) + map_path = os.fspath(archiver.tmpdir / "input.map") + with open(map_path, "w") as f: + f.write("0 4096 same\n") + + def expect_error(exc_class, *args): + if archiver.FORK_DEFAULT: + cmd(archiver, *args, exit_code=exc_class().exit_code) + else: + with pytest.raises(exc_class): + cmd(archiver, *args) + + cmd(archiver, "create", "ref2", "input") # two file items + # --reuse-from requires --map + expect_error(CommandError, "create", "--reuse-from", "ref2", "test", "input/file") + # --reuse-path requires --reuse-from + expect_error(CommandError, "create", "--map", map_path, "--reuse-path", "input/file", "test", "input/file") + # "same" state requires --reuse-from + expect_error(Error, "create", "--map", map_path, "test", "input/file") + # ambiguous reference archive (two file items, no --reuse-path) + expect_error(CommandError, "create", "--map", map_path, "--reuse-from", "ref2", "test", "input/file") + # --reuse-path selecting a non-existing item + expect_error( + CommandError, + "create", + "--map", + map_path, + "--reuse-from", + "ref2", + "--reuse-path", + "nosuch", + "test", + "input/file", + ) + # disambiguated via --reuse-path: works + cmd( + archiver, + "create", + "--map", + map_path, + "--reuse-from", + "ref2", + "--reuse-path", + "input/file", + "test", + "input/file", + ) diff --git a/src/borg/testsuite/helpers/fs_test.py b/src/borg/testsuite/helpers/fs_test.py index e67a230013..d93ddb1c6c 100644 --- a/src/borg/testsuite/helpers/fs_test.py +++ b/src/borg/testsuite/helpers/fs_test.py @@ -24,7 +24,13 @@ make_path_safe, map_chars, SpecialFileReader, + MAP_DATA, + MAP_ZERO, + MAP_SAME, + read_input_map, + input_map_check_size, ) +from ...helpers.errors import Error from ...platform import is_win32, is_darwin from .. import are_fifos_supported, are_hardlinks_supported from .. import rejected_dotdot_paths @@ -606,3 +612,48 @@ def test_special_file_reader_not_seekable(): with pytest.raises(OSError) as exc_info: reader.seek(0) assert exc_info.value.errno == errno.ESPIPE + + +def test_read_input_map(tmp_path): + p = tmp_path / "input.map" + p.write_text("# a comment\n\n0 0x1000 data\n0x1000 4096 zero # trailing comment\n8192 100 data\n") + assert read_input_map(os.fspath(p)) == [(0, 4096, MAP_DATA), (4096, 4096, MAP_ZERO), (8192, 100, MAP_DATA)] + + +def test_read_input_map_same_state(tmp_path): + p = tmp_path / "input.map" + p.write_text("0 100 same\n") + with pytest.raises(Error, match="invalid state"): + read_input_map(os.fspath(p)) + assert read_input_map(os.fspath(p), allow_same=True) == [(0, 100, MAP_SAME)] + + +@pytest.mark.parametrize( + "content, match", + [ + ("", "empty input map"), + ("0 100\n", "expected 'START LENGTH STATE'"), + ("0 100 data extra\n", "expected 'START LENGTH STATE'"), + ("x 100 data\n", "expected 'START LENGTH STATE'"), + ("0 100 nodata\n", "invalid state"), + ("0 0 data\n", "LENGTH must be positive"), + ("0 -5 data\n", "LENGTH must be positive"), + ("100 100 data\n", "expected START 0"), + ("0 100 data\n50 100 zero\n", "expected START 100"), + ("0 100 data\n200 100 zero\n", "expected START 100"), + ], +) +def test_read_input_map_invalid(tmp_path, content, match): + p = tmp_path / "input.map" + p.write_text(content) + with pytest.raises(Error, match=match): + read_input_map(os.fspath(p)) + + +def test_input_map_check_size(): + input_map = [(0, 100, MAP_DATA), (100, 50, MAP_ZERO)] + input_map_check_size(input_map, 150) # exact coverage - ok + with pytest.raises(Error, match="covers 150 bytes, but the input has 151"): + input_map_check_size(input_map, 151) + with pytest.raises(Error, match="covers 150 bytes, but the input has 149"): + input_map_check_size(input_map, 149)