archive: use BytesIO to store data file

When opening a sigmf archive with SigMFArchiveReader(), the data file is
currently set to the full archive (including metadata).

This causes issues when writing the archive back to disk and invalidates
the metadata hash since the data_file is now a tar archive and not just
the set of samples.

To work around the issue, carry around a BytesIO buffer with the content
of the data_file and write it to the tmpdir just before saving a new
archive to disk.

Signed-off-by: Liam Beguin <[email protected]>

Author: liambeguin

sigmf/archive.py

@@ -7,6 +7,7 @@
 """Create and extract SigMF archives."""
 
 import os
+import io
 import shutil
 import tarfile
 import tempfile
@@ -72,7 +73,12 @@ def __init__(self, sigmffile, name=None, fileobj=None):
         with open(sigmf_md_path, "w") as mdfile:
             self.sigmffile.dump(mdfile, pretty=True)
 
-        shutil.copy(self.sigmffile.data_file, sigmf_data_path)
+        if isinstance(self.sigmffile.data_buffer, io.BytesIO):
+            self.sigmffile.data_file = sigmf_data_path
+            with open(sigmf_data_path, 'wb') as f:
+                f.write(self.sigmffile.data_buffer.getbuffer())
+        else:
+            shutil.copy(self.sigmffile.data_file, sigmf_data_path)
 
         def chmod(tarinfo):
             if tarinfo.isdir():
@@ -110,7 +116,7 @@ def _ensure_name_has_correct_extension(self):
         self.name = name if has_correct_extension else name + SIGMF_ARCHIVE_EXT
 
     def _ensure_data_file_set(self):
-        if not self.sigmffile.data_file:
+        if not self.sigmffile.data_file and not isinstance(self.sigmffile.data_buffer, io.BytesIO):
             err = "no data file - use `set_data_file`"
             raise SigMFFileError(err)
 

sigmf/archivereader.py

@@ -7,9 +7,11 @@
 """Access SigMF archives without extracting them."""
 
 import os
+import io
 import shutil
 import tarfile
 import tempfile
+from pathlib import Path
 
 from . import __version__
 from .archive import SIGMF_ARCHIVE_EXT, SIGMF_DATASET_EXT, SIGMF_METADATA_EXT, SigMFArchive
@@ -64,6 +66,8 @@ def __init__(self, name=None, skip_checksum=False, map_readonly=True, archive_bu
                 elif memb.name.endswith(SIGMF_DATASET_EXT):
                     data_offset = memb.offset_data
                     data_size_bytes = memb.size
+                    with tar_obj.extractfile(memb) as memb_fid:
+                        data_buffer = io.BytesIO(memb_fid.read())
 
                 else:
                     print('A regular file', memb.name, 'was found but ignored in the archive')
@@ -77,10 +81,8 @@ def __init__(self, name=None, skip_checksum=False, map_readonly=True, archive_bu
         valid_md = self.sigmffile.validate()
 
         self.sigmffile.set_data_file(
-            self.name,
-            data_buffer=archive_buffer,
+            data_buffer=data_buffer,
             skip_checksum=skip_checksum,
-            offset=data_offset,
             size_bytes=data_size_bytes,
             map_readonly=map_readonly,
         )

sigmf/sigmffile.py

@@ -15,6 +15,7 @@
 from os import path
 
 import numpy as np
+import io
 
 from . import __specification__, __version__, schema, sigmf_hash, validate
 from .archive import SIGMF_ARCHIVE_EXT, SIGMF_COLLECTION_EXT, SIGMF_DATASET_EXT, SIGMF_METADATA_EXT, SigMFArchive
@@ -246,7 +247,7 @@ def _is_conforming_dataset(self):
             # check for any non-zero `header_bytes` fields in captures segments
             if capture.get(self.HEADER_BYTES_KEY, 0):
                 return False
-        if not path.isfile(self.data_file):
+        if self.data_file is not None and not path.isfile(self.data_file):
             return False
         # if we get here, the file exists and is conforming
         return True
@@ -631,7 +632,7 @@ def read_samples(self, start_index=0, count=-1, autoscale=True, raw_components=F
             raise IOError('Number of samples must be greater than zero, or -1 for all samples.')
         elif start_index + count > self.sample_count:
             raise IOError("Cannot read beyond EOF.")
-        if self.data_file is None:
+        if self.data_file is None and not isinstance(self.data_buffer, io.BytesIO):
             if self.get_global_field(self.METADATA_ONLY_KEY, False):
                 # only if data_file is `None` allows access to dynamically generated datsets
                 raise SigMFFileError("Cannot read samples from a metadata only distribution.")
@@ -658,9 +659,15 @@ def _read_datafile(self, first_byte, nitems, autoscale, raw_components):
         data_type_out = np.dtype("f4") if not self.is_complex_data else np.dtype("f4, f4")
         num_channels = self.get_num_channels()
 
-        fp = open(self.data_file, "rb")
-        fp.seek(first_byte, 0)
-        data = np.fromfile(fp, dtype=data_type_in, count=nitems)
+        if self.data_file is not None:
+            fp = open(self.data_file, "rb")
+            fp.seek(first_byte, 0)
+            data = np.fromfile(fp, dtype=data_type_in, count=nitems)
+        elif self.data_buffer is not None:
+            data = np.frombuffer(self.data_buffer.getbuffer(), dtype=data_type_in, count=nitems)
+        else:
+            data = self._memmap
+
         if num_channels != 1:
             # return reshaped view for num_channels
             # first dimension will be double size if `is_complex_data`
@@ -678,7 +685,9 @@ def _read_datafile(self, first_byte, nitems, autoscale, raw_components):
         else:
             data = data.view(component_type_in)
 
-        fp.close()
+        if self.data_file is not None:
+            fp.close()
+
         return data