Merge branch 'hotfix-2.0.2'

CHANGELOG.md

@@ -1,6 +1,31 @@
 # Changelog
 
 
+## Version 2.0.2 (2021-12-09)
+
+RocketLogger software:
+* [FIXED] Corrupt RLD files when logging no binary channels (#57)
+* [FIXED] Corrupt channel info in split RLD files (#62)
+
+Python support library:
+* [ADDED] Standalone script to recover RLD files corrupted by #57
+* [FIXED] Missing documentation of file revision changes (#61)
+* [CHANGED] Cross validate file headers and tolerate channel info corruption of split files (relates to #62)
+
+
+_Notes:_
+
+This bugfix release resolves RLD file corruption related issues. Details on how to fully recover affected measurement files are given below.
+A timely update is highly recommended to avoid generating corrupt data files.
+
+
+_Recover Corrupt Data Files:_
+
+Measurement files affected by bugs #57 or #62 can be recovered without data loss:
+* To recover files affected by #57 use the `bug57_recover.py` script provided in the Python source package (or available directly from the repository).
+* The updated Python library can tolerate channel info header corruption of split files due to bug #62 if all files of the same measurement are imported in one batch.
+
+
 ## Version 2.0.1 (2021-11-29)
 
 RocketLogger software:
@@ -83,6 +108,7 @@ RocketLogger software:
 * [CHANGED] rework low level hardware interfacing for Debian *buster* compatibility (#2)
 * [CHANGED] update and reorganize internal software API and headers for increased consistency
 * [CHANGED] update to latest and official compiler tools (#18)
+* [CHANGED] Binary file format: increment file version to `0x04`
 * [REMOVED] legacy web control interface
 
 Python support library:

script/python/LICENSE

@@ -1,4 +1,5 @@
 Copyright (c) 2016-2020, ETH Zurich, Computer Engineering Group
+Copyright (c) 2020-2021, Lukas Sigrist <[email protected]>
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

script/python/README.md

@@ -12,6 +12,10 @@ support to generate calibration data from measurements.
 * Matplotlib: for plotting data overview
 * pandas: for pandas DataFrame export
 
+**Compatibility**
+* Data processing: supports all officially specified RLD file version (versions 2-4)
+* Calibration: compatible with RocketLogger calibration file version 2
+
 
 ## Installation
 

script/python/bug57_recover.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+"""
+Recover RLD measurement files suffering from bug #57.
+
+Usage: ./bug57_recovery.sh <filename>
+* <filename> the filename of the RLD file to recover
+
+This script attempts to patch the file header of RLD files that do not
+contain binary data channels, by adding a dummy "BUG57_PATCH" binary channel.
+Adding this dummy channel makes files suffering from bug #57 again compliant
+with the RLD file specification.
+The script does not modify the original data, but stores the patched data in
+a new data file with the input filename suffixed with "_patched".
+
+See also: <https://github.com/ETHZ-TEC/RocketLogger/issues/57>
+"""
+
+import os
+import re
+import sys
+
+_RLD_FILE_REGEX = re.compile(
+    r"(?P<basename>.*?)(?P<part>_p\d+)?(?P<ext>\.rld)", re.IGNORECASE
+)
+
+# file header length offset and size in bytes
+_HEADER_LENGTH_OFFSET = 0x06
+_HEADER_LENGTH_BYTES = 2
+
+# file comment length offset and size in bytes
+_COMMENT_LENGTH_OFFSET = 0x30
+_COMMENT_LENGTH_BYTES = 4
+
+# file binary channel count offset and size in bytes
+_BINARY_CHANNEL_COUNT_OFFSET = 0x34
+_BINARY_CHANNEL_COUNT_BYTES = 2
+
+# file analog channel count offset and size in bytes
+_ANALOG_CHANNEL_COUNT_OFFSET = 0x36
+_ANALOG_CHANNEL_COUNT_BYTES = 2
+
+# Data Valid (binary) channel info with name "BUG57_PATCH"
+_DUMMY_BINARY_CHANNEL_DATA = bytes([0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF,
+    0x42, 0x55, 0x47, 0x35, 0x37, 0x5F, 0x50, 0x41, 0x54, 0x43, 0x48, 0x00, 0x00, 0x00, 0x00, 0x00])
+
+
+class FilePatchingError(Exception):
+    """File patching related errors."""
+
+    pass
+
+
+if __name__ == "__main__":
+
+    # get filename argument
+    if len(sys.argv) != 2:
+        raise TypeError("the script takes exactly one filename argument")
+    input_file = str(sys.argv[1])
+
+    # output file name and check filename extension
+    basename, ext = os.path.splitext(input_file)
+    match = _RLD_FILE_REGEX.match(input_file)
+
+    if not match.group("ext"):
+        raise ValueError("the script only processes RLD files")
+
+    if match.group("part"):
+        output_file = (
+            match.group("basename")
+            + "_patched"
+            + match.group("part")
+            + match.group("ext")
+        )
+    else:
+        output_file = match.group("basename") + "_patched" + match.group("ext")
+
+    try:
+        with open(input_file, "r+b") as file_in, open(output_file, "x+b") as file_out:
+            print(f"Attempt to create patched file: {output_file}")
+
+            # copy start of original header
+            file_out.write(file_in.read(_HEADER_LENGTH_OFFSET))
+
+            # update header length
+            header_length = int.from_bytes(
+                file_in.read(_HEADER_LENGTH_BYTES), byteorder="little", signed=False
+            )
+            patched_header_length = header_length + len(_DUMMY_BINARY_CHANNEL_DATA)
+            file_out.write(
+                int.to_bytes(
+                    patched_header_length,
+                    length=_HEADER_LENGTH_BYTES,
+                    byteorder="little",
+                    signed=False,
+                )
+            )
+
+            # copy header up to comment length
+            file_out.write(
+                file_in.read(
+                    _COMMENT_LENGTH_OFFSET
+                    - _HEADER_LENGTH_BYTES
+                    - _HEADER_LENGTH_OFFSET
+                )
+            )
+
+            # get and copy comment length
+            comment_length_bytes = file_in.read(_COMMENT_LENGTH_BYTES)
+            file_out.write(comment_length_bytes)
+            comment_length = int.from_bytes(
+                comment_length_bytes, byteorder="little", signed=False
+            )
+
+            # update binary channel count
+            binary_channel_count = int.from_bytes(
+                file_in.read(_BINARY_CHANNEL_COUNT_BYTES),
+                byteorder="little",
+                signed=False,
+            )
+            if binary_channel_count != 0:
+                raise FilePatchingError(
+                    "Provided file contains binary therefore cannot (and never should) be patched"
+                )
+
+            binary_channel_count += 1
+            file_out.write(
+                int.to_bytes(
+                    binary_channel_count,
+                    length=_HEADER_LENGTH_BYTES,
+                    byteorder="little",
+                    signed=False,
+                )
+            )
+
+            # copy remaining analog channel count and comment
+            file_out.write(file_in.read(_ANALOG_CHANNEL_COUNT_BYTES + comment_length))
+
+            # append dummy binary channel
+            file_out.write(_DUMMY_BINARY_CHANNEL_DATA)
+
+            # copy remaining file content
+            file_out.write(file_in.read())
+    except FileExistsError:
+        print(f"A patched file `{output_file}` already exists! Skip patching...")
+        sys.exit(1)
+    except FilePatchingError as e:
+        print(f"Failed to patch file: {e}")
+        print(f"Removing incomplete file: {output_file}")
+        os.remove(output_file)
+        sys.exit(1)
+
+    print(f"Patched file was successfully written to: {output_file}")

script/python/docs/conf.py

@@ -22,7 +22,7 @@
 author = 'ETH Zurich, Computer Engineering Group'
 
 # The full version, including alpha/beta/rc tags
-release = '2.0.1'
+release = '2.0.2'
 
 
 # -- General configuration ---------------------------------------------------

script/python/docs/index.rst

@@ -19,16 +19,15 @@ generation of RocketLogger device calibration file from measurements.
 Features
 --------
 
-Python support for RocketLogger Data (``*.rld``) files by the 
-:mod:`rocketlogger.data` module.
+Python support for RocketLogger Data (``*.rld``) files by the :mod:`rocketlogger.data` module.
+Supports any officially specified RocketLogger data file version (versions 2-4).
 
 - Import
 - Channel merging
 - Data extraction
 - Plotting of file data
 
-Support for RocketLogger Device Calibration by the
-:mod:`rocketlogger.calibration` module.
+Support for RocketLogger Device Calibration by the :mod:`rocketlogger.calibration` module.
 
 - Generate new calibrations from measurements
 - Read/write calibration files

script/python/rocketlogger/data.py

@@ -314,14 +314,14 @@ def __init__(
         else:
             raise FileNotFoundError(f"File '{filename}' does not exist.")
 
-    def _read_file_header(self, file_handle):
+    def _read_file_header_lead_in(self, file_handle):
         """
-        Read a RocketLogger data file's header, including comment and channels.
+        Read a RocketLogger data file's header fixed size lead-in
 
         :param file_handle: The file handle to read from, with pointer
             positioned at file start
 
-        :returns: Dictionary containing the read file header data
+        :returns: Dictionary containing the read file header lead-in data
         """
         header = {}
 
@@ -386,6 +386,19 @@ def _read_file_header(self, file_handle):
                 )
             )
 
+        return header
+
+    def _read_file_header(self, file_handle):
+        """
+        Read a RocketLogger data file's header, including comment and channels.
+
+        :param file_handle: The file handle to read from, with pointer
+            positioned after lead-in/at start of comment
+
+        :returns: Dictionary containing the read file header data
+        """
+        header = self._read_file_header_lead_in(file_handle)
+
         # read comment field
         header["comment"] = _read_str(file_handle, header["comment_length"])
 
@@ -421,8 +434,41 @@ def _read_file_header(self, file_handle):
             # add channel to header
             header["channels"].append(channel)
 
+        # consistency check: file stream position matches header size
+        stream_position = file_handle.tell()
+        if stream_position != header["header_length"]:
+            raise RocketLoggerFileError(
+                f"File position {stream_position} does not match "
+                f"header size {header['header_length']}"
+            )
+
         return header
 
+    def _validate_matching_header(self, header1, header2):
+        """
+        Validate that file headers match, i.e. correspond to the same measurement.
+
+        :param header1: First file header data to compare
+
+        :param header2: Second file header data to compare
+        """
+        required_matches = [
+            "file_version",
+            "header_length",
+            "data_block_size",
+            "sample_rate",
+            "mac_address",
+            "start_time",
+            "comment_length",
+            "channel_binary_count",
+            "channel_analog_count",
+        ]
+        for header_field in required_matches:
+            if header1[header_field] != header2[header_field]:
+                raise RocketLoggerDataError(
+                    f"File header not matching at field: {header_field}"
+                )
+
     def _read_file_data(
         self, file_handle, file_header, decimation_factor=1, memory_mapped=True
     ):
@@ -783,33 +829,34 @@ def load_file(
                 break
 
             with open(file_name, "rb") as file_handle:
-                # read file header
-                header = self._read_file_header(file_handle)
-
-                # consistency check: file stream position matches header size
-                if "header_length" not in header:
-                    raise RocketLoggerFileError("Invalid file header read.")
-
-                stream_position = file_handle.tell()
-                if stream_position != header["header_length"]:
-                    raise RocketLoggerFileError(
-                        f"File position {stream_position} does not match "
-                        f"header size {header['header_length']}"
-                    )
+                if files_loaded == 0:
+                    # read full header for first file
+                    header = self._read_file_header(file_handle)
+                else:
+                    # read header lead-in only for continuation file
+                    header = self._read_file_header_lead_in(file_handle)
+                    file_handle.seek(header["header_length"])
+
+                    # consistency check against first file
+                    self._validate_matching_header(header, self._header)
 
+                    # reuse previously read header fields
+                    header["comment"] = self._header["comment"]
+                    header["channels"] = self._header["channels"]
+
+                # validate decimation factor argument
                 if (header["data_block_size"] % decimation_factor) > 0:
                     raise ValueError(
                         "Decimation factor needs to be divider of the buffer size."
                     )
 
-                if header_only is True:
+                if header_only:
                     # set data arrays to None on header only import
                     self._timestamps_realtime = None
                     self._timestamps_monotonic = None
                     self._data = None
                 else:
-                    # calculate file and data block sizes
-                    file_size = file_handle.seek(0, os.SEEK_END)
+                    # calculate data block size
                     block_bin_bytes = _BINARY_CHANNEL_STUFF_BYTES * ceil(
                         header["channel_binary_count"]
                         / (_BINARY_CHANNEL_STUFF_BYTES * 8)
@@ -826,11 +873,12 @@ def load_file(
                     ] * (block_bin_bytes + block_analog_bytes)
 
                     # file size and header consistency check and recovery
-                    if (
-                        file_size
-                        != header["header_length"]
+                    file_size = file_handle.seek(0, os.SEEK_END)
+                    file_data_bytes = (
+                        header["header_length"]
                         + header["data_block_count"] * block_size_bytes
-                    ):
+                    )
+                    if file_size != file_data_bytes:
                         data_blocks_recovered = floor(
                             (file_size - header["header_length"]) / block_size_bytes
                         )

script/python/setup.py

@@ -39,7 +39,7 @@
 
 setup(
     name="rocketlogger",
-    version="2.0.1",
+    version="2.0.2",
     author="ETH Zurich, Computer Engineering Group",
     description="RocketLogger Python Support",
     long_description=long_description,