diff --git a/CHANGELOG.md b/CHANGELOG.md index 0dba4bde..b69c3273 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,31 @@ # Changelog +## Version 2.0.2 (2021-12-09) + +RocketLogger software: +* [FIXED] Corrupt RLD files when logging no binary channels (#57) +* [FIXED] Corrupt channel info in split RLD files (#62) + +Python support library: +* [ADDED] Standalone script to recover RLD files corrupted by #57 +* [FIXED] Missing documentation of file revision changes (#61) +* [CHANGED] Cross validate file headers and tolerate channel info corruption of split files (relates to #62) + + +_Notes:_ + +This bugfix release resolves RLD file corruption related issues. Details on how to fully recover affected measurement files are given below. +A timely update is highly recommended to avoid generating corrupt data files. + + +_Recover Corrupt Data Files:_ + +Measurement files affected by bugs #57 or #62 can be recovered without data loss: +* To recover files affected by #57 use the `bug57_recover.py` script provided in the Python source package (or available directly from the repository). +* The updated Python library can tolerate channel info header corruption of split files due to bug #62 if all files of the same measurement are imported in one batch. + + ## Version 2.0.1 (2021-11-29) RocketLogger software: @@ -83,6 +108,7 @@ RocketLogger software: * [CHANGED] rework low level hardware interfacing for Debian *buster* compatibility (#2) * [CHANGED] update and reorganize internal software API and headers for increased consistency * [CHANGED] update to latest and official compiler tools (#18) +* [CHANGED] Binary file format: increment file version to `0x04` * [REMOVED] legacy web control interface Python support library: diff --git a/script/python/LICENSE b/script/python/LICENSE index 115a8ed8..733094e4 100644 --- a/script/python/LICENSE +++ b/script/python/LICENSE @@ -1,4 +1,5 @@ Copyright (c) 2016-2020, ETH Zurich, Computer Engineering Group +Copyright (c) 2020-2021, Lukas Sigrist All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/script/python/README.md b/script/python/README.md index c8f9f945..b29fb22e 100644 --- a/script/python/README.md +++ b/script/python/README.md @@ -12,6 +12,10 @@ support to generate calibration data from measurements. * Matplotlib: for plotting data overview * pandas: for pandas DataFrame export +**Compatibility** +* Data processing: supports all officially specified RLD file version (versions 2-4) +* Calibration: compatible with RocketLogger calibration file version 2 + ## Installation diff --git a/script/python/bug57_recover.py b/script/python/bug57_recover.py new file mode 100755 index 00000000..bf39044a --- /dev/null +++ b/script/python/bug57_recover.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +""" +Recover RLD measurement files suffering from bug #57. + +Usage: ./bug57_recovery.sh +* the filename of the RLD file to recover + +This script attempts to patch the file header of RLD files that do not +contain binary data channels, by adding a dummy "BUG57_PATCH" binary channel. +Adding this dummy channel makes files suffering from bug #57 again compliant +with the RLD file specification. +The script does not modify the original data, but stores the patched data in +a new data file with the input filename suffixed with "_patched". + +See also: +""" + +import os +import re +import sys + +_RLD_FILE_REGEX = re.compile( + r"(?P.*?)(?P_p\d+)?(?P\.rld)", re.IGNORECASE +) + +# file header length offset and size in bytes +_HEADER_LENGTH_OFFSET = 0x06 +_HEADER_LENGTH_BYTES = 2 + +# file comment length offset and size in bytes +_COMMENT_LENGTH_OFFSET = 0x30 +_COMMENT_LENGTH_BYTES = 4 + +# file binary channel count offset and size in bytes +_BINARY_CHANNEL_COUNT_OFFSET = 0x34 +_BINARY_CHANNEL_COUNT_BYTES = 2 + +# file analog channel count offset and size in bytes +_ANALOG_CHANNEL_COUNT_OFFSET = 0x36 +_ANALOG_CHANNEL_COUNT_BYTES = 2 + +# Data Valid (binary) channel info with name "BUG57_PATCH" +_DUMMY_BINARY_CHANNEL_DATA = bytes([0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, + 0x42, 0x55, 0x47, 0x35, 0x37, 0x5F, 0x50, 0x41, 0x54, 0x43, 0x48, 0x00, 0x00, 0x00, 0x00, 0x00]) + + +class FilePatchingError(Exception): + """File patching related errors.""" + + pass + + +if __name__ == "__main__": + + # get filename argument + if len(sys.argv) != 2: + raise TypeError("the script takes exactly one filename argument") + input_file = str(sys.argv[1]) + + # output file name and check filename extension + basename, ext = os.path.splitext(input_file) + match = _RLD_FILE_REGEX.match(input_file) + + if not match.group("ext"): + raise ValueError("the script only processes RLD files") + + if match.group("part"): + output_file = ( + match.group("basename") + + "_patched" + + match.group("part") + + match.group("ext") + ) + else: + output_file = match.group("basename") + "_patched" + match.group("ext") + + try: + with open(input_file, "r+b") as file_in, open(output_file, "x+b") as file_out: + print(f"Attempt to create patched file: {output_file}") + + # copy start of original header + file_out.write(file_in.read(_HEADER_LENGTH_OFFSET)) + + # update header length + header_length = int.from_bytes( + file_in.read(_HEADER_LENGTH_BYTES), byteorder="little", signed=False + ) + patched_header_length = header_length + len(_DUMMY_BINARY_CHANNEL_DATA) + file_out.write( + int.to_bytes( + patched_header_length, + length=_HEADER_LENGTH_BYTES, + byteorder="little", + signed=False, + ) + ) + + # copy header up to comment length + file_out.write( + file_in.read( + _COMMENT_LENGTH_OFFSET + - _HEADER_LENGTH_BYTES + - _HEADER_LENGTH_OFFSET + ) + ) + + # get and copy comment length + comment_length_bytes = file_in.read(_COMMENT_LENGTH_BYTES) + file_out.write(comment_length_bytes) + comment_length = int.from_bytes( + comment_length_bytes, byteorder="little", signed=False + ) + + # update binary channel count + binary_channel_count = int.from_bytes( + file_in.read(_BINARY_CHANNEL_COUNT_BYTES), + byteorder="little", + signed=False, + ) + if binary_channel_count != 0: + raise FilePatchingError( + "Provided file contains binary therefore cannot (and never should) be patched" + ) + + binary_channel_count += 1 + file_out.write( + int.to_bytes( + binary_channel_count, + length=_HEADER_LENGTH_BYTES, + byteorder="little", + signed=False, + ) + ) + + # copy remaining analog channel count and comment + file_out.write(file_in.read(_ANALOG_CHANNEL_COUNT_BYTES + comment_length)) + + # append dummy binary channel + file_out.write(_DUMMY_BINARY_CHANNEL_DATA) + + # copy remaining file content + file_out.write(file_in.read()) + except FileExistsError: + print(f"A patched file `{output_file}` already exists! Skip patching...") + sys.exit(1) + except FilePatchingError as e: + print(f"Failed to patch file: {e}") + print(f"Removing incomplete file: {output_file}") + os.remove(output_file) + sys.exit(1) + + print(f"Patched file was successfully written to: {output_file}") diff --git a/script/python/data/test_non_split.rld b/script/python/data/test_non_split.rld new file mode 100644 index 00000000..1c7dfb56 Binary files /dev/null and b/script/python/data/test_non_split.rld differ diff --git a/script/python/data/test_non_split_p1.rld b/script/python/data/test_non_split_p1.rld new file mode 100644 index 00000000..c9727a02 Binary files /dev/null and b/script/python/data/test_non_split_p1.rld differ diff --git a/script/python/docs/conf.py b/script/python/docs/conf.py index d852efb1..71b52dd3 100644 --- a/script/python/docs/conf.py +++ b/script/python/docs/conf.py @@ -22,7 +22,7 @@ author = 'ETH Zurich, Computer Engineering Group' # The full version, including alpha/beta/rc tags -release = '2.0.1' +release = '2.0.2' # -- General configuration --------------------------------------------------- diff --git a/script/python/docs/index.rst b/script/python/docs/index.rst index 18ff4333..6c877467 100644 --- a/script/python/docs/index.rst +++ b/script/python/docs/index.rst @@ -19,16 +19,15 @@ generation of RocketLogger device calibration file from measurements. Features -------- -Python support for RocketLogger Data (``*.rld``) files by the -:mod:`rocketlogger.data` module. +Python support for RocketLogger Data (``*.rld``) files by the :mod:`rocketlogger.data` module. +Supports any officially specified RocketLogger data file version (versions 2-4). - Import - Channel merging - Data extraction - Plotting of file data -Support for RocketLogger Device Calibration by the -:mod:`rocketlogger.calibration` module. +Support for RocketLogger Device Calibration by the :mod:`rocketlogger.calibration` module. - Generate new calibrations from measurements - Read/write calibration files diff --git a/script/python/rocketlogger/data.py b/script/python/rocketlogger/data.py index f83c09ee..44c49098 100644 --- a/script/python/rocketlogger/data.py +++ b/script/python/rocketlogger/data.py @@ -314,14 +314,14 @@ def __init__( else: raise FileNotFoundError(f"File '{filename}' does not exist.") - def _read_file_header(self, file_handle): + def _read_file_header_lead_in(self, file_handle): """ - Read a RocketLogger data file's header, including comment and channels. + Read a RocketLogger data file's header fixed size lead-in :param file_handle: The file handle to read from, with pointer positioned at file start - :returns: Dictionary containing the read file header data + :returns: Dictionary containing the read file header lead-in data """ header = {} @@ -386,6 +386,19 @@ def _read_file_header(self, file_handle): ) ) + return header + + def _read_file_header(self, file_handle): + """ + Read a RocketLogger data file's header, including comment and channels. + + :param file_handle: The file handle to read from, with pointer + positioned after lead-in/at start of comment + + :returns: Dictionary containing the read file header data + """ + header = self._read_file_header_lead_in(file_handle) + # read comment field header["comment"] = _read_str(file_handle, header["comment_length"]) @@ -421,8 +434,41 @@ def _read_file_header(self, file_handle): # add channel to header header["channels"].append(channel) + # consistency check: file stream position matches header size + stream_position = file_handle.tell() + if stream_position != header["header_length"]: + raise RocketLoggerFileError( + f"File position {stream_position} does not match " + f"header size {header['header_length']}" + ) + return header + def _validate_matching_header(self, header1, header2): + """ + Validate that file headers match, i.e. correspond to the same measurement. + + :param header1: First file header data to compare + + :param header2: Second file header data to compare + """ + required_matches = [ + "file_version", + "header_length", + "data_block_size", + "sample_rate", + "mac_address", + "start_time", + "comment_length", + "channel_binary_count", + "channel_analog_count", + ] + for header_field in required_matches: + if header1[header_field] != header2[header_field]: + raise RocketLoggerDataError( + f"File header not matching at field: {header_field}" + ) + def _read_file_data( self, file_handle, file_header, decimation_factor=1, memory_mapped=True ): @@ -783,33 +829,34 @@ def load_file( break with open(file_name, "rb") as file_handle: - # read file header - header = self._read_file_header(file_handle) - - # consistency check: file stream position matches header size - if "header_length" not in header: - raise RocketLoggerFileError("Invalid file header read.") - - stream_position = file_handle.tell() - if stream_position != header["header_length"]: - raise RocketLoggerFileError( - f"File position {stream_position} does not match " - f"header size {header['header_length']}" - ) + if files_loaded == 0: + # read full header for first file + header = self._read_file_header(file_handle) + else: + # read header lead-in only for continuation file + header = self._read_file_header_lead_in(file_handle) + file_handle.seek(header["header_length"]) + + # consistency check against first file + self._validate_matching_header(header, self._header) + # reuse previously read header fields + header["comment"] = self._header["comment"] + header["channels"] = self._header["channels"] + + # validate decimation factor argument if (header["data_block_size"] % decimation_factor) > 0: raise ValueError( "Decimation factor needs to be divider of the buffer size." ) - if header_only is True: + if header_only: # set data arrays to None on header only import self._timestamps_realtime = None self._timestamps_monotonic = None self._data = None else: - # calculate file and data block sizes - file_size = file_handle.seek(0, os.SEEK_END) + # calculate data block size block_bin_bytes = _BINARY_CHANNEL_STUFF_BYTES * ceil( header["channel_binary_count"] / (_BINARY_CHANNEL_STUFF_BYTES * 8) @@ -826,11 +873,12 @@ def load_file( ] * (block_bin_bytes + block_analog_bytes) # file size and header consistency check and recovery - if ( - file_size - != header["header_length"] + file_size = file_handle.seek(0, os.SEEK_END) + file_data_bytes = ( + header["header_length"] + header["data_block_count"] * block_size_bytes - ): + ) + if file_size != file_data_bytes: data_blocks_recovered = floor( (file_size - header["header_length"]) / block_size_bytes ) diff --git a/script/python/setup.py b/script/python/setup.py index 26f8c15a..fe6434ca 100644 --- a/script/python/setup.py +++ b/script/python/setup.py @@ -39,7 +39,7 @@ setup( name="rocketlogger", - version="2.0.1", + version="2.0.2", author="ETH Zurich, Computer Engineering Group", description="RocketLogger Python Support", long_description=long_description, diff --git a/script/python/tests/test_data.py b/script/python/tests/test_data.py index 3f731958..06631075 100644 --- a/script/python/tests/test_data.py +++ b/script/python/tests/test_data.py @@ -64,6 +64,7 @@ _HEADER_UNALIGNED_TEST_FILE = os.path.join(_TEST_FILE_DIR, "test_header_unaligned.rld") _TRUNCATED_TEST_FILE = os.path.join(_TEST_FILE_DIR, "test_truncated.rld") _SPLIT_TEST_FILE = os.path.join(_TEST_FILE_DIR, "test_split.rld") +_NON_SPLIT_TEST_FILE = os.path.join(_TEST_FILE_DIR, "test_non_split.rld") _SPLIT_TRUNCATED_TEST_FILE = os.path.join(_TEST_FILE_DIR, "test_split_truncated.rld") _TEMP_FILE = os.path.join(_TEST_FILE_DIR, "temp_data.rld") @@ -352,6 +353,12 @@ def test_channel_names(self): ) +class TestJoinMissmatch(TestCase): + def test_exclude_all(self): + with self.assertRaisesRegex(RocketLoggerDataError, "header not matching at field: start_time"): + RocketLoggerData(_NON_SPLIT_TEST_FILE) + + class TestJoinExclude(TestCase): def test_exclude_all(self): with self.assertRaisesRegex(RocketLoggerDataError, "Could not load valid data"): diff --git a/software/node_server/package-lock.json b/software/node_server/package-lock.json index d9b01edd..cda780c4 100644 --- a/software/node_server/package-lock.json +++ b/software/node_server/package-lock.json @@ -1,6 +1,6 @@ { "name": "rocketlogger", - "version": "2.0.1", + "version": "2.0.2", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/software/node_server/package.json b/software/node_server/package.json index d65cac73..ed479dde 100644 --- a/software/node_server/package.json +++ b/software/node_server/package.json @@ -1,6 +1,6 @@ { "name": "rocketlogger", - "version": "2.0.1", + "version": "2.0.2", "description": "RocketLogger web interface", "homepage": "https://github.com/ETHZ-TEC/RocketLogger#readme", "author": "ETH Zurich, Computer Engineering Group", diff --git a/software/rocketlogger/Doxyfile b/software/rocketlogger/Doxyfile index bbaa9c97..871e7fce 100644 --- a/software/rocketlogger/Doxyfile +++ b/software/rocketlogger/Doxyfile @@ -38,7 +38,7 @@ PROJECT_NAME = RocketLogger # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 2.0.1 +PROJECT_NUMBER = 2.0.2 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/software/rocketlogger/meson.build b/software/rocketlogger/meson.build index b5efb064..72dc5a01 100644 --- a/software/rocketlogger/meson.build +++ b/software/rocketlogger/meson.build @@ -1,5 +1,5 @@ project('rocketlogger', 'c', - version : '2.0.1', + version : '2.0.2', default_options : ['buildtype=debugoptimized', 'c_std=c11', 'cpp_std=c++11', 'warning_level=3', 'prefix=/usr', 'install_umask=0022'], diff --git a/software/rocketlogger/pru.c b/software/rocketlogger/pru.c index 5b737636..3e5b9759 100644 --- a/software/rocketlogger/pru.c +++ b/software/rocketlogger/pru.c @@ -176,11 +176,10 @@ int pru_sample(FILE *data_file, FILE *ambient_file, // data file header lead-in rl_file_setup_data_lead_in(&(data_file_header.lead_in), config); - // channel array + // allocate channel info array int total_channel_count = data_file_header.lead_in.channel_bin_count + data_file_header.lead_in.channel_count; - rl_file_channel_t file_channel[total_channel_count]; - data_file_header.channel = file_channel; + data_file_header.channel = malloc(total_channel_count * sizeof(rl_file_channel_t)); // complete file header rl_file_setup_data_header(&data_file_header, config); @@ -592,16 +591,21 @@ int pru_sample(FILE *data_file, FILE *ambient_file, } // FILE FINISH (flush) - // flush ambient data and cleanup file header - if (config->ambient_enable) { - fflush(ambient_file); - free(ambient_file_header.channel); - } - - if (config->file_enable && !(rl_status.error)) { + if (config->file_enable) { + // flush data file and clean up file header fflush(data_file); - rl_log(RL_LOG_INFO, "stored %llu samples to file", - rl_status.sample_count); + free(data_file_header.channel); + + // flush ambient file and clean up file header + if (config->ambient_enable) { + fflush(ambient_file); + free(ambient_file_header.channel); + } + + if (rl_status.error == false) { + rl_log(RL_LOG_INFO, "stored %llu samples to file", + rl_status.sample_count); + } } // STATE diff --git a/software/rocketlogger/rl_file.c b/software/rocketlogger/rl_file.c index 03faf871..d9884ab1 100644 --- a/software/rocketlogger/rl_file.c +++ b/software/rocketlogger/rl_file.c @@ -466,8 +466,10 @@ int rl_file_add_data_block(FILE *data_file, int32_t const *analog_buffer, index++; } - // write digital data to file - fwrite(&data, sizeof(data), 1, data_file); + // write digital data to file if any channel available + if (index > 0) { + fwrite(&data, sizeof(data), 1, data_file); + } } else if (config->file_format == RL_FILE_FORMAT_CSV) { if (config->digital_enable) { uint32_t binary_mask = PRU_DIGITAL_INPUT1_MASK; diff --git a/software/system/setup/system/issue.net b/software/system/setup/system/issue.net index 7d4682dc..b94ed488 100644 --- a/software/system/setup/system/issue.net +++ b/software/system/setup/system/issue.net @@ -1 +1 @@ -RocketLogger 2.0.1 +RocketLogger 2.0.2