feat: offline cross-endian decoding and golden fixtures

.gitattributes

@@ -0,0 +1,2 @@
+# Golden test fixtures are committed binaries; never apply text/EOL normalization
+tests/golden/** binary

.gitignore

@@ -29,3 +29,6 @@ output.xml
 report.html
 tmp/
 /*.clltk_trace
+
+# golden test fixtures are committed binaries, not build artifacts
+!tests/golden/*.o

VERSION.md

@@ -1,6 +1,15 @@
-1.3.0
+1.4.0
 
 # Change log
+## 1.4.0
+- feat: cross-endian decoding. Trace files and ELF binaries written on a machine with the
+  opposite byte order (e.g. s390x files read on x86_64/aarch64 and vice versa) are
+  byte-swapped while reading; the byte order is detected from the file magic / ELF
+  identification. Works through the decoder library, the clltk CLI, and the python decoder
+  (which additionally fixes float/double arguments from foreign-endian files).
+- test: golden trace file fixtures generated by real library builds at every format
+  boundary in the public history (1.2.39, 1.2.49, 1.2.64, 1.3.0) plus big-endian s390x
+  trace file and ELF fixtures; decoded through both decoders on every test run.
 ## 1.3.0
 - fix: tracepoints in inline functions, templates, and class members no longer fail with
   "causes a section type conflict" on GCC >= 15.2. Meta entries are now ordinary statics;

decoder_tool/cpp/source/Tracebuffer.cpp

@@ -108,7 +108,12 @@ TracepointPtr SyncTbInternal::next(const TracepointFilterFunc &filter) noexcept
 		auto &e = std::get<Ringbuffer::EntryPtr>(ringbuffer_entry);
 		if (e == nullptr) return {};
 
-		const uint64_t fileoffset = get<uint64_t>(e->body()) & ((1ULL << 48) - 1);
+		// the file offset is stored as 6 bytes in the writer's byte order at
+		// the start of the entry body; the remaining 2 bytes of the uint64
+		// read belong to the next field and are discarded
+		const uint64_t fileoffset_raw = get<uint64_t>(e->body(), 0, e->foreignEndian());
+		const uint64_t fileoffset =
+			e->foreignEndian() ? (fileoffset_raw >> 16) : (fileoffset_raw & ((1ULL << 48) - 1));
 		if (fileoffset == 0x01) {
 			auto tp = make_tracepoint<TracepointDynamic>(std::string(name()), std::move(e),
 														 m_source_type);
@@ -147,7 +152,12 @@ TracepointPtr SyncTbInternal::next_pooled(TracepointPool &pool,
 		auto &e = std::get<Ringbuffer::EntryPtr>(ringbuffer_entry);
 		if (e == nullptr) return {};
 
-		const uint64_t fileoffset = get<uint64_t>(e->body()) & ((1ULL << 48) - 1);
+		// the file offset is stored as 6 bytes in the writer's byte order at
+		// the start of the entry body; the remaining 2 bytes of the uint64
+		// read belong to the next field and are discarded
+		const uint64_t fileoffset_raw = get<uint64_t>(e->body(), 0, e->foreignEndian());
+		const uint64_t fileoffset =
+			e->foreignEndian() ? (fileoffset_raw >> 16) : (fileoffset_raw & ((1ULL << 48) - 1));
 		if (fileoffset == 0x01) {
 			auto tp = make_pooled_tracepoint<TracepointDynamic>(pool, std::string(name()),
 																std::move(e), m_source_type);
@@ -201,7 +211,7 @@ bool Tracebuffer::is_tracebuffer(const fs::path &path) {
 
 	const std::string_view magic{fileHead.data(), fileHead.size()};
 	if (magic == little_endian_magic) return true;
-	if (magic == big_endian_magic) return false; // currently not supported
+	if (magic == big_endian_magic) return true; // foreign byte order, byte-swapped while reading
 	return false;
 }
 bool SnapTracebuffer::is_formattable(const fs::path &path) {

decoder_tool/cpp/source/Tracepoint.cpp

@@ -32,10 +32,10 @@ void TracepointDeleter::operator()(Tracepoint *ptr) const noexcept {
 }
 
 TraceEntryHead::TraceEntryHead(uint64_t n, uint64_t t, const std::span<const uint8_t> &body,
-							   SourceType src)
+							   bool foreign_endian, SourceType src)
 	: Tracepoint(n, t, src)
-	, m_pid(get<uint32_t>(body, 6))
-	, m_tid(get<uint32_t>(body, 10)) {}
+	, m_pid(get<uint32_t>(body, 6, foreign_endian))
+	, m_tid(get<uint32_t>(body, 10, foreign_endian)) {}
 
 TraceEntryHead::TraceEntryHead(uint64_t n, uint64_t t, uint32_t pid, uint32_t tid, SourceType src)
 	: Tracepoint(n, t, src)
@@ -44,7 +44,8 @@ TraceEntryHead::TraceEntryHead(uint64_t n, uint64_t t, uint32_t pid, uint32_t ti
 
 TracepointDynamic::TracepointDynamic(std::string tb_name, source::Ringbuffer::EntryPtr entry,
 									 SourceType src)
-	: TraceEntryHead(entry->nr, get<uint64_t>(entry->body(), 14), entry->body(), src)
+	: TraceEntryHead(entry->nr, get<uint64_t>(entry->body(), 14, entry->foreignEndian()),
+					 entry->body(), entry->foreignEndian(), src)
 	, m_tracebuffer(std::move(tb_name))
 	, e(std::move(entry)) {
 	const size_t size = e->body().size();
@@ -78,6 +79,7 @@ TracepointDynamic::TracepointDynamic(std::string tb_name, source::Ringbuffer::En
 	}
 	const char *const line_start = current;
 	m_line = *std::bit_cast<const size_t *>(line_start);
+	if (e->foreignEndian()) { m_line = source::internal::byteswapValue(m_line); }
 	current += line_len;
 	remaining -= line_len;
 
@@ -90,13 +92,14 @@ using FilePtr = source::internal::FilePtr;
 TracepointStatic::TracepointStatic(std::string tb_name, source::Ringbuffer::EntryPtr &&entry,
 								   const std::span<const uint8_t> &arg_m, const FilePtr &&f,
 								   SourceType src)
-	: TraceEntryHead(entry->nr, get<uint64_t>(entry->body(), 14), entry->body(), src)
+	: TraceEntryHead(entry->nr, get<uint64_t>(entry->body(), 14, entry->foreignEndian()),
+					 entry->body(), entry->foreignEndian(), src)
 	, m_tracebuffer(std::move(tb_name))
 	, m(arg_m)
 	, e(std::move(entry))
 	, m_keep_memory(f)
 	, m_type(toMetaType(get<uint8_t>(m, 5)))
-	, m_line(get<uint32_t>(m, 6))
+	, m_line(get<uint32_t>(m, 6, e->foreignEndian()))
 	, m_arg_count(std::min((uint8_t)10, get<uint8_t>(m, 10)))
 	, m_arg_types((const char *)&m[11], m_arg_count)
 	, m_file()
@@ -132,9 +135,9 @@ const std::string_view TracepointStatic::msg() const {
 		const size_t arg_size = (e->size() > 22) ? (e->size() - 22) : 0;
 		std::span<const uint8_t> args_raw{arg_start, arg_size};
 		if (m_type == MetaType::printf) [[likely]] {
-			m_msg = source::formatter::printf(format(), m_arg_types, args_raw);
+			m_msg = source::formatter::printf(format(), m_arg_types, args_raw, e->foreignEndian());
 		} else if (m_type == MetaType::dump) {
-			m_msg = source::formatter::dump(format(), m_arg_types, args_raw);
+			m_msg = source::formatter::dump(format(), m_arg_types, args_raw, e->foreignEndian());
 		} else {
 			CLLTK_DECODER_THROW(
 				exception::InvalidMeta,

decoder_tool/cpp/source/TracepointInternal.hpp

@@ -13,11 +13,17 @@ template <typename T>
 concept PODType = std::is_standard_layout_v<T> && std::is_trivial_v<T>;
 
 template <PODType T, std::ranges::contiguous_range R>
-INLINE static constexpr T get(R r, size_t offset = 0) {
+INLINE static constexpr T get(R r, size_t offset = 0, bool foreign_endian = false) {
 	const uintptr_t base = std::bit_cast<uintptr_t>(r.data());
 	const uintptr_t position = base + offset;
 	const T *const ptr = reinterpret_cast<T *>(position);
-	return *ptr;
+	T value = *ptr;
+	if constexpr (CommonLowLevelTracingKit::decoder::source::internal::ByteSwappable<T>) {
+		if (foreign_endian) {
+			value = CommonLowLevelTracingKit::decoder::source::internal::byteswapValue(value);
+		}
+	}
+	return value;
 }
 
 namespace CommonLowLevelTracingKit::decoder {
@@ -68,7 +74,7 @@ namespace CommonLowLevelTracingKit::decoder {
 		const uint32_t m_pid;
 		const uint32_t m_tid;
 		TraceEntryHead(uint64_t n, uint64_t t, const std::span<const uint8_t> &body,
-					   SourceType src = SourceType::Unknown);
+					   bool foreign_endian, SourceType src = SourceType::Unknown);
 		TraceEntryHead(uint64_t n, uint64_t t, uint32_t pid, uint32_t tid,
 					   SourceType src = SourceType::Unknown);
 		uint32_t pid() const noexcept override { return m_pid; };
@@ -139,7 +145,7 @@ namespace CommonLowLevelTracingKit::decoder {
 			, m_line() {}
 		VirtualTracepoint(std::string tb_name, const source::Ringbuffer::Entry &e,
 						  const std::string &msg, SourceType src = SourceType::Unknown)
-			: TraceEntryHead(e.nr, get<uint64_t>(e.body(), 14), 0, 0, src)
+			: TraceEntryHead(e.nr, get<uint64_t>(e.body(), 14, e.foreignEndian()), 0, 0, src)
 			, m_tracebuffer(std::move(tb_name))
 			, m_msg(msg)
 			, m_file()

decoder_tool/cpp/source/low_level/elf_reader.cpp

@@ -2,9 +2,11 @@
 // SPDX-License-Identifier: BSD-2-Clause-Patent
 
 #include "elf_reader.hpp"
+#include "file.hpp"
 #include "meta_parser.hpp"
 
 #include <algorithm>
+#include <bit>
 #include <cstring>
 #include <elf.h>
 #include <fstream>
@@ -31,6 +33,21 @@ namespace CommonLowLevelTracingKit::decoder::source {
 			return data[EI_CLASS] == ELFCLASS64;
 		}
 
+		// an ELF file declares its byte order in the identification bytes;
+		// foreign means the opposite of this host's order
+		bool isForeignElf(const std::vector<uint8_t> &data) {
+			if (data.size() < EI_NIDENT) { return false; }
+			constexpr uint8_t host_order =
+				(std::endian::native == std::endian::little) ? ELFDATA2LSB : ELFDATA2MSB;
+			const uint8_t file_order = data[EI_DATA];
+			return (file_order == ELFDATA2LSB || file_order == ELFDATA2MSB) &&
+				   (file_order != host_order);
+		}
+
+		template <typename T> T swapped(T value, bool foreign) {
+			return foreign ? internal::byteswapValue(value) : value;
+		}
+
 		std::string getSectionName(const std::vector<uint8_t> &data, uint64_t strtab_offset,
 								   uint32_t name_index) {
 			if (strtab_offset + name_index >= data.size()) { return ""; }
@@ -47,27 +64,28 @@ namespace CommonLowLevelTracingKit::decoder::source {
 
 			if (data.size() < sizeof(Elf64_Ehdr)) { return sections; }
 
+			const bool foreign = isForeignElf(data);
 			const auto *ehdr = reinterpret_cast<const Elf64_Ehdr *>(data.data());
-			if (ehdr->e_shoff == 0 || ehdr->e_shnum == 0) { return sections; }
-			if (ehdr->e_shoff + ehdr->e_shnum * sizeof(Elf64_Shdr) > data.size()) {
-				return sections;
-			}
+			const uint64_t e_shoff = swapped(ehdr->e_shoff, foreign);
+			const uint16_t e_shnum = swapped(ehdr->e_shnum, foreign);
+			const uint16_t e_shstrndx = swapped(ehdr->e_shstrndx, foreign);
+			if (e_shoff == 0 || e_shnum == 0) { return sections; }
+			if (e_shoff + e_shnum * sizeof(Elf64_Shdr) > data.size()) { return sections; }
 
-			if (ehdr->e_shstrndx >= ehdr->e_shnum) { return sections; }
+			if (e_shstrndx >= e_shnum) { return sections; }
 
-			const auto *shdr_base =
-				reinterpret_cast<const Elf64_Shdr *>(data.data() + ehdr->e_shoff);
-			const auto &shstrtab_hdr = shdr_base[ehdr->e_shstrndx];
+			const auto *shdr_base = reinterpret_cast<const Elf64_Shdr *>(data.data() + e_shoff);
+			const uint64_t shstrtab_offset = swapped(shdr_base[e_shstrndx].sh_offset, foreign);
 
-			for (uint16_t i = 0; i < ehdr->e_shnum; ++i) {
+			for (uint16_t i = 0; i < e_shnum; ++i) {
 				const auto &shdr = shdr_base[i];
 
 				ElfSectionInfo info;
-				info.name = getSectionName(data, shstrtab_hdr.sh_offset, shdr.sh_name);
-				info.offset = shdr.sh_offset;
-				info.size = shdr.sh_size;
-				info.addr = shdr.sh_addr;
-				info.type = shdr.sh_type;
+				info.name = getSectionName(data, shstrtab_offset, swapped(shdr.sh_name, foreign));
+				info.offset = swapped(shdr.sh_offset, foreign);
+				info.size = swapped(shdr.sh_size, foreign);
+				info.addr = swapped(shdr.sh_addr, foreign);
+				info.type = swapped(shdr.sh_type, foreign);
 
 				sections.push_back(std::move(info));
 			}
@@ -80,26 +98,27 @@ namespace CommonLowLevelTracingKit::decoder::source {
 
 			if (data.size() < sizeof(Elf32_Ehdr)) { return sections; }
 
+			const bool foreign = isForeignElf(data);
 			const auto *ehdr = reinterpret_cast<const Elf32_Ehdr *>(data.data());
-			if (ehdr->e_shoff == 0 || ehdr->e_shnum == 0) { return sections; }
-			if (ehdr->e_shoff + ehdr->e_shnum * sizeof(Elf32_Shdr) > data.size()) {
-				return sections;
-			}
-			if (ehdr->e_shstrndx >= ehdr->e_shnum) { return sections; }
+			const uint32_t e_shoff = swapped(ehdr->e_shoff, foreign);
+			const uint16_t e_shnum = swapped(ehdr->e_shnum, foreign);
+			const uint16_t e_shstrndx = swapped(ehdr->e_shstrndx, foreign);
+			if (e_shoff == 0 || e_shnum == 0) { return sections; }
+			if (e_shoff + e_shnum * sizeof(Elf32_Shdr) > data.size()) { return sections; }
+			if (e_shstrndx >= e_shnum) { return sections; }
 
-			const auto *shdr_base =
-				reinterpret_cast<const Elf32_Shdr *>(data.data() + ehdr->e_shoff);
-			const auto &shstrtab_hdr = shdr_base[ehdr->e_shstrndx];
+			const auto *shdr_base = reinterpret_cast<const Elf32_Shdr *>(data.data() + e_shoff);
+			const uint32_t shstrtab_offset = swapped(shdr_base[e_shstrndx].sh_offset, foreign);
 
-			for (uint16_t i = 0; i < ehdr->e_shnum; ++i) {
+			for (uint16_t i = 0; i < e_shnum; ++i) {
 				const auto &shdr = shdr_base[i];
 
 				ElfSectionInfo info;
-				info.name = getSectionName(data, shstrtab_hdr.sh_offset, shdr.sh_name);
-				info.offset = shdr.sh_offset;
-				info.size = shdr.sh_size;
-				info.addr = shdr.sh_addr;
-				info.type = shdr.sh_type;
+				info.name = getSectionName(data, shstrtab_offset, swapped(shdr.sh_name, foreign));
+				info.offset = swapped(shdr.sh_offset, foreign);
+				info.size = swapped(shdr.sh_size, foreign);
+				info.addr = swapped(shdr.sh_addr, foreign);
+				info.type = swapped(shdr.sh_type, foreign);
 
 				sections.push_back(std::move(info));
 			}
@@ -134,19 +153,20 @@ namespace CommonLowLevelTracingKit::decoder::source {
 		// parse one self-describing meta entry (magic + size prefix) located
 		// at a file offset and append the result
 		void appendMetaEntryAt(const std::vector<uint8_t> &data, uint64_t entry_offset,
-							   MetaEntryInfoCollection &entries) {
+							   MetaEntryInfoCollection &entries, bool foreign) {
 			if (entry_offset + MetaParser::MIN_ENTRY_SIZE > data.size()) { return; }
 
 			uint32_t entry_size = 0;
 			std::memcpy(&entry_size, data.data() + entry_offset + MetaParser::OFFSET_SIZE,
 						sizeof(entry_size));
+			entry_size = swapped(entry_size, foreign);
 			if (entry_size < MetaParser::MIN_ENTRY_SIZE ||
 				entry_offset + entry_size > data.size()) {
 				return;
 			}
 
 			const std::span<const uint8_t> entry_data(data.data() + entry_offset, entry_size);
-			auto parsed = MetaParser::parse(entry_data, entry_offset);
+			auto parsed = MetaParser::parse(entry_data, entry_offset, foreign);
 			entries.insert(entries.end(), std::make_move_iterator(parsed.begin()),
 						   std::make_move_iterator(parsed.end()));
 		}
@@ -156,7 +176,7 @@ namespace CommonLowLevelTracingKit::decoder::source {
 			// e_type sits at the same offset for ELFCLASS32 and ELFCLASS64
 			uint16_t e_type = 0;
 			std::memcpy(&e_type, data.data() + offsetof(Elf64_Ehdr, e_type), sizeof(e_type));
-			return e_type == ET_REL;
+			return swapped(e_type, isForeignElf(data)) == ET_REL;
 		}
 
 		// In relocatable objects (.o) the pointer slots are zero; the meta
@@ -182,13 +202,17 @@ namespace CommonLowLevelTracingKit::decoder::source {
 			if (symtab == nullptr || rela == nullptr) { return entries; }
 			if (rela->offset + rela->size > data.size()) { return entries; }
 
+			const bool foreign = isForeignElf(data);
 			std::vector<uint64_t> seen;
 			const size_t entry_stride = 2 * pointer_size;
 			const size_t count = rela->size / sizeof(Elf64_Rela);
 			for (size_t i = 0; i < count; ++i) {
 				Elf64_Rela relocation = {};
 				std::memcpy(&relocation, data.data() + rela->offset + i * sizeof(relocation),
 							sizeof(relocation));
+				relocation.r_offset = swapped(relocation.r_offset, foreign);
+				relocation.r_info = swapped(relocation.r_info, foreign);
+				relocation.r_addend = swapped(relocation.r_addend, foreign);
 				if (relocation.r_offset % entry_stride != 0) { continue; } // offset cache slot
 
 				const uint64_t sym_index = ELF64_R_SYM(relocation.r_info);
@@ -199,6 +223,8 @@ namespace CommonLowLevelTracingKit::decoder::source {
 				}
 				Elf64_Sym symbol = {};
 				std::memcpy(&symbol, data.data() + sym_offset, sizeof(symbol));
+				symbol.st_shndx = swapped(symbol.st_shndx, foreign);
+				symbol.st_value = swapped(symbol.st_value, foreign);
 				if (symbol.st_shndx == SHN_UNDEF || symbol.st_shndx >= sections.size()) {
 					continue;
 				}
@@ -208,7 +234,7 @@ namespace CommonLowLevelTracingKit::decoder::source {
 					target.offset + symbol.st_value + (uint64_t)relocation.r_addend;
 				if (std::find(seen.begin(), seen.end(), entry_offset) != seen.end()) { continue; }
 				seen.push_back(entry_offset);
-				appendMetaEntryAt(data, entry_offset, entries);
+				appendMetaEntryAt(data, entry_offset, entries, foreign);
 			}
 
 			return entries;
@@ -227,6 +253,7 @@ namespace CommonLowLevelTracingKit::decoder::source {
 													size_t pointer_size) {
 			MetaEntryInfoCollection entries;
 
+			const bool foreign = isForeignElf(data);
 			std::vector<uint64_t> seen;
 			const size_t entry_stride = 2 * pointer_size;
 			const size_t count = ptr_section.size / entry_stride;
@@ -235,7 +262,15 @@ namespace CommonLowLevelTracingKit::decoder::source {
 				if (ptr_offset + pointer_size > data.size()) { break; }
 
 				uint64_t address = 0;
-				std::memcpy(&address, data.data() + ptr_offset, pointer_size);
+				if (pointer_size == sizeof(uint64_t)) {
+					uint64_t raw = 0;
+					std::memcpy(&raw, data.data() + ptr_offset, sizeof(raw));
+					address = swapped(raw, foreign);
+				} else {
+					uint32_t raw = 0;
+					std::memcpy(&raw, data.data() + ptr_offset, sizeof(raw));
+					address = swapped(raw, foreign);
+				}
 				if (address == 0) { continue; }
 				if (std::find(seen.begin(), seen.end(), address) != seen.end()) { continue; }
 				seen.push_back(address);
@@ -245,7 +280,8 @@ namespace CommonLowLevelTracingKit::decoder::source {
 					if (address < section.addr || address >= section.addr + section.size) {
 						continue;
 					}
-					appendMetaEntryAt(data, section.offset + (address - section.addr), entries);
+					appendMetaEntryAt(data, section.offset + (address - section.addr), entries,
+									  foreign);
 					break;
 				}
 			}

decoder_tool/cpp/source/low_level/file.cpp

@@ -94,7 +94,8 @@ size_t internal::File::grow() const {
 source::FilePart::FilePart(const FilePart &a_filePart, const size_t a_offset)
 	: m_file(a_filePart.m_file)
 	, m_offset(a_filePart.m_offset + a_offset)
-	, m_base(m_file->data()) {
+	, m_base(m_file->data())
+	, m_foreign_endian(a_filePart.m_foreign_endian) {
 	const size_t max_access = m_offset;
 	if (max_access >= m_file->size()) [[unlikely]] {
 		m_file->grow();