From b379dc2a0ea6f2e81c61a7a7ef3635a8d401a271 Mon Sep 17 00:00:00 2001 From: Jo5ta Date: Fri, 10 Jul 2026 20:42:19 +0200 Subject: [PATCH 1/8] fix: python decoder float/double arguments in foreign-endian files Integer reads honor the byte order detected from the file magic, but struct.unpack used native byte order for float and double tracepoint arguments, so they decoded as garbage from foreign-endian trace files. Select the unpack byte order from the detected endianness. Signed-off-by: Jo5ta --- decoder_tool/python/clltk_decoder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/decoder_tool/python/clltk_decoder.py b/decoder_tool/python/clltk_decoder.py index acc0c8b..b10b133 100755 --- a/decoder_tool/python/clltk_decoder.py +++ b/decoder_tool/python/clltk_decoder.py @@ -163,11 +163,11 @@ def get_int(raw, offset, size, signed=False): def get_float(raw, offset): - return struct.unpack('f', get(raw, offset, 4))[0] + return struct.unpack('f', get(raw, offset, 4))[0] def get_double(raw, offset): - return struct.unpack('d', get(raw, offset, 8))[0] + return struct.unpack('d', get(raw, offset, 8))[0] class Definition: From f952839defa0b8d3cb1456dedc02691bec61b95f Mon Sep 17 00:00:00 2001 From: Jo5ta Date: Fri, 10 Jul 2026 20:42:19 +0200 Subject: [PATCH 2/8] test: add golden trace file fixtures across versions and byte orders Committed .clltk_trace fixtures produced by the real library: 1.2.64 and 1.3.0 on aarch64 (little endian) and 1.3.0 on s390x (big endian, generated under qemu-user emulation). Each contains one tracepoint per argument type with fixed values, so decoder output is deterministic. tests/test_golden.py decodes every fixture with the python decoder and compares the formatted messages - guarding backward compatibility with old file versions and cross-endian decoding. Generator source and instructions live in tests/golden/generator/. Signed-off-by: Jo5ta --- tests/golden/README.md | 32 ++++++++ tests/golden/generator/gen.sh | 17 ++++ tests/golden/generator/writer.c | 26 ++++++ .../golden-1.2.64-le-aarch64.clltk_trace | Bin 0 -> 4919 bytes .../golden/golden-1.3.0-be-s390x.clltk_trace | Bin 0 -> 5184 bytes .../golden-1.3.0-le-aarch64.clltk_trace | Bin 0 -> 5184 bytes tests/test_golden.py | 75 ++++++++++++++++++ 7 files changed, 150 insertions(+) create mode 100644 tests/golden/README.md create mode 100755 tests/golden/generator/gen.sh create mode 100644 tests/golden/generator/writer.c create mode 100644 tests/golden/golden-1.2.64-le-aarch64.clltk_trace create mode 100644 tests/golden/golden-1.3.0-be-s390x.clltk_trace create mode 100644 tests/golden/golden-1.3.0-le-aarch64.clltk_trace create mode 100644 tests/test_golden.py diff --git a/tests/golden/README.md b/tests/golden/README.md new file mode 100644 index 0000000..b41e2ca --- /dev/null +++ b/tests/golden/README.md @@ -0,0 +1,32 @@ +# Golden trace file fixtures + +Committed `.clltk_trace` files used to test the decoders against known +inputs: different library versions, both byte orders, all argument types. + +Every fixture was produced by `generator/writer.c` (one tracepoint per +argument type with fixed values, plus one never-fired tracepoint) via +`generator/gen.sh`. The values inside a fixture (pids, tids, timestamps) +are frozen in the committed file, so decoder output is fully deterministic. + +| fixture | produced by | +| --- | --- | +| `golden-1.2.64-le-aarch64.clltk_trace` | library 1.2.64 (commit 1fa19d2), aarch64 | +| `golden-1.3.0-le-aarch64.clltk_trace` | library 1.3.0, aarch64 | +| `golden-1.3.0-be-s390x.clltk_trace` | library 1.3.0, s390x (big endian) | + +`tests/test_golden.py` decodes every fixture and asserts the formatted +tracepoint messages. When the trace file format changes, add a new fixture +for the new version instead of replacing old ones — old files must stay +decodable. + +## Regenerating / adding fixtures + +```bash +mkdir out +podman run --rm -v "$(git rev-parse --show-toplevel):/src:ro" -v "$PWD/out:/out" \ + --arch registry.fedoraproject.org/fedora:43 bash /src/tests/golden/generator/gen.sh +``` + +For big-endian fixtures use `--arch s390x`; this needs qemu-user-static +binfmt support (on a podman machine VM, register the handler in the VM and +run rootful: `podman machine ssh sudo podman run ...`). diff --git a/tests/golden/generator/gen.sh b/tests/golden/generator/gen.sh new file mode 100755 index 0000000..e2ab4a0 --- /dev/null +++ b/tests/golden/generator/gen.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# builds the tracing library from /src and produces a golden trace file in /out +set -e +dnf install -y -q gcc gcc-c++ cmake make git gettext-envsubst rsync > /dev/null 2>&1 +git config --global --add safe.directory "*"; cmake -S /src -B /tmp/b \ + -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=OFF \ + -DCLLTK_TRACING=ON -DCLLTK_COMMAND_LINE_TOOL=OFF -DCLLTK_CPP_DECODER=OFF \ + -DCLLTK_SNAPSHOT=OFF -DCLLTK_PYTHON_DECODER=OFF -DCLLTK_EXAMPLES=OFF \ + -DCLLTK_TESTS=OFF > /tmp/cfg.log 2>&1 || { tail -5 /tmp/cfg.log; exit 1; } +cmake --build /tmp/b --target clltk_tracing_static -- -j4 > /tmp/build.log 2>&1 || { tail -30 /tmp/build.log; exit 1; } +LIB=$(find /tmp/b -name 'libclltk_tracing*.a' | head -1) +gcc -std=c11 -O1 -I/src/tracing_library/include /src/.fixtures/writer.c "$LIB" -pthread -o /tmp/writer +rm -rf /tmp/traces && mkdir /tmp/traces +CLLTK_TRACING_PATH=/tmp/traces /tmp/writer +cp /tmp/traces/GOLDEN.clltk_trace /out/ +uname -m > /out/arch.txt +echo "fixture written: $(ls -l /out/GOLDEN.clltk_trace)" diff --git a/tests/golden/generator/writer.c b/tests/golden/generator/writer.c new file mode 100644 index 0000000..4a57b53 --- /dev/null +++ b/tests/golden/generator/writer.c @@ -0,0 +1,26 @@ +// Golden fixture writer: exercises every tracepoint argument type once with +// deterministic values. The produced .clltk_trace file is committed as a +// golden test fixture; pid/tid/timestamps are frozen inside the file. +#include "CommonLowLevelTracingKit/tracing/tracing.h" +#include + +CLLTK_TRACEBUFFER(GOLDEN, 4096) + +volatile int never = 0; + +int main(void) +{ + CLLTK_TRACEPOINT(GOLDEN, "plain int %d", 42); + CLLTK_TRACEPOINT(GOLDEN, "u8 %u u16 %u u32 %u", (uint8_t)8, (uint16_t)1616, (uint32_t)323232); + CLLTK_TRACEPOINT(GOLDEN, "i64 %ld u64 %lu", (int64_t)-64646464, + (uint64_t)18446744073709551615ull); + CLLTK_TRACEPOINT(GOLDEN, "float %f double %f", 3.5f, -2.25); + CLLTK_TRACEPOINT(GOLDEN, "string %s", "golden string"); + CLLTK_TRACEPOINT(GOLDEN, "pointer %p", (void *)0x123456789abcull); + const uint8_t dump_data[8] = {0xde, 0xad, 0xbe, 0xef, 0x01, 0x02, 0x03, 0x04}; + CLLTK_TRACEPOINT_DUMP(GOLDEN, "golden dump", dump_data, sizeof(dump_data)); + if (never) { + CLLTK_TRACEPOINT(GOLDEN, "never fired %d", 1); + } + return 0; +} diff --git a/tests/golden/golden-1.2.64-le-aarch64.clltk_trace b/tests/golden/golden-1.2.64-le-aarch64.clltk_trace new file mode 100644 index 0000000000000000000000000000000000000000..b122d4ac651e0c1d8f60bc10cd2f3301b23fcbdd GIT binary patch literal 4919 zcmcClR;eo~N=!~oDosmEEn;wBVq^dT3oyyR5CNqf1fdKF)h-U@y8HXMxcV_T`}l-- z$Gb*Ab$2m9Oo1puW=}#fhA_uN0J|D?s4y3@;nD9FmX^sH41@BlG{OZ&gW}6Cu%!U~-vjL*wz$Dc_|B--9of5xTGjEFP%ZNPLAQ! zUKGWaoe;%)W>til0A+YV);>j1vZ@=Rgac^ay|w$^Gcqx=l!0s?1*0J_8UmvsFd71* zAut*OqaiRF0;3@?8UmvsFd71*Aut*O_(A}A_yL(c37;kkj0gxV=m6z!gwo>f{yr|Qehkh&J|W)m zt`SU(V2LiMGy{YSG!Z5Mq$eSM{1bIV= z;9o+S(&u89Q_)UWW=%KbLDFfou^PxT0y%}3(UU>HxF}g)FDk&Lr;1{r4!GLDZqxjYl6fu$A- zs-+60hGt;O*a%252y`Ck|JJG7*x!`q6`)cPhh(nVS&+GAAanVdd~$Fan`vgEpqi7S zPzt6?8R|c!8@HrQooJ-^FDZ9j#a$#LEq{QFv;-L`z?7DP)5x@({6wI)(-czjOOtX^ zfh>lUwCQX3)QYaAGrlxERs6CU$yEJ&AXD{0rV26^<1`d9GNYhc%y5o<`P35mrTmfw za=SM^IJFkZECVi(Sq31pgcu8OnpKbwjF;3R1=RuuO|?`Ha}GsT`9zo0*0_wnNQN0! zfebSQjUS0HR^T)YI&zd!np?myTO{$-YlFVrb{FP}a>mCxkW4e-0hwk3a+(;PWRsU# P26R|jW>IPiDB%DAYy1qC literal 0 HcmV?d00001 diff --git a/tests/golden/golden-1.3.0-le-aarch64.clltk_trace b/tests/golden/golden-1.3.0-le-aarch64.clltk_trace new file mode 100644 index 0000000000000000000000000000000000000000..a7fb114cd480dabb960640debe52a176042d76b9 GIT binary patch literal 5184 zcmcClR;eo~N=!~oDosmEEn;9`W@G>X3oyyR5CNqf1fdKFWgrgay8HXMxcV_T`}l-- z$Gb)_L3DR9Kxl{}WcDN!V+eCR1hA`NhYE8c8;)HmR?#{shIv=Pe!32n2jPm-J3Qtv z9+c1mvVYXcGJI7L0_lWFKe`H$<_KV0@Q#IHVx1bp>>3mmzivQO9GIl~=RXpdTc^Zu z{yd7(skb3Y85j~AK$b8tFde9^(_rvb6$Wu&c1GTVDB}Z)rRV3Qq~<9UmlS2@r88`) zlVdnlh@yDkBZ%TXvns+&fVQ=QtlfvAME@y72?x-;du#W-XJle#0hv7tMnhmU1V%$( zGz3ONU^E0qLtr!nMnhmU1V%$(Gz3ONU^E2qh5+*L12TIOUR~r%b}&Ko+U!iK4w zfzvSP$WcmZZUMtK#gyQfgM7_{ list: + """Decode one fixture with the python decoder, return formatted messages.""" + with tempfile.TemporaryDirectory() as tmp: + out_csv = pathlib.Path(tmp) / "out.csv" + result = subprocess.run( + [sys.executable, str(PYTHON_DECODER), "-o", str(out_csv), str(fixture)], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + with open(out_csv, newline="") as fh: + rows = list(csv.DictReader(fh)) + messages = [row["formatted"].strip() for row in rows] + return [m for m in messages if not m.startswith('{"tracebuffer info')] + + +class golden_python_decoder(unittest.TestCase): + def test_little_endian_fixtures(self): + for name in LITTLE_ENDIAN_FIXTURES: + with self.subTest(fixture=name): + messages = decode_with_python(GOLDEN_DIR / name) + self.assertEqual(EXPECTED_MESSAGES, messages) + + def test_big_endian_fixtures(self): + for name in BIG_ENDIAN_FIXTURES: + with self.subTest(fixture=name): + messages = decode_with_python(GOLDEN_DIR / name) + self.assertEqual(EXPECTED_MESSAGES, messages) + + +if __name__ == "__main__": + unittest.main() From ed37e73b45e305c20bc9054cb10c7c35866b1cba Mon Sep 17 00:00:00 2001 From: Jo5ta Date: Fri, 10 Jul 2026 20:42:20 +0200 Subject: [PATCH 3/8] decoder: endianness-aware FilePart reads (foundation) FilePart gains a foreign-endian flag: when set, get() and getLimted() byte-swap arithmetic values, and sub-FileParts inherit the flag. getReference() stays raw for byte arrays and native-only live paths. Nothing sets the flag yet - behavior is unchanged until the tracebuffer magic check is wired up to it (foreign-endian files are still rejected there). Groundwork for decoding big-endian s390x trace files on little-endian hosts. Signed-off-by: Jo5ta --- decoder_tool/cpp/source/low_level/file.cpp | 3 +- decoder_tool/cpp/source/low_level/file.hpp | 45 ++++++++++++++++++++-- 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/decoder_tool/cpp/source/low_level/file.cpp b/decoder_tool/cpp/source/low_level/file.cpp index 8eb3e57..87acb39 100644 --- a/decoder_tool/cpp/source/low_level/file.cpp +++ b/decoder_tool/cpp/source/low_level/file.cpp @@ -94,7 +94,8 @@ size_t internal::File::grow() const { source::FilePart::FilePart(const FilePart &a_filePart, const size_t a_offset) : m_file(a_filePart.m_file) , m_offset(a_filePart.m_offset + a_offset) - , m_base(m_file->data()) { + , m_base(m_file->data()) + , m_foreign_endian(a_filePart.m_foreign_endian) { const size_t max_access = m_offset; if (max_access >= m_file->size()) [[unlikely]] { m_file->grow(); diff --git a/decoder_tool/cpp/source/low_level/file.hpp b/decoder_tool/cpp/source/low_level/file.hpp index 3af28c7..5c95f4e 100644 --- a/decoder_tool/cpp/source/low_level/file.hpp +++ b/decoder_tool/cpp/source/low_level/file.hpp @@ -3,6 +3,7 @@ #include "CommonLowLevelTracingKit/decoder/Inline.hpp" #include #include +#include #include #include #include @@ -21,6 +22,24 @@ namespace CommonLowLevelTracingKit::decoder::source { using FilePtr = std::shared_ptr; static constexpr size_t s_max_file_size = 1024 * 1024 * 1024; + + // byte-swap one arithmetic value; used to read files written on a + // machine with the opposite byte order (detected via the file magic) + template INLINE T byteswapValue(T value) { + static_assert(std::is_arithmetic_v); + if constexpr (sizeof(T) == 2) { + return std::bit_cast(__builtin_bswap16(std::bit_cast(value))); + } else if constexpr (sizeof(T) == 4) { + return std::bit_cast(__builtin_bswap32(std::bit_cast(value))); + } else if constexpr (sizeof(T) == 8) { + return std::bit_cast(__builtin_bswap64(std::bit_cast(value))); + } else { + return value; + } + } + + template + concept ByteSwappable = std::is_arithmetic_v && (sizeof(T) > 1); } // namespace internal class FilePart final { friend struct internal::File; @@ -33,20 +52,31 @@ namespace CommonLowLevelTracingKit::decoder::source { FilePart &operator=(const FilePart &) = delete; FilePart &operator=(FilePart &&) = delete; + /// raw zero-copy access; never byte-swapped. Use only for byte arrays + /// and for native-endian live paths (foreign-endian files are + /// restricted to offline decoding). template INLINE const T &getReference(size_t offset = 0) const { const T &value = *(const T *)getPtr(offset, sizeof(T)); return value; } template INLINE T get(size_t offset = 0) const { - return *(const T *)getPtr(offset, sizeof(T)); + T value = *(const T *)getPtr(offset, sizeof(T)); + if constexpr (internal::ByteSwappable) { + if (m_foreign_endian) { value = internal::byteswapValue(value); } + } + return value; } template requires(internal::POD) INLINE T getLimted(size_t limit, size_t offset = 0) const noexcept { - std::array value = {}; - getLimtedRaw(value.data(), offset, value.size(), limit); - return *reinterpret_cast(value.data()); + std::array raw = {}; + getLimtedRaw(raw.data(), offset, raw.size(), limit); + T value = *reinterpret_cast(raw.data()); + if constexpr (internal::ByteSwappable) { + if (m_foreign_endian) { value = internal::byteswapValue(value); } + } + return value; } template INLINE void copyOut(R &out, size_t offset, size_t size, size_t limit) const { @@ -66,6 +96,12 @@ namespace CommonLowLevelTracingKit::decoder::source { const std::filesystem::path &path() const noexcept; + /// mark this file as written with the opposite byte order (decided by + /// the tracebuffer magic); all get/getLimted calls then byte-swap. + /// Sub-FileParts created afterwards inherit the flag. + INLINE void setForeignEndian(bool foreign) noexcept { m_foreign_endian = foreign; } + INLINE bool isForeignEndian() const noexcept { return m_foreign_endian; } + private: uintptr_t getPtr(size_t offset, size_t object_size) const; void getLimtedRaw(uint8_t *const out, size_t offset, size_t size, @@ -74,6 +110,7 @@ namespace CommonLowLevelTracingKit::decoder::source { const internal::FilePtr m_file; const size_t m_offset; const uintptr_t m_base; + bool m_foreign_endian = false; }; template <> FilePart FilePart::get(size_t offset) const; From d2850799cad180b5c0a98b735281497750b3554a Mon Sep 17 00:00:00 2001 From: Jo5ta Date: Fri, 10 Jul 2026 20:51:03 +0200 Subject: [PATCH 4/8] test: add golden fixtures for historic format-boundary versions Extend the golden set with fixtures generated from git worktrees of the versions where the trace file format actually changed: 1.2.39 (oldest buildable state, predates the 1.2.40 dump-format change) and 1.2.49 (predates the 1.2.50 definition-V2 format). Both decode identically to current files with today's python decoder, pinning backward compatibility across every format boundary in the public history. The generator now adapts the writer to the header layout of the checked-out tree, so future boundary versions can be added the same way. The minor <= 1 layout (32-byte mutex) predates the public repository and cannot be regenerated; it stays covered only by the decoder's version branch. Signed-off-by: Jo5ta --- tests/golden/README.md | 13 ++++++++++++- tests/golden/generator/gen.sh | 18 ++++++++++++------ .../golden-1.2.39-le-aarch64.clltk_trace | Bin 0 -> 4909 bytes .../golden-1.2.49-le-aarch64.clltk_trace | Bin 0 -> 4903 bytes tests/test_golden.py | 2 ++ 5 files changed, 26 insertions(+), 7 deletions(-) create mode 100644 tests/golden/golden-1.2.39-le-aarch64.clltk_trace create mode 100644 tests/golden/golden-1.2.49-le-aarch64.clltk_trace diff --git a/tests/golden/README.md b/tests/golden/README.md index b41e2ca..837a686 100644 --- a/tests/golden/README.md +++ b/tests/golden/README.md @@ -3,13 +3,24 @@ Committed `.clltk_trace` files used to test the decoders against known inputs: different library versions, both byte orders, all argument types. -Every fixture was produced by `generator/writer.c` (one tracepoint per +Old-version fixtures are generated from a git worktree of the respective +commit; `generator/gen.sh` adapts the writer to the header layout of the +checked-out tree. Every fixture was produced by `generator/writer.c` (one tracepoint per argument type with fixed values, plus one never-fired tracepoint) via `generator/gen.sh`. The values inside a fixture (pids, tids, timestamps) are frozen in the committed file, so decoder output is fully deterministic. +The versions were chosen at real format boundaries: 1.2.39 is the oldest +buildable state and predates the dump-format change (1.2.40); 1.2.49 +predates the definition-V2 format (1.2.50); 1.2.64 is the last version +before the metaptr discovery rework; 1.3.0 is current. The minor <= 1 +layout (32-byte mutex) predates the public history and cannot be +regenerated from source. + | fixture | produced by | | --- | --- | +| `golden-1.2.39-le-aarch64.clltk_trace` | library 1.2.39 (commit 312369e), aarch64 | +| `golden-1.2.49-le-aarch64.clltk_trace` | library 1.2.49 (commit c15038b), aarch64 | | `golden-1.2.64-le-aarch64.clltk_trace` | library 1.2.64 (commit 1fa19d2), aarch64 | | `golden-1.3.0-le-aarch64.clltk_trace` | library 1.3.0, aarch64 | | `golden-1.3.0-be-s390x.clltk_trace` | library 1.3.0, s390x (big endian) | diff --git a/tests/golden/generator/gen.sh b/tests/golden/generator/gen.sh index e2ab4a0..dc7f658 100755 --- a/tests/golden/generator/gen.sh +++ b/tests/golden/generator/gen.sh @@ -1,17 +1,23 @@ #!/bin/bash -# builds the tracing library from /src and produces a golden trace file in /out +# era-adaptive golden fixture generator: adapts the writer include to the +# header layout of the checked-out source tree in /src set -e dnf install -y -q gcc gcc-c++ cmake make git gettext-envsubst rsync > /dev/null 2>&1 -git config --global --add safe.directory "*"; cmake -S /src -B /tmp/b \ +git config --global --add safe.directory "*" +cp /fix/writer.c /tmp/writer.c +if [ ! -f /src/tracing_library/include/CommonLowLevelTracingKit/tracing/tracing.h ]; then + # pre-1.2.48 layout: header lives directly under CommonLowLevelTracingKit/ + sed -i 's|CommonLowLevelTracingKit/tracing/tracing.h|CommonLowLevelTracingKit/tracing.h|' /tmp/writer.c +fi +cmake -S /src -B /tmp/b \ -DCMAKE_BUILD_TYPE=Debug -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=OFF \ -DCLLTK_TRACING=ON -DCLLTK_COMMAND_LINE_TOOL=OFF -DCLLTK_CPP_DECODER=OFF \ - -DCLLTK_SNAPSHOT=OFF -DCLLTK_PYTHON_DECODER=OFF -DCLLTK_EXAMPLES=OFF \ - -DCLLTK_TESTS=OFF > /tmp/cfg.log 2>&1 || { tail -5 /tmp/cfg.log; exit 1; } + -DCLLTK_SNAPSHOT=OFF -DCLLTK_PYTHON_DECODER=OFF -DCLLTK_DECODER=OFF \ + -DCLLTK_KERNEL_TRACING=OFF -DCLLTK_EXAMPLES=OFF -DCLLTK_TESTS=OFF > /tmp/cfg.log 2>&1 || { tail -10 /tmp/cfg.log; exit 1; } cmake --build /tmp/b --target clltk_tracing_static -- -j4 > /tmp/build.log 2>&1 || { tail -30 /tmp/build.log; exit 1; } LIB=$(find /tmp/b -name 'libclltk_tracing*.a' | head -1) -gcc -std=c11 -O1 -I/src/tracing_library/include /src/.fixtures/writer.c "$LIB" -pthread -o /tmp/writer +gcc -std=c11 -O1 -I/src/tracing_library/include /tmp/writer.c "$LIB" -pthread -o /tmp/writer rm -rf /tmp/traces && mkdir /tmp/traces CLLTK_TRACING_PATH=/tmp/traces /tmp/writer cp /tmp/traces/GOLDEN.clltk_trace /out/ -uname -m > /out/arch.txt echo "fixture written: $(ls -l /out/GOLDEN.clltk_trace)" diff --git a/tests/golden/golden-1.2.39-le-aarch64.clltk_trace b/tests/golden/golden-1.2.39-le-aarch64.clltk_trace new file mode 100644 index 0000000000000000000000000000000000000000..83a5c6c679718d62177fb96a0d4f046de665de03 GIT binary patch literal 4909 zcmcClR;eo~N=!~oDosmEEn-k-Vq^dT3oyyR-~pu#1fdKFb(syyb@%siarI+hfarn< zBC{u{>)P zCP0}mkhND)l+>!_l!);ENURzN5N#H__git^p-Ci*El!=YX6&@gBh6tl=rOG z0CE|5L4+rReo1bDetA)5NotW^GDAU5VrHH~W?qScY6?TOAxH^mbb&d!JQGc2sfB`S zsY0ot8JIFQ0#ZPQU{I|Cq!{^u1d~q=nzl?c69v_r6opbS4bf7q4^j^rPGL$*K~tZW zlb;B5OqxPUerZxpDv-s%P^}754;s2)EJjlf8UIjFEe1+5RD%XD7==MZ0h+pke4rmw zixgA~K&_sTkwvBLBCFoVYaFe$ZVJalN=x_4op)0^B)O3tW#n*ABv)s*%_jgfg!;G zWCQ~P(}4qZ8VtUzD9TD)A*N?ttwm8B;|Wo`XI4d+ z2~b7{WbIWHB~{)KB^*HW?ycSTo{@=}Wi!b3Q7{?;qaiRF0;3@?8UmvsFd71*Aut*O zqaiRF0;3@?8UmvsfG-4a4L?l6r-=f&dZ>k>e0_|En?+`^+;Ur%%(3 Date: Fri, 10 Jul 2026 21:37:02 +0200 Subject: [PATCH 5/8] decoder: decode trace files written with the opposite byte order The C++ decoder recognized the big-endian file magic but rejected such files. The byte order is now detected from the magic when the file is opened and flagged on the FilePart, which byte-swaps all reads transparently. Places that read outside the FilePart swap explicitly: the atomic ringbuffer head is loaded by value and swapped, ringbuffer entries carry the flag for pid/tid/timestamp/line extraction, the formatter swaps every argument by its type (including float/double and the string/dump size prefixes), and the meta parser swaps its size and line fields. The 48 bit file offset at the start of each entry body is stored as six writer-order bytes, so a foreign value sits in the upper bits of the swapped read and is shifted instead of masked. With this, trace files from s390x decode correctly on little-endian hosts and vice versa, through both the library and the clltk CLI. The golden tests now decode every fixture through the CLI as well; the big-endian fixture is real s390x output. The byte-order marker in the file magic is now documented in the file specification. Foreign-endian files are static by nature (their producer cannot run on this host), so all read paths simply work; no live/offline split is needed. Signed-off-by: Jo5ta --- VERSION.md | 4 ++ decoder_tool/cpp/source/Tracebuffer.cpp | 16 ++++- decoder_tool/cpp/source/Tracepoint.cpp | 19 +++--- .../cpp/source/TracepointInternal.hpp | 14 +++-- .../cpp/source/low_level/formatter.cpp | 59 +++++++++++-------- .../cpp/source/low_level/formatter.hpp | 4 +- .../cpp/source/low_level/meta_parser.cpp | 13 ++-- .../cpp/source/low_level/meta_parser.hpp | 5 +- .../cpp/source/low_level/ringbuffer.cpp | 4 +- .../cpp/source/low_level/ringbuffer.hpp | 16 ++++- .../cpp/source/low_level/stack_section.cpp | 3 +- .../cpp/source/low_level/tracebufferfile.cpp | 30 ++++++++-- docs/file_specification.asciidoc | 9 +++ tests/test_golden.py | 57 ++++++++++++++++++ 14 files changed, 199 insertions(+), 54 deletions(-) diff --git a/VERSION.md b/VERSION.md index ebce79e..2cf5275 100644 --- a/VERSION.md +++ b/VERSION.md @@ -11,6 +11,10 @@ first execution of a tracepoint needs no lookup. - feat: decoder/CLI read both the new `_metaptr` pointer sections and legacy inline `_meta` sections from ELF binaries. +- feat: cross-endian decoding. Trace files and ELF binaries written on a machine with the + opposite byte order (e.g. s390x files read on x86_64/aarch64) are byte-swapped while + reading; the byte order is detected from the file magic / ELF identification. The python + decoder additionally fixes float/double arguments from foreign-endian files. - BREAKING (link-time): objects compiled with older headers cannot be mixed with objects compiled with these headers in one binary; rebuild all translation units. - ci: add non-LTO build leg (`unittests-nolto` preset); LTO had masked the section conflict. diff --git a/decoder_tool/cpp/source/Tracebuffer.cpp b/decoder_tool/cpp/source/Tracebuffer.cpp index ec841da..d2d7f8d 100644 --- a/decoder_tool/cpp/source/Tracebuffer.cpp +++ b/decoder_tool/cpp/source/Tracebuffer.cpp @@ -108,7 +108,12 @@ TracepointPtr SyncTbInternal::next(const TracepointFilterFunc &filter) noexcept auto &e = std::get(ringbuffer_entry); if (e == nullptr) return {}; - const uint64_t fileoffset = get(e->body()) & ((1ULL << 48) - 1); + // the file offset is stored as 6 bytes in the writer's byte order at + // the start of the entry body; the remaining 2 bytes of the uint64 + // read belong to the next field and are discarded + const uint64_t fileoffset_raw = get(e->body(), 0, e->foreignEndian()); + const uint64_t fileoffset = + e->foreignEndian() ? (fileoffset_raw >> 16) : (fileoffset_raw & ((1ULL << 48) - 1)); if (fileoffset == 0x01) { auto tp = make_tracepoint(std::string(name()), std::move(e), m_source_type); @@ -147,7 +152,12 @@ TracepointPtr SyncTbInternal::next_pooled(TracepointPool &pool, auto &e = std::get(ringbuffer_entry); if (e == nullptr) return {}; - const uint64_t fileoffset = get(e->body()) & ((1ULL << 48) - 1); + // the file offset is stored as 6 bytes in the writer's byte order at + // the start of the entry body; the remaining 2 bytes of the uint64 + // read belong to the next field and are discarded + const uint64_t fileoffset_raw = get(e->body(), 0, e->foreignEndian()); + const uint64_t fileoffset = + e->foreignEndian() ? (fileoffset_raw >> 16) : (fileoffset_raw & ((1ULL << 48) - 1)); if (fileoffset == 0x01) { auto tp = make_pooled_tracepoint(pool, std::string(name()), std::move(e), m_source_type); @@ -201,7 +211,7 @@ bool Tracebuffer::is_tracebuffer(const fs::path &path) { const std::string_view magic{fileHead.data(), fileHead.size()}; if (magic == little_endian_magic) return true; - if (magic == big_endian_magic) return false; // currently not supported + if (magic == big_endian_magic) return true; // foreign byte order, byte-swapped while reading return false; } bool SnapTracebuffer::is_formattable(const fs::path &path) { diff --git a/decoder_tool/cpp/source/Tracepoint.cpp b/decoder_tool/cpp/source/Tracepoint.cpp index 95089c8..aa02757 100644 --- a/decoder_tool/cpp/source/Tracepoint.cpp +++ b/decoder_tool/cpp/source/Tracepoint.cpp @@ -32,10 +32,10 @@ void TracepointDeleter::operator()(Tracepoint *ptr) const noexcept { } TraceEntryHead::TraceEntryHead(uint64_t n, uint64_t t, const std::span &body, - SourceType src) + bool foreign_endian, SourceType src) : Tracepoint(n, t, src) - , m_pid(get(body, 6)) - , m_tid(get(body, 10)) {} + , m_pid(get(body, 6, foreign_endian)) + , m_tid(get(body, 10, foreign_endian)) {} TraceEntryHead::TraceEntryHead(uint64_t n, uint64_t t, uint32_t pid, uint32_t tid, SourceType src) : Tracepoint(n, t, src) @@ -44,7 +44,8 @@ TraceEntryHead::TraceEntryHead(uint64_t n, uint64_t t, uint32_t pid, uint32_t ti TracepointDynamic::TracepointDynamic(std::string tb_name, source::Ringbuffer::EntryPtr entry, SourceType src) - : TraceEntryHead(entry->nr, get(entry->body(), 14), entry->body(), src) + : TraceEntryHead(entry->nr, get(entry->body(), 14, entry->foreignEndian()), + entry->body(), entry->foreignEndian(), src) , m_tracebuffer(std::move(tb_name)) , e(std::move(entry)) { const size_t size = e->body().size(); @@ -78,6 +79,7 @@ TracepointDynamic::TracepointDynamic(std::string tb_name, source::Ringbuffer::En } const char *const line_start = current; m_line = *std::bit_cast(line_start); + if (e->foreignEndian()) { m_line = source::internal::byteswapValue(m_line); } current += line_len; remaining -= line_len; @@ -90,13 +92,14 @@ using FilePtr = source::internal::FilePtr; TracepointStatic::TracepointStatic(std::string tb_name, source::Ringbuffer::EntryPtr &&entry, const std::span &arg_m, const FilePtr &&f, SourceType src) - : TraceEntryHead(entry->nr, get(entry->body(), 14), entry->body(), src) + : TraceEntryHead(entry->nr, get(entry->body(), 14, entry->foreignEndian()), + entry->body(), entry->foreignEndian(), src) , m_tracebuffer(std::move(tb_name)) , m(arg_m) , e(std::move(entry)) , m_keep_memory(f) , m_type(toMetaType(get(m, 5))) - , m_line(get(m, 6)) + , m_line(get(m, 6, e->foreignEndian())) , m_arg_count(std::min((uint8_t)10, get(m, 10))) , m_arg_types((const char *)&m[11], m_arg_count) , m_file() @@ -132,9 +135,9 @@ const std::string_view TracepointStatic::msg() const { const size_t arg_size = (e->size() > 22) ? (e->size() - 22) : 0; std::span args_raw{arg_start, arg_size}; if (m_type == MetaType::printf) [[likely]] { - m_msg = source::formatter::printf(format(), m_arg_types, args_raw); + m_msg = source::formatter::printf(format(), m_arg_types, args_raw, e->foreignEndian()); } else if (m_type == MetaType::dump) { - m_msg = source::formatter::dump(format(), m_arg_types, args_raw); + m_msg = source::formatter::dump(format(), m_arg_types, args_raw, e->foreignEndian()); } else { CLLTK_DECODER_THROW( exception::InvalidMeta, diff --git a/decoder_tool/cpp/source/TracepointInternal.hpp b/decoder_tool/cpp/source/TracepointInternal.hpp index e89e479..978dd94 100644 --- a/decoder_tool/cpp/source/TracepointInternal.hpp +++ b/decoder_tool/cpp/source/TracepointInternal.hpp @@ -13,11 +13,17 @@ template concept PODType = std::is_standard_layout_v && std::is_trivial_v; template -INLINE static constexpr T get(R r, size_t offset = 0) { +INLINE static constexpr T get(R r, size_t offset = 0, bool foreign_endian = false) { const uintptr_t base = std::bit_cast(r.data()); const uintptr_t position = base + offset; const T *const ptr = reinterpret_cast(position); - return *ptr; + T value = *ptr; + if constexpr (CommonLowLevelTracingKit::decoder::source::internal::ByteSwappable) { + if (foreign_endian) { + value = CommonLowLevelTracingKit::decoder::source::internal::byteswapValue(value); + } + } + return value; } namespace CommonLowLevelTracingKit::decoder { @@ -68,7 +74,7 @@ namespace CommonLowLevelTracingKit::decoder { const uint32_t m_pid; const uint32_t m_tid; TraceEntryHead(uint64_t n, uint64_t t, const std::span &body, - SourceType src = SourceType::Unknown); + bool foreign_endian, SourceType src = SourceType::Unknown); TraceEntryHead(uint64_t n, uint64_t t, uint32_t pid, uint32_t tid, SourceType src = SourceType::Unknown); uint32_t pid() const noexcept override { return m_pid; }; @@ -139,7 +145,7 @@ namespace CommonLowLevelTracingKit::decoder { , m_line() {} VirtualTracepoint(std::string tb_name, const source::Ringbuffer::Entry &e, const std::string &msg, SourceType src = SourceType::Unknown) - : TraceEntryHead(e.nr, get(e.body(), 14), 0, 0, src) + : TraceEntryHead(e.nr, get(e.body(), 14, e.foreignEndian()), 0, 0, src) , m_tracebuffer(std::move(tb_name)) , m_msg(msg) , m_file() diff --git a/decoder_tool/cpp/source/low_level/formatter.cpp b/decoder_tool/cpp/source/low_level/formatter.cpp index d1bb892..9ae6e5f 100644 --- a/decoder_tool/cpp/source/low_level/formatter.cpp +++ b/decoder_tool/cpp/source/low_level/formatter.cpp @@ -1,6 +1,7 @@ #include "formatter.hpp" #include "CommonLowLevelTracingKit/decoder/Common.hpp" +#include "file.hpp" #include #include #include @@ -42,8 +43,9 @@ CONST_INLINE static constexpr ffi_type *clltk_type_to_ffi_type(const char clltk_ default: CLLTK_DECODER_THROW(FormattingFailed, "unknown type"); } } -CONST_INLINE static constexpr size_t -clltk_arg_to_size(const char clltk_type, const uintptr_t clltk_arg, size_t remaining) { +CONST_INLINE static constexpr size_t clltk_arg_to_size(const char clltk_type, + const uintptr_t clltk_arg, size_t remaining, + bool foreign_endian) { switch (clltk_type) { case 'c': return sizeof(uint8_t); case 'C': return sizeof(int8_t); @@ -60,6 +62,9 @@ clltk_arg_to_size(const char clltk_type, const uintptr_t clltk_arg, size_t remai if (sizeof(size) > remaining) [[unlikely]] CLLTK_DECODER_THROW(FormattingFailed, "no space for string arg size left"); memcpy(&size, std::bit_cast(clltk_arg), sizeof(size)); + if (foreign_endian) { + size = CommonLowLevelTracingKit::decoder::source::internal::byteswapValue(size); + } size += sizeof(uint32_t); if (size > remaining) [[unlikely]] CLLTK_DECODER_THROW(FormattingFailed, "string arg bigger than raw args"); @@ -75,31 +80,36 @@ clltk_arg_to_size(const char clltk_type, const uintptr_t clltk_arg, size_t remai } } template -static INLINE const any get_native(uintptr_t p, size_t remaining) { +static INLINE const any get_native(uintptr_t p, size_t remaining, bool foreign_endian) { if (sizeof(T) > remaining) [[unlikely]] CLLTK_DECODER_THROW(FormattingFailed, "out of range access for formatter"); T value{}; memcpy(&value, std::bit_cast(p), sizeof(T)); + if constexpr (CommonLowLevelTracingKit::decoder::source::internal::ByteSwappable) { + if (foreign_endian) { + value = CommonLowLevelTracingKit::decoder::source::internal::byteswapValue(value); + } + } static_assert(sizeof(ProxyT) == sizeof(void *)); ProxyT xvalue = (ProxyT)value; return xvalue; } INLINE static constexpr any clltk_arg_to_native(const char clltk_type, const uintptr_t clltk_arg, - size_t remaining) { + size_t remaining, bool foreign_endian) { switch (clltk_type) { - case 'c': return get_native(clltk_arg, remaining); - case 'C': return get_native(clltk_arg, remaining); - case 'w': return get_native(clltk_arg, remaining); - case 'W': return get_native(clltk_arg, remaining); - case 'i': return get_native(clltk_arg, remaining); - case 'I': return get_native(clltk_arg, remaining); - case 'l': return get_native(clltk_arg, remaining); - case 'L': return get_native(clltk_arg, remaining); + case 'c': return get_native(clltk_arg, remaining, foreign_endian); + case 'C': return get_native(clltk_arg, remaining, foreign_endian); + case 'w': return get_native(clltk_arg, remaining, foreign_endian); + case 'W': return get_native(clltk_arg, remaining, foreign_endian); + case 'i': return get_native(clltk_arg, remaining, foreign_endian); + case 'I': return get_native(clltk_arg, remaining, foreign_endian); + case 'l': return get_native(clltk_arg, remaining, foreign_endian); + case 'L': return get_native(clltk_arg, remaining, foreign_endian); case 'f': - return get_native( - clltk_arg, remaining); // use double as a proxy type to get valid data from raw args - case 'd': return get_native(clltk_arg, remaining); - case 'p': return get_native(clltk_arg, remaining); + // use double as a proxy type to get valid data from raw args + return get_native(clltk_arg, remaining, foreign_endian); + case 'd': return get_native(clltk_arg, remaining, foreign_endian); + case 'p': return get_native(clltk_arg, remaining, foreign_endian); case 's': return std::bit_cast(clltk_arg + sizeof(uint32_t)); case InvalidStringArgType: // the value in clltk_arg is a pointer to a now invalid/unusable memory address. @@ -111,7 +121,7 @@ INLINE static constexpr any clltk_arg_to_native(const char clltk_type, const uin INLINE static std::array clltk_args_to_native_args(const std::string_view format, const std::span &clltk_types, - const std::span &raw_clltk_args) { + const std::span &raw_clltk_args, bool foreign_endian) { std::array args{}; args[0] = std::bit_cast(nullptr); args[1] = std::bit_cast(0lu); @@ -123,9 +133,9 @@ clltk_args_to_native_args(const std::string_view format, const std::span(&raw_clltk_args[raw_arg_offset]); const size_t remaining = raw_clltk_args.size() - raw_arg_offset; - const any value = clltk_arg_to_native(type, current, remaining); + const any value = clltk_arg_to_native(type, current, remaining, foreign_endian); args[fix_arg_count + i] = value; - const size_t arg_size = clltk_arg_to_size(type, current, remaining); + const size_t arg_size = clltk_arg_to_size(type, current, remaining, foreign_endian); raw_arg_offset += arg_size; } @@ -246,7 +256,7 @@ static INLINE std::string clean_up_str_view(const std::string_view str) { // call snprintf with ffi std::string formatter::printf(const std::string_view format, const std::span &types_raw, - const std::span &args_raw) { + const std::span &args_raw, bool foreign_endian) { if (format.empty()) return ""; if (format.data()[format.size()] != '\0') CLLTK_DECODER_THROW(FormattingFailed, "missing format termination"); @@ -260,7 +270,7 @@ std::string formatter::printf(const std::string_view format, const std::span &types_raw, - const std::span &args_raw) { + const std::span &args_raw, bool foreign_endian) { if (types_raw.size() != 1 || types_raw[0] != 'x') CLLTK_DECODER_THROW(InvalidMeta, "wrong meta for drump tracepoint"); const size_t format_size = format.size(); if (args_raw.size() < sizeof(uint32_t)) CLLTK_DECODER_THROW(FormattingFailed, "args_raw too small for dump size"); - const uint32_t dump_size = (*std::bit_cast(args_raw.data())); + uint32_t dump_size = (*std::bit_cast(args_raw.data())); + if (foreign_endian) { + dump_size = CommonLowLevelTracingKit::decoder::source::internal::byteswapValue(dump_size); + } if (args_raw.size() < sizeof(uint32_t) + dump_size) CLLTK_DECODER_THROW(FormattingFailed, "args_raw too small for dump body"); const std::span dump_body{&args_raw[sizeof(uint32_t)], dump_size}; diff --git a/decoder_tool/cpp/source/low_level/formatter.hpp b/decoder_tool/cpp/source/low_level/formatter.hpp index 05ca3c9..955b53c 100644 --- a/decoder_tool/cpp/source/low_level/formatter.hpp +++ b/decoder_tool/cpp/source/low_level/formatter.hpp @@ -10,8 +10,8 @@ namespace CommonLowLevelTracingKit::decoder::source::formatter { std::string printf(const std::string_view format, const std::span &types, - const std::span &args_raw); + const std::span &args_raw, bool foreign_endian = false); std::string dump(const std::string_view format, const std::span &types, - const std::span &args_raw); + const std::span &args_raw, bool foreign_endian = false); } // namespace CommonLowLevelTracingKit::decoder::source::formatter \ No newline at end of file diff --git a/decoder_tool/cpp/source/low_level/meta_parser.cpp b/decoder_tool/cpp/source/low_level/meta_parser.cpp index b486248..f07fb11 100644 --- a/decoder_tool/cpp/source/low_level/meta_parser.cpp +++ b/decoder_tool/cpp/source/low_level/meta_parser.cpp @@ -3,11 +3,14 @@ #include "meta_parser.hpp" +#include "file.hpp" + #include namespace CommonLowLevelTracingKit::decoder::source { - MetaEntryInfoCollection MetaParser::parse(std::span data, uint64_t base_offset) { + MetaEntryInfoCollection MetaParser::parse(std::span data, uint64_t base_offset, + bool foreign_endian) { MetaEntryInfoCollection result; size_t offset = 0; @@ -20,7 +23,7 @@ namespace CommonLowLevelTracingKit::decoder::source { MetaEntryInfo entry; entry.offset = base_offset + offset; - const size_t consumed = parseOne(data, offset, entry); + const size_t consumed = parseOne(data, offset, entry, foreign_endian); if (consumed == 0) { ++offset; continue; @@ -33,8 +36,8 @@ namespace CommonLowLevelTracingKit::decoder::source { return result; } - size_t MetaParser::parseOne(std::span data, size_t offset, - MetaEntryInfo &entry) { + size_t MetaParser::parseOne(std::span data, size_t offset, MetaEntryInfo &entry, + bool foreign_endian) { const size_t remaining = data.size() - offset; if (remaining < MIN_ENTRY_SIZE) { return 0; } @@ -43,6 +46,7 @@ namespace CommonLowLevelTracingKit::decoder::source { uint32_t entry_size = 0; std::memcpy(&entry_size, base + OFFSET_SIZE, sizeof(entry_size)); + if (foreign_endian) { entry_size = internal::byteswapValue(entry_size); } if (entry_size < MIN_ENTRY_SIZE || entry_size > remaining) { return 0; } entry.size = entry_size; @@ -57,6 +61,7 @@ namespace CommonLowLevelTracingKit::decoder::source { } std::memcpy(&entry.line, base + OFFSET_LINE, sizeof(entry.line)); + if (foreign_endian) { entry.line = internal::byteswapValue(entry.line); } entry.arg_count = base[OFFSET_ARG_COUNT]; const size_t arg_types_array_size = entry.arg_count + 1; diff --git a/decoder_tool/cpp/source/low_level/meta_parser.hpp b/decoder_tool/cpp/source/low_level/meta_parser.hpp index 7dd75f4..7171449 100644 --- a/decoder_tool/cpp/source/low_level/meta_parser.hpp +++ b/decoder_tool/cpp/source/low_level/meta_parser.hpp @@ -15,8 +15,9 @@ namespace CommonLowLevelTracingKit::decoder::source { class MetaParser final { public: static MetaEntryInfoCollection parse(std::span data, - uint64_t base_offset = 0); - static size_t parseOne(std::span data, size_t offset, MetaEntryInfo &entry); + uint64_t base_offset = 0, bool foreign_endian = false); + static size_t parseOne(std::span data, size_t offset, MetaEntryInfo &entry, + bool foreign_endian = false); CONST_INLINE static bool isValidMagic(uint8_t byte) { return byte == MAGIC_BYTE; } static constexpr uint8_t MAGIC_BYTE = '{'; diff --git a/decoder_tool/cpp/source/low_level/ringbuffer.cpp b/decoder_tool/cpp/source/low_level/ringbuffer.cpp index afc8465..3ee959c 100644 --- a/decoder_tool/cpp/source/low_level/ringbuffer.cpp +++ b/decoder_tool/cpp/source/low_level/ringbuffer.cpp @@ -17,7 +17,8 @@ EntryPtr Entry::make(const uint64_t entry_nr, const uint64_t body_start, const s Entry::Entry(const uint64_t entry_nr, const uint64_t body_start, const size_t body_size, const FilePart &rb_body, const uint64_t rb_size, const bool skip_crc) noexcept : nr(entry_nr) - , m_valid(false) { + , m_valid(false) + , m_foreign_endian(rb_body.isForeignEndian()) { if (body_size <= static_body_size) [[likely]] { rb_body.copyOut(m_static_body, body_start, body_size, rb_size); @@ -112,6 +113,7 @@ std::variant Ringbuffer::getNextEntry() noexcept { } Ringbuffer::Ringbuffer(FilePart &&file) : m_file(file) + , m_foreign_endian(file.isForeignEndian()) , m_version(file.get(0)) , m_headpart(&file.getReference>(72)) , m_read(capture()) diff --git a/decoder_tool/cpp/source/low_level/ringbuffer.hpp b/decoder_tool/cpp/source/low_level/ringbuffer.hpp index 7e882ea..5f36332 100644 --- a/decoder_tool/cpp/source/low_level/ringbuffer.hpp +++ b/decoder_tool/cpp/source/low_level/ringbuffer.hpp @@ -106,7 +106,16 @@ namespace CommonLowLevelTracingKit::decoder::source { } INLINE uint64_t getAvailable() const noexcept { return getSize() - getUsed(); } INLINE const HeadPart capture() const noexcept { - return m_headpart->load(std::memory_order_relaxed); + HeadPart c = m_headpart->load(std::memory_order_relaxed); + if (m_foreign_endian) [[unlikely]] { + c.size = internal::byteswapValue(c.size); + c.wrapped = internal::byteswapValue(c.wrapped); + c.dropped = internal::byteswapValue(c.dropped); + c.entries = internal::byteswapValue(c.entries); + c.next_free = internal::byteswapValue(c.next_free); + c.last_valid = internal::byteswapValue(c.last_valid); + } + return c; } INLINE void reset() noexcept { m_read.reset(capture()); } @@ -131,6 +140,7 @@ namespace CommonLowLevelTracingKit::decoder::source { private: std::mutex m_this_lock; const FilePart m_file; + const bool m_foreign_endian; const uint64_t m_version; const std::atomic *const m_headpart; State m_read; @@ -142,6 +152,9 @@ namespace CommonLowLevelTracingKit::decoder::source { public: CONST_INLINE const std::span &body() const noexcept { return m_body; } CONST_INLINE size_t size() const noexcept { return m_body.size(); } + /// true when the trace file was written with the opposite byte order; + /// multi-byte fields inside body() must then be byte-swapped + CONST_INLINE bool foreignEndian() const noexcept { return m_foreign_endian; } static constexpr uint8_t header_size = 4; @@ -158,6 +171,7 @@ namespace CommonLowLevelTracingKit::decoder::source { friend class Ringbuffer; bool m_valid; + const bool m_foreign_endian; std::array m_static_body; std::vector m_dynamic_body; std::span m_body; diff --git a/decoder_tool/cpp/source/low_level/stack_section.cpp b/decoder_tool/cpp/source/low_level/stack_section.cpp index ffd6129..9285362 100644 --- a/decoder_tool/cpp/source/low_level/stack_section.cpp +++ b/decoder_tool/cpp/source/low_level/stack_section.cpp @@ -75,7 +75,8 @@ namespace CommonLowLevelTracingKit::decoder::source { const uint64_t body_file_offset = stack_entry.file_offset + stack_layout::ENTRY_HEADER_SIZE; - auto parsed = MetaParser::parse(body_span, body_file_offset); + auto parsed = + MetaParser::parse(body_span, body_file_offset, file_part.isForeignEndian()); for (auto &entry : parsed) { result.push_back(std::move(entry)); } } diff --git a/decoder_tool/cpp/source/low_level/tracebufferfile.cpp b/decoder_tool/cpp/source/low_level/tracebufferfile.cpp index a2afe3c..975a22e 100644 --- a/decoder_tool/cpp/source/low_level/tracebufferfile.cpp +++ b/decoder_tool/cpp/source/low_level/tracebufferfile.cpp @@ -2,13 +2,36 @@ #include "tracebufferfile.hpp" #include +#include #include namespace source = CommonLowLevelTracingKit::decoder::source; using namespace std::string_literals; +using MagicType = std::array; +// the writer stores the magic as two native-endian uint64 words, so the byte +// pattern in the file identifies the writer's byte order +static constexpr MagicType little_endian_magic = {'?', '#', '$', '~', 't', 'r', 'a', 'c', + 'e', 'b', 'u', 'f', 'f', 'e', 'r', '\0'}; +static constexpr MagicType big_endian_magic = {'c', 'a', 'r', 't', '~', '$', '#', '?', + '\0', 'r', 'e', 'f', 'f', 'u', 'b', 'e'}; + +// open the file and mark it foreign-endian when the magic byte pattern shows +// it was written on a machine with the opposite byte order; all multi-byte +// reads through the FilePart then byte-swap transparently +static source::FilePart makeFilePart(const std::string &path) { + source::FilePart file{path}; + const auto magic = file.get(0); + if constexpr (std::endian::native == std::endian::little) { + file.setForeignEndian(magic == big_endian_magic); + } else { + file.setForeignEndian(magic == little_endian_magic); + } + return file; +} + source::TracebufferFile::TracebufferFile(const std::string &p_path) - : m_file(p_path) + : m_file(makeFilePart(p_path)) , m_definition(getFilePart(getFileHeader().get(24))) , m_ringbuffer(getFilePart(getFileHeader().get(32))) { if (!validFile()) { throw std::runtime_error("In valid tracebuffer " + p_path); } @@ -16,11 +39,8 @@ source::TracebufferFile::TracebufferFile(const std::string &p_path) } bool source::TracebufferFile::getFileHeaderMagicValid() const { - using MagicType = std::array; - const MagicType expected = {'?', '#', '$', '~', 't', 'r', 'a', 'c', - 'e', 'b', 'u', 'f', 'f', 'e', 'r', '\0'}; const auto magic = getFilePart().get(); - return expected == magic; + return (magic == little_endian_magic) || (magic == big_endian_magic); } source::TracebufferFile::VersionType source::TracebufferFile::getVersion() const { const auto rawVersion = getFilePart().get(16); diff --git a/docs/file_specification.asciidoc b/docs/file_specification.asciidoc index d123882..0da1777 100644 --- a/docs/file_specification.asciidoc +++ b/docs/file_specification.asciidoc @@ -31,6 +31,15 @@ The following tables shows the location of different values, with offsets to sta The file header is exactly 56 bytes. The crc8 is calculated over the first 55 bytes (offset 0-54). +==== Byte order +All multi-byte fields are stored in the native byte order of the writing +machine. The byte order is identified by the magic: the writer stores the +magic as two native-endian 64 bit words, so a little-endian file starts with +the bytes `?#$~tracebuffer\0` and a big-endian file (e.g. written on s390x) +starts with `cart~$#?\0reffube`. Decoders detect the byte order from the +magic and byte-swap all multi-byte fields when reading a file written with +the opposite byte order. + === Definition Section The definition section is located at a file offset defined inside the file header and consists of the path and name of the tracebuffer, along with optional extended metadata. Thereby, even after a file is renamed, the original tracebuffer name could be recovered. diff --git a/tests/test_golden.py b/tests/test_golden.py index 102ce8f..64cb40e 100644 --- a/tests/test_golden.py +++ b/tests/test_golden.py @@ -18,6 +18,15 @@ sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) +from helpers.base import run_command +from helpers.clltk_cmd import clltk + + +def setUpModule(): + """Build clltk-cmd before running the CLI golden tests.""" + run_command("cmake --preset default") + run_command("cmake --build --preset default --target clltk-cmd") + GOLDEN_DIR = pathlib.Path(__file__).parent / "golden" PYTHON_DECODER = pathlib.Path(__file__).parent.parent / "decoder_tool" / "python" / "clltk_decoder.py" @@ -59,6 +68,22 @@ def decode_with_python(fixture: pathlib.Path) -> list: return [m for m in messages if not m.startswith('{"tracebuffer info')] +def decode_with_cli(fixture: pathlib.Path) -> list: + """Decode one fixture with the clltk CLI, return formatted messages.""" + result = clltk("decode", str(fixture)) + assert result.returncode == 0, result.stderr + messages = [] + for line in result.stdout.splitlines(): + # column layout: timestamp | time | tracebuffer | pid | tid | formatted | file | line + columns = line.split("|") + if len(columns) < 8: + continue + formatted = columns[5].strip() + if formatted in EXPECTED_MESSAGES: + messages.append(formatted) + return messages + + class golden_python_decoder(unittest.TestCase): def test_little_endian_fixtures(self): for name in LITTLE_ENDIAN_FIXTURES: @@ -73,5 +98,37 @@ def test_big_endian_fixtures(self): self.assertEqual(EXPECTED_MESSAGES, messages) +class golden_elf_meta(unittest.TestCase): + """clltk meta extracts tracepoint metadata from committed big-endian + s390x ELF objects: the .so through virtual addresses, the .o through + relocation records.""" + + def _assert_meta(self, name: str): + result = clltk("meta", str(GOLDEN_DIR / name)) + self.assertEqual(result.returncode, 0, msg=result.stderr) + self.assertIn("BE_ELF_TEST", result.stdout) + self.assertIn("big endian elf tracepoint %d", result.stdout) + + def test_big_endian_shared_object(self): + self._assert_meta("golden-1.3.0-be-s390x.so") + + def test_big_endian_relocatable_object(self): + self._assert_meta("golden-1.3.0-be-s390x.o") + + +class golden_cli_decoder(unittest.TestCase): + def test_little_endian_fixtures(self): + for name in LITTLE_ENDIAN_FIXTURES: + with self.subTest(fixture=name): + messages = decode_with_cli(GOLDEN_DIR / name) + self.assertEqual(EXPECTED_MESSAGES, messages) + + def test_big_endian_fixtures(self): + for name in BIG_ENDIAN_FIXTURES: + with self.subTest(fixture=name): + messages = decode_with_cli(GOLDEN_DIR / name) + self.assertEqual(EXPECTED_MESSAGES, messages) + + if __name__ == "__main__": unittest.main() From 45b52c83284b8eb4980df5d3072a75100d09bb62 Mon Sep 17 00:00:00 2001 From: Jo5ta Date: Fri, 10 Jul 2026 21:37:02 +0200 Subject: [PATCH 6/8] decoder: read tracepoint meta from foreign-endian ELF binaries The ELF reader interpreted section headers, relocation records, symbol entries, discovery pointers, and meta entry sizes in host byte order. Swap them based on the byte order declared in the ELF identification (EI_DATA), so clltk meta works on s390x binaries inspected from little-endian hosts: shared objects through virtual addresses and relocatable objects through relocation records (whose e_type check also needed the swap to even take the relocation path). Committed big-endian s390x ELF objects join the golden fixtures and are exercised by tests. Signed-off-by: Jo5ta --- .../cpp/source/low_level/elf_reader.cpp | 104 ++++++++++++------ tests/golden/README.md | 1 + tests/golden/golden-1.3.0-be-s390x.so | Bin 0 -> 9368 bytes 3 files changed, 71 insertions(+), 34 deletions(-) create mode 100755 tests/golden/golden-1.3.0-be-s390x.so diff --git a/decoder_tool/cpp/source/low_level/elf_reader.cpp b/decoder_tool/cpp/source/low_level/elf_reader.cpp index ec4da48..ee93c1f 100644 --- a/decoder_tool/cpp/source/low_level/elf_reader.cpp +++ b/decoder_tool/cpp/source/low_level/elf_reader.cpp @@ -2,9 +2,11 @@ // SPDX-License-Identifier: BSD-2-Clause-Patent #include "elf_reader.hpp" +#include "file.hpp" #include "meta_parser.hpp" #include +#include #include #include #include @@ -31,6 +33,21 @@ namespace CommonLowLevelTracingKit::decoder::source { return data[EI_CLASS] == ELFCLASS64; } + // an ELF file declares its byte order in the identification bytes; + // foreign means the opposite of this host's order + bool isForeignElf(const std::vector &data) { + if (data.size() < EI_NIDENT) { return false; } + constexpr uint8_t host_order = + (std::endian::native == std::endian::little) ? ELFDATA2LSB : ELFDATA2MSB; + const uint8_t file_order = data[EI_DATA]; + return (file_order == ELFDATA2LSB || file_order == ELFDATA2MSB) && + (file_order != host_order); + } + + template T swapped(T value, bool foreign) { + return foreign ? internal::byteswapValue(value) : value; + } + std::string getSectionName(const std::vector &data, uint64_t strtab_offset, uint32_t name_index) { if (strtab_offset + name_index >= data.size()) { return ""; } @@ -47,27 +64,28 @@ namespace CommonLowLevelTracingKit::decoder::source { if (data.size() < sizeof(Elf64_Ehdr)) { return sections; } + const bool foreign = isForeignElf(data); const auto *ehdr = reinterpret_cast(data.data()); - if (ehdr->e_shoff == 0 || ehdr->e_shnum == 0) { return sections; } - if (ehdr->e_shoff + ehdr->e_shnum * sizeof(Elf64_Shdr) > data.size()) { - return sections; - } + const uint64_t e_shoff = swapped(ehdr->e_shoff, foreign); + const uint16_t e_shnum = swapped(ehdr->e_shnum, foreign); + const uint16_t e_shstrndx = swapped(ehdr->e_shstrndx, foreign); + if (e_shoff == 0 || e_shnum == 0) { return sections; } + if (e_shoff + e_shnum * sizeof(Elf64_Shdr) > data.size()) { return sections; } - if (ehdr->e_shstrndx >= ehdr->e_shnum) { return sections; } + if (e_shstrndx >= e_shnum) { return sections; } - const auto *shdr_base = - reinterpret_cast(data.data() + ehdr->e_shoff); - const auto &shstrtab_hdr = shdr_base[ehdr->e_shstrndx]; + const auto *shdr_base = reinterpret_cast(data.data() + e_shoff); + const uint64_t shstrtab_offset = swapped(shdr_base[e_shstrndx].sh_offset, foreign); - for (uint16_t i = 0; i < ehdr->e_shnum; ++i) { + for (uint16_t i = 0; i < e_shnum; ++i) { const auto &shdr = shdr_base[i]; ElfSectionInfo info; - info.name = getSectionName(data, shstrtab_hdr.sh_offset, shdr.sh_name); - info.offset = shdr.sh_offset; - info.size = shdr.sh_size; - info.addr = shdr.sh_addr; - info.type = shdr.sh_type; + info.name = getSectionName(data, shstrtab_offset, swapped(shdr.sh_name, foreign)); + info.offset = swapped(shdr.sh_offset, foreign); + info.size = swapped(shdr.sh_size, foreign); + info.addr = swapped(shdr.sh_addr, foreign); + info.type = swapped(shdr.sh_type, foreign); sections.push_back(std::move(info)); } @@ -80,26 +98,27 @@ namespace CommonLowLevelTracingKit::decoder::source { if (data.size() < sizeof(Elf32_Ehdr)) { return sections; } + const bool foreign = isForeignElf(data); const auto *ehdr = reinterpret_cast(data.data()); - if (ehdr->e_shoff == 0 || ehdr->e_shnum == 0) { return sections; } - if (ehdr->e_shoff + ehdr->e_shnum * sizeof(Elf32_Shdr) > data.size()) { - return sections; - } - if (ehdr->e_shstrndx >= ehdr->e_shnum) { return sections; } + const uint32_t e_shoff = swapped(ehdr->e_shoff, foreign); + const uint16_t e_shnum = swapped(ehdr->e_shnum, foreign); + const uint16_t e_shstrndx = swapped(ehdr->e_shstrndx, foreign); + if (e_shoff == 0 || e_shnum == 0) { return sections; } + if (e_shoff + e_shnum * sizeof(Elf32_Shdr) > data.size()) { return sections; } + if (e_shstrndx >= e_shnum) { return sections; } - const auto *shdr_base = - reinterpret_cast(data.data() + ehdr->e_shoff); - const auto &shstrtab_hdr = shdr_base[ehdr->e_shstrndx]; + const auto *shdr_base = reinterpret_cast(data.data() + e_shoff); + const uint32_t shstrtab_offset = swapped(shdr_base[e_shstrndx].sh_offset, foreign); - for (uint16_t i = 0; i < ehdr->e_shnum; ++i) { + for (uint16_t i = 0; i < e_shnum; ++i) { const auto &shdr = shdr_base[i]; ElfSectionInfo info; - info.name = getSectionName(data, shstrtab_hdr.sh_offset, shdr.sh_name); - info.offset = shdr.sh_offset; - info.size = shdr.sh_size; - info.addr = shdr.sh_addr; - info.type = shdr.sh_type; + info.name = getSectionName(data, shstrtab_offset, swapped(shdr.sh_name, foreign)); + info.offset = swapped(shdr.sh_offset, foreign); + info.size = swapped(shdr.sh_size, foreign); + info.addr = swapped(shdr.sh_addr, foreign); + info.type = swapped(shdr.sh_type, foreign); sections.push_back(std::move(info)); } @@ -134,19 +153,20 @@ namespace CommonLowLevelTracingKit::decoder::source { // parse one self-describing meta entry (magic + size prefix) located // at a file offset and append the result void appendMetaEntryAt(const std::vector &data, uint64_t entry_offset, - MetaEntryInfoCollection &entries) { + MetaEntryInfoCollection &entries, bool foreign) { if (entry_offset + MetaParser::MIN_ENTRY_SIZE > data.size()) { return; } uint32_t entry_size = 0; std::memcpy(&entry_size, data.data() + entry_offset + MetaParser::OFFSET_SIZE, sizeof(entry_size)); + entry_size = swapped(entry_size, foreign); if (entry_size < MetaParser::MIN_ENTRY_SIZE || entry_offset + entry_size > data.size()) { return; } const std::span entry_data(data.data() + entry_offset, entry_size); - auto parsed = MetaParser::parse(entry_data, entry_offset); + auto parsed = MetaParser::parse(entry_data, entry_offset, foreign); entries.insert(entries.end(), std::make_move_iterator(parsed.begin()), std::make_move_iterator(parsed.end())); } @@ -156,7 +176,7 @@ namespace CommonLowLevelTracingKit::decoder::source { // e_type sits at the same offset for ELFCLASS32 and ELFCLASS64 uint16_t e_type = 0; std::memcpy(&e_type, data.data() + offsetof(Elf64_Ehdr, e_type), sizeof(e_type)); - return e_type == ET_REL; + return swapped(e_type, isForeignElf(data)) == ET_REL; } // In relocatable objects (.o) the pointer slots are zero; the meta @@ -182,6 +202,7 @@ namespace CommonLowLevelTracingKit::decoder::source { if (symtab == nullptr || rela == nullptr) { return entries; } if (rela->offset + rela->size > data.size()) { return entries; } + const bool foreign = isForeignElf(data); std::vector seen; const size_t entry_stride = 2 * pointer_size; const size_t count = rela->size / sizeof(Elf64_Rela); @@ -189,6 +210,9 @@ namespace CommonLowLevelTracingKit::decoder::source { Elf64_Rela relocation = {}; std::memcpy(&relocation, data.data() + rela->offset + i * sizeof(relocation), sizeof(relocation)); + relocation.r_offset = swapped(relocation.r_offset, foreign); + relocation.r_info = swapped(relocation.r_info, foreign); + relocation.r_addend = swapped(relocation.r_addend, foreign); if (relocation.r_offset % entry_stride != 0) { continue; } // offset cache slot const uint64_t sym_index = ELF64_R_SYM(relocation.r_info); @@ -199,6 +223,8 @@ namespace CommonLowLevelTracingKit::decoder::source { } Elf64_Sym symbol = {}; std::memcpy(&symbol, data.data() + sym_offset, sizeof(symbol)); + symbol.st_shndx = swapped(symbol.st_shndx, foreign); + symbol.st_value = swapped(symbol.st_value, foreign); if (symbol.st_shndx == SHN_UNDEF || symbol.st_shndx >= sections.size()) { continue; } @@ -208,7 +234,7 @@ namespace CommonLowLevelTracingKit::decoder::source { target.offset + symbol.st_value + (uint64_t)relocation.r_addend; if (std::find(seen.begin(), seen.end(), entry_offset) != seen.end()) { continue; } seen.push_back(entry_offset); - appendMetaEntryAt(data, entry_offset, entries); + appendMetaEntryAt(data, entry_offset, entries, foreign); } return entries; @@ -227,6 +253,7 @@ namespace CommonLowLevelTracingKit::decoder::source { size_t pointer_size) { MetaEntryInfoCollection entries; + const bool foreign = isForeignElf(data); std::vector seen; const size_t entry_stride = 2 * pointer_size; const size_t count = ptr_section.size / entry_stride; @@ -235,7 +262,15 @@ namespace CommonLowLevelTracingKit::decoder::source { if (ptr_offset + pointer_size > data.size()) { break; } uint64_t address = 0; - std::memcpy(&address, data.data() + ptr_offset, pointer_size); + if (pointer_size == sizeof(uint64_t)) { + uint64_t raw = 0; + std::memcpy(&raw, data.data() + ptr_offset, sizeof(raw)); + address = swapped(raw, foreign); + } else { + uint32_t raw = 0; + std::memcpy(&raw, data.data() + ptr_offset, sizeof(raw)); + address = swapped(raw, foreign); + } if (address == 0) { continue; } if (std::find(seen.begin(), seen.end(), address) != seen.end()) { continue; } seen.push_back(address); @@ -245,7 +280,8 @@ namespace CommonLowLevelTracingKit::decoder::source { if (address < section.addr || address >= section.addr + section.size) { continue; } - appendMetaEntryAt(data, section.offset + (address - section.addr), entries); + appendMetaEntryAt(data, section.offset + (address - section.addr), entries, + foreign); break; } } diff --git a/tests/golden/README.md b/tests/golden/README.md index 837a686..8dace7a 100644 --- a/tests/golden/README.md +++ b/tests/golden/README.md @@ -24,6 +24,7 @@ regenerated from source. | `golden-1.2.64-le-aarch64.clltk_trace` | library 1.2.64 (commit 1fa19d2), aarch64 | | `golden-1.3.0-le-aarch64.clltk_trace` | library 1.3.0, aarch64 | | `golden-1.3.0-be-s390x.clltk_trace` | library 1.3.0, s390x (big endian) | +| `golden-1.3.0-be-s390x.o` / `.so` | ELF objects compiled with library 1.3.0 headers on s390x | `tests/test_golden.py` decodes every fixture and asserts the formatted tracepoint messages. When the trace file format changes, add a new fixture diff --git a/tests/golden/golden-1.3.0-be-s390x.so b/tests/golden/golden-1.3.0-be-s390x.so new file mode 100755 index 0000000000000000000000000000000000000000..55b1956ef985a4edcdc520bc16f40a99e61d556a GIT binary patch literal 9368 zcmd5>e{5Y<9Y6QA+iPFj?QMWox6$jIqzv}bZW9Vd?S8cD$OdziWDxcCwePll;q|@p z-W{}1wI%~L1X&1>nJ7*J7Bt32quH3o7c%Am!C}Bm0%2Na(9YO}L1w@`Ki_lC_x8No z`!*8!!_&O`J>Spw$N8S$Kl+W1EuE2wCHzz+K8CyjNJ+brFFmfgHW3x=qDfo|o^viioedk?+2?_X!ri(XuJ>7i>!fmcj$NcKq4vx>^^o3ghIKxaFb&&3Fd+u^tsj$4bu z{ab6paWPW?zcSpv_+bV8rz_xh#j8Z4s)9YDsQlh!;N!En^|g3JP&w-mvy_7y%|@@ z!X-}MNYd#~XOo%qolu0k+_2;HWisBaj+amNxxG94``x@Vn9Qa!pu=8XYRN(`pf*>W z&Za#P;+F&6(o5KsVQJ4^^^V7YQ1I0%JrFVI#G`E{AfBGz{eGDca`IH z$2GpA9OwGmJpsH~>yMPU(S3F;d zrDWIOL*?puOc*7b+MD)@Al?Wp8iL!v$X8IG>P@i(@fMBw_;hZjMXWejEt)3YFZE2t zQIQqJA`JgjL@agFQ*n^$gAuW4Vwb3D_{&uL>A0wd)iF^WJtG7x+F(JtzxiWyrX^kj z3ur`7M-gyTNXaY8OZ0R@9JzXNaqR62=Rl>BI@{8fk>rTwo%<Z!f^ zF&d`E_oE$7?JLdfcQMvh%5pnSHJ9ENW8k~+yT->iPn;>ejuK8#JgQ#0B*c8~HCdl& z8Ogqh!(uU&AA57krqZO}**G4a*jReEyk=^=!{4^jZPL@ze#a9A$rpTwwq0*lUW;(Ew?|p z)7hU%31MtT;w}{2YT;95Z5G=6{MOKwiT4wGx3#3|bAafaTU{!BKyA}iD*fjyXb_kxay?%nl!>h6nu ztM>ky2cqAe|KPmN1;t3@%YS(3#oxd1{BuVaC~vgh4Z`LX(Cb8Jx%QY;)|x_~&nQM2 z^JAj(o(17joZz3+ z86!p&uOEgaXO$qA&CI4KoRPjQ}bER|dD&CsW_&i0cikjE=p$PRGCB1K2 zzfi2dZ#8P3_DZCt1B%K$m9kH2UXC&3X^*7zaYdrr6lHuK+t(=E((T3f7CyJjyRSBt zgzmSL=u9AT7~n*46p;4dONv(m8N9CdJdbej|JHKd&YJtk2;X}zK|On1AvOJ) z-lOS{G(Do}-xRH#ujyqNSesSe{PHNR%rb~j~rB|R-zb@k=Sl==__SYp$<71o+$t9Dr1t}>EV(BW}k(+u4T zIy@KZ9-nz6Kg)pYaiivqf$Mqe&rd0WKD8;5KaV8`oX*-v+>eR_ zuGf*8Hih_joyu{4`8eN~sNMYpKfE56wIgu8r#Je_$N*oc`10`$Tu*@d9RdACiuc!7 z$wHs*c1ZrZDmma6$&BE8l;VKvb+3H>1HVLR`0FM6p*KAwTYmWieZ5}V!FJR2EMGUE zulEODH|4D}ydK@S=gYVUP#oq;bCy zA8u!J{NFn3hxb$I?GEU3e;qLVKUyLF?-_n=fY2f=bejVsD@XzDzEsfJ+_B?Qo zpT{T7hu44y`-|-VZs1+oPYdV=z=Qn=Kh{MCKCbmatfk@gvS_W>W zMSVl&mvt@tgp6lvljZv{$FsE!WrfTytIvp^+E*tlH|xh7|B=@bPsqAq-K{Id4%pA5 z%L9mFt)JQ9_zT8+UGuDp^-ts7&xkW}ak!tzr^DlkbXLGK7|eRU81I21QmCN+fDxx2 z7Zhz|vO+x1SMc-KaJ(vN)OVab%hZ?mdhr;Uy)ls!v+t6QH-wiwyiDRX&PnB*flRJ9 znQ>BHE?;nxJ4ZxcZfH2;dTuJw60Sk7m1(@^<&&e1oAvUeqCcM;a-G!9p`lTj7#v>q z(q5UWz69yV#J*g%;N^Grp>TdaOu4fNJx1c$PAjX|!9*+b-srGfpdjZw6NW(Q-P9c# z_C^zH1eyjHIBFQj>D<0(_1Fya6cAN=`?YHt5>8 z9S~l5*>uyE8`iJe;@r^Ld1FVn)4gu}mJT@7kEXM)sAccH?Xz!_=Y2jddlfb=m}T#w zALd0k@MNu```=$|%U^)aXKVdL{9&GS>HVDFq03*-bt4+D#`4{n8`r?uo*RY+`G+FO(s3+NA3T~NG&eUvE&NxpF zsdV_3ktT=IeTXiHGVw{sK+HnT4mAJokmxNGL;?%M5azwAo@|pu()04^-kly6m5(CP z*@f?w*g_};|8MC3pHPYQJyA0W-2dLwdA6TXynoJa1CQgApJ<=wL^8d;^ZA_5;Y{T{ zmd@hZKcAcBzTXaSPpG2gA6$iqBp@_-_BZi|@C(}SP>TM&?3m*HuUY=HK=r+k$2i~H`2X@1<0n5a5^%&=SU<1c(#!D_u{vY7=xb@3FPD&_WQ|km6v(MR{`sE$`0_F3$df@@c zD0L~~f7zbsS^NU_wpR4Jy-fp5*dDw^wn2NXA7{IEHq_L#r?oq1Kce*FnwGT}iA?*; z3|_atJx)sKwsA=+_aDU874*@1Et+cL+b7C#UmteXTF7|+(Q15;j!!DUvbBJT*_Twd zIUiXEbkQ-z%-;*T48N-t@7KQ(Jl&bhwN Date: Fri, 10 Jul 2026 22:09:19 +0200 Subject: [PATCH 7/8] chore: bump version to 1.4.0 Signed-off-by: Jo5ta --- VERSION.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/VERSION.md b/VERSION.md index 2cf5275..77b6dff 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1,6 +1,15 @@ -1.3.0 +1.4.0 # Change log +## 1.4.0 +- feat: cross-endian decoding. Trace files and ELF binaries written on a machine with the + opposite byte order (e.g. s390x files read on x86_64/aarch64 and vice versa) are + byte-swapped while reading; the byte order is detected from the file magic / ELF + identification. Works through the decoder library, the clltk CLI, and the python decoder + (which additionally fixes float/double arguments from foreign-endian files). +- test: golden trace file fixtures generated by real library builds at every format + boundary in the public history (1.2.39, 1.2.49, 1.2.64, 1.3.0) plus big-endian s390x + trace file and ELF fixtures; decoded through both decoders on every test run. ## 1.3.0 - fix: tracepoints in inline functions, templates, and class members no longer fail with "causes a section type conflict" on GCC >= 15.2. Meta entries are now ordinary statics; @@ -11,10 +20,6 @@ first execution of a tracepoint needs no lookup. - feat: decoder/CLI read both the new `_metaptr` pointer sections and legacy inline `_meta` sections from ELF binaries. -- feat: cross-endian decoding. Trace files and ELF binaries written on a machine with the - opposite byte order (e.g. s390x files read on x86_64/aarch64) are byte-swapped while - reading; the byte order is detected from the file magic / ELF identification. The python - decoder additionally fixes float/double arguments from foreign-endian files. - BREAKING (link-time): objects compiled with older headers cannot be mixed with objects compiled with these headers in one binary; rebuild all translation units. - ci: add non-LTO build leg (`unittests-nolto` preset); LTO had masked the section conflict. From e2bf2a6ddebda61ec4b2c1d3faf8515a07c001fa Mon Sep 17 00:00:00 2001 From: Jo5ta Date: Sat, 11 Jul 2026 11:27:03 +0200 Subject: [PATCH 8/8] test: commit the big-endian s390x ELF golden object test_golden.py references golden-1.3.0-be-s390x.o, but **/*.o in .gitignore kept it out of the repo, so CI had no file to decode ('No meta information found'). It only passed locally because the generated object lingered in the working tree. Un-ignore golden object files and add a .gitattributes marking golden fixtures binary so git never applies EOL/text normalization to them. Signed-off-by: Jo5ta --- .gitattributes | 2 ++ .gitignore | 3 +++ tests/golden/golden-1.3.0-be-s390x.o | Bin 0 -> 5448 bytes 3 files changed, 5 insertions(+) create mode 100644 .gitattributes create mode 100644 tests/golden/golden-1.3.0-be-s390x.o diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..80f605d --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Golden test fixtures are committed binaries; never apply text/EOL normalization +tests/golden/** binary diff --git a/.gitignore b/.gitignore index 2b7f736..bb83939 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,6 @@ output.xml report.html tmp/ /*.clltk_trace + +# golden test fixtures are committed binaries, not build artifacts +!tests/golden/*.o diff --git a/tests/golden/golden-1.3.0-be-s390x.o b/tests/golden/golden-1.3.0-be-s390x.o new file mode 100644 index 0000000000000000000000000000000000000000..067ce3f3c7edfed71761547945a7eb92f24bab07 GIT binary patch literal 5448 zcmb_fTZ|mV6|J6KytY}d%_HF8h%{ix0--ar4(5@3Y%pHdykvnRC4A63p6T(9yfd@R zG>HwC@CW2U5^NzpP(IM)0~RR?FM)V{Fh~vr62N>Qlt_^*t%#J^QWgnCjud4$r@C%W zO>H+wQLg4z-Fxc3s=B(W=YgHK&J+rsO372_LAHQ&1j5F(TAMcQSJmc|M^>iQ*5#G{ zT>qZlCgMGx?WwcXbT@i+RQ=}IX6mo5Qx}{(OO2@UQ_my%pA+=03$*P^KxURcQ)`!g z3w~LRLcVCZ*HvnC`6!}}E+6h~R->mL?|-UeezW(9_M7bgT^&M>S@3mWgD-v7|10V` zp+?94d-9EyAEfnZFQ>{&bj0V=Dz}&_-&|GY?^)%t{~W4H@-MxPYTxdkMvl|WH3mA_ zS}E#$yJbC=H(^-{OAo5aBy*}?`N{0c(`ro6m9E;_o8!EeeyB!z)0B=~%QjijBq!qQ zdD+;h$^O68A*R(PFFKjQvZiO5LzC#J&vo?7VYNPNw^Tmr$G<@pdT_^owIUfo|Hd z<9dJcrrW=NxnH_Emv1u{5>KpS&Zg`m^QO3|w{1<=XdLw%&brDDS|8w`F`| zm+3)V@2_b|sjiy&+4#@Te0cpY*8Otruf`r(bL5PfiC&@b=m+n=_oqL;`_3OG60b?v z?>0UQYR0j?TU#4b^@XRJoG@tO>LZDDo%F6_2|v+R8fPtG3zpi9AGKeUlik8P)o(OhBW?HX_a zJ=a&_=(Ea!UXY~j>Nx^5T`t$Q-0j=yC%||)-f3k7!+#LM69Q@`Syow?*Ws)Z*Pe9%;N-`#`_ec)@ zqK}Qcn!JxQadQ@im4$_Pe^_ZXJ8}EoO5AFz;MN^QHYGMw%2-q*+h7~VzRl)p6*i){ z9HfS=TCEesf%d+J7TYC+eOjBS@xjHY6BIKn!o|3)!l3+%nO zR*TxZrhKuh8X2+R=4)X3o^A)gbKL;!;ehs)C$E>r*`|L8XTmOAI_J@IIJTLDt;QSxp z*7Iv%S$Vs_zX5jhJ_RhJexS)6!5k#kOyUtZ2^O3fiT5(U!d#qV+z;pV4G{ow&uUz2 z$=;^O_&!+T$UVgPS#fBa6^Au)%#8y`@mUB?UC73}0-~p_YtOp4ZY~Z;ufL1?z!~5D zKG@^nUFf94V9Or&Q_wlsh|7D?$L*w_7pWaBl!G`r5UW6o)pA@`!QM{C=(>1#~N@4PF#_xgqucNnY$qCj0=)XLw5G5Zr>R}rPXLeP1M|M z#ZiC{{i|?NSN5AyqWQ4aE;k}B#{Y?Y#PLk5HBQXwME9O^aM^!^`TfghPMX$j+N?XC zh=neCK6}X{{@lO(mnS&iQH#74qfA>MMe(Q1cQ7aa_#zEJvtZ_z_~Os}{6-aB#sQHE z#iUQijXI=%2cXpNeBY)1636vt-tFML`d={)N71u>c|KAv>%R@mJAvfZKc~^b{EV8` z&p5N-octR&Mq(c3m%P#wuY%>f+|56aF_PFP+CV2Bg7fmv83)NP|5GF$@gIR@{ms2( zyi`G+zk};Dghupx*!0wSwpdhn+9;bWkfP+L{17;={sS}s@%eN8gU)C;5`2o8h|8Uq7Cc61DIF8%%63#epv^JNBl#uo)xCs=f6{YEB@v#g_0ht}gC0Q2VGGY+%pIe+<1u;S6(2N~VB=6sXm|o^f^(=6X|Agep_5U2H>QLwa literal 0 HcmV?d00001