Skip to content

Commit d4b7c0d

Browse files
tobias-stadlertobias-stadler
authored
[Remarks] Auto-detect remark parser format (#144554)
Add remark format 'Auto', which performs automatic detection of the remark format using the magic numbers at the beginning of the remarks files. The RemarkLinker already did something similar, so we streamlined this and exposed this to llvm-remarkutil.
1 parent 67c52aa commit d4b7c0d

File tree

15 files changed

+76
-33
lines changed

15 files changed

+76
-33
lines changed

‎llvm/include/llvm/Remarks/RemarkFormat.h

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,14 +23,17 @@ namespace remarks {
2323
constexpr StringLiteral Magic("REMARKS");
2424

2525
/// The format used for serializing/deserializing remarks.
26-
enum class Format { Unknown, YAML, Bitstream };
26+
enum class Format { Unknown, Auto, YAML, Bitstream };
2727

2828
/// Parse and validate a string for the remark format.
2929
LLVM_ABI Expected<Format> parseFormat(StringRef FormatStr);
3030

3131
/// Parse and validate a magic number to a remark format.
3232
LLVM_ABI Expected<Format> magicToFormat(StringRef Magic);
3333

34+
/// Detect format based on selected format and magic number
35+
LLVM_ABI Expected<Format> detectFormat(Format Selected, StringRef Magic);
36+
3437
} // end namespace remarks
3538
} // end namespace llvm
3639

‎llvm/include/llvm/Remarks/RemarkLinker.h

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -80,13 +80,12 @@ struct RemarkLinker {
8080
/// \p Buffer.
8181
/// \p Buffer can be either a standalone remark container or just
8282
/// metadata. This takes care of uniquing and merging the remarks.
83-
LLVM_ABI Error link(StringRef Buffer,
84-
std::optional<Format> RemarkFormat = std::nullopt);
83+
LLVM_ABI Error link(StringRef Buffer, Format RemarkFormat = Format::Auto);
8584

8685
/// Link the remarks found in \p Obj by looking for the right section and
8786
/// calling the method above.
8887
LLVM_ABI Error link(const object::ObjectFile &Obj,
89-
std::optional<Format> RemarkFormat = std::nullopt);
88+
Format RemarkFormat = Format::Auto);
9089

9190
/// Serialize the linked remarks to the stream \p OS, using the format \p
9291
/// RemarkFormat.

‎llvm/lib/Remarks/RemarkFormat.cpp

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,22 @@ Expected<Format> llvm::remarks::magicToFormat(StringRef MagicStr) {
4242

4343
if (Result == Format::Unknown)
4444
return createStringError(std::make_error_code(std::errc::invalid_argument),
45-
"Unknown remark magic: '%s'", MagicStr.data());
45+
"Automatic detection of remark format failed. "
46+
"Unknown magic number: '%.4s'",
47+
MagicStr.data());
4648
return Result;
4749
}
50+
51+
Expected<Format> llvm::remarks::detectFormat(Format Selected,
52+
StringRef MagicStr) {
53+
if (Selected == Format::Unknown)
54+
return createStringError(std::make_error_code(std::errc::invalid_argument),
55+
"Unknown remark parser format.");
56+
if (Selected != Format::Auto)
57+
return Selected;
58+
59+
// Empty files are valid bitstream files
60+
if (MagicStr.empty())
61+
return Format::Bitstream;
62+
return magicToFormat(MagicStr);
63+
}

‎llvm/lib/Remarks/RemarkLinker.cpp

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -66,17 +66,10 @@ void RemarkLinker::setExternalFilePrependPath(StringRef PrependPathIn) {
6666
PrependPath = std::string(PrependPathIn);
6767
}
6868

69-
Error RemarkLinker::link(StringRef Buffer, std::optional<Format> RemarkFormat) {
70-
if (!RemarkFormat) {
71-
Expected<Format> ParserFormat = magicToFormat(Buffer);
72-
if (!ParserFormat)
73-
return ParserFormat.takeError();
74-
RemarkFormat = *ParserFormat;
75-
}
76-
69+
Error RemarkLinker::link(StringRef Buffer, Format RemarkFormat) {
7770
Expected<std::unique_ptr<RemarkParser>> MaybeParser =
7871
createRemarkParserFromMeta(
79-
*RemarkFormat, Buffer,
72+
RemarkFormat, Buffer,
8073
PrependPath ? std::optional<StringRef>(StringRef(*PrependPath))
8174
: std::optional<StringRef>());
8275
if (!MaybeParser)
@@ -102,8 +95,7 @@ Error RemarkLinker::link(StringRef Buffer, std::optional<Format> RemarkFormat) {
10295
return Error::success();
10396
}
10497

105-
Error RemarkLinker::link(const object::ObjectFile &Obj,
106-
std::optional<Format> RemarkFormat) {
98+
Error RemarkLinker::link(const object::ObjectFile &Obj, Format RemarkFormat) {
10799
Expected<std::optional<StringRef>> SectionOrErr =
108100
getRemarksSectionContents(Obj);
109101
if (!SectionOrErr)

‎llvm/lib/Remarks/RemarkParser.cpp

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
#include "BitstreamRemarkParser.h"
1616
#include "YAMLRemarkParser.h"
1717
#include "llvm-c/Remarks.h"
18+
#include "llvm/Remarks/RemarkFormat.h"
1819
#include "llvm/Support/CBindingWrapping.h"
1920
#include <optional>
2021

@@ -50,14 +51,18 @@ Expected<StringRef> ParsedStringTable::operator[](size_t Index) const {
5051

5152
Expected<std::unique_ptr<RemarkParser>>
5253
llvm::remarks::createRemarkParser(Format ParserFormat, StringRef Buf) {
53-
switch (ParserFormat) {
54+
auto DetectedFormat = detectFormat(ParserFormat, Buf);
55+
if (!DetectedFormat)
56+
return DetectedFormat.takeError();
57+
58+
switch (*DetectedFormat) {
5459
case Format::YAML:
5560
return std::make_unique<YAMLRemarkParser>(Buf);
5661
case Format::Bitstream:
5762
return std::make_unique<BitstreamRemarkParser>(Buf);
5863
case Format::Unknown:
59-
return createStringError(std::make_error_code(std::errc::invalid_argument),
60-
"Unknown remark parser format.");
64+
case Format::Auto:
65+
break;
6166
}
6267
llvm_unreachable("unhandled ParseFormat");
6368
}
@@ -66,15 +71,19 @@ Expected<std::unique_ptr<RemarkParser>>
6671
llvm::remarks::createRemarkParserFromMeta(
6772
Format ParserFormat, StringRef Buf,
6873
std::optional<StringRef> ExternalFilePrependPath) {
69-
switch (ParserFormat) {
74+
auto DetectedFormat = detectFormat(ParserFormat, Buf);
75+
if (!DetectedFormat)
76+
return DetectedFormat.takeError();
77+
78+
switch (*DetectedFormat) {
7079
case Format::YAML:
7180
return createYAMLParserFromMeta(Buf, std::move(ExternalFilePrependPath));
7281
case Format::Bitstream:
7382
return createBitstreamParserFromMeta(Buf,
7483
std::move(ExternalFilePrependPath));
7584
case Format::Unknown:
76-
return createStringError(std::make_error_code(std::errc::invalid_argument),
77-
"Unknown remark parser format.");
85+
case Format::Auto:
86+
break;
7887
}
7988
llvm_unreachable("unhandled ParseFormat");
8089
}

‎llvm/lib/Remarks/RemarkSerializer.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,9 @@ remarks::createRemarkSerializer(Format RemarksFormat, SerializerMode Mode,
2222
raw_ostream &OS) {
2323
switch (RemarksFormat) {
2424
case Format::Unknown:
25+
case Format::Auto:
2526
return createStringError(std::errc::invalid_argument,
26-
"Unknown remark serializer format.");
27+
"Invalid remark serializer format.");
2728
case Format::YAML:
2829
return std::make_unique<YAMLRemarkSerializer>(OS, Mode);
2930
case Format::Bitstream:
@@ -37,8 +38,9 @@ remarks::createRemarkSerializer(Format RemarksFormat, SerializerMode Mode,
3738
raw_ostream &OS, remarks::StringTable StrTab) {
3839
switch (RemarksFormat) {
3940
case Format::Unknown:
41+
case Format::Auto:
4042
return createStringError(std::errc::invalid_argument,
41-
"Unknown remark serializer format.");
43+
"Invalid remark serializer format.");
4244
case Format::YAML:
4345
return std::make_unique<YAMLRemarkSerializer>(OS, Mode, std::move(StrTab));
4446
case Format::Bitstream:

‎llvm/test/tools/llvm-remarkutil/Inputs/broken-remark-magic.bitstream

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
12345678

‎llvm/test/tools/llvm-remarkutil/annotation-count.test

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
RUN: llvm-remarkutil annotation-count --parser=yaml --annotation-type=remark %p/Inputs/annotation-count.yaml | FileCheck %s
2+
RUN: llvm-remarkutil annotation-count --annotation-type=remark %p/Inputs/annotation-count.yaml | FileCheck %s
23
RUN: llvm-remarkutil yaml2bitstream %p/Inputs/annotation-count.yaml | llvm-remarkutil annotation-count --parser=bitstream --annotation-type=remark | FileCheck %s
4+
RUN: llvm-remarkutil yaml2bitstream %p/Inputs/annotation-count.yaml | llvm-remarkutil annotation-count --annotation-type=remark | FileCheck %s
35
RUN: llvm-remarkutil count --parser=yaml --count-by=arg --group-by=function --remark-name="AnnotationSummary" %p/Inputs/annotation-count.yaml | FileCheck %s --check-prefix=COUNT-CHECK
46
RUN: llvm-remarkutil yaml2bitstream %p/Inputs/annotation-count.yaml | llvm-remarkutil count --parser=bitstream --count-by=arg --group-by=function --remark-name="AnnotationSummary" | FileCheck %s --check-prefix=COUNT-CHECK
57

‎llvm/test/tools/llvm-remarkutil/broken-bitstream-remark-magic.test

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
RUN: not llvm-remarkutil instruction-count %p/Inputs/broken-remark-magic.bitstream -o - 2>&1 | FileCheck %s
2+
RUN: not llvm-remarkutil instruction-mix %p/Inputs/broken-remark-magic.bitstream -o - 2>&1 | FileCheck %s
3+
RUN: not llvm-remarkutil annotation-count --annotation-type=remark %p/Inputs/broken-remark-magic.bitstream -o - 2>&1 | FileCheck %s
4+
RUN: not llvm-remarkutil count %p/Inputs/broken-remark-magic.bitstream -o - 2>&1 | FileCheck %s
5+
6+
CHECK: error: Automatic detection of remark format failed. Unknown magic number: '1234'

‎llvm/test/tools/llvm-remarkutil/empty-file.test

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,11 @@ RUN: llvm-remarkutil instruction-count --parser=bitstream %p/Inputs/empty-file -
88
RUN: llvm-remarkutil instruction-mix --parser=bitstream %p/Inputs/empty-file --report_style=csv -o - 2>&1 | FileCheck %s --allow-empty --check-prefix=MIXBITSTREAM
99
RUN: llvm-remarkutil annotation-count --parser=bitstream --annotation-type=remark %p/Inputs/empty-file -o - 2>&1 | FileCheck %s --allow-empty --check-prefix=ANNOTATIONBITSTREAM
1010
RUN: llvm-remarkutil count --parser=bitstream %p/Inputs/empty-file -o - 2>&1 | FileCheck %s --allow-empty --check-prefix=COUNTBITSTREAM
11+
; Parser format auto-detection should treat empty files as bitstream files
12+
RUN: llvm-remarkutil instruction-count %p/Inputs/empty-file -o - 2>&1 | FileCheck %s --allow-empty --check-prefix=SIZEBITSTREAM
13+
RUN: llvm-remarkutil instruction-mix %p/Inputs/empty-file --report_style=csv -o - 2>&1 | FileCheck %s --allow-empty --check-prefix=MIXBITSTREAM
14+
RUN: llvm-remarkutil annotation-count --annotation-type=remark %p/Inputs/empty-file -o - 2>&1 | FileCheck %s --allow-empty --check-prefix=ANNOTATIONBITSTREAM
15+
RUN: llvm-remarkutil count %p/Inputs/empty-file -o - 2>&1 | FileCheck %s --allow-empty --check-prefix=COUNTBITSTREAM
1116

1217
; YAMLPARSER: error: document root is not of mapping type.
1318

‎llvm/test/tools/llvm-remarkutil/instruction-count.test

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
RUN: llvm-remarkutil instruction-count --parser=yaml %p/Inputs/instruction-count.yaml | FileCheck %s
2+
RUN: llvm-remarkutil instruction-count %p/Inputs/instruction-count.yaml | FileCheck %s
23
RUN: llvm-remarkutil yaml2bitstream %p/Inputs/instruction-count.yaml | llvm-remarkutil instruction-count --parser=bitstream | FileCheck %s
4+
RUN: llvm-remarkutil yaml2bitstream %p/Inputs/instruction-count.yaml | llvm-remarkutil instruction-count | FileCheck %s
35
RUN: llvm-remarkutil count --parser=yaml --count-by=arg --group-by=function --remark-name="InstructionCount" %p/Inputs/instruction-count.yaml | FileCheck %s --check-prefix=COUNT-CHECK
46
RUN: llvm-remarkutil yaml2bitstream %p/Inputs/instruction-count.yaml | llvm-remarkutil count --parser=bitstream --count-by=arg --group-by=function --remark-name="InstructionCount" | FileCheck %s --check-prefix=COUNT-CHECK
57
RUN: not llvm-remarkutil count --parser=yaml --count-by=arg --group-by=function --rremark-name=* %p/Inputs/instruction-count.yaml 2>&1 | FileCheck %s --check-prefix=ERROR-REPOPERATOR -DARG=rremark-name
@@ -18,4 +20,4 @@ RUN: not llvm-remarkutil count --parser=yaml --count-by=arg --group-by=function
1820
; COUNT-CHECK: func3,3
1921

2022
; ERROR-REPOPERATOR: error: invalid argument '--[[ARG]]=*': repetition-operator operand invalid
21-
; ERROR-BOTHFILTERS: error: conflicting arguments: --remark-name and --rremark-name
23+
; ERROR-BOTHFILTERS: error: conflicting arguments: --remark-name and --rremark-name

‎llvm/test/tools/llvm-remarkutil/instruction-mix.test

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
RUN: llvm-remarkutil instruction-mix --parser=yaml %p/Inputs/instruction-mix.yaml | FileCheck %s
2+
RUN: llvm-remarkutil instruction-mix %p/Inputs/instruction-mix.yaml | FileCheck %s
23
RUN: llvm-remarkutil yaml2bitstream %p/Inputs/instruction-mix.yaml | llvm-remarkutil instruction-mix --parser=bitstream | FileCheck %s
4+
RUN: llvm-remarkutil yaml2bitstream %p/Inputs/instruction-mix.yaml | llvm-remarkutil instruction-mix | FileCheck %s
35
RUN: llvm-remarkutil instruction-mix --parser=yaml %p/Inputs/instruction-mix.yaml --report_style=human | FileCheck %s
46
RUN: llvm-remarkutil instruction-mix --parser=yaml %p/Inputs/instruction-mix.yaml --report_style=csv | FileCheck %s --check-prefix=CSV
57
RUN: llvm-remarkutil instruction-mix --parser=yaml %p/Inputs/instruction-mix.yaml --rfilter=meow | FileCheck %s --check-prefix=MEOW-RE
@@ -34,4 +36,4 @@ RUN: not llvm-remarkutil instruction-mix --parser=yaml %p/Inputs/instruction-mix
3436
; NONE-EXACT: ----------- -----
3537
; NONE-NOT: {{.*}}
3638

37-
; ERROR: error: invalid argument '--rfilter=*': repetition-operator operand invalid
39+
; ERROR: error: invalid argument '--rfilter=*': repetition-operator operand invalid

‎llvm/test/tools/llvm-remarkutil/size-diff/no-difference.test

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
RUN: llvm-remarkutil size-diff %p/Inputs/1-func-1-instr-1-stack.yaml %p/Inputs/1-func-1-instr-1-stack.yaml --parser=yaml | FileCheck -strict-whitespace %s
2+
RUN: llvm-remarkutil size-diff %p/Inputs/1-func-1-instr-1-stack.yaml %p/Inputs/1-func-1-instr-1-stack.yaml | FileCheck -strict-whitespace %s
3+
RUN: llvm-remarkutil yaml2bitstream -o %t.bitstream %p/Inputs/1-func-1-instr-1-stack.yaml
4+
RUN: llvm-remarkutil size-diff %t.bitstream %p/Inputs/1-func-1-instr-1-stack.yaml | FileCheck -strict-whitespace %s
25

36
; Same file passed twice -> no changes reported.
47

‎llvm/tools/llvm-remarkutil/RemarkUtilHelpers.h

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,12 @@
3535
// Keep Input format and names consistent accross the modes via a macro.
3636
#define INPUT_FORMAT_COMMAND_LINE_OPTIONS(SUBOPT) \
3737
static cl::opt<Format> InputFormat( \
38-
"parser", cl::desc("Input remark format to parse"), \
39-
cl::values(clEnumValN(Format::YAML, "yaml", "YAML"), \
40-
clEnumValN(Format::Bitstream, "bitstream", "Bitstream")), \
38+
"parser", cl::init(Format::Auto), \
39+
cl::desc("Input remark format to parse"), \
40+
cl::values( \
41+
clEnumValN(Format::Auto, "auto", "Automatic detection (default)"), \
42+
clEnumValN(Format::YAML, "yaml", "YAML"), \
43+
clEnumValN(Format::Bitstream, "bitstream", "Bitstream")), \
4144
cl::sub(SUBOPT));
4245

4346
#define DEBUG_LOC_INFO_COMMAND_LINE_OPTIONS(SUBOPT) \

‎llvm/unittests/Remarks/RemarksLinkingTest.cpp

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -243,10 +243,8 @@ TEST(Remarks, LinkingError) {
243243
// Check that the prepend path is propagated and fails with the full path.
244244
// Also ensures that the remark format is correctly auto-detected.
245245
RL.setExternalFilePrependPath("/baddir/");
246-
Error E = RL.link(
247-
StringRef("REMARKS\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0badfile.opt.yaml",
248-
40),
249-
/*RemarkFormat=*/std::nullopt);
246+
Error E = RL.link(StringRef(
247+
"REMARKS\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0badfile.opt.yaml", 40));
250248
EXPECT_TRUE(static_cast<bool>(E));
251249
std::string ErrorMessage = toString(std::move(E));
252250
EXPECT_EQ(StringRef(ErrorMessage).lower(),

0 commit comments

Comments
 (0)