diff --git a/lld/ELF/Arch/X86_64.cpp b/lld/ELF/Arch/X86_64.cpp index dc3cb4ee9d6b7..4df776c92e981 100644 --- a/lld/ELF/Arch/X86_64.cpp +++ b/lld/ELF/Arch/X86_64.cpp @@ -354,6 +354,9 @@ void X86_64::relaxCFIJumpTables() const { sec->file, sec->name, sec->type, sec->flags, sec->entsize, sec->entsize, sec->contentMaybeDecompress().slice(begin, end - begin)); + // Ensure that --preferred-function-alignment does not mess with the + // placement of this section. + slice->retainAlignment = true; for (const Relocation &r : ArrayRef(rbegin, rend)) { slice->relocations.push_back( Relocation{r.expr, r.type, r.offset - begin, r.addend, r.sym}); @@ -421,6 +424,9 @@ void X86_64::relaxCFIJumpTables() const { // table. First add a slice for the unmodified jump table entries // before this one. addSectionSlice(begin, cur, rbegin, rcur); + // Ensure that --preferred-function-alignment does not mess with the + // placement of this section. + target->retainAlignment = true; // Add the target to our replacement list, and set the target's // replacement list to the empty list. This removes it from its // original position and adds it here, as well as causing @@ -441,6 +447,7 @@ void X86_64::relaxCFIJumpTables() const { // jump table where it is and keep the last entry. if (lastSec) { addSectionSlice(begin, cur, rbegin, rcur); + lastSec->retainAlignment = true; replacements.push_back(lastSec); sectionReplacements[sec] = {}; sectionReplacements[lastSec] = replacements; diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h index d9639b06ca4bf..26b75d7d7beba 100644 --- a/lld/ELF/Config.h +++ b/lld/ELF/Config.h @@ -358,6 +358,7 @@ struct Config { bool optRemarksWithHotness; bool picThunk; bool pie; + std::optional preferredFunctionAlignment; bool printGcSections; bool printIcfSections; bool printMemoryUsage; diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp index 21d228eda6470..105b5d5a8460f 100644 --- a/lld/ELF/Driver.cpp +++ b/lld/ELF/Driver.cpp @@ -1508,6 +1508,9 @@ static void readConfigs(Ctx &ctx, opt::InputArgList &args) { if (auto *arg = args.getLastArg(OPT_package_metadata)) parsePackageMetadata(ctx, *arg); ctx.arg.pie = args.hasFlag(OPT_pie, OPT_no_pie, false); + if (args.hasArg(OPT_preferred_function_alignment)) + ctx.arg.preferredFunctionAlignment = + args::getInteger(args, OPT_preferred_function_alignment, 0); ctx.arg.printIcfSections = args.hasFlag(OPT_print_icf_sections, OPT_no_print_icf_sections, false); ctx.arg.printGcSections = diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp index 68e3feb1bd048..299aba8cb7d2f 100644 --- a/lld/ELF/InputSection.cpp +++ b/lld/ELF/InputSection.cpp @@ -58,7 +58,7 @@ InputSectionBase::InputSectionBase(InputFile *file, StringRef name, Kind sectionKind) : SectionBase(sectionKind, file, name, type, flags, link, info, addralign, entsize), - bss(0), decodedCrel(0), keepUnique(0), nopFiller(0), + bss(0), decodedCrel(0), keepUnique(0), nopFiller(0), retainAlignment(0), content_(data.data()), size(data.size()) { // In order to reduce memory allocation, we assume that mergeable // sections are smaller than 4 GiB, which is not an unreasonable diff --git a/lld/ELF/InputSection.h b/lld/ELF/InputSection.h index 98e7d5d4ff0cd..0fb437d8e4d7b 100644 --- a/lld/ELF/InputSection.h +++ b/lld/ELF/InputSection.h @@ -183,6 +183,11 @@ class InputSectionBase : public SectionBase { LLVM_PREFERRED_TYPE(bool) uint8_t nopFiller : 1; + // If true, --preferred-function-alignment has no effect on this section. + // Set by the CFI jump table relaxation pass. + LLVM_PREFERRED_TYPE(bool) + uint8_t retainAlignment : 1; + mutable bool compressed = false; // Input sections are part of an output section. Special sections diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td index cc91680550b4b..510fd22b6be37 100644 --- a/lld/ELF/Options.td +++ b/lld/ELF/Options.td @@ -406,6 +406,9 @@ def pop_state: F<"pop-state">, def push_state: F<"push-state">, HelpText<"Save the current state of --as-needed, -static and --whole-archive">; +def preferred_function_alignment: JJ<"preferred-function-alignment=">, + HelpText<"Align functions to the given alignment if possible">; + def print_map: F<"print-map">, HelpText<"Print a link map to the standard output">; diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp index abc156cb93bdc..556dda54e0db7 100644 --- a/lld/ELF/Writer.cpp +++ b/lld/ELF/Writer.cpp @@ -1499,6 +1499,20 @@ static void randomizeSectionPadding(Ctx &ctx) { } } +static void applyPreferredFunctionAlignment(Ctx &ctx) { + if (!ctx.arg.preferredFunctionAlignment) + return; + SmallVector storage; + for (OutputSection *osec : ctx.outputSections) { + if (!(osec->flags & SHF_EXECINSTR)) + continue; + for (InputSection *sec : getInputSections(*osec, storage)) + if (!isa(sec) && !sec->retainAlignment) + sec->addralign = std::max( + sec->addralign, *ctx.arg.preferredFunctionAlignment); + } +} + // We need to generate and finalize the content that depends on the address of // InputSections. As the generation of the content may also alter InputSection // addresses we must converge to a fixed point. We do that here. See the comment @@ -1530,6 +1544,9 @@ template void Writer::finalizeAddressDependentContent() { ctx.target->relaxCFIJumpTables(); + if (ctx.arg.preferredFunctionAlignment) + applyPreferredFunctionAlignment(ctx); + uint32_t pass = 0, assignPasses = 0; for (;;) { bool changed = ctx.target->needsThunks diff --git a/lld/docs/ld.lld.1 b/lld/docs/ld.lld.1 index 7edc522b4f6a4..3bedc16e19178 100644 --- a/lld/docs/ld.lld.1 +++ b/lld/docs/ld.lld.1 @@ -520,6 +520,13 @@ Currently the same as yes. Don't use. .El +.It Fl -preferred-function-alignment Ns = Ns Ar align +Specify the preferred function alignment. This flag increases the +alignment of all user-provided executable (SHF_EXECINSTR) sections to the +specified value unless retaining the original alignment is beneficial +for performance reasons. For example, when using LLVM's control flow +integrity feature, functions may retain their original alignment if +necessary in order to move them into the jump table. .It Fl -print-gc-sections List removed unused sections. .It Fl -print-icf-sections diff --git a/lld/test/ELF/preferred-function-alignment.s b/lld/test/ELF/preferred-function-alignment.s new file mode 100644 index 0000000000000..cb0d58f8ff186 --- /dev/null +++ b/lld/test/ELF/preferred-function-alignment.s @@ -0,0 +1,15 @@ +// REQUIRES: x86 +// RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o +// RUN: ld.lld --preferred-function-alignment=32 -o %t %t.o +// RUN: llvm-nm -n %t | FileCheck %s + +// CHECK: 0000000000201120 t f1 +.section .text.f1,"ax",@progbits +f1: +ret + +// CHECK: 0000000000201140 t f2 +.section .text.f2,"ax",@progbits +f2: +ret + diff --git a/lld/test/ELF/x86_64-relax-jump-tables.s b/lld/test/ELF/x86_64-relax-jump-tables.s index 782d1be655a7d..8f4738888114b 100644 --- a/lld/test/ELF/x86_64-relax-jump-tables.s +++ b/lld/test/ELF/x86_64-relax-jump-tables.s @@ -2,6 +2,8 @@ // RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o // RUN: ld.lld %t.o -shared -o %t // RUN: llvm-objdump -d --show-all-symbols %t | FileCheck %s +// RUN: ld.lld %t.o -shared -o %t32 --preferred-function-alignment=32 +// RUN: llvm-objdump -d --show-all-symbols %t32 | FileCheck %s // Mostly positive cases, except for f2. .section .text.jt1,"ax",@llvm_cfi_jump_table,8