diff options
author | Fangrui Song <i@maskray.me> | 2022-10-17 21:01:10 +0300 |
---|---|---|
committer | Fangrui Song <i@maskray.me> | 2022-10-17 21:01:11 +0300 |
commit | 685b21255315e699aa839d93fe71b37d806c90c2 (patch) | |
tree | 19809b26ea23dce00973114f09c378269fe451c4 /lld | |
parent | 5b9597f59a445523bd59b5251ab1c2865e74919f (diff) |
[ELF] Make relocateAlloc target specific. NFC
The target-specific code (AArch64, PPC64) does not fit into the generic code and
adds virtual function overhead. Move relocateAlloc into ELF/Arch/ instead. This
removes many virtual functions (relaxTls*). In addition, this helps get rid of
getRelocTargetVA dispatch and many RelExpr members in the future.
Diffstat (limited to 'lld')
-rw-r--r-- | lld/ELF/AArch64ErrataFix.cpp | 2 | ||||
-rw-r--r-- | lld/ELF/ARMErrataFix.cpp | 2 | ||||
-rw-r--r-- | lld/ELF/Arch/AArch64.cpp | 80 | ||||
-rw-r--r-- | lld/ELF/Arch/PPC.cpp | 45 | ||||
-rw-r--r-- | lld/ELF/Arch/PPC64.cpp | 106 | ||||
-rw-r--r-- | lld/ELF/Arch/X86.cpp | 53 | ||||
-rw-r--r-- | lld/ELF/Arch/X86_64.cpp | 105 | ||||
-rw-r--r-- | lld/ELF/InputSection.cpp | 129 | ||||
-rw-r--r-- | lld/ELF/InputSection.h | 1 | ||||
-rw-r--r-- | lld/ELF/SyntheticSections.cpp | 6 | ||||
-rw-r--r-- | lld/ELF/Target.cpp | 37 | ||||
-rw-r--r-- | lld/ELF/Target.h | 24 |
12 files changed, 322 insertions, 268 deletions
diff --git a/lld/ELF/AArch64ErrataFix.cpp b/lld/ELF/AArch64ErrataFix.cpp index ae473284ef0b..41a29a7328c2 100644 --- a/lld/ELF/AArch64ErrataFix.cpp +++ b/lld/ELF/AArch64ErrataFix.cpp @@ -412,7 +412,7 @@ void Patch843419Section::writeTo(uint8_t *buf) { write32le(buf, read32le(patchee->rawData.begin() + patcheeOffset)); // Apply any relocation transferred from the original patchee section. - relocateAlloc(buf, buf + getSize()); + target->relocateAlloc(*this, buf); // Return address is the next instruction after the one we have just copied. uint64_t s = getLDSTAddr() + 4; diff --git a/lld/ELF/ARMErrataFix.cpp b/lld/ELF/ARMErrataFix.cpp index 375d138da625..ce4093925d6c 100644 --- a/lld/ELF/ARMErrataFix.cpp +++ b/lld/ELF/ARMErrataFix.cpp @@ -182,7 +182,7 @@ void Patch657417Section::writeTo(uint8_t *buf) { write32le(buf, 0x9000f000); // If we have a relocation then apply it. if (!relocations.empty()) { - relocateAlloc(buf, buf + getSize()); + target->relocateAlloc(*this, buf); return; } diff --git a/lld/ELF/Arch/AArch64.cpp b/lld/ELF/Arch/AArch64.cpp index c521a96d978e..95b06bffa8d0 100644 --- a/lld/ELF/Arch/AArch64.cpp +++ b/lld/ELF/Arch/AArch64.cpp @@ -6,6 +6,7 @@ // //===----------------------------------------------------------------------===// +#include "OutputSections.h" #include "Symbols.h" #include "SyntheticSections.h" #include "Target.h" @@ -48,12 +49,18 @@ public: void relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const override; RelExpr adjustTlsExpr(RelType type, RelExpr expr) const override; - void relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, - uint64_t val) const override; - void relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, - uint64_t val) const override; - void relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, - uint64_t val) const override; + void relocateAlloc(InputSectionBase &sec, uint8_t *buf) const override; + +private: + void relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const; + void relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, uint64_t val) const; + void relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const; + void initRelaxer(ArrayRef<Relocation> relocs) const; + bool tryRelaxAdrpAdd(const Relocation &adrpRel, const Relocation &addRel, + uint64_t secAddr, uint8_t *buf) const; + bool tryRelaxAdrpLdr(const Relocation &adrpRel, const Relocation &ldrRel, + uint64_t secAddr, uint8_t *buf) const; + mutable bool safeToRelaxAdrpLdr = false; }; } // namespace @@ -587,11 +594,9 @@ void AArch64::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, llvm_unreachable("invalid relocation for TLS IE to LE relaxation"); } -AArch64Relaxer::AArch64Relaxer(ArrayRef<Relocation> relocs) { - if (!config->relax || config->emachine != EM_AARCH64) { - safeToRelaxAdrpLdr = false; +void AArch64::initRelaxer(ArrayRef<Relocation> relocs) const { + if (!config->relax) return; - } // Check if R_AARCH64_ADR_GOT_PAGE and R_AARCH64_LD64_GOT_LO12_NC // always appear in pairs. size_t i = 0; @@ -610,9 +615,9 @@ AArch64Relaxer::AArch64Relaxer(ArrayRef<Relocation> relocs) { safeToRelaxAdrpLdr = i == size; } -bool AArch64Relaxer::tryRelaxAdrpAdd(const Relocation &adrpRel, - const Relocation &addRel, uint64_t secAddr, - uint8_t *buf) const { +bool AArch64::tryRelaxAdrpAdd(const Relocation &adrpRel, + const Relocation &addRel, uint64_t secAddr, + uint8_t *buf) const { // When the address of sym is within the range of ADR then // we may relax // ADRP xn, sym @@ -659,9 +664,9 @@ bool AArch64Relaxer::tryRelaxAdrpAdd(const Relocation &adrpRel, return true; } -bool AArch64Relaxer::tryRelaxAdrpLdr(const Relocation &adrpRel, - const Relocation &ldrRel, uint64_t secAddr, - uint8_t *buf) const { +bool AArch64::tryRelaxAdrpLdr(const Relocation &adrpRel, + const Relocation &ldrRel, uint64_t secAddr, + uint8_t *buf) const { if (!safeToRelaxAdrpLdr) return false; @@ -734,6 +739,49 @@ bool AArch64Relaxer::tryRelaxAdrpLdr(const Relocation &adrpRel, return true; } +void AArch64::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const { + uint64_t secAddr = sec.getOutputSection()->addr; + if (auto *s = dyn_cast<InputSection>(&sec)) + secAddr += s->outSecOff; + initRelaxer(sec.relocations); + for (size_t i = 0, size = sec.relocations.size(); i != size; ++i) { + const Relocation &rel = sec.relocations[i]; + uint8_t *loc = buf + rel.offset; + const uint64_t val = + sec.getRelocTargetVA(sec.file, rel.type, rel.addend, + secAddr + rel.offset, *rel.sym, rel.expr); + switch (rel.expr) { + case R_AARCH64_GOT_PAGE_PC: + if (i + 1 < size && + tryRelaxAdrpLdr(rel, sec.relocations[i + 1], secAddr, buf)) { + ++i; + continue; + } + break; + case R_AARCH64_PAGE_PC: + if (i + 1 < size && + tryRelaxAdrpAdd(rel, sec.relocations[i + 1], secAddr, buf)) { + ++i; + continue; + } + break; + case R_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC: + case R_RELAX_TLS_GD_TO_IE_ABS: + relaxTlsGdToIe(loc, rel, val); + continue; + case R_RELAX_TLS_GD_TO_LE: + relaxTlsGdToLe(loc, rel, val); + continue; + case R_RELAX_TLS_IE_TO_LE: + relaxTlsIeToLe(loc, rel, val); + continue; + default: + break; + } + relocate(loc, rel, val); + } +} + // AArch64 may use security features in variant PLT sequences. These are: // Pointer Authentication (PAC), introduced in armv8.3-a and Branch Target // Indicator (BTI) introduced in armv8.5-a. The additional instructions used diff --git a/lld/ELF/Arch/PPC.cpp b/lld/ELF/Arch/PPC.cpp index 47c31e3a3b94..cbf25453b2fc 100644 --- a/lld/ELF/Arch/PPC.cpp +++ b/lld/ELF/Arch/PPC.cpp @@ -51,14 +51,13 @@ public: uint64_t val) const override; RelExpr adjustTlsExpr(RelType type, RelExpr expr) const override; int getTlsGdRelaxSkip(RelType type) const override; - void relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, - uint64_t val) const override; - void relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, - uint64_t val) const override; - void relaxTlsLdToLe(uint8_t *loc, const Relocation &rel, - uint64_t val) const override; - void relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, - uint64_t val) const override; + void relocateAlloc(InputSectionBase &sec, uint8_t *buf) const override; + +private: + void relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, uint64_t val) const; + void relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const; + void relaxTlsLdToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const; + void relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const; }; } // namespace @@ -483,6 +482,36 @@ void PPC::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, } } +void PPC::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const { + uint64_t secAddr = sec.getOutputSection()->addr; + if (auto *s = dyn_cast<InputSection>(&sec)) + secAddr += s->outSecOff; + for (const Relocation &rel : sec.relocations) { + uint8_t *loc = buf + rel.offset; + const uint64_t val = SignExtend64( + sec.getRelocTargetVA(sec.file, rel.type, rel.addend, + secAddr + rel.offset, *rel.sym, rel.expr), + 32); + switch (rel.expr) { + case R_RELAX_TLS_GD_TO_IE_GOT_OFF: + relaxTlsGdToIe(loc, rel, val); + break; + case R_RELAX_TLS_GD_TO_LE: + relaxTlsGdToLe(loc, rel, val); + break; + case R_RELAX_TLS_LD_TO_LE_ABS: + relaxTlsLdToLe(loc, rel, val); + break; + case R_RELAX_TLS_IE_TO_LE: + relaxTlsIeToLe(loc, rel, val); + break; + default: + relocate(loc, rel, val); + break; + } + } +} + TargetInfo *elf::getPPCTargetInfo() { static PPC target; return ⌖ diff --git a/lld/ELF/Arch/PPC64.cpp b/lld/ELF/Arch/PPC64.cpp index 312954649596..cdd96065a4de 100644 --- a/lld/ELF/Arch/PPC64.cpp +++ b/lld/ELF/Arch/PPC64.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "InputFiles.h" +#include "OutputSections.h" #include "SymbolTable.h" #include "Symbols.h" #include "SyntheticSections.h" @@ -174,19 +175,17 @@ public: RelExpr adjustTlsExpr(RelType type, RelExpr expr) const override; RelExpr adjustGotPcExpr(RelType type, int64_t addend, const uint8_t *loc) const override; - void relaxGot(uint8_t *loc, const Relocation &rel, - uint64_t val) const override; - void relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, - uint64_t val) const override; - void relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, - uint64_t val) const override; - void relaxTlsLdToLe(uint8_t *loc, const Relocation &rel, - uint64_t val) const override; - void relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, - uint64_t val) const override; + void relaxGot(uint8_t *loc, const Relocation &rel, uint64_t val) const; + void relocateAlloc(InputSectionBase &sec, uint8_t *buf) const override; bool adjustPrologueForCrossSplitStack(uint8_t *loc, uint8_t *end, uint8_t stOther) const override; + +private: + void relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, uint64_t val) const; + void relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const; + void relaxTlsLdToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const; + void relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, uint64_t val) const; }; } // namespace @@ -362,7 +361,8 @@ getRelaTocSymAndAddend(InputSectionBase *tocSec, uint64_t offset) { // ld/lwa 3, 0(3) # load the value from the address // // Returns true if the relaxation is performed. -bool elf::tryRelaxPPC64TocIndirection(const Relocation &rel, uint8_t *bufLoc) { +static bool tryRelaxPPC64TocIndirection(const Relocation &rel, + uint8_t *bufLoc) { assert(config->tocOptimize); if (rel.addend < 0) return false; @@ -393,7 +393,8 @@ bool elf::tryRelaxPPC64TocIndirection(const Relocation &rel, uint8_t *bufLoc) { return false; // Add PPC64TocOffset that will be subtracted by PPC64::relocate(). - target->relaxGot(bufLoc, rel, tocRelative + ppc64TocOffset); + static_cast<const PPC64 &>(*target).relaxGot(bufLoc, rel, + tocRelative + ppc64TocOffset); return true; } @@ -1512,6 +1513,87 @@ void PPC64::relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, } } +void PPC64::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const { + uint64_t secAddr = sec.getOutputSection()->addr; + if (auto *s = dyn_cast<InputSection>(&sec)) + secAddr += s->outSecOff; + uint64_t lastPPCRelaxedRelocOff = -1; + for (const Relocation &rel : sec.relocations) { + uint8_t *loc = buf + rel.offset; + const uint64_t val = + sec.getRelocTargetVA(sec.file, rel.type, rel.addend, + secAddr + rel.offset, *rel.sym, rel.expr); + switch (rel.expr) { + case R_PPC64_RELAX_GOT_PC: { + // The R_PPC64_PCREL_OPT relocation must appear immediately after + // R_PPC64_GOT_PCREL34 in the relocations table at the same offset. + // We can only relax R_PPC64_PCREL_OPT if we have also relaxed + // the associated R_PPC64_GOT_PCREL34 since only the latter has an + // associated symbol. So save the offset when relaxing R_PPC64_GOT_PCREL34 + // and only relax the other if the saved offset matches. + if (rel.type == R_PPC64_GOT_PCREL34) + lastPPCRelaxedRelocOff = rel.offset; + if (rel.type == R_PPC64_PCREL_OPT && rel.offset != lastPPCRelaxedRelocOff) + break; + relaxGot(loc, rel, val); + break; + } + case R_PPC64_RELAX_TOC: + // rel.sym refers to the STT_SECTION symbol associated to the .toc input + // section. If an R_PPC64_TOC16_LO (.toc + addend) references the TOC + // entry, there may be R_PPC64_TOC16_HA not paired with + // R_PPC64_TOC16_LO_DS. Don't relax. This loses some relaxation + // opportunities but is safe. + if (ppc64noTocRelax.count({rel.sym, rel.addend}) || + !tryRelaxPPC64TocIndirection(rel, loc)) + relocate(loc, rel, val); + break; + case R_PPC64_CALL: + // If this is a call to __tls_get_addr, it may be part of a TLS + // sequence that has been relaxed and turned into a nop. In this + // case, we don't want to handle it as a call. + if (read32(loc) == 0x60000000) // nop + break; + + // Patch a nop (0x60000000) to a ld. + if (rel.sym->needsTocRestore) { + // gcc/gfortran 5.4, 6.3 and earlier versions do not add nop for + // recursive calls even if the function is preemptible. This is not + // wrong in the common case where the function is not preempted at + // runtime. Just ignore. + if ((rel.offset + 8 > sec.rawData.size() || + read32(loc + 4) != 0x60000000) && + rel.sym->file != sec.file) { + // Use substr(6) to remove the "__plt_" prefix. + errorOrWarn(getErrorLocation(loc) + "call to " + + lld::toString(*rel.sym).substr(6) + + " lacks nop, can't restore toc"); + break; + } + write32(loc + 4, 0xe8410018); // ld %r2, 24(%r1) + } + relocate(loc, rel, val); + break; + case R_RELAX_TLS_GD_TO_IE: + case R_RELAX_TLS_GD_TO_IE_GOT_OFF: + relaxTlsGdToIe(loc, rel, val); + break; + case R_RELAX_TLS_GD_TO_LE: + relaxTlsGdToLe(loc, rel, val); + break; + case R_RELAX_TLS_LD_TO_LE_ABS: + relaxTlsLdToLe(loc, rel, val); + break; + case R_RELAX_TLS_IE_TO_LE: + relaxTlsIeToLe(loc, rel, val); + break; + default: + relocate(loc, rel, val); + break; + } + } +} + // The prologue for a split-stack function is expected to look roughly // like this: // .Lglobal_entry_point: diff --git a/lld/ELF/Arch/X86.cpp b/lld/ELF/Arch/X86.cpp index d136d45d9e55..79ad7db34641 100644 --- a/lld/ELF/Arch/X86.cpp +++ b/lld/ELF/Arch/X86.cpp @@ -6,6 +6,7 @@ // //===----------------------------------------------------------------------===// +#include "OutputSections.h" #include "Symbols.h" #include "SyntheticSections.h" #include "Target.h" @@ -37,14 +38,7 @@ public: uint64_t val) const override; RelExpr adjustTlsExpr(RelType type, RelExpr expr) const override; - void relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, - uint64_t val) const override; - void relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, - uint64_t val) const override; - void relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, - uint64_t val) const override; - void relaxTlsLdToLe(uint8_t *loc, const Relocation &rel, - uint64_t val) const override; + void relocateAlloc(InputSectionBase &sec, uint8_t *buf) const override; }; } // namespace @@ -350,8 +344,7 @@ void X86::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const { } } -void X86::relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, - uint64_t val) const { +static void relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, uint64_t val) { if (rel.type == R_386_TLS_GD) { // Convert // leal x@tlsgd(, %ebx, 1), %eax @@ -384,8 +377,7 @@ void X86::relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, } } -void X86::relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, - uint64_t val) const { +static void relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, uint64_t val) { if (rel.type == R_386_TLS_GD) { // Convert // leal x@tlsgd(, %ebx, 1), %eax @@ -418,8 +410,7 @@ void X86::relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, // In some conditions, relocations can be optimized to avoid using GOT. // This function does that for Initial Exec to Local Exec case. -void X86::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, - uint64_t val) const { +static void relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, uint64_t val) { // Ulrich's document section 6.2 says that @gotntpoff can // be used with MOVL or ADDL instructions. // @indntpoff is similar to @gotntpoff, but for use in @@ -456,8 +447,7 @@ void X86::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, write32le(loc, val); } -void X86::relaxTlsLdToLe(uint8_t *loc, const Relocation &rel, - uint64_t val) const { +static void relaxTlsLdToLe(uint8_t *loc, const Relocation &rel, uint64_t val) { if (rel.type == R_386_TLS_LDO_32) { write32le(loc, val); return; @@ -478,6 +468,37 @@ void X86::relaxTlsLdToLe(uint8_t *loc, const Relocation &rel, memcpy(loc - 2, inst, sizeof(inst)); } +void X86::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const { + uint64_t secAddr = sec.getOutputSection()->addr; + if (auto *s = dyn_cast<InputSection>(&sec)) + secAddr += s->outSecOff; + for (const Relocation &rel : sec.relocations) { + uint8_t *loc = buf + rel.offset; + const uint64_t val = SignExtend64( + sec.getRelocTargetVA(sec.file, rel.type, rel.addend, + secAddr + rel.offset, *rel.sym, rel.expr), + 32); + switch (rel.expr) { + case R_RELAX_TLS_GD_TO_IE_GOTPLT: + relaxTlsGdToIe(loc, rel, val); + continue; + case R_RELAX_TLS_GD_TO_LE: + case R_RELAX_TLS_GD_TO_LE_NEG: + relaxTlsGdToLe(loc, rel, val); + continue; + case R_RELAX_TLS_LD_TO_LE: + relaxTlsLdToLe(loc, rel, val); + break; + case R_RELAX_TLS_IE_TO_LE: + relaxTlsIeToLe(loc, rel, val); + continue; + default: + relocate(loc, rel, val); + break; + } + } +} + // If Intel Indirect Branch Tracking is enabled, we have to emit special PLT // entries containing endbr32 instructions. A PLT entry will be split into two // parts, one in .plt.sec (writePlt), and the other in .plt (writeIBTPlt). diff --git a/lld/ELF/Arch/X86_64.cpp b/lld/ELF/Arch/X86_64.cpp index 3fe73aeb2754..adb1637fd423 100644 --- a/lld/ELF/Arch/X86_64.cpp +++ b/lld/ELF/Arch/X86_64.cpp @@ -40,19 +40,9 @@ public: int64_t getImplicitAddend(const uint8_t *buf, RelType type) const override; void applyJumpInstrMod(uint8_t *loc, JumpModType type, unsigned size) const override; - RelExpr adjustGotPcExpr(RelType type, int64_t addend, const uint8_t *loc) const override; - void relaxGot(uint8_t *loc, const Relocation &rel, - uint64_t val) const override; - void relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, - uint64_t val) const override; - void relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, - uint64_t val) const override; - void relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, - uint64_t val) const override; - void relaxTlsLdToLe(uint8_t *loc, const Relocation &rel, - uint64_t val) const override; + void relocateAlloc(InputSectionBase &sec, uint8_t *buf) const override; bool adjustPrologueForCrossSplitStack(uint8_t *loc, uint8_t *end, uint8_t stOther) const override; bool deleteFallThruJmpInsn(InputSection &is, InputFile *file, @@ -424,8 +414,7 @@ RelType X86_64::getDynRel(RelType type) const { return R_X86_64_NONE; } -void X86_64::relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, - uint64_t val) const { +static void relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, uint64_t val) { if (rel.type == R_X86_64_TLSGD) { // Convert // .byte 0x66 @@ -465,8 +454,7 @@ void X86_64::relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, } } -void X86_64::relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, - uint64_t val) const { +static void relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, uint64_t val) { if (rel.type == R_X86_64_TLSGD) { // Convert // .byte 0x66 @@ -507,8 +495,7 @@ void X86_64::relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, // In some conditions, R_X86_64_GOTTPOFF relocation can be optimized to // R_X86_64_TPOFF32 so that it does not use GOT. -void X86_64::relaxTlsIeToLe(uint8_t *loc, const Relocation &, - uint64_t val) const { +static void relaxTlsIeToLe(uint8_t *loc, const Relocation &, uint64_t val) { uint8_t *inst = loc - 3; uint8_t reg = loc[-1] >> 3; uint8_t *regSlot = loc - 1; @@ -549,17 +536,7 @@ void X86_64::relaxTlsIeToLe(uint8_t *loc, const Relocation &, write32le(loc, val + 4); } -void X86_64::relaxTlsLdToLe(uint8_t *loc, const Relocation &rel, - uint64_t val) const { - if (rel.type == R_X86_64_DTPOFF64) { - write64le(loc, val); - return; - } - if (rel.type == R_X86_64_DTPOFF32) { - write32le(loc, val); - return; - } - +static void relaxTlsLdToLe(uint8_t *loc, const Relocation &rel, uint64_t val) { const uint8_t inst[] = { 0x66, 0x66, // .word 0x6666 0x66, // .byte 0x66 @@ -738,6 +715,8 @@ int64_t X86_64::getImplicitAddend(const uint8_t *buf, RelType type) const { } } +static void relaxGot(uint8_t *loc, const Relocation &rel, uint64_t val); + void X86_64::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const { switch (rel.type) { case R_X86_64_8: @@ -761,18 +740,11 @@ void X86_64::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const { write32le(loc, val); break; case R_X86_64_32S: - case R_X86_64_TPOFF32: case R_X86_64_GOT32: case R_X86_64_GOTPC32: - case R_X86_64_GOTPC32_TLSDESC: case R_X86_64_GOTPCREL: - case R_X86_64_GOTPCRELX: - case R_X86_64_REX_GOTPCRELX: case R_X86_64_PC32: - case R_X86_64_GOTTPOFF: case R_X86_64_PLT32: - case R_X86_64_TLSGD: - case R_X86_64_TLSLD: case R_X86_64_DTPOFF32: case R_X86_64_SIZE32: checkInt(loc, val, 32, rel); @@ -788,6 +760,48 @@ void X86_64::relocate(uint8_t *loc, const Relocation &rel, uint64_t val) const { case R_X86_64_PLTOFF64: write64le(loc, val); break; + case R_X86_64_GOTPCRELX: + case R_X86_64_REX_GOTPCRELX: + if (rel.expr != R_GOT_PC) { + relaxGot(loc, rel, val); + } else { + checkInt(loc, val, 32, rel); + write32le(loc, val); + } + break; + case R_X86_64_GOTPC32_TLSDESC: + case R_X86_64_TLSDESC_CALL: + case R_X86_64_TLSGD: + if (rel.expr == R_RELAX_TLS_GD_TO_LE) { + relaxTlsGdToLe(loc, rel, val); + } else if (rel.expr == R_RELAX_TLS_GD_TO_IE) { + relaxTlsGdToIe(loc, rel, val); + } else { + checkInt(loc, val, 32, rel); + write32le(loc, val); + } + break; + case R_X86_64_TLSLD: + if (rel.expr == R_RELAX_TLS_LD_TO_LE) { + relaxTlsLdToLe(loc, rel, val); + } else { + checkInt(loc, val, 32, rel); + write32le(loc, val); + } + break; + case R_X86_64_GOTTPOFF: + if (rel.expr == R_RELAX_TLS_IE_TO_LE) { + relaxTlsIeToLe(loc, rel, val); + } else { + checkInt(loc, val, 32, rel); + write32le(loc, val); + } + break; + case R_X86_64_TPOFF32: + checkInt(loc, val, 32, rel); + write32le(loc, val); + break; + case R_X86_64_TLSDESC: // The addend is stored in the second 64-bit word. write64le(loc + 8, val); @@ -897,7 +911,7 @@ static void relaxGotNoPic(uint8_t *loc, uint64_t val, uint8_t op, write32le(loc, val); } -void X86_64::relaxGot(uint8_t *loc, const Relocation &rel, uint64_t val) const { +static void relaxGot(uint8_t *loc, const Relocation &rel, uint64_t val) { checkInt(loc, val, 32, rel); const uint8_t op = loc[-2]; const uint8_t modRm = loc[-1]; @@ -971,6 +985,25 @@ bool X86_64::adjustPrologueForCrossSplitStack(uint8_t *loc, uint8_t *end, return false; } +void X86_64::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const { + uint64_t secAddr = sec.getOutputSection()->addr; + if (auto *s = dyn_cast<InputSection>(&sec)) + secAddr += s->outSecOff; + for (const Relocation &rel : sec.relocations) { + if (rel.expr == R_NONE) // See deleteFallThruJmpInsn + continue; + uint8_t *loc = buf + rel.offset; + const uint64_t val = + sec.getRelocTargetVA(sec.file, rel.type, rel.addend, + secAddr + rel.offset, *rel.sym, rel.expr); + relocate(loc, rel, val); + } + if (sec.jumpInstrMod) { + applyJumpInstrMod(buf + sec.jumpInstrMod->offset, + sec.jumpInstrMod->original, sec.jumpInstrMod->size); + } +} + // If Intel Indirect Branch Tracking is enabled, we have to emit special PLT // entries containing endbr64 instructions. A PLT entry will be split into two // parts, one in .plt.sec (writePlt), and the other in .plt (writeIBTPlt). diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp index aa41da623e17..53daefb7a9e5 100644 --- a/lld/ELF/InputSection.cpp +++ b/lld/ELF/InputSection.cpp @@ -965,7 +965,7 @@ void InputSectionBase::relocate(uint8_t *buf, uint8_t *bufEnd) { adjustSplitStackFunctionPrologues<ELFT>(buf, bufEnd); if (flags & SHF_ALLOC) { - relocateAlloc(buf, bufEnd); + target->relocateAlloc(*this, buf); return; } @@ -981,133 +981,6 @@ void InputSectionBase::relocate(uint8_t *buf, uint8_t *bufEnd) { sec->relocateNonAlloc<ELFT>(buf, rels.relas); } -void InputSectionBase::relocateAlloc(uint8_t *buf, uint8_t *bufEnd) { - assert(flags & SHF_ALLOC); - const unsigned bits = config->wordsize * 8; - const TargetInfo &target = *elf::target; - uint64_t lastPPCRelaxedRelocOff = UINT64_C(-1); - AArch64Relaxer aarch64relaxer(relocations); - for (size_t i = 0, size = relocations.size(); i != size; ++i) { - const Relocation &rel = relocations[i]; - if (rel.expr == R_NONE) - continue; - uint64_t offset = rel.offset; - uint8_t *bufLoc = buf + offset; - - uint64_t secAddr = getOutputSection()->addr; - if (auto *sec = dyn_cast<InputSection>(this)) - secAddr += sec->outSecOff; - const uint64_t addrLoc = secAddr + offset; - const uint64_t targetVA = - SignExtend64(getRelocTargetVA(file, rel.type, rel.addend, addrLoc, - *rel.sym, rel.expr), - bits); - switch (rel.expr) { - case R_RELAX_HINT: - continue; - case R_RELAX_GOT_PC: - case R_RELAX_GOT_PC_NOPIC: - target.relaxGot(bufLoc, rel, targetVA); - break; - case R_AARCH64_GOT_PAGE_PC: - if (i + 1 < size && aarch64relaxer.tryRelaxAdrpLdr( - rel, relocations[i + 1], secAddr, buf)) { - ++i; - continue; - } - target.relocate(bufLoc, rel, targetVA); - break; - case R_AARCH64_PAGE_PC: - if (i + 1 < size && aarch64relaxer.tryRelaxAdrpAdd( - rel, relocations[i + 1], secAddr, buf)) { - ++i; - continue; - } - target.relocate(bufLoc, rel, targetVA); - break; - case R_PPC64_RELAX_GOT_PC: { - // The R_PPC64_PCREL_OPT relocation must appear immediately after - // R_PPC64_GOT_PCREL34 in the relocations table at the same offset. - // We can only relax R_PPC64_PCREL_OPT if we have also relaxed - // the associated R_PPC64_GOT_PCREL34 since only the latter has an - // associated symbol. So save the offset when relaxing R_PPC64_GOT_PCREL34 - // and only relax the other if the saved offset matches. - if (rel.type == R_PPC64_GOT_PCREL34) - lastPPCRelaxedRelocOff = offset; - if (rel.type == R_PPC64_PCREL_OPT && offset != lastPPCRelaxedRelocOff) - break; - target.relaxGot(bufLoc, rel, targetVA); - break; - } - case R_PPC64_RELAX_TOC: - // rel.sym refers to the STT_SECTION symbol associated to the .toc input - // section. If an R_PPC64_TOC16_LO (.toc + addend) references the TOC - // entry, there may be R_PPC64_TOC16_HA not paired with - // R_PPC64_TOC16_LO_DS. Don't relax. This loses some relaxation - // opportunities but is safe. - if (ppc64noTocRelax.count({rel.sym, rel.addend}) || - !tryRelaxPPC64TocIndirection(rel, bufLoc)) - target.relocate(bufLoc, rel, targetVA); - break; - case R_RELAX_TLS_IE_TO_LE: - target.relaxTlsIeToLe(bufLoc, rel, targetVA); - break; - case R_RELAX_TLS_LD_TO_LE: - case R_RELAX_TLS_LD_TO_LE_ABS: - target.relaxTlsLdToLe(bufLoc, rel, targetVA); - break; - case R_RELAX_TLS_GD_TO_LE: - case R_RELAX_TLS_GD_TO_LE_NEG: - target.relaxTlsGdToLe(bufLoc, rel, targetVA); - break; - case R_AARCH64_RELAX_TLS_GD_TO_IE_PAGE_PC: - case R_RELAX_TLS_GD_TO_IE: - case R_RELAX_TLS_GD_TO_IE_ABS: - case R_RELAX_TLS_GD_TO_IE_GOT_OFF: - case R_RELAX_TLS_GD_TO_IE_GOTPLT: - target.relaxTlsGdToIe(bufLoc, rel, targetVA); - break; - case R_PPC64_CALL: - // If this is a call to __tls_get_addr, it may be part of a TLS - // sequence that has been relaxed and turned into a nop. In this - // case, we don't want to handle it as a call. - if (read32(bufLoc) == 0x60000000) // nop - break; - - // Patch a nop (0x60000000) to a ld. - if (rel.sym->needsTocRestore) { - // gcc/gfortran 5.4, 6.3 and earlier versions do not add nop for - // recursive calls even if the function is preemptible. This is not - // wrong in the common case where the function is not preempted at - // runtime. Just ignore. - if ((bufLoc + 8 > bufEnd || read32(bufLoc + 4) != 0x60000000) && - rel.sym->file != file) { - // Use substr(6) to remove the "__plt_" prefix. - errorOrWarn(getErrorLocation(bufLoc) + "call to " + - lld::toString(*rel.sym).substr(6) + - " lacks nop, can't restore toc"); - break; - } - write32(bufLoc + 4, 0xe8410018); // ld %r2, 24(%r1) - } - target.relocate(bufLoc, rel, targetVA); - break; - default: - target.relocate(bufLoc, rel, targetVA); - break; - } - } - - // Apply jumpInstrMods. jumpInstrMods are created when the opcode of - // a jmp insn must be modified to shrink the jmp insn or to flip the jmp - // insn. This is primarily used to relax and optimize jumps created with - // basic block sections. - if (jumpInstrMod) { - target.applyJumpInstrMod(buf + jumpInstrMod->offset, jumpInstrMod->original, - jumpInstrMod->size); - } -} - // For each function-defining prologue, find any calls to __morestack, // and replace them with calls to __morestack_non_split. static void switchMorestackCallsToMorestackNonSplit( diff --git a/lld/ELF/InputSection.h b/lld/ELF/InputSection.h index 6cb20818d9c4..19915249c5c7 100644 --- a/lld/ELF/InputSection.h +++ b/lld/ELF/InputSection.h @@ -195,7 +195,6 @@ public: // relocations, assuming that Buf points to this section's copy in // the mmap'ed output buffer. template <class ELFT> void relocate(uint8_t *buf, uint8_t *bufEnd); - void relocateAlloc(uint8_t *buf, uint8_t *bufEnd); static uint64_t getRelocTargetVA(const InputFile *File, RelType Type, int64_t A, uint64_t P, const Symbol &Sym, RelExpr Expr); diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp index 1297811c7b81..12cfc97bf584 100644 --- a/lld/ELF/SyntheticSections.cpp +++ b/lld/ELF/SyntheticSections.cpp @@ -605,7 +605,7 @@ void EhFrameSection::writeTo(uint8_t *buf) { // in the output buffer, but relocateAlloc() still works because // getOffset() takes care of discontiguous section pieces. for (EhInputSection *s : sections) - s->relocateAlloc(buf, nullptr); + target->relocateAlloc(*s, buf); if (getPartition().ehFrameHdr && getPartition().ehFrameHdr->getParent()) getPartition().ehFrameHdr->write(); @@ -682,7 +682,7 @@ void GotSection::writeTo(uint8_t *buf) { if (size == 0) return; target->writeGotHeader(buf); - relocateAlloc(buf, buf + size); + target->relocateAlloc(*this, buf); } static uint64_t getMipsPageAddr(uint64_t addr) { @@ -3523,7 +3523,7 @@ void ARMExidxSyntheticSection::writeTo(uint8_t *buf) { assert(isec->getParent() != nullptr); if (InputSection *d = findExidxSection(isec)) { memcpy(buf + offset, d->rawData.data(), d->rawData.size()); - d->relocateAlloc(buf + d->outSecOff, buf + d->outSecOff + d->getSize()); + target->relocateAlloc(*d, buf + d->outSecOff); offset += d->getSize(); } else { // A Linker generated CANTUNWIND section. diff --git a/lld/ELF/Target.cpp b/lld/ELF/Target.cpp index cb3ce6d5ddb4..9c366ce0c7f3 100644 --- a/lld/ELF/Target.cpp +++ b/lld/ELF/Target.cpp @@ -152,29 +152,20 @@ RelExpr TargetInfo::adjustGotPcExpr(RelType type, int64_t addend, return R_GOT_PC; } -void TargetInfo::relaxGot(uint8_t *loc, const Relocation &rel, - uint64_t val) const { - llvm_unreachable("Should not have claimed to be relaxable"); -} - -void TargetInfo::relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, - uint64_t val) const { - llvm_unreachable("Should not have claimed to be relaxable"); -} - -void TargetInfo::relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, - uint64_t val) const { - llvm_unreachable("Should not have claimed to be relaxable"); -} - -void TargetInfo::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, - uint64_t val) const { - llvm_unreachable("Should not have claimed to be relaxable"); -} - -void TargetInfo::relaxTlsLdToLe(uint8_t *loc, const Relocation &rel, - uint64_t val) const { - llvm_unreachable("Should not have claimed to be relaxable"); +void TargetInfo::relocateAlloc(InputSectionBase &sec, uint8_t *buf) const { + const unsigned bits = config->is64 ? 64 : 32; + uint64_t secAddr = sec.getOutputSection()->addr; + if (auto *s = dyn_cast<InputSection>(&sec)) + secAddr += s->outSecOff; + for (const Relocation &rel : sec.relocations) { + uint8_t *loc = buf + rel.offset; + const uint64_t val = SignExtend64( + sec.getRelocTargetVA(sec.file, rel.type, rel.addend, + secAddr + rel.offset, *rel.sym, rel.expr), + bits); + if (rel.expr != R_RELAX_HINT) + relocate(loc, rel, val); + } } uint64_t TargetInfo::getImageBase() const { diff --git a/lld/ELF/Target.h b/lld/ELF/Target.h index cfe288037f3e..9959e30f3e54 100644 --- a/lld/ELF/Target.h +++ b/lld/ELF/Target.h @@ -89,6 +89,7 @@ public: void relocateNoSym(uint8_t *loc, RelType type, uint64_t val) const { relocate(loc, Relocation{R_NONE, type, 0, 0, nullptr}, val); } + virtual void relocateAlloc(InputSectionBase &sec, uint8_t *buf) const; // Do a linker relaxation pass and return true if we changed something. virtual bool relaxOnce(int pass) const { return false; } @@ -157,16 +158,6 @@ public: virtual RelExpr adjustTlsExpr(RelType type, RelExpr expr) const; virtual RelExpr adjustGotPcExpr(RelType type, int64_t addend, const uint8_t *loc) const; - virtual void relaxGot(uint8_t *loc, const Relocation &rel, - uint64_t val) const; - virtual void relaxTlsGdToIe(uint8_t *loc, const Relocation &rel, - uint64_t val) const; - virtual void relaxTlsGdToLe(uint8_t *loc, const Relocation &rel, - uint64_t val) const; - virtual void relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, - uint64_t val) const; - virtual void relaxTlsLdToLe(uint8_t *loc, const Relocation &rel, - uint64_t val) const; protected: // On FreeBSD x86_64 the first page cannot be mmaped. @@ -205,7 +196,6 @@ static inline std::string getErrorLocation(const uint8_t *loc) { void writePPC32GlinkSection(uint8_t *buf, size_t numEntries); -bool tryRelaxPPC64TocIndirection(const Relocation &rel, uint8_t *bufLoc); unsigned getPPCDFormOp(unsigned secondaryOp); // In the PowerPC64 Elf V2 abi a function can have 2 entry points. The first @@ -227,18 +217,6 @@ uint64_t getPPC64TocBase(); uint64_t getAArch64Page(uint64_t expr); void riscvFinalizeRelax(int passes); -class AArch64Relaxer { - bool safeToRelaxAdrpLdr = true; - -public: - explicit AArch64Relaxer(ArrayRef<Relocation> relocs); - - bool tryRelaxAdrpAdd(const Relocation &adrpRel, const Relocation &addRel, - uint64_t secAddr, uint8_t *buf) const; - bool tryRelaxAdrpLdr(const Relocation &adrpRel, const Relocation &ldrRel, - uint64_t secAddr, uint8_t *buf) const; -}; - LLVM_LIBRARY_VISIBILITY extern const TargetInfo *target; TargetInfo *getTarget(); |