[llvm-rtdyld] Precalculate required memory upfront

[sjoerdmeijer]

This changes the behaviour of llvm-rtdyld to calculate and allocate all required memory for objects and its data/code section upfront as opposed to doing this on-demand.

Allocation of the memory upfront avoids fragmentation of the memory, which is a workaround for relocations getting out of range as explained and discussed here:

https://discourse.llvm.org/t/llvm-rtdyld-aarch64-abi-relocation-restrictions/74616

[github-actions]

⚠️ C/C++ code formatter, clang-format found issues in your code. ⚠️

You can test this locally with the following command:

git-clang-format --diff da843aa09f4cb5caab1cf0c802f2d203ada84c54 8442c0fd63c42558534f9b33f4f6efdc8305aca7 -- llvm/include/llvm/ExecutionEngine/RuntimeDyld.h llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h llvm/tools/llvm-rtdyld/llvm-rtdyld.cpp

View the diff from clang-format here.

diff --git a/llvm/include/llvm/ExecutionEngine/RuntimeDyld.h b/llvm/include/llvm/ExecutionEngine/RuntimeDyld.h index 423715453b..c715ec8884 100644 --- a/llvm/include/llvm/ExecutionEngine/RuntimeDyld.h +++ b/llvm/include/llvm/ExecutionEngine/RuntimeDyld.h @@ -196,8 +196,9 @@ public: /// TODO Error precalculateMemorySize(const object::ObjectFile &Obj, - uint64_t &CodeSize, Align &CodeAlign, uint64_t &RODataSize, Align - &RODataAlign, uint64_t &RWDataSize, Align &RWDataAlign); + uint64_t &CodeSize, Align &CodeAlign, + uint64_t &RODataSize, Align &RODataAlign, + uint64_t &RWDataSize, Align &RWDataAlign); /// Add the referenced object file to the list of objects to be loaded and /// relocated. diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp index 83cafa2d4e..f95acafe9b 100644 --- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp +++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp @@ -1345,18 +1345,19 @@ createRuntimeDyldMachO( return Dyld; } -Error RuntimeDyld::precalculateMemorySize( - const ObjectFile &Obj, uint64_t &CodeSize, Align &CodeAlign, - uint64_t &RODataSize, Align &RODataAlign, uint64_t &RWDataSize, - Align &RWDataAlign) { +Error RuntimeDyld::precalculateMemorySize(const ObjectFile &Obj, + uint64_t &CodeSize, Align &CodeAlign, + uint64_t &RODataSize, + Align &RODataAlign, + uint64_t &RWDataSize, + Align &RWDataAlign) { if (!Dyld) { if (!Obj.isELF()) report_fatal_error("Incompatible object format!"); - Dyld = - createRuntimeDyldELF(static_cast<Triple::ArchType>(Obj.getArch()), - MemMgr, Resolver, ProcessAllSections, - std::move(NotifyStubEmitted)); + Dyld = createRuntimeDyldELF(static_cast<Triple::ArchType>(Obj.getArch()), + MemMgr, Resolver, ProcessAllSections, + std::move(NotifyStubEmitted)); } if (!Dyld->isCompatibleFile(Obj)) report_fatal_error("Incompatible object format!"); @@ -1366,7 +1367,6 @@ Error RuntimeDyld::precalculateMemorySize( return Err; } - std::unique_ptr<RuntimeDyld::LoadedObjectInfo> RuntimeDyld::loadObject(const ObjectFile &Obj) { if (!Dyld) { diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h index 75caf01955..285de95698 100644 --- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h +++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h @@ -428,9 +428,6 @@ protected: unsigned computeSectionStubBufSize(const ObjectFile &Obj, const SectionRef &Section); - - - // Implementation of the generic part of the loadObject algorithm. Expected<ObjSectionToIDMap> loadObjectImpl(const object::ObjectFile &Obj); @@ -460,14 +457,13 @@ public: virtual ~RuntimeDyldImpl(); -// Compute an upper bound of the memory that is required to load all + // Compute an upper bound of the memory that is required to load all // sections Error computeTotalAllocSize(const ObjectFile &Obj, uint64_t &CodeSize, Align &CodeAlign, uint64_t &RODataSize, Align &RODataAlign, uint64_t &RWDataSize, Align &RWDataAlign); - void setProcessAllSections(bool ProcessAllSections) { this->ProcessAllSections = ProcessAllSections; } diff --git a/llvm/tools/llvm-rtdyld/llvm-rtdyld.cpp b/llvm/tools/llvm-rtdyld/llvm-rtdyld.cpp index a89c9ce486..fc7a654c11 100644 --- a/llvm/tools/llvm-rtdyld/llvm-rtdyld.cpp +++ b/llvm/tools/llvm-rtdyld/llvm-rtdyld.cpp @@ -556,7 +556,6 @@ static int executeInput() { uint64_t TotalRODataSize = 0; uint64_t TotalRWDataSize = 0; - // If we don't have any input files, read from stdin. if (!InputFileList.size()) { InputFileList.push_back("-"); @@ -590,8 +589,9 @@ static int executeInput() { uint64_t CodeSize, RODataSize, RWDataSize; Align CodeAlign, RODataAlign, RWDataAlign; - Error Err = Dyld.precalculateMemorySize(Obj, CodeSize, CodeAlign, - RODataSize, RODataAlign, RWDataSize, RWDataAlign); + Error Err = + Dyld.precalculateMemorySize(Obj, CodeSize, CodeAlign, RODataSize, + RODataAlign, RWDataSize, RWDataAlign); if (Err) ErrorAndExit("Can't compute total size"); @@ -600,7 +600,7 @@ static int executeInput() { TotalRWDataSize += RWDataSize; } - if (!PreallocMemory) + if (!PreallocMemory) PreallocMemory = TotalCodeSize + TotalRODataSize + TotalRWDataSize; doPreallocation(MemMgr); 

[sjoerdmeijer]

This depends on #71409