diff --git a/include/llvm/ExecutionEngine/SectionMemoryManager.h b/include/llvm/ExecutionEngine/SectionMemoryManager.h index 0b0dcb0..7bb96eb 100644 --- a/include/llvm/ExecutionEngine/SectionMemoryManager.h +++ b/include/llvm/ExecutionEngine/SectionMemoryManager.h @@ -83,10 +83,28 @@ public: virtual void invalidateInstructionCache(); private: + struct FreeMemBlock { + // The actual block of free memory + sys::MemoryBlock Free; + // If there is a pending allocation from the same reservation right before + // this block, store it's index in PendingMem, to be able to update the + // pending region if part of this block is allocated, rather than having to + // create a new one + unsigned PendingPrefixIndex; + }; + struct MemoryGroup { - SmallVector AllocatedMem; - SmallVector FreeMem; - sys::MemoryBlock Near; + // PendingMem contains all blocks of memory (subblocks of AllocatedMem) + // which have not yet had their permissions applied, but have been given + // out to the user. FreeMem contains all block of memory, which have + // neither had their permissions applied, nor been given out to the user. + SmallVector PendingMem; + SmallVector FreeMem; + + // All memory blocks that have been requested from the system + SmallVector AllocatedMem; + + sys::MemoryBlock Near; }; uint8_t *allocateSection(MemoryGroup &MemGroup, uintptr_t Size, @@ -103,4 +121,3 @@ private: } #endif // LLVM_EXECUTION_ENGINE_SECTION_MEMORY_MANAGER_H - diff --git a/include/llvm/Support/raw_ostream.h b/include/llvm/Support/raw_ostream.h index 28e512c..686e8c8 100644 --- a/include/llvm/Support/raw_ostream.h +++ b/include/llvm/Support/raw_ostream.h @@ -485,6 +485,9 @@ public: /// A raw_ostream that writes to an SmallVector or SmallString. This is a /// simple adaptor class. This class does not encounter output errors. +/// raw_svector_ostream operates without a buffer, delegating all memory +/// management to the SmallString. Thus the SmallString is always up-to-date, +/// may be used directly and there is no need to call flush(). class raw_svector_ostream : public raw_pwrite_stream { SmallVectorImpl &OS; @@ -493,32 +496,24 @@ class raw_svector_ostream : public raw_pwrite_stream { void pwrite_impl(const char *Ptr, size_t Size, uint64_t Offset) override; - /// Return the current position within the stream, not counting the bytes - /// currently in the buffer. + /// Return the current position within the stream. uint64_t current_pos() const override; -protected: - // Like the regular constructor, but doesn't call init. - explicit raw_svector_ostream(SmallVectorImpl &O, unsigned); - void init(); - public: /// Construct a new raw_svector_ostream. /// /// \param O The vector to write to; this should generally have at least 128 /// bytes free to avoid any extraneous memory overhead. - explicit raw_svector_ostream(SmallVectorImpl &O); - ~raw_svector_ostream() override; - + explicit raw_svector_ostream(SmallVectorImpl &O) : OS(O) { + SetUnbuffered(); + } + ~raw_svector_ostream() override {} - /// This is called when the SmallVector we're appending to is changed outside - /// of the raw_svector_ostream's control. It is only safe to do this if the - /// raw_svector_ostream has previously been flushed. - void resync(); + // FIXME: resync is no-op. Remove it and its users. + void resync() {} - /// Flushes the stream contents to the target vector and return a StringRef - /// for the vector contents. - StringRef str(); + /// Return a StringRef for the vector contents. + StringRef str() { return StringRef(OS.data(), OS.size()); } }; /// A raw_ostream that discards all output. @@ -541,9 +536,7 @@ class buffer_ostream : public raw_svector_ostream { SmallVector Buffer; public: - buffer_ostream(raw_ostream &OS) : raw_svector_ostream(Buffer, 0), OS(OS) { - init(); - } + buffer_ostream(raw_ostream &OS) : raw_svector_ostream(Buffer), OS(OS) {} ~buffer_ostream() { OS << str(); } }; diff --git a/lib/CodeGen/MachineModuleInfo.cpp b/lib/CodeGen/MachineModuleInfo.cpp index 6a20624..53a9ea7 100644 --- a/lib/CodeGen/MachineModuleInfo.cpp +++ b/lib/CodeGen/MachineModuleInfo.cpp @@ -227,6 +227,7 @@ bool MachineModuleInfo::doFinalization(Module &M) { AddrLabelSymbols = nullptr; Context.reset(); + FuncInfoMap.clear(); delete ObjFileMMI; ObjFileMMI = nullptr; diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp index 93287a3..93bbd6f 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp @@ -85,20 +85,25 @@ void RuntimeDyldImpl::resolveRelocations() { // First, resolve relocations associated with external symbols. resolveExternalSymbols(); - // Just iterate over the sections we have and resolve all the relocations - // in them. Gross overkill, but it gets the job done. - for (int i = 0, e = Sections.size(); i != e; ++i) { + // Iterate over all outstanding relocations + for (auto it = Relocations.begin(), e = Relocations.end(); it != e; ++it) { // The Section here (Sections[i]) refers to the section in which the // symbol for the relocation is located. The SectionID in the relocation // entry provides the section to which the relocation will be applied. - uint64_t Addr = Sections[i].LoadAddress; - DEBUG(dbgs() << "Resolving relocations Section #" << i << "\t" + int Idx = it->first; + uint64_t Addr = Sections[Idx].LoadAddress; + DEBUG(dbgs() << "Resolving relocations Section #" << Idx << "\t" << format("%p", (uintptr_t)Addr) << "\n"); - DEBUG(dumpSectionMemory(Sections[i], "before relocations")); - resolveRelocationList(Relocations[i], Addr); - DEBUG(dumpSectionMemory(Sections[i], "after relocations")); - Relocations.erase(i); + resolveRelocationList(it->second, Addr); } + Relocations.clear(); + + // Print out sections after relocation. + DEBUG( + for (int i = 0, e = Sections.size(); i != e; ++i) + dumpSectionMemory(Sections[i], "after relocations"); + ); + } void RuntimeDyldImpl::mapSectionAddress(const void *LocalAddress, diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h index e085a92..cc12093 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h @@ -30,6 +30,7 @@ #include "llvm/Support/SwapByteOrder.h" #include "llvm/Support/raw_ostream.h" #include +#include #include using namespace llvm; @@ -224,7 +225,7 @@ protected: // Relocations to sections already loaded. Indexed by SectionID which is the // source of the address. The target where the address will be written is // SectionID/Offset in the relocation itself. - DenseMap Relocations; + std::unordered_map Relocations; // Relocations to external symbols that are not yet resolved. Symbols are // external when they aren't found in the global symbol table of all loaded diff --git a/lib/ExecutionEngine/SectionMemoryManager.cpp b/lib/ExecutionEngine/SectionMemoryManager.cpp index 5986084..e2f2208 100644 --- a/lib/ExecutionEngine/SectionMemoryManager.cpp +++ b/lib/ExecutionEngine/SectionMemoryManager.cpp @@ -15,6 +15,7 @@ #include "llvm/Config/config.h" #include "llvm/ExecutionEngine/SectionMemoryManager.h" #include "llvm/Support/MathExtras.h" +#include "llvm/Support/Process.h" namespace llvm { @@ -48,16 +49,27 @@ uint8_t *SectionMemoryManager::allocateSection(MemoryGroup &MemGroup, // Look in the list of free memory regions and use a block there if one // is available. - for (int i = 0, e = MemGroup.FreeMem.size(); i != e; ++i) { - sys::MemoryBlock &MB = MemGroup.FreeMem[i]; - if (MB.size() >= RequiredSize) { - Addr = (uintptr_t)MB.base(); - uintptr_t EndOfBlock = Addr + MB.size(); + for (FreeMemBlock &FreeMB : MemGroup.FreeMem) { + if (FreeMB.Free.size() >= RequiredSize) { + Addr = (uintptr_t)FreeMB.Free.base(); + uintptr_t EndOfBlock = Addr + FreeMB.Free.size(); // Align the address. Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1); - // Store cutted free memory block. - MemGroup.FreeMem[i] = sys::MemoryBlock((void*)(Addr + Size), - EndOfBlock - Addr - Size); + + if (FreeMB.PendingPrefixIndex == (unsigned)-1) { + // The part of the block we're giving out to the user is now pending + MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size)); + + // Remember this pending block, such that future allocations can just + // modify it rather than creating a new one + FreeMB.PendingPrefixIndex = MemGroup.PendingMem.size() - 1; + } else { + sys::MemoryBlock &PendingMB = MemGroup.PendingMem[FreeMB.PendingPrefixIndex]; + PendingMB = sys::MemoryBlock(PendingMB.base(), Addr + Size - (uintptr_t)PendingMB.base()); + } + + // Remember how much free space is now left in this block + FreeMB.Free = sys::MemoryBlock((void *)(Addr + Size), EndOfBlock - Addr - Size); return (uint8_t*)Addr; } } @@ -85,6 +97,7 @@ uint8_t *SectionMemoryManager::allocateSection(MemoryGroup &MemGroup, // Save this address as the basis for our next request MemGroup.Near = MB; + // Remember that we allocated this memory MemGroup.AllocatedMem.push_back(MB); Addr = (uintptr_t)MB.base(); uintptr_t EndOfBlock = Addr + MB.size(); @@ -92,11 +105,18 @@ uint8_t *SectionMemoryManager::allocateSection(MemoryGroup &MemGroup, // Align the address. Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1); + // The part of the block we're giving out to the user is now pending + MemGroup.PendingMem.push_back(sys::MemoryBlock((void *)Addr, Size)); + // The allocateMappedMemory may allocate much more memory than we need. In // this case, we store the unused memory as a free memory block. unsigned FreeSize = EndOfBlock-Addr-Size; - if (FreeSize > 16) - MemGroup.FreeMem.push_back(sys::MemoryBlock((void*)(Addr + Size), FreeSize)); + if (FreeSize > 16) { + FreeMemBlock FreeMB; + FreeMB.Free = sys::MemoryBlock((void*)(Addr + Size), FreeSize); + FreeMB.PendingPrefixIndex = (unsigned)-1; + MemGroup.FreeMem.push_back(FreeMB); + } // Return aligned address return (uint8_t*)Addr; @@ -107,9 +127,6 @@ bool SectionMemoryManager::finalizeMemory(std::string *ErrMsg) // FIXME: Should in-progress permissions be reverted if an error occurs? std::error_code ec; - // Don't allow free memory blocks to be used after setting protection flags. - CodeMem.FreeMem.clear(); - // Make code memory executable. ec = applyMemoryGroupPermissions(CodeMem, sys::Memory::MF_READ | sys::Memory::MF_EXEC); @@ -143,36 +160,62 @@ bool SectionMemoryManager::finalizeMemory(std::string *ErrMsg) return false; } +static sys::MemoryBlock trimBlockToPageSize(sys::MemoryBlock M) { + static const size_t PageSize = sys::Process::getPageSize(); + + size_t StartOverlap = + (PageSize - ((uintptr_t)M.base() % PageSize)) % PageSize; + + size_t TrimmedSize = M.size(); + TrimmedSize -= StartOverlap; + TrimmedSize -= TrimmedSize % PageSize; + + sys::MemoryBlock Trimmed((void *)((uintptr_t)M.base() + StartOverlap), TrimmedSize); + + assert(((uintptr_t)Trimmed.base() % PageSize) == 0); + assert((Trimmed.size() % PageSize) == 0); + assert(M.base() <= Trimmed.base() && Trimmed.size() <= M.size()); + + return Trimmed; +} + + std::error_code SectionMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup, unsigned Permissions) { - - for (int i = 0, e = MemGroup.AllocatedMem.size(); i != e; ++i) { - std::error_code ec; - ec = - sys::Memory::protectMappedMemory(MemGroup.AllocatedMem[i], Permissions); - if (ec) { - return ec; - } + for (sys::MemoryBlock &MB : MemGroup.PendingMem) + if (std::error_code EC = sys::Memory::protectMappedMemory(MB, Permissions)) + return EC; + + MemGroup.PendingMem.clear(); + + // Now go through free blocks and trim any of them that don't span the entire + // page because one of the pending blocks may have overlapped it. + for (FreeMemBlock &FreeMB : MemGroup.FreeMem) { + FreeMB.Free = trimBlockToPageSize(FreeMB.Free); + // We cleared the PendingMem list, so all these pointers are now invalid + FreeMB.PendingPrefixIndex = (unsigned)-1; } + // Remove all blocks which are now empty + MemGroup.FreeMem.erase( + std::remove_if(MemGroup.FreeMem.begin(), MemGroup.FreeMem.end(), + [](FreeMemBlock &FreeMB) { return FreeMB.Free.size() == 0; }), + MemGroup.FreeMem.end()); + return std::error_code(); } void SectionMemoryManager::invalidateInstructionCache() { - for (int i = 0, e = CodeMem.AllocatedMem.size(); i != e; ++i) - sys::Memory::InvalidateInstructionCache(CodeMem.AllocatedMem[i].base(), - CodeMem.AllocatedMem[i].size()); + for (sys::MemoryBlock &Block : CodeMem.PendingMem) + sys::Memory::InvalidateInstructionCache(Block.base(), Block.size()); } SectionMemoryManager::~SectionMemoryManager() { - for (unsigned i = 0, e = CodeMem.AllocatedMem.size(); i != e; ++i) - sys::Memory::releaseMappedMemory(CodeMem.AllocatedMem[i]); - for (unsigned i = 0, e = RWDataMem.AllocatedMem.size(); i != e; ++i) - sys::Memory::releaseMappedMemory(RWDataMem.AllocatedMem[i]); - for (unsigned i = 0, e = RODataMem.AllocatedMem.size(); i != e; ++i) - sys::Memory::releaseMappedMemory(RODataMem.AllocatedMem[i]); + for (MemoryGroup *Group : {&CodeMem, &RWDataMem, &RODataMem}) { + for (sys::MemoryBlock &Block : Group->AllocatedMem) + sys::Memory::releaseMappedMemory(Block); + } } } // namespace llvm - diff --git a/lib/IR/Verifier.cpp b/lib/IR/Verifier.cpp index 2a0a4ff..228c09f 100644 --- a/lib/IR/Verifier.cpp +++ b/lib/IR/Verifier.cpp @@ -1754,6 +1754,14 @@ void Verifier::visitFunction(const Function &F) { assert(F.hasMetadata() != MDs.empty() && "Bit out-of-sync"); VerifyFunctionMetadata(MDs); + // Check validity of the personality function + if (F.hasPersonalityFn()) { + auto *Per = dyn_cast(F.getPersonalityFn()->stripPointerCasts()); + if (Per) + Assert(Per->getParent() == F.getParent(), + "Referencing personality function in another module!", &F, Per); + } + if (F.isMaterializable()) { // Function has a body somewhere we can't see. Assert(MDs.empty(), "unmaterialized function cannot have metadata", &F, diff --git a/lib/Support/Unix/Memory.inc b/lib/Support/Unix/Memory.inc index c421ee8..33beff5 100644 --- a/lib/Support/Unix/Memory.inc +++ b/lib/Support/Unix/Memory.inc @@ -153,6 +153,7 @@ Memory::releaseMappedMemory(MemoryBlock &M) { std::error_code Memory::protectMappedMemory(const MemoryBlock &M, unsigned Flags) { + static const size_t PageSize = Process::getPageSize(); if (M.Address == nullptr || M.Size == 0) return std::error_code(); @@ -161,7 +162,7 @@ Memory::protectMappedMemory(const MemoryBlock &M, unsigned Flags) { int Protect = getPosixProtectionFlags(Flags); - int Result = ::mprotect(M.Address, M.Size, Protect); + int Result = ::mprotect((void*)((uintptr_t)M.Address & ~(PageSize-1)), PageSize*((M.Size+PageSize-1)/PageSize), Protect); if (Result != 0) return std::error_code(errno, std::generic_category()); @@ -181,7 +182,7 @@ Memory::AllocateRWX(size_t NumBytes, const MemoryBlock* NearBlock, std::string *ErrMsg) { if (NumBytes == 0) return MemoryBlock(); - size_t PageSize = Process::getPageSize(); + static const size_t PageSize = Process::getPageSize(); size_t NumPages = (NumBytes+PageSize-1)/PageSize; int fd = -1; diff --git a/lib/Support/raw_ostream.cpp b/lib/Support/raw_ostream.cpp index 42f830b..3a37987 100644 --- a/lib/Support/raw_ostream.cpp +++ b/lib/Support/raw_ostream.cpp @@ -755,72 +755,15 @@ void raw_string_ostream::write_impl(const char *Ptr, size_t Size) { // raw_svector_ostream //===----------------------------------------------------------------------===// -// The raw_svector_ostream implementation uses the SmallVector itself as the -// buffer for the raw_ostream. We guarantee that the raw_ostream buffer is -// always pointing past the end of the vector, but within the vector -// capacity. This allows raw_ostream to write directly into the correct place, -// and we only need to set the vector size when the data is flushed. +uint64_t raw_svector_ostream::current_pos() const { return OS.size(); } -raw_svector_ostream::raw_svector_ostream(SmallVectorImpl &O, unsigned) - : OS(O) {} - -raw_svector_ostream::raw_svector_ostream(SmallVectorImpl &O) : OS(O) { - init(); -} - -void raw_svector_ostream::init() { - // Set up the initial external buffer. We make sure that the buffer has at - // least 128 bytes free; raw_ostream itself only requires 64, but we want to - // make sure that we don't grow the buffer unnecessarily on destruction (when - // the data is flushed). See the FIXME below. - OS.reserve(OS.size() + 128); - SetBuffer(OS.end(), OS.capacity() - OS.size()); -} - -raw_svector_ostream::~raw_svector_ostream() { - // FIXME: Prevent resizing during this flush(). - flush(); -} +void raw_svector_ostream::write_impl(const char *Ptr, size_t Size) { + OS.append(Ptr, Ptr + Size); +}; void raw_svector_ostream::pwrite_impl(const char *Ptr, size_t Size, uint64_t Offset) { - flush(); - memcpy(OS.begin() + Offset, Ptr, Size); -} - -/// resync - This is called when the SmallVector we're appending to is changed -/// outside of the raw_svector_ostream's control. It is only safe to do this -/// if the raw_svector_ostream has previously been flushed. -void raw_svector_ostream::resync() { - assert(GetNumBytesInBuffer() == 0 && "Didn't flush before mutating vector"); - - if (OS.capacity() - OS.size() < 64) - OS.reserve(OS.capacity() * 2); - SetBuffer(OS.end(), OS.capacity() - OS.size()); -} - -void raw_svector_ostream::write_impl(const char *Ptr, size_t Size) { - if (Ptr == OS.end()) { - // Grow the buffer to include the scratch area without copying. - size_t NewSize = OS.size() + Size; - assert(NewSize <= OS.capacity() && "Invalid write_impl() call!"); - OS.set_size(NewSize); - } else { - assert(!GetNumBytesInBuffer()); - OS.append(Ptr, Ptr + Size); - } - - OS.reserve(OS.size() + 64); - SetBuffer(OS.end(), OS.capacity() - OS.size()); -} - -uint64_t raw_svector_ostream::current_pos() const { - return OS.size(); -} - -StringRef raw_svector_ostream::str() { - flush(); - return StringRef(OS.begin(), OS.size()); + memcpy(OS.data() + Offset, Ptr, Size); } //===----------------------------------------------------------------------===// diff --git a/lib/Transforms/Scalar/Sink.cpp b/lib/Transforms/Scalar/Sink.cpp index f49f4ea..1bdbf11 100644 --- a/lib/Transforms/Scalar/Sink.cpp +++ b/lib/Transforms/Scalar/Sink.cpp @@ -169,7 +169,8 @@ static bool isSafeToMove(Instruction *Inst, AliasAnalysis *AA, return false; } - if (isa(Inst) || isa(Inst)) + if (isa(Inst) || isa(Inst) || + isa(Inst)) return false; // Convergent operations can only be moved to control equivalent blocks. diff --git a/lib/Transforms/Utils/CloneFunction.cpp b/lib/Transforms/Utils/CloneFunction.cpp index cc4d6c6..404b007 100644 --- a/lib/Transforms/Utils/CloneFunction.cpp +++ b/lib/Transforms/Utils/CloneFunction.cpp @@ -96,6 +96,13 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc, NewFunc->copyAttributesFrom(OldFunc); NewFunc->setAttributes(NewAttrs); + // Fix up the personality function that got copied over. + if (OldFunc->hasPersonalityFn()) + NewFunc->setPersonalityFn( + MapValue(OldFunc->getPersonalityFn(), VMap, + ModuleLevelChanges ? RF_None : RF_NoModuleLevelChanges, + TypeMapper, Materializer)); + AttributeSet OldAttrs = OldFunc->getAttributes(); // Clone any argument attributes that are present in the VMap. for (const Argument &OldArg : OldFunc->args()) diff --git a/test/Transforms/Sink/landingpad.ll b/test/Transforms/Sink/landingpad.ll new file mode 100644 index 0000000..10548fd --- /dev/null +++ b/test/Transforms/Sink/landingpad.ll @@ -0,0 +1,33 @@ +; Test that we don't sink landingpads +; RUN: opt -sink -S < %s | FileCheck %s + +declare hidden void @g() +declare void @h() +declare i32 @__gxx_personality_v0(...) + +define void @f() personality i8* bitcast (i32 (...)* @__gxx_personality_v0 to i8*) { +entry: + invoke void @g() + to label %invoke.cont.15 unwind label %lpad + +invoke.cont.15: + unreachable + +; CHECK: lpad: +; CHECK: %0 = landingpad { i8*, i32 } +lpad: + %0 = landingpad { i8*, i32 } + catch i8* null + invoke void @h() + to label %invoke.cont unwind label %lpad.1 + +; CHECK: invoke.cont +; CHECK-NOT: %0 = landingpad { i8*, i32 } +invoke.cont: + ret void + +lpad.1: + %1 = landingpad { i8*, i32 } + cleanup + resume { i8*, i32 } %1 +}