diff options
author | Petr Hosek <phosek@google.com> | 2021-10-18 22:23:52 +0300 |
---|---|---|
committer | Petr Hosek <phosek@google.com> | 2021-10-18 22:24:05 +0300 |
commit | 8e46e34d243524b9a1f9487718ea60e990b35fa3 (patch) | |
tree | 142c081e7fac5e7db1c014af9527b622ffa6049d /llvm/lib/LTO | |
parent | 25107cc8b1f77cddd5eaddddcbe27ad387abd9d8 (diff) |
Revert "[Support][ThinLTO] Move ThinLTO caching to LLVM Support library"
This reverts commit 92b8cc52bbc8194f2cd6a5f742b874969421afca since
it broke the gold plugin.
Diffstat (limited to 'llvm/lib/LTO')
-rw-r--r-- | llvm/lib/LTO/CMakeLists.txt | 1 | ||||
-rw-r--r-- | llvm/lib/LTO/Caching.cpp | 151 | ||||
-rw-r--r-- | llvm/lib/LTO/LTOCodeGenerator.cpp | 7 |
3 files changed, 156 insertions, 3 deletions
diff --git a/llvm/lib/LTO/CMakeLists.txt b/llvm/lib/LTO/CMakeLists.txt index 3abeceab055c..824e2e4e84c7 100644 --- a/llvm/lib/LTO/CMakeLists.txt +++ b/llvm/lib/LTO/CMakeLists.txt @@ -1,4 +1,5 @@ add_llvm_component_library(LLVMLTO + Caching.cpp LTO.cpp LTOBackend.cpp LTOModule.cpp diff --git a/llvm/lib/LTO/Caching.cpp b/llvm/lib/LTO/Caching.cpp new file mode 100644 index 000000000000..75a89e729f43 --- /dev/null +++ b/llvm/lib/LTO/Caching.cpp @@ -0,0 +1,151 @@ +//===-Caching.cpp - LLVM Link Time Optimizer Cache Handling ---------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the Caching for ThinLTO. +// +//===----------------------------------------------------------------------===// + +#include "llvm/LTO/Caching.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/Support/Errc.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Process.h" +#include "llvm/Support/raw_ostream.h" + +#if !defined(_MSC_VER) && !defined(__MINGW32__) +#include <unistd.h> +#else +#include <io.h> +#endif + +using namespace llvm; +using namespace llvm::lto; + +Expected<NativeObjectCache> lto::localCache(StringRef CacheDirectoryPath, + AddBufferFn AddBuffer) { + if (std::error_code EC = sys::fs::create_directories(CacheDirectoryPath)) + return errorCodeToError(EC); + + return [=](unsigned Task, StringRef Key) -> AddStreamFn { + // This choice of file name allows the cache to be pruned (see pruneCache() + // in include/llvm/Support/CachePruning.h). + SmallString<64> EntryPath; + sys::path::append(EntryPath, CacheDirectoryPath, "llvmcache-" + Key); + // First, see if we have a cache hit. + SmallString<64> ResultPath; + Expected<sys::fs::file_t> FDOrErr = sys::fs::openNativeFileForRead( + Twine(EntryPath), sys::fs::OF_UpdateAtime, &ResultPath); + std::error_code EC; + if (FDOrErr) { + ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr = + MemoryBuffer::getOpenFile(*FDOrErr, EntryPath, + /*FileSize=*/-1, + /*RequiresNullTerminator=*/false); + sys::fs::closeFile(*FDOrErr); + if (MBOrErr) { + AddBuffer(Task, std::move(*MBOrErr)); + return AddStreamFn(); + } + EC = MBOrErr.getError(); + } else { + EC = errorToErrorCode(FDOrErr.takeError()); + } + + // On Windows we can fail to open a cache file with a permission denied + // error. This generally means that another process has requested to delete + // the file while it is still open, but it could also mean that another + // process has opened the file without the sharing permissions we need. + // Since the file is probably being deleted we handle it in the same way as + // if the file did not exist at all. + if (EC != errc::no_such_file_or_directory && EC != errc::permission_denied) + report_fatal_error(Twine("Failed to open cache file ") + EntryPath + + ": " + EC.message() + "\n"); + + // This native object stream is responsible for commiting the resulting + // file to the cache and calling AddBuffer to add it to the link. + struct CacheStream : NativeObjectStream { + AddBufferFn AddBuffer; + sys::fs::TempFile TempFile; + std::string EntryPath; + unsigned Task; + + CacheStream(std::unique_ptr<raw_pwrite_stream> OS, AddBufferFn AddBuffer, + sys::fs::TempFile TempFile, std::string EntryPath, + unsigned Task) + : NativeObjectStream(std::move(OS)), AddBuffer(std::move(AddBuffer)), + TempFile(std::move(TempFile)), EntryPath(std::move(EntryPath)), + Task(Task) {} + + ~CacheStream() { + // Make sure the stream is closed before committing it. + OS.reset(); + + // Open the file first to avoid racing with a cache pruner. + ErrorOr<std::unique_ptr<MemoryBuffer>> MBOrErr = + MemoryBuffer::getOpenFile( + sys::fs::convertFDToNativeFile(TempFile.FD), TempFile.TmpName, + /*FileSize=*/-1, /*RequiresNullTerminator=*/false); + if (!MBOrErr) + report_fatal_error(Twine("Failed to open new cache file ") + + TempFile.TmpName + ": " + + MBOrErr.getError().message() + "\n"); + + // On POSIX systems, this will atomically replace the destination if + // it already exists. We try to emulate this on Windows, but this may + // fail with a permission denied error (for example, if the destination + // is currently opened by another process that does not give us the + // sharing permissions we need). Since the existing file should be + // semantically equivalent to the one we are trying to write, we give + // AddBuffer a copy of the bytes we wrote in that case. We do this + // instead of just using the existing file, because the pruner might + // delete the file before we get a chance to use it. + Error E = TempFile.keep(EntryPath); + E = handleErrors(std::move(E), [&](const ECError &E) -> Error { + std::error_code EC = E.convertToErrorCode(); + if (EC != errc::permission_denied) + return errorCodeToError(EC); + + auto MBCopy = MemoryBuffer::getMemBufferCopy((*MBOrErr)->getBuffer(), + EntryPath); + MBOrErr = std::move(MBCopy); + + // FIXME: should we consume the discard error? + consumeError(TempFile.discard()); + + return Error::success(); + }); + + if (E) + report_fatal_error(Twine("Failed to rename temporary file ") + + TempFile.TmpName + " to " + EntryPath + ": " + + toString(std::move(E)) + "\n"); + + AddBuffer(Task, std::move(*MBOrErr)); + } + }; + + return [=](size_t Task) -> std::unique_ptr<NativeObjectStream> { + // Write to a temporary to avoid race condition + SmallString<64> TempFilenameModel; + sys::path::append(TempFilenameModel, CacheDirectoryPath, "Thin-%%%%%%.tmp.o"); + Expected<sys::fs::TempFile> Temp = sys::fs::TempFile::create( + TempFilenameModel, sys::fs::owner_read | sys::fs::owner_write); + if (!Temp) { + errs() << "Error: " << toString(Temp.takeError()) << "\n"; + report_fatal_error("ThinLTO: Can't get a temporary file"); + } + + // This CacheStream will move the temporary file into the cache when done. + return std::make_unique<CacheStream>( + std::make_unique<raw_fd_ostream>(Temp->FD, /* ShouldClose */ false), + AddBuffer, std::move(*Temp), std::string(EntryPath.str()), Task); + }; + }; +} diff --git a/llvm/lib/LTO/LTOCodeGenerator.cpp b/llvm/lib/LTO/LTOCodeGenerator.cpp index 7dca994e735c..412d6f312650 100644 --- a/llvm/lib/LTO/LTOCodeGenerator.cpp +++ b/llvm/lib/LTO/LTOCodeGenerator.cpp @@ -245,7 +245,8 @@ bool LTOCodeGenerator::compileOptimizedToFile(const char **Name) { // make unique temp output file to put generated code SmallString<128> Filename; - auto AddStream = [&](size_t Task) -> std::unique_ptr<NativeObjectStream> { + auto AddStream = + [&](size_t Task) -> std::unique_ptr<lto::NativeObjectStream> { StringRef Extension(Config.CGFileType == CGFT_AssemblyFile ? "s" : "o"); int FD; @@ -254,7 +255,7 @@ bool LTOCodeGenerator::compileOptimizedToFile(const char **Name) { if (EC) emitError(EC.message()); - return std::make_unique<NativeObjectStream>( + return std::make_unique<lto::NativeObjectStream>( std::make_unique<llvm::raw_fd_ostream>(FD, true)); }; @@ -556,7 +557,7 @@ bool LTOCodeGenerator::optimize() { return true; } -bool LTOCodeGenerator::compileOptimized(AddStreamFn AddStream, +bool LTOCodeGenerator::compileOptimized(lto::AddStreamFn AddStream, unsigned ParallelismLevel) { if (!this->determineTarget()) return false; |