From 6f8fc217eaa12bf657be1c6468ed9938d10168b3 Mon Sep 17 00:00:00 2001 From: Dimitry Andric Date: Thu, 27 Jan 2022 23:06:42 +0100 Subject: Vendor import of llvm-project main llvmorg-14-init-17616-g024a1fab5c35. --- lld/ELF/OutputSections.cpp | 140 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 108 insertions(+), 32 deletions(-) (limited to 'lld/ELF/OutputSections.cpp') diff --git a/lld/ELF/OutputSections.cpp b/lld/ELF/OutputSections.cpp index 4a03ac387814..c73d6e439238 100644 --- a/lld/ELF/OutputSections.cpp +++ b/lld/ELF/OutputSections.cpp @@ -15,7 +15,7 @@ #include "lld/Common/Memory.h" #include "lld/Common/Strings.h" #include "llvm/BinaryFormat/Dwarf.h" -#include "llvm/Support/Compression.h" +#include "llvm/Config/config.h" // LLVM_ENABLE_ZLIB #include "llvm/Support/MD5.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/Parallel.h" @@ -23,6 +23,9 @@ #include "llvm/Support/TimeProfiler.h" #include #include +#if LLVM_ENABLE_ZLIB +#include +#endif using namespace llvm; using namespace llvm::dwarf; @@ -284,38 +287,94 @@ static void fill(uint8_t *buf, size_t size, memcpy(buf + i, filler.data(), size - i); } +#if LLVM_ENABLE_ZLIB +static SmallVector deflateShard(ArrayRef in, int level, + int flush) { + // 15 and 8 are default. windowBits=-15 is negative to generate raw deflate + // data with no zlib header or trailer. + z_stream s = {}; + deflateInit2(&s, level, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY); + s.next_in = const_cast(in.data()); + s.avail_in = in.size(); + + // Allocate a buffer of half of the input size, and grow it by 1.5x if + // insufficient. + SmallVector out; + size_t pos = 0; + out.resize_for_overwrite(std::max(in.size() / 2, 64)); + do { + if (pos == out.size()) + out.resize_for_overwrite(out.size() * 3 / 2); + s.next_out = out.data() + pos; + s.avail_out = out.size() - pos; + (void)deflate(&s, flush); + pos = s.next_out - out.data(); + } while (s.avail_out == 0); + assert(s.avail_in == 0); + + out.truncate(pos); + deflateEnd(&s); + return out; +} +#endif + // Compress section contents if this section contains debug info. template void OutputSection::maybeCompress() { +#if LLVM_ENABLE_ZLIB using Elf_Chdr = typename ELFT::Chdr; // Compress only DWARF debug sections. if (!config->compressDebugSections || (flags & SHF_ALLOC) || - !name.startswith(".debug_")) + !name.startswith(".debug_") || size == 0) return; llvm::TimeTraceScope timeScope("Compress debug sections"); - // Create a section header. - zDebugHeader.resize(sizeof(Elf_Chdr)); - auto *hdr = reinterpret_cast(zDebugHeader.data()); - hdr->ch_type = ELFCOMPRESS_ZLIB; - hdr->ch_size = size; - hdr->ch_addralign = alignment; - - // Write section contents to a temporary buffer and compress it. - std::vector buf(size); - writeTo(buf.data()); - // We chose 1 as the default compression level because it is the fastest. If - // -O2 is given, we use level 6 to compress debug info more by ~15%. We found - // that level 7 to 9 doesn't make much difference (~1% more compression) while - // they take significant amount of time (~2x), so level 6 seems enough. - if (Error e = zlib::compress(toStringRef(buf), compressedData, - config->optimize >= 2 ? 6 : 1)) - fatal("compress failed: " + llvm::toString(std::move(e))); - - // Update section headers. - size = sizeof(Elf_Chdr) + compressedData.size(); + // Write uncompressed data to a temporary zero-initialized buffer. + auto buf = std::make_unique(size); + writeTo(buf.get()); + // We chose 1 (Z_BEST_SPEED) as the default compression level because it is + // the fastest. If -O2 is given, we use level 6 to compress debug info more by + // ~15%. We found that level 7 to 9 doesn't make much difference (~1% more + // compression) while they take significant amount of time (~2x), so level 6 + // seems enough. + const int level = config->optimize >= 2 ? 6 : Z_BEST_SPEED; + + // Split input into 1-MiB shards. + constexpr size_t shardSize = 1 << 20; + const size_t numShards = (size + shardSize - 1) / shardSize; + auto shardsIn = std::make_unique[]>(numShards); + for (size_t i = 0, start = 0, end; start != size; ++i, start = end) { + end = std::min(start + shardSize, (size_t)size); + shardsIn[i] = makeArrayRef(buf.get() + start, end - start); + } + + // Compress shards and compute Alder-32 checksums. Use Z_SYNC_FLUSH for all + // shards but the last to flush the output to a byte boundary to be + // concatenated with the next shard. + auto shardsOut = std::make_unique[]>(numShards); + auto shardsAdler = std::make_unique(numShards); + parallelForEachN(0, numShards, [&](size_t i) { + shardsOut[i] = deflateShard(shardsIn[i], level, + i != numShards - 1 ? Z_SYNC_FLUSH : Z_FINISH); + shardsAdler[i] = adler32(1, shardsIn[i].data(), shardsIn[i].size()); + }); + + // Update section size and combine Alder-32 checksums. + uint32_t checksum = 1; // Initial Adler-32 value + compressed.uncompressedSize = size; + size = sizeof(Elf_Chdr) + 2; // Elf_Chdir and zlib header + for (size_t i = 0; i != numShards; ++i) { + size += shardsOut[i].size(); + checksum = adler32_combine(checksum, shardsAdler[i], shardsIn[i].size()); + } + size += 4; // checksum + + compressed.shards = std::move(shardsOut); + compressed.numShards = numShards; + compressed.checksum = checksum; flags |= SHF_COMPRESSED; +#endif } static void writeInt(uint8_t *buf, uint64_t data, uint64_t size) { @@ -339,15 +398,32 @@ template void OutputSection::writeTo(uint8_t *buf) { // If --compress-debug-section is specified and if this is a debug section, // we've already compressed section contents. If that's the case, // just write it down. - if (!compressedData.empty()) { - memcpy(buf, zDebugHeader.data(), zDebugHeader.size()); - memcpy(buf + zDebugHeader.size(), compressedData.data(), - compressedData.size()); + if (compressed.shards) { + auto *chdr = reinterpret_cast(buf); + chdr->ch_type = ELFCOMPRESS_ZLIB; + chdr->ch_size = compressed.uncompressedSize; + chdr->ch_addralign = alignment; + buf += sizeof(*chdr); + + // Compute shard offsets. + auto offsets = std::make_unique(compressed.numShards); + offsets[0] = 2; // zlib header + for (size_t i = 1; i != compressed.numShards; ++i) + offsets[i] = offsets[i - 1] + compressed.shards[i - 1].size(); + + buf[0] = 0x78; // CMF + buf[1] = 0x01; // FLG: best speed + parallelForEachN(0, compressed.numShards, [&](size_t i) { + memcpy(buf + offsets[i], compressed.shards[i].data(), + compressed.shards[i].size()); + }); + + write32be(buf + (size - sizeof(*chdr) - 4), compressed.checksum); return; } // Write leading padding. - std::vector sections = getInputSections(this); + SmallVector sections = getInputSections(*this); std::array filler = getFiller(); bool nonZeroFiller = read32(filler.data()) != 0; if (nonZeroFiller) @@ -355,7 +431,7 @@ template void OutputSection::writeTo(uint8_t *buf) { parallelForEachN(0, sections.size(), [&](size_t i) { InputSection *isec = sections[i]; - isec->writeTo(buf); + isec->writeTo(buf + isec->outSecOff); // Fill gaps between sections. if (nonZeroFiller) { @@ -520,9 +596,9 @@ InputSection *elf::getFirstInputSection(const OutputSection *os) { return nullptr; } -std::vector elf::getInputSections(const OutputSection *os) { - std::vector ret; - for (SectionCommand *cmd : os->commands) +SmallVector elf::getInputSections(const OutputSection &os) { + SmallVector ret; + for (SectionCommand *cmd : os.commands) if (auto *isd = dyn_cast(cmd)) ret.insert(ret.end(), isd->sections.begin(), isd->sections.end()); return ret; @@ -550,7 +626,7 @@ std::array OutputSection::getFiller() { void OutputSection::checkDynRelAddends(const uint8_t *bufStart) { assert(config->writeAddends && config->checkDynamicRelocs); assert(type == SHT_REL || type == SHT_RELA); - std::vector sections = getInputSections(this); + SmallVector sections = getInputSections(*this); parallelForEachN(0, sections.size(), [&](size_t i) { // When linking with -r or --emit-relocs we might also call this function // for input .rel[a]. sections which we simply pass through to the -- cgit v1.3