MaskRay created this revision. MaskRay added reviewers: aaron.ballman, jansvoboda11, rsmith. Herald added a subscriber: StephenFan. Herald added a project: All. MaskRay requested review of this revision. Herald added a project: clang. Herald added a subscriber: cfe-commits.
Extend SM_SLOC_BUFFER_BLOB_COMPRESSED to allow zstd, which is much faster (compression/decompression) than zlib and has better compression ratio. An alternative is to add a value beside SM_SLOC_BUFFER_BLOB_COMPRESSED, but reusing SM_SLOC_BUFFER_BLOB_COMPRESSED slightly simplifies the implementation and leads to better diagnostics when a slightly older Clang consumes zstd compressed blob. Repository: rG LLVM Github Monorepo https://reviews.llvm.org/D137885 Files: clang/lib/Serialization/ASTReader.cpp clang/lib/Serialization/ASTWriter.cpp clang/test/Modules/embed-files-compressed.cpp Index: clang/test/Modules/embed-files-compressed.cpp =================================================================== --- clang/test/Modules/embed-files-compressed.cpp +++ clang/test/Modules/embed-files-compressed.cpp @@ -1,4 +1,4 @@ -// REQUIRES: zlib +// REQUIRES: zlib || zstd // REQUIRES: shell // // RUN: rm -rf %t Index: clang/lib/Serialization/ASTWriter.cpp =================================================================== --- clang/lib/Serialization/ASTWriter.cpp +++ clang/lib/Serialization/ASTWriter.cpp @@ -1980,6 +1980,14 @@ // Compress the buffer if possible. We expect that almost all PCM // consumers will not want its contents. SmallVector<uint8_t, 0> CompressedBuffer; + if (llvm::compression::zstd::isAvailable()) { + llvm::compression::zstd::compress( + llvm::arrayRefFromStringRef(Blob.drop_back(1)), CompressedBuffer); + RecordDataType Record[] = {SM_SLOC_BUFFER_BLOB_COMPRESSED, Blob.size() - 1}; + Stream.EmitRecordWithBlob(SLocBufferBlobCompressedAbbrv, Record, + llvm::toStringRef(CompressedBuffer)); + return; + } if (llvm::compression::zlib::isAvailable()) { llvm::compression::zlib::compress( llvm::arrayRefFromStringRef(Blob.drop_back(1)), CompressedBuffer); Index: clang/lib/Serialization/ASTReader.cpp =================================================================== --- clang/lib/Serialization/ASTReader.cpp +++ clang/lib/Serialization/ASTReader.cpp @@ -1452,19 +1452,23 @@ unsigned RecCode = MaybeRecCode.get(); if (RecCode == SM_SLOC_BUFFER_BLOB_COMPRESSED) { - if (!llvm::compression::zlib::isAvailable()) { - Error("zlib is not available"); + const llvm::compression::Format F = + Blob.size() >= 2 && memcmp(Blob.data(), "\x1f\x8b", 2) == 0 + ? llvm::compression::Format::Zlib + : llvm::compression::Format::Zstd; + if (const char *Reason = llvm::compression::getReasonIfUnsupported(F)) { + Error(Reason); return nullptr; } - SmallVector<uint8_t, 0> Uncompressed; - if (llvm::Error E = llvm::compression::zlib::decompress( - llvm::arrayRefFromStringRef(Blob), Uncompressed, Record[0])) { + SmallVector<uint8_t, 0> Decompressed; + if (llvm::Error E = llvm::compression::decompress( + F, llvm::arrayRefFromStringRef(Blob), Decompressed, Record[0])) { Error("could not decompress embedded file contents: " + llvm::toString(std::move(E))); return nullptr; } return llvm::MemoryBuffer::getMemBufferCopy( - llvm::toStringRef(Uncompressed), Name); + llvm::toStringRef(Decompressed), Name); } else if (RecCode == SM_SLOC_BUFFER_BLOB) { return llvm::MemoryBuffer::getMemBuffer(Blob.drop_back(1), Name, true); } else {
Index: clang/test/Modules/embed-files-compressed.cpp =================================================================== --- clang/test/Modules/embed-files-compressed.cpp +++ clang/test/Modules/embed-files-compressed.cpp @@ -1,4 +1,4 @@ -// REQUIRES: zlib +// REQUIRES: zlib || zstd // REQUIRES: shell // // RUN: rm -rf %t Index: clang/lib/Serialization/ASTWriter.cpp =================================================================== --- clang/lib/Serialization/ASTWriter.cpp +++ clang/lib/Serialization/ASTWriter.cpp @@ -1980,6 +1980,14 @@ // Compress the buffer if possible. We expect that almost all PCM // consumers will not want its contents. SmallVector<uint8_t, 0> CompressedBuffer; + if (llvm::compression::zstd::isAvailable()) { + llvm::compression::zstd::compress( + llvm::arrayRefFromStringRef(Blob.drop_back(1)), CompressedBuffer); + RecordDataType Record[] = {SM_SLOC_BUFFER_BLOB_COMPRESSED, Blob.size() - 1}; + Stream.EmitRecordWithBlob(SLocBufferBlobCompressedAbbrv, Record, + llvm::toStringRef(CompressedBuffer)); + return; + } if (llvm::compression::zlib::isAvailable()) { llvm::compression::zlib::compress( llvm::arrayRefFromStringRef(Blob.drop_back(1)), CompressedBuffer); Index: clang/lib/Serialization/ASTReader.cpp =================================================================== --- clang/lib/Serialization/ASTReader.cpp +++ clang/lib/Serialization/ASTReader.cpp @@ -1452,19 +1452,23 @@ unsigned RecCode = MaybeRecCode.get(); if (RecCode == SM_SLOC_BUFFER_BLOB_COMPRESSED) { - if (!llvm::compression::zlib::isAvailable()) { - Error("zlib is not available"); + const llvm::compression::Format F = + Blob.size() >= 2 && memcmp(Blob.data(), "\x1f\x8b", 2) == 0 + ? llvm::compression::Format::Zlib + : llvm::compression::Format::Zstd; + if (const char *Reason = llvm::compression::getReasonIfUnsupported(F)) { + Error(Reason); return nullptr; } - SmallVector<uint8_t, 0> Uncompressed; - if (llvm::Error E = llvm::compression::zlib::decompress( - llvm::arrayRefFromStringRef(Blob), Uncompressed, Record[0])) { + SmallVector<uint8_t, 0> Decompressed; + if (llvm::Error E = llvm::compression::decompress( + F, llvm::arrayRefFromStringRef(Blob), Decompressed, Record[0])) { Error("could not decompress embedded file contents: " + llvm::toString(std::move(E))); return nullptr; } return llvm::MemoryBuffer::getMemBufferCopy( - llvm::toStringRef(Uncompressed), Name); + llvm::toStringRef(Decompressed), Name); } else if (RecCode == SM_SLOC_BUFFER_BLOB) { return llvm::MemoryBuffer::getMemBuffer(Blob.drop_back(1), Name, true); } else {
_______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits