ckissane updated this revision to Diff 441114.
ckissane edited the summary of this revision.
ckissane added a comment.
- feat: add zstd compression
- Merge branch 'ckissane.refactor-compression-namespace' into
ckissane.add-zstd-compression
- Merge branch 'ckissane.refactor-compression-namespace' into
ckissane.add-zstd-compression
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D128465/new/
https://reviews.llvm.org/D128465
Files:
clang-tools-extra/clangd/CMakeLists.txt
clang-tools-extra/clangd/test/lit.site.cfg.py.in
clang/test/CMakeLists.txt
clang/test/lit.site.cfg.py.in
compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
compiler-rt/test/lit.common.configured.in
flang/CMakeLists.txt
lld/ELF/CMakeLists.txt
lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
llvm/CMakeLists.txt
llvm/cmake/config-ix.cmake
llvm/cmake/modules/FindZSTD.cmake
llvm/cmake/modules/LLVMConfig.cmake.in
llvm/include/llvm/Config/llvm-config.h.cmake
llvm/include/llvm/Support/Compression.h
llvm/lib/Support/Compression.cpp
utils/bazel/llvm_configs/llvm-config.h.cmake
Index: utils/bazel/llvm_configs/llvm-config.h.cmake
===================================================================
--- utils/bazel/llvm_configs/llvm-config.h.cmake
+++ utils/bazel/llvm_configs/llvm-config.h.cmake
@@ -92,6 +92,9 @@
/* Define if zlib compression is available */
#cmakedefine01 LLVM_ENABLE_ZLIB
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
/* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
#cmakedefine LLVM_HAVE_TF_API
Index: llvm/lib/Support/Compression.cpp
===================================================================
--- llvm/lib/Support/Compression.cpp
+++ llvm/lib/Support/Compression.cpp
@@ -20,6 +20,9 @@
#if LLVM_ENABLE_ZLIB
#include <zlib.h>
#endif
+#if LLVM_ENABLE_ZSTD
+#include <zstd.h>
+#endif
using namespace llvm;
@@ -102,3 +105,63 @@
llvm_unreachable("zlib::uncompress is unavailable");
}
#endif
+
+#if LLVM_ENABLE_ZSTD
+
+bool zstd::isAvailable() { return true; }
+
+void zstd::compress(StringRef InputBuffer,
+ SmallVectorImpl<char> &CompressedBuffer, int Level) {
+ unsigned long CompressedBufferSize = ::ZSTD_compressBound(InputBuffer.size());
+ CompressedBuffer.resize_for_overwrite(CompressedBufferSize);
+ unsigned long CompressedSize = ::ZSTD_compress(
+ (Bytef *)CompressedBuffer.data(), CompressedBufferSize,
+ (const Bytef *)InputBuffer.data(), InputBuffer.size(), Level);
+ if (ZSTD_isError(CompressedSize))
+ report_bad_alloc_error("Allocation failed");
+ // Tell MemorySanitizer that zstd output buffer is fully initialized.
+ // This avoids a false report when running LLVM with uninstrumented ZLib.
+ __msan_unpoison(CompressedBuffer.data(), CompressedSize);
+ CompressedBuffer.truncate(CompressedSize);
+}
+
+Error zstd::uncompress(StringRef InputBuffer, char *UncompressedBuffer,
+ size_t &UncompressedSize) {
+ unsigned long long const rSize = ZSTD_getFrameContentSize(
+ (const Bytef *)InputBuffer.data(), InputBuffer.size());
+ size_t const Res =
+ ::ZSTD_decompress((Bytef *)UncompressedBuffer, rSize,
+ (const Bytef *)InputBuffer.data(), InputBuffer.size());
+ UncompressedSize = Res;
+ // Tell MemorySanitizer that zstd output buffer is fully initialized.
+ // This avoids a false report when running LLVM with uninstrumented ZLib.
+ __msan_unpoison(UncompressedBuffer, UncompressedSize);
+ return Res != rSize ? createError(ZSTD_getErrorName(Res)) : Error::success();
+}
+
+Error zstd::uncompress(StringRef InputBuffer,
+ SmallVectorImpl<char> &UncompressedBuffer,
+ size_t UncompressedSize) {
+ UncompressedBuffer.resize_for_overwrite(UncompressedSize);
+ Error E = zstd::uncompress(InputBuffer, UncompressedBuffer.data(),
+ UncompressedSize);
+ UncompressedBuffer.truncate(UncompressedSize);
+ return E;
+}
+
+#else
+bool zstd::isAvailable() { return false; }
+void zstd::compress(StringRef InputBuffer,
+ SmallVectorImpl<char> &CompressedBuffer, int Level) {
+ llvm_unreachable("zstd::compress is unavailable");
+}
+Error zstd::uncompress(StringRef InputBuffer, char *UncompressedBuffer,
+ size_t &UncompressedSize) {
+ llvm_unreachable("zstd::uncompress is unavailable");
+}
+Error zstd::uncompress(StringRef InputBuffer,
+ SmallVectorImpl<char> &UncompressedBuffer,
+ size_t UncompressedSize) {
+ llvm_unreachable("zstd::uncompress is unavailable");
+}
+#endif
\ No newline at end of file
Index: llvm/include/llvm/Support/Compression.h
===================================================================
--- llvm/include/llvm/Support/Compression.h
+++ llvm/include/llvm/Support/Compression.h
@@ -44,7 +44,33 @@
} // End of namespace zlib
+namespace zstd {
+
+static constexpr std::string AlgorithmName = "zstd";
+static constexpr int NoCompression = -5;
+static constexpr int BestSpeedCompression = 1;
+static constexpr int DefaultCompression = 5;
+static constexpr int BestSizeCompression = 12;
+
+bool isAvailable();
+
+void compress(StringRef InputBuffer, SmallVectorImpl<char> &CompressedBuffer,
+ int Level = DefaultCompression);
+
+Error uncompress(StringRef InputBuffer, char *UncompressedBuffer,
+ size_t &UncompressedSize);
+
+Error uncompress(StringRef InputBuffer,
+ SmallVectorImpl<char> &UncompressedBuffer,
+ size_t UncompressedSize);
+
+} // End of namespace zstd
+
+#if LLVM_ENABLE_ZSTD
+namespace tooling = llvm::compression::zstd;
+#else
namespace tooling = llvm::compression::zlib;
+#endif
namespace profile = llvm::compression::tooling;
Index: llvm/include/llvm/Config/llvm-config.h.cmake
===================================================================
--- llvm/include/llvm/Config/llvm-config.h.cmake
+++ llvm/include/llvm/Config/llvm-config.h.cmake
@@ -92,6 +92,9 @@
/* Define if zlib compression is available */
#cmakedefine01 LLVM_ENABLE_ZLIB
+/* Define if zstd compression is available */
+#cmakedefine01 LLVM_ENABLE_ZSTD
+
/* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
#cmakedefine LLVM_HAVE_TF_API
Index: llvm/cmake/modules/LLVMConfig.cmake.in
===================================================================
--- llvm/cmake/modules/LLVMConfig.cmake.in
+++ llvm/cmake/modules/LLVMConfig.cmake.in
@@ -73,6 +73,12 @@
find_package(ZLIB)
endif()
+set(LLVM_ENABLE_ZSTD @LLVM_ENABLE_ZSTD@)
+if(LLVM_ENABLE_ZSTD)
+ set(ZSTD_ROOT @ZSTD_ROOT@)
+ find_package(ZSTD)
+endif()
+
set(LLVM_ENABLE_LIBXML2 @LLVM_ENABLE_LIBXML2@)
if(LLVM_ENABLE_LIBXML2)
find_package(LibXml2)
Index: llvm/cmake/modules/FindZSTD.cmake
===================================================================
--- /dev/null
+++ llvm/cmake/modules/FindZSTD.cmake
@@ -0,0 +1,41 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# - Try to find Facebook zstd library
+# This will define
+# ZSTD_FOUND
+# ZSTD_INCLUDE_DIR
+# ZSTD_LIBRARY
+#
+
+find_path(ZSTD_INCLUDE_DIR NAMES zstd.h)
+
+find_library(ZSTD_LIBRARY_DEBUG NAMES zstdd zstd_staticd)
+find_library(ZSTD_LIBRARY_RELEASE NAMES zstd zstd_static)
+
+include(SelectLibraryConfigurations)
+SELECT_LIBRARY_CONFIGURATIONS(ZSTD)
+
+include(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(
+ ZSTD DEFAULT_MSG
+ ZSTD_LIBRARY ZSTD_INCLUDE_DIR
+)
+
+if (ZSTD_FOUND)
+ message(STATUS "Found Zstd: ${ZSTD_LIBRARY}")
+endif()
+
+mark_as_advanced(ZSTD_INCLUDE_DIR ZSTD_LIBRARY)
Index: llvm/cmake/config-ix.cmake
===================================================================
--- llvm/cmake/config-ix.cmake
+++ llvm/cmake/config-ix.cmake
@@ -135,6 +135,27 @@
set(LLVM_ENABLE_ZLIB "${HAVE_ZLIB}")
endif()
+if(LLVM_ENABLE_ZSTD)
+ if(LLVM_ENABLE_ZSTD STREQUAL FORCE_ON)
+ find_package(ZSTD REQUIRED)
+ elseif(NOT LLVM_USE_SANITIZER MATCHES "Memory.*")
+ find_package(ZSTD)
+ endif()
+ if(ZSTD_FOUND)
+ # Check if zstd we found is usable; for example, we may have found a 32-bit
+ # library on a 64-bit system which would result in a link-time failure.
+ cmake_push_check_state()
+ list(APPEND CMAKE_REQUIRED_INCLUDES ${ZSTD_INCLUDE_DIR})
+ list(APPEND CMAKE_REQUIRED_LIBRARIES ${ZSTD_LIBRARY})
+ check_symbol_exists(ZSTD_compress zstd.h HAVE_ZSTD)
+ cmake_pop_check_state()
+ if(LLVM_ENABLE_ZSTD STREQUAL FORCE_ON AND NOT HAVE_ZSTD)
+ message(FATAL_ERROR "Failed to configure zstd")
+ endif()
+ endif()
+ set(LLVM_ENABLE_ZSTD "${HAVE_ZSTD}")
+endif()
+
if(LLVM_ENABLE_LIBXML2)
if(LLVM_ENABLE_LIBXML2 STREQUAL FORCE_ON)
find_package(LibXml2 REQUIRED)
Index: llvm/CMakeLists.txt
===================================================================
--- llvm/CMakeLists.txt
+++ llvm/CMakeLists.txt
@@ -438,6 +438,8 @@
set(LLVM_ENABLE_ZLIB "ON" CACHE STRING "Use zlib for compression/decompression if available. Can be ON, OFF, or FORCE_ON")
+set(LLVM_ENABLE_ZSTD "ON" CACHE STRING "Use zstd for compression/decompression if available. Can be ON, OFF, or FORCE_ON")
+
set(LLVM_ENABLE_CURL "OFF" CACHE STRING "Use libcurl for the HTTP client if available. Can be ON, OFF, or FORCE_ON")
set(LLVM_Z3_INSTALL_DIR "" CACHE STRING "Install directory of the Z3 solver.")
Index: lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
===================================================================
--- lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
+++ lldb/source/Plugins/Process/gdb-remote/CMakeLists.txt
@@ -18,6 +18,10 @@
list(APPEND LLDB_SYSTEM_LIBS ZLIB::ZLIB)
endif()
+if(LLVM_ENABLE_ZSTD)
+ list(APPEND LLDB_SYSTEM_LIBS zstd)
+endif()
+
add_lldb_library(lldbPluginProcessGDBRemote PLUGIN
GDBRemoteClientBase.cpp
GDBRemoteCommunication.cpp
Index: lld/ELF/CMakeLists.txt
===================================================================
--- lld/ELF/CMakeLists.txt
+++ lld/ELF/CMakeLists.txt
@@ -6,6 +6,10 @@
list(APPEND imported_libs ZLIB::ZLIB)
endif()
+if(LLVM_ENABLE_ZSTD)
+ list(APPEND imported_libs zstd)
+endif()
+
add_lld_library(lldELF
AArch64ErrataFix.cpp
Arch/AArch64.cpp
Index: flang/CMakeLists.txt
===================================================================
--- flang/CMakeLists.txt
+++ flang/CMakeLists.txt
@@ -70,6 +70,12 @@
if(LLVM_ENABLE_ZLIB)
find_package(ZLIB REQUIRED)
endif()
+
+ # If LLVM links to zstd we need the imported targets so we can too.
+ if(LLVM_ENABLE_ZSTD)
+ find_package(ZSTD REQUIRED)
+ endif()
+
option(LLVM_ENABLE_PEDANTIC "Compile with pedantic enabled." ON)
if(CMAKE_COMPILER_IS_GNUCXX)
set(USE_NO_MAYBE_UNINITIALIZED 1)
Index: compiler-rt/test/lit.common.configured.in
===================================================================
--- compiler-rt/test/lit.common.configured.in
+++ compiler-rt/test/lit.common.configured.in
@@ -65,6 +65,7 @@
set_default("target_suffix", "-%s" % config.target_arch)
set_default("have_zlib", "@LLVM_ENABLE_ZLIB@")
+set_default("have_zstd", "@LLVM_ENABLE_ZSTD@")
set_default("libcxx_used", "@LLVM_LIBCXX_USED@")
# LLVM tools dir can be passed in lit parameters, so try to
Index: compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
===================================================================
--- compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
+++ compiler-rt/lib/sanitizer_common/symbolizer/scripts/build_symbolizer.sh
@@ -140,6 +140,7 @@
-DLLVM_TABLEGEN=$TBLGEN \
-DLLVM_DEFAULT_TARGET_TRIPLE="${TARGET_TRIPLE}" \
-DLLVM_ENABLE_ZLIB=ON \
+ -DLLVM_ENABLE_ZSTD=ON \
-DLLVM_ENABLE_TERMINFO=OFF \
-DLLVM_ENABLE_THREADS=OFF \
$LLVM_SRC
Index: clang/test/lit.site.cfg.py.in
===================================================================
--- clang/test/lit.site.cfg.py.in
+++ clang/test/lit.site.cfg.py.in
@@ -21,6 +21,7 @@
config.host_cxx = "@CMAKE_CXX_COMPILER@"
config.llvm_use_sanitizer = "@LLVM_USE_SANITIZER@"
config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
config.clang_arcmt = @CLANG_ENABLE_ARCMT@
config.clang_default_pie_on_linux = @CLANG_DEFAULT_PIE_ON_LINUX@
config.clang_enable_opaque_pointers = @CLANG_ENABLE_OPAQUE_POINTERS_INTERNAL@
Index: clang/test/CMakeLists.txt
===================================================================
--- clang/test/CMakeLists.txt
+++ clang/test/CMakeLists.txt
@@ -11,6 +11,7 @@
CLANG_SPAWN_CC1
ENABLE_BACKTRACES
LLVM_ENABLE_ZLIB
+ LLVM_ENABLE_ZSTD
LLVM_ENABLE_PER_TARGET_RUNTIME_DIR
LLVM_ENABLE_THREADS
LLVM_WITH_Z3
Index: clang-tools-extra/clangd/test/lit.site.cfg.py.in
===================================================================
--- clang-tools-extra/clangd/test/lit.site.cfg.py.in
+++ clang-tools-extra/clangd/test/lit.site.cfg.py.in
@@ -17,6 +17,7 @@
config.clangd_enable_remote = @CLANGD_ENABLE_REMOTE@
config.clangd_tidy_checks = @CLANGD_TIDY_CHECKS@
config.have_zlib = @LLVM_ENABLE_ZLIB@
+config.have_zstd = @LLVM_ENABLE_ZSTD@
# Delegate logic to lit.cfg.py.
lit_config.load_config(config, "@CMAKE_CURRENT_SOURCE_DIR@/lit.cfg.py")
Index: clang-tools-extra/clangd/CMakeLists.txt
===================================================================
--- clang-tools-extra/clangd/CMakeLists.txt
+++ clang-tools-extra/clangd/CMakeLists.txt
@@ -29,6 +29,7 @@
CLANGD_MALLOC_TRIM
CLANGD_TIDY_CHECKS
LLVM_ENABLE_ZLIB
+ LLVM_ENABLE_ZSTD
)
configure_file(
_______________________________________________
lldb-commits mailing list
lldb-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits
