This is an automated email from the ASF dual-hosted git repository.
marong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new d76865ab73 [GLUTEN-10452][VL] Remove IAA support (#10480)
d76865ab73 is described below
commit d76865ab73f39d074806600e209da6ae8ed10fa3
Author: Rong Ma <[email protected]>
AuthorDate: Mon Aug 18 19:13:14 2025 +0100
[GLUTEN-10452][VL] Remove IAA support (#10480)
---
cpp/CMake/BuildQpl.cmake | 60 -----
cpp/CMakeLists.txt | 5 -
cpp/core/CMakeLists.txt | 7 -
cpp/core/config/GlutenConfig.h | 1 -
cpp/core/jni/JniCommon.h | 3 -
cpp/core/utils/Compression.cc | 12 -
cpp/core/utils/Compression.h | 2 +-
cpp/core/utils/qpl/QplCodec.cc | 255 ---------------------
cpp/core/utils/qpl/QplCodec.h | 35 ---
cpp/core/utils/qpl/QplJobPool.cc | 122 ----------
cpp/core/utils/qpl/QplJobPool.h | 85 -------
cpp/velox/benchmarks/GenericBenchmark.cc | 14 +-
cpp/velox/compute/VeloxBackend.cc | 3 -
dev/builddeps-veloxbe.sh | 6 -
docs/get-started/Velox.md | 2 +-
docs/get-started/VeloxIAA.md | 75 ------
docs/get-started/build-guide.md | 2 -
.../org/apache/gluten/config/GlutenConfig.scala | 11 +-
.../apache/spark/shuffle/GlutenShuffleUtils.scala | 2 -
19 files changed, 4 insertions(+), 698 deletions(-)
diff --git a/cpp/CMake/BuildQpl.cmake b/cpp/CMake/BuildQpl.cmake
deleted file mode 100644
index 7715bb8e76..0000000000
--- a/cpp/CMake/BuildQpl.cmake
+++ /dev/null
@@ -1,60 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-include(ExternalProject)
-
-macro(build_qpl)
- message(STATUS "Building QPL from source")
- set(QPL_BUILD_VERSION "v1.1.0")
- set(QPL_BUILD_SHA256_CHECKSUM
- "00306000035621dfbc21007481395c46ba9723fc8add8ca5142847b94dc564c5")
- set(QPL_SOURCE_URL
- "https://github.com/intel/qpl/archive/refs/tags/v1.1.0.tar.gz";)
- set(QPL_LIB_NAME "qpl")
-
- set(QPL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/qpl_ep-install")
- set(QPL_SOURCE_DIR "${QPL_PREFIX}/src/qpl_ep")
- set(QPL_INCLUDE_DIR "${QPL_PREFIX}/include")
- set(QPL_LIB_DIR "${QPL_PREFIX}/lib")
- set(QPL_STATIC_LIB_NAME
-
"${CMAKE_STATIC_LIBRARY_PREFIX}${QPL_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}${QPL_STATIC_LIB_MAJOR_VERSION}"
- )
- set(QPL_STATIC_LIB_TARGETS "${QPL_LIB_DIR}/${QPL_STATIC_LIB_NAME}")
- ExternalProject_Add(
- qpl_ep
- PREFIX ${QPL_PREFIX}
- URL ${QPL_SOURCE_URL}
- URL_HASH "SHA256=${QPL_BUILD_SHA256_CHECKSUM}"
- SOURCE_DIR ${QPL_SOURCE_DIR}
- CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${QPL_PREFIX}
- -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DQPL_BUILD_TESTS=OFF
- -DLOG_HW_INIT=ON
- BUILD_BYPRODUCTS ${QPL_STATIC_LIB_TARGETS})
-
- # The include directory must exist before it is referenced by a target.
- file(MAKE_DIRECTORY "${QPL_INCLUDE_DIR}")
-
- add_library(qpl::qpl STATIC IMPORTED)
- set_target_properties(
- qpl::qpl
- PROPERTIES IMPORTED_LOCATION "${QPL_LIB_DIR}/${QPL_STATIC_LIB_NAME}"
- INTERFACE_INCLUDE_DIRECTORIES "${QPL_INCLUDE_DIR}")
-
- add_dependencies(qpl::qpl qpl_ep)
-endmacro()
-
-build_qpl()
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index acec63b4fe..1399525311 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -49,7 +49,6 @@ option(BUILD_BENCHMARKS "Build Benchmarks" OFF)
option(ENABLE_JEMALLOC_STATS "Prints Jemalloc stats for debugging" OFF)
option(BUILD_GLOG "Build Glog from Source" OFF)
option(ENABLE_QAT "Enable QAT for de/compression" OFF)
-option(ENABLE_IAA "Enable IAA for de/compression" OFF)
option(ENABLE_GCS "Enable GCS" OFF)
option(ENABLE_S3 "Enable S3" OFF)
option(ENABLE_HDFS "Enable HDFS" OFF)
@@ -233,10 +232,6 @@ if(ENABLE_QAT)
add_definitions(-DGLUTEN_ENABLE_QAT)
endif()
-if(ENABLE_IAA)
- add_definitions(-DGLUTEN_ENABLE_IAA)
-endif()
-
if(ENABLE_GPU)
add_definitions(-DGLUTEN_ENABLE_GPU)
endif()
diff --git a/cpp/core/CMakeLists.txt b/cpp/core/CMakeLists.txt
index d628a880dd..58dd301b69 100644
--- a/cpp/core/CMakeLists.txt
+++ b/cpp/core/CMakeLists.txt
@@ -187,13 +187,6 @@ if(ENABLE_QAT)
target_link_libraries(gluten PUBLIC qatzip::qatzip qatzstd::qatzstd)
endif()
-if(ENABLE_IAA)
- include(BuildQpl)
- target_include_directories(gluten PUBLIC ${QPL_INCLUDE_DIR})
- target_sources(gluten PRIVATE utils/qpl/QplJobPool.cc utils/qpl/QplCodec.cc)
- target_link_libraries(gluten PUBLIC qpl::qpl)
-endif()
-
find_protobuf()
message(STATUS "Found Protobuf: ${PROTOBUF_LIBRARY}")
target_link_libraries(gluten LINK_PUBLIC ${PROTOBUF_LIBRARY})
diff --git a/cpp/core/config/GlutenConfig.h b/cpp/core/config/GlutenConfig.h
index fe24260306..a1aeb5ac35 100644
--- a/cpp/core/config/GlutenConfig.h
+++ b/cpp/core/config/GlutenConfig.h
@@ -79,7 +79,6 @@ const std::string kShuffleSpillDiskWriteBufferSize =
"spark.shuffle.spill.diskWr
const std::string kSortShuffleReaderDeserializerBufferSize =
"spark.gluten.sql.columnar.shuffle.sort.deserializerBufferSize";
const std::string kQatBackendName = "qat";
-const std::string kIaaBackendName = "iaa";
const std::string kSparkRedactionRegex = "spark.redaction.regex";
const std::string kSparkRedactionString = "*********(redacted)";
diff --git a/cpp/core/jni/JniCommon.h b/cpp/core/jni/JniCommon.h
index d777d39fa7..bc59a2a485 100644
--- a/cpp/core/jni/JniCommon.h
+++ b/cpp/core/jni/JniCommon.h
@@ -362,9 +362,6 @@ static inline gluten::CodecBackend getCodecBackend(JNIEnv*
env, jstring codecBac
if (codecBackend == "qat") {
return gluten::CodecBackend::QAT;
}
- if (codecBackend == "iaa") {
- return gluten::CodecBackend::IAA;
- }
throw std::invalid_argument("Not support this codec backend " +
codecBackend);
}
diff --git a/cpp/core/utils/Compression.cc b/cpp/core/utils/Compression.cc
index 76e3331267..913ee3b55f 100644
--- a/cpp/core/utils/Compression.cc
+++ b/cpp/core/utils/Compression.cc
@@ -23,10 +23,6 @@
#include "utils/qat/QatCodec.h"
#endif
-#ifdef GLUTEN_ENABLE_IAA
-#include "utils/qpl/QplCodec.h"
-#endif
-
namespace gluten {
std::unique_ptr<arrow::util::Codec>
@@ -45,8 +41,6 @@ createArrowIpcCodec(arrow::Compression::type compressedType,
CodecBackend codecB
#else
throw GlutenException("Backend QAT but not compile with option
GLUTEN_ENABLE_QAT");
#endif
- } else {
- throw GlutenException("Backend IAA not support zstd compression");
}
} break;
case arrow::Compression::GZIP: {
@@ -57,12 +51,6 @@ createArrowIpcCodec(arrow::Compression::type compressedType,
CodecBackend codecB
codec = qat::makeDefaultQatGZipCodec();
#else
throw GlutenException("Backend QAT but not compile with option
GLUTEN_ENABLE_QAT");
-#endif
- } else {
-#if defined(GLUTEN_ENABLE_IAA)
- codec = qpl::MakeDefaultQplGZipCodec();
-#else
- throw GlutenException("Backend IAA but not compile with option
GLUTEN_ENABLE_IAA");
#endif
}
} break;
diff --git a/cpp/core/utils/Compression.h b/cpp/core/utils/Compression.h
index aebce00f34..5a3e170fb4 100644
--- a/cpp/core/utils/Compression.h
+++ b/cpp/core/utils/Compression.h
@@ -21,7 +21,7 @@
namespace gluten {
-enum CodecBackend { NONE, QAT, IAA };
+enum CodecBackend { NONE, QAT };
// BUFFER mode will preallocate max compressed buffer, and then compress each
buffer to the max compressed buffer
// ROWVECTOR mode will copy the buffers to a big buffer and then compress the
big buffer
diff --git a/cpp/core/utils/qpl/QplCodec.cc b/cpp/core/utils/qpl/QplCodec.cc
deleted file mode 100644
index b41a00585e..0000000000
--- a/cpp/core/utils/qpl/QplCodec.cc
+++ /dev/null
@@ -1,255 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <arrow/util/compression.h>
-#include <arrow/util/logging.h>
-#include <utils/qpl/QplCodec.h>
-#include <utils/qpl/QplJobPool.h>
-#include <iostream>
-#include <map>
-
-namespace gluten {
-namespace qpl {
-
-class HardwareCodecDeflateQpl {
- public:
- /// RET_ERROR stands for hardware codec fail,need fallback to software codec.
- static constexpr int64_t RET_ERROR = -1;
-
- explicit HardwareCodecDeflateQpl(qpl_compression_levels compressionLevel) :
compressionLevel_(compressionLevel){};
-
- int64_t doCompressData(const uint8_t* source, uint32_t source_size, uint8_t*
dest, uint32_t dest_size) const {
- uint32_t job_id;
- qpl_job* jobPtr;
- if (!(jobPtr = QplJobHWPool::GetInstance().AcquireJob(job_id))) {
- ARROW_LOG(WARNING)
- << "DeflateQpl HW codec failed, falling back to SW codec. (Details:
doCompressData->AcquireJob fail, probably job pool exhausted)";
- return RET_ERROR;
- }
-
- jobPtr->op = qpl_op_compress;
- jobPtr->next_in_ptr = const_cast<uint8_t*>(source);
- jobPtr->next_out_ptr = dest;
- jobPtr->available_in = source_size;
- jobPtr->level = compressionLevel_;
- jobPtr->available_out = dest_size;
- jobPtr->flags = QPL_FLAG_FIRST | QPL_FLAG_DYNAMIC_HUFFMAN | QPL_FLAG_LAST
| QPL_FLAG_OMIT_VERIFY;
-
- if (auto status = qpl_execute_job(jobPtr); status == QPL_STS_OK) {
- auto compressed_size = jobPtr->total_out;
- QplJobHWPool::GetInstance().ReleaseJob(job_id);
- return compressed_size;
- } else {
- ARROW_LOG(WARNING)
- << "DeflateQpl HW codec failed, falling back to SW codec. (Details:
doCompressData->qpl_execute_job with error code: "
- << status << " - please refer to qpl_status in
./contrib/qpl/include/qpl/c_api/status.h)";
- QplJobHWPool::GetInstance().ReleaseJob(job_id);
- return RET_ERROR;
- }
- }
-
- /// Submit job request to the IAA hardware and then busy waiting till it
complete.
- int64_t doDecompressData(const uint8_t* source, uint32_t source_size,
uint8_t* dest, uint32_t uncompressed_size) {
- uint32_t job_id = 0;
- qpl_job* jobPtr;
- if (!(jobPtr = QplJobHWPool::GetInstance().AcquireJob(job_id))) {
- ARROW_LOG(WARNING)
- << "DeflateQpl HW codec failed, falling back to SW codec.(Details:
doDecompressData->AcquireJob fail, probably job pool exhausted)";
- return RET_ERROR;
- }
-
- // Performing a decompression operation
- jobPtr->op = qpl_op_decompress;
- jobPtr->next_in_ptr = const_cast<uint8_t*>(source);
- jobPtr->next_out_ptr = dest;
- jobPtr->available_in = source_size;
- jobPtr->available_out = uncompressed_size;
- jobPtr->flags = QPL_FLAG_FIRST | QPL_FLAG_LAST;
-
- if (auto status = qpl_execute_job(jobPtr); status == QPL_STS_OK) {
- auto decompressed_size = jobPtr->total_out;
- QplJobHWPool::GetInstance().ReleaseJob(job_id);
- return decompressed_size;
- } else {
- ARROW_LOG(WARNING)
- << "DeflateQpl HW codec failed, falling back to SW codec. (Details:
doDeCompressData->qpl_execute_job with error code: "
- << status << " - please refer to qpl_status in
./contrib/qpl/include/qpl/c_api/status.h)";
- QplJobHWPool::GetInstance().ReleaseJob(job_id);
- return RET_ERROR;
- }
- }
-
- private:
- qpl_compression_levels compressionLevel_ = qpl_default_level;
-};
-
-class SoftwareCodecDeflateQpl final {
- public:
- explicit SoftwareCodecDeflateQpl(qpl_compression_levels compressionLevel) :
compressionLevel_(compressionLevel){};
-
- ~SoftwareCodecDeflateQpl() {
- if (swJob) {
- qpl_fini_job(swJob);
- }
- }
-
- int64_t doCompressData(const uint8_t* source, uint32_t source_size, uint8_t*
dest, uint32_t dest_size) {
- qpl_job* jobPtr = getJobCodecPtr();
- // Performing a compression operation
- jobPtr->op = qpl_op_compress;
- jobPtr->next_in_ptr = const_cast<uint8_t*>(source);
- jobPtr->next_out_ptr = dest;
- jobPtr->available_in = source_size;
- jobPtr->level = compressionLevel_;
- jobPtr->available_out = dest_size;
- jobPtr->flags = QPL_FLAG_FIRST | QPL_FLAG_DYNAMIC_HUFFMAN | QPL_FLAG_LAST
| QPL_FLAG_OMIT_VERIFY;
-
- if (auto status = qpl_execute_job(jobPtr); status != QPL_STS_OK) {
- throw GlutenException(
- "Execution of DeflateQpl software fallback codec failed. (Details:
qpl_init_job with error code: " +
- std::to_string(status) + " - please refer to qpl_status in
./contrib/qpl/include/qpl/c_api/status.h)");
- }
-
- return jobPtr->total_out;
- }
-
- int64_t doDecompressData(const uint8_t* source, uint32_t source_size,
uint8_t* dest, uint32_t uncompressed_size) {
- qpl_job* jobPtr = getJobCodecPtr();
-
- // Performing a decompression operation
- jobPtr->op = qpl_op_decompress;
- jobPtr->next_in_ptr = const_cast<uint8_t*>(source);
- jobPtr->next_out_ptr = dest;
- jobPtr->available_in = source_size;
- jobPtr->available_out = uncompressed_size;
- jobPtr->flags = QPL_FLAG_FIRST | QPL_FLAG_LAST;
-
- if (auto status = qpl_execute_job(jobPtr); status != QPL_STS_OK) {
- throw GlutenException(
- "Execution of DeflateQpl software fallback codec failed. (Details:
qpl_init_job with error code: " +
- std::to_string(status) + " - please refer to qpl_status in
./contrib/qpl/include/qpl/c_api/status.h)");
- }
-
- return jobPtr->total_out;
- }
-
- private:
- qpl_job* swJob = nullptr;
- std::unique_ptr<uint8_t[]> sw_buffer;
- qpl_compression_levels compressionLevel_ = qpl_default_level;
-
- qpl_job* getJobCodecPtr() {
- if (!swJob) {
- uint32_t size = 0;
- qpl_get_job_size(qpl_path_software, &size);
-
- sw_buffer = std::make_unique<uint8_t[]>(size);
- swJob = reinterpret_cast<qpl_job*>(sw_buffer.get());
-
- // Job initialization
- if (auto status = qpl_init_job(qpl_path_software, swJob); status !=
QPL_STS_OK)
- throw GlutenException(
- "Initialization of DeflateQpl software fallback codec failed.
(Details: qpl_init_job with error code: " +
- std::to_string(status) + " - please refer to qpl_status in
./contrib/qpl/include/qpl/c_api/status.h)");
- }
- return swJob;
- }
-};
-
-class QplGzipCodec final : public arrow::util::Codec {
- public:
- explicit QplGzipCodec(qpl_compression_levels compressionLevel)
- : hwCodec_(std::make_unique<HardwareCodecDeflateQpl>(compressionLevel)),
- swCodec_(std::make_unique<SoftwareCodecDeflateQpl>(compressionLevel))
{}
-
- arrow::Result<int64_t>
- Compress(int64_t input_len, const uint8_t* input, int64_t output_buffer_len,
uint8_t* output_buffer) override {
- auto res = HardwareCodecDeflateQpl::RET_ERROR;
- if (QplJobHWPool::GetInstance().IsJobPoolReady()) {
- res = hwCodec_->doCompressData(input, input_len, output_buffer,
output_buffer_len);
- }
- if (res == HardwareCodecDeflateQpl::RET_ERROR) {
- return swCodec_->doCompressData(input, input_len, output_buffer,
output_buffer_len);
- }
- return res;
- }
-
- arrow::Result<int64_t>
- Decompress(int64_t input_len, const uint8_t* input, int64_t
output_buffer_len, uint8_t* output_buffer) override {
- auto res = HardwareCodecDeflateQpl::RET_ERROR;
- if (QplJobHWPool::GetInstance().IsJobPoolReady()) {
- res = hwCodec_->doDecompressData(input, input_len, output_buffer,
output_buffer_len);
- }
- if (res == HardwareCodecDeflateQpl::RET_ERROR) {
- return swCodec_->doDecompressData(input, input_len, output_buffer,
output_buffer_len);
- }
- return res;
- }
-
- int64_t MaxCompressedLen(int64_t input_len, const uint8_t*
ARROW_ARG_UNUSED(input)) override {
- ARROW_DCHECK_GE(input_len, 0);
- /// Aligned with ZLIB
- return ((input_len) + ((input_len) >> 12) + ((input_len) >> 14) +
((input_len) >> 25) + 13LL);
- }
-
- arrow::Result<std::shared_ptr<arrow::util::Compressor>> MakeCompressor()
override {
- return arrow::Status::NotImplemented("Streaming compression unsupported
with QAT");
- }
-
- arrow::Result<std::shared_ptr<arrow::util::Decompressor>> MakeDecompressor()
override {
- return arrow::Status::NotImplemented("Streaming decompression unsupported
with QAT");
- }
-
- arrow::Compression::type compression_type() const override {
- return arrow::Compression::GZIP;
- }
-
- int minimum_compression_level() const override {
- return qpl_level_1;
- }
- int maximum_compression_level() const override {
- return qpl_high_level;
- }
- int default_compression_level() const override {
- return qpl_default_level;
- }
-
- private:
- std::unique_ptr<HardwareCodecDeflateQpl> hwCodec_;
- std::unique_ptr<SoftwareCodecDeflateQpl> swCodec_;
-};
-
-bool SupportsCodec(const std::string& codec) {
- if (std::any_of(qpl_supported_codec.begin(), qpl_supported_codec.end(),
[&](const auto& qat_codec) {
- return qat_codec == codec;
- })) {
- return true;
- }
- return false;
-}
-
-std::unique_ptr<arrow::util::Codec> MakeQplGZipCodec(int compressionLevel) {
- auto qplCompressionLevel =
static_cast<qpl_compression_levels>(compressionLevel);
- return std::unique_ptr<arrow::util::Codec>(new
QplGzipCodec(qplCompressionLevel));
-}
-
-std::unique_ptr<arrow::util::Codec> MakeDefaultQplGZipCodec() {
- return MakeQplGZipCodec(qpl_default_level);
-}
-
-} // namespace qpl
-} // namespace gluten
diff --git a/cpp/core/utils/qpl/QplCodec.h b/cpp/core/utils/qpl/QplCodec.h
deleted file mode 100644
index 249b42cd00..0000000000
--- a/cpp/core/utils/qpl/QplCodec.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-#include <arrow/util/compression.h>
-#include <utils/qpl/QplJobPool.h>
-
-namespace gluten {
-namespace qpl {
-
-static const std::vector<std::string> qpl_supported_codec = {"gzip"};
-
-bool SupportsCodec(const std::string& codec);
-
-std::unique_ptr<arrow::util::Codec> MakeQplGZipCodec(int compressionLevel);
-
-std::unique_ptr<arrow::util::Codec> MakeDefaultQplGZipCodec();
-
-} // namespace qpl
-} // namespace gluten
diff --git a/cpp/core/utils/qpl/QplJobPool.cc b/cpp/core/utils/qpl/QplJobPool.cc
deleted file mode 100644
index 87a4b76872..0000000000
--- a/cpp/core/utils/qpl/QplJobPool.cc
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "utils/qpl/QplJobPool.h"
-#include "utils/Macros.h"
-
-#include <arrow/util/logging.h>
-#include <iostream>
-
-namespace gluten {
-namespace qpl {
-
-std::array<qpl_job*, QplJobHWPool::MAX_JOB_NUMBER> QplJobHWPool::jobPool;
-std::array<std::atomic_bool, QplJobHWPool::MAX_JOB_NUMBER>
QplJobHWPool::jobLocks;
-bool QplJobHWPool::jobPoolReady = false;
-std::unique_ptr<uint8_t[]> QplJobHWPool::hwJobsBuffer;
-
-QplJobHWPool& QplJobHWPool::GetInstance() {
- static QplJobHWPool pool;
- return pool;
-}
-
-QplJobHWPool::QplJobHWPool() : randomEngine(std::random_device()()),
distribution(0, MAX_JOB_NUMBER - 1) {
- uint64_t initTime = 0;
- TIME_NANO(initTime, InitJobPool());
- DLOG(INFO) << "Init job pool took " << 1.0 * initTime / 1e6 << "ms";
-}
-
-QplJobHWPool::~QplJobHWPool() {
- for (uint32_t i = 0; i < MAX_JOB_NUMBER; ++i) {
- if (jobPool[i]) {
- while (!tryLockJob(i))
- ;
- qpl_fini_job(jobPool[i]);
- unLockJob(i);
- jobPool[i] = nullptr;
- }
- }
- jobPoolReady = false;
-}
-
-void QplJobHWPool::InitJobPool() {
- uint32_t jobSize = 0;
- const char* qpl_version = qpl_get_library_version();
-
- // Get size required for saving a single qpl job object
- qpl_get_job_size(qpl_path_hardware, &jobSize);
- // Allocate entire buffer for storing all job objects
- hwJobsBuffer = std::make_unique<uint8_t[]>(jobSize * MAX_JOB_NUMBER);
- // Initialize pool for storing all job object pointers
- // Reallocate buffer by shifting address offset for each job object.
- for (uint32_t index = 0; index < MAX_JOB_NUMBER; ++index) {
- qpl_job* qplJobPtr = reinterpret_cast<qpl_job*>(hwJobsBuffer.get() + index
* jobSize);
- if (auto status = qpl_init_job(qpl_path_hardware, qplJobPtr); status !=
QPL_STS_OK) {
- jobPoolReady = false;
- ARROW_LOG(WARNING)
- << "Initialization of hardware-assisted DeflateQpl codec failed at
index: " << index
- << ", falling back to SW codec. (Details: QplJobHWPool->qpl_init_job
with error code: " << status
- << " - please refer to qpl_status in
./contrib/qpl/include/qpl/c_api/status.h). "
- << "Please check if Intel In-Memory Analytics Accelerator (IAA) is
properly set up. QPL Version: "
- << qpl_version;
- return;
- }
- jobPool[index] = qplJobPtr;
- jobLocks[index].store(false);
- }
- ARROW_LOG(WARNING) << "Initialization of hardware-assisted DeflateQpl codec
succeeded.";
- jobPoolReady = true;
-}
-
-qpl_job* QplJobHWPool::AcquireJob(uint32_t& jobId) {
- if (!IsJobPoolReady()) {
- return nullptr;
- }
- uint32_t retry = 0;
- auto index = distribution(randomEngine);
- while (!tryLockJob(index)) {
- index = distribution(randomEngine);
- retry++;
- if (retry > MAX_JOB_NUMBER) {
- return nullptr;
- }
- }
- jobId = MAX_JOB_NUMBER - index;
- DLOG(INFO) << "Acquired job index " << index << " after " << retry << "
retries.";
- return jobPool[index];
-}
-
-void QplJobHWPool::ReleaseJob(uint32_t jobId) {
- if (IsJobPoolReady()) {
- auto index = MAX_JOB_NUMBER - jobId;
- unLockJob(index);
- }
-}
-
-bool QplJobHWPool::tryLockJob(uint32_t index) {
- CheckJobIndex(index);
- bool expected = false;
- return jobLocks[index].compare_exchange_strong(expected, true);
-}
-
-void QplJobHWPool::unLockJob(uint32_t index) {
- CheckJobIndex(index);
- jobLocks[index].store(false);
-}
-
-} // namespace qpl
-} // namespace gluten
diff --git a/cpp/core/utils/qpl/QplJobPool.h b/cpp/core/utils/qpl/QplJobPool.h
deleted file mode 100644
index 929f7691fa..0000000000
--- a/cpp/core/utils/qpl/QplJobPool.h
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-#include <atomic>
-#include <cstdint>
-#include <memory>
-#include <random>
-#include <thread>
-#include <utility>
-#include <vector>
-
-#include <qpl/qpl.h>
-#include "utils/Exception.h"
-
-namespace gluten {
-namespace qpl {
-
-/// QplJobHWPool is resource pool to provide the job objects, which is
-/// used for storing context information during.
-/// Memory for QPL job will be allocated when the QPLJobHWPool instance is
created
-///
-// QPL job can offload RLE-decoding/Filter/(De)compression works to hardware
accelerator.
-class QplJobHWPool {
- public:
- static QplJobHWPool& GetInstance();
-
- /// Acquire QPL job
- ///
- /// @param jobId QPL job id, used when release QPL job
- /// \return Pointer to the QPL job. If acquire job failed, return nullptr.
- qpl_job* AcquireJob(uint32_t& jobId);
-
- /// \brief Release QPL job by the jobId.
- void ReleaseJob(uint32_t jobId);
-
- /// \brief Return if the QPL job is allocated sucessfully.
- static const bool& IsJobPoolReady() {
- return jobPoolReady;
- }
-
- private:
- QplJobHWPool();
- ~QplJobHWPool();
- static void InitJobPool();
- bool tryLockJob(uint32_t index);
- void unLockJob(uint32_t index);
-
- static inline void CheckJobIndex(uint32_t index) {
- if (index >= MAX_JOB_NUMBER) {
- throw GlutenException("Index exceeds MAX_JOB_NUMBER 512: " +
std::to_string(index));
- }
- }
-
- /// Max jobs in QPL_JOB_POOL
- static constexpr auto MAX_JOB_NUMBER = 64;
- /// Entire buffer for storing all job objects
- static std::unique_ptr<uint8_t[]> hwJobsBuffer;
- /// Job pool for storing all job object pointers
- static std::array<qpl_job*, MAX_JOB_NUMBER> jobPool;
- /// Locks for accessing each job object pointers
- static std::array<std::atomic_bool, MAX_JOB_NUMBER> jobLocks;
-
- static bool jobPoolReady;
- std::mt19937 randomEngine;
- std::uniform_int_distribution<int> distribution;
-};
-
-} // namespace qpl
-} // namespace gluten
diff --git a/cpp/velox/benchmarks/GenericBenchmark.cc
b/cpp/velox/benchmarks/GenericBenchmark.cc
index 24e8d3b62b..98c92b732b 100644
--- a/cpp/velox/benchmarks/GenericBenchmark.cc
+++ b/cpp/velox/benchmarks/GenericBenchmark.cc
@@ -58,10 +58,7 @@ DEFINE_string(
"rr",
"Short partitioning name. Valid options are rr, hash, range, single,
random (only for test purpose)");
DEFINE_bool(rss, false, "Mocking rss.");
-DEFINE_string(
- compression,
- "lz4",
- "Specify the compression codec. Valid options are none, lz4, zstd,
qat_gzip, qat_zstd, iaa_gzip");
+DEFINE_string(compression, "lz4", "Specify the compression codec. Valid
options are none, lz4, zstd");
DEFINE_int32(shuffle_partitions, 200, "Number of shuffle split (reducer)
partitions");
DEFINE_bool(shuffle_dictionary, false, "Whether to enable dictionary encoding
for shuffle write.");
@@ -172,15 +169,6 @@ void setCompressionTypeFromFlag(arrow::Compression::type&
compressionType, Codec
compressionType = arrow::Compression::LZ4_FRAME;
} else if (FLAGS_compression == "zstd") {
compressionType = arrow::Compression::ZSTD;
- } else if (FLAGS_compression == "qat_gzip") {
- codecBackend = CodecBackend::QAT;
- compressionType = arrow::Compression::GZIP;
- } else if (FLAGS_compression == "qat_zstd") {
- codecBackend = CodecBackend::QAT;
- compressionType = arrow::Compression::ZSTD;
- } else if (FLAGS_compression == "iaa_gzip") {
- codecBackend = CodecBackend::IAA;
- compressionType = arrow::Compression::GZIP;
} else {
throw GlutenException("Unrecognized compression type: " +
FLAGS_compression);
}
diff --git a/cpp/velox/compute/VeloxBackend.cc
b/cpp/velox/compute/VeloxBackend.cc
index 25ad26a10d..96fa3e5f4c 100644
--- a/cpp/velox/compute/VeloxBackend.cc
+++ b/cpp/velox/compute/VeloxBackend.cc
@@ -27,9 +27,6 @@
#ifdef GLUTEN_ENABLE_QAT
#include "utils/qat/QatCodec.h"
#endif
-#ifdef GLUTEN_ENABLE_IAA
-#include "utils/qpl/QplCodec.h"
-#endif
#ifdef GLUTEN_ENABLE_GPU
#include "velox/experimental/cudf/exec/ToCudf.h"
#endif
diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh
index 309c3dc3e3..aeaf872804 100755
--- a/dev/builddeps-veloxbe.sh
+++ b/dev/builddeps-veloxbe.sh
@@ -32,7 +32,6 @@ ENABLE_JEMALLOC_STATS=OFF
BUILD_VELOX_TESTS=OFF
BUILD_VELOX_BENCHMARKS=OFF
ENABLE_QAT=OFF
-ENABLE_IAA=OFF
ENABLE_GCS=OFF
ENABLE_S3=OFF
ENABLE_HDFS=OFF
@@ -88,10 +87,6 @@ do
ENABLE_QAT=("${arg#*=}")
shift # Remove argument name from processing
;;
- --enable_iaa=*)
- ENABLE_IAA=("${arg#*=}")
- shift # Remove argument name from processing
- ;;
--enable_gcs=*)
ENABLE_GCS=("${arg#*=}")
shift # Remove argument name from processing
@@ -241,7 +236,6 @@ function build_gluten_cpp {
-DBUILD_BENCHMARKS=$BUILD_BENCHMARKS \
-DENABLE_JEMALLOC_STATS=$ENABLE_JEMALLOC_STATS \
-DENABLE_QAT=$ENABLE_QAT \
- -DENABLE_IAA=$ENABLE_IAA \
-DENABLE_GCS=$ENABLE_GCS \
-DENABLE_S3=$ENABLE_S3 \
-DENABLE_HDFS=$ENABLE_HDFS \
diff --git a/docs/get-started/Velox.md b/docs/get-started/Velox.md
index 862bea2970..64d35bba1c 100644
--- a/docs/get-started/Velox.md
+++ b/docs/get-started/Velox.md
@@ -594,4 +594,4 @@ This feature has been tested through a series of tests, and
we are collecting mo
# Accelerators
-Please refer [QAT](VeloxQAT.md) [IAA](VeloxIAA.md) for details
+Please refer [QAT](VeloxQAT.md) for details
diff --git a/docs/get-started/VeloxIAA.md b/docs/get-started/VeloxIAA.md
deleted file mode 100644
index ffd74bf6c6..0000000000
--- a/docs/get-started/VeloxIAA.md
+++ /dev/null
@@ -1,75 +0,0 @@
----
-layout: page
-title: IAA Support in Velox Backend
-nav_order: 1
-parent: Getting-Started
----
-
-
-# Intel® In-memory Analytics Accelerator (IAA/IAX) support
-
-Similar to Intel® QAT, Gluten supports using Intel® In-memory Analytics
Accelerator (IAA, also called IAX) for data compression during Spark Shuffle.
It benefits from IAA Hardware-based acceleration on compression/decompression,
and uses Gzip as compression format for higher compression ratio to reduce the
pressure on disks and network transmission.
-
-This feature is based on Intel® [QPL](https://github.com/intel/qpl).
-
-## Build Gluten with IAA
-
-Gluten will internally build and link to a specific version of QPL library,
but extra environment setup is still required. Please refer to [QPL
Installation
Guide](https://intel.github.io/qpl/documentation/get_started_docs/installation.html)
to install dependencies and configure accelerators.
-
-**This step is required if your application is running as Non-root user.**
-Create a group for the users who have privilege to use IAA, and grant group
iaa read/write access to the IAA Work-Queues.
-
-```bash
-sudo groupadd iaa
-sudo usermod -aG iaa username # need to relogin
-sudo chgrp -R iaa /dev/iax
-sudo chmod -R g+rw /dev/iax
-```
-
-After the set-up, you can now build Gluten with QAT. Below command is used to
enable this feature
-
-```bash
-cd /path/to/gluten
-
-## The script builds four jars for spark 3.2.2, 3.3.1, 3.4.3 and 3.5.1.
-./dev/buildbundle-veloxbe.sh --enable_iaa=ON
-```
-
-## Enable IAA with Gzip Compression for shuffle compression
-
-1. To enable QAT at run-time, first make sure you have configured the IAA
Work-Queues correctly, and the file permissions of /dev/iax/wqX.0 are correct.
-
-```bash
-sudo ls -l /dev/iax
-```
-
-The output should be like:
-
-```
-total 0
-crw-rw---- 1 root iaa 509, 0 Apr 5 18:54 wq1.0
-crw-rw---- 1 root iaa 509, 5 Apr 5 18:54 wq11.0
-crw-rw---- 1 root iaa 509, 6 Apr 5 18:54 wq13.0
-crw-rw---- 1 root iaa 509, 7 Apr 5 18:54 wq15.0
-crw-rw---- 1 root iaa 509, 1 Apr 5 18:54 wq3.0
-crw-rw---- 1 root iaa 509, 2 Apr 5 18:54 wq5.0
-crw-rw---- 1 root iaa 509, 3 Apr 5 18:54 wq7.0
-crw-rw---- 1 root iaa 509, 4 Apr 5 18:54 wq9.0
-```
-
-2. Extra Gluten configurations are required when starting Spark application
-
-```
---conf spark.gluten.sql.columnar.shuffle.codec=gzip
---conf spark.gluten.sql.columnar.shuffle.codecBackend=iaa
-```
-
-## IAA references
-
-**Intel® IAA Enabling Guide**
-
-Check out the [Intel® In-Memory Analytics Accelerator (Intel® IAA) Enabling
Guide](https://www.intel.com/content/www/us/en/developer/articles/technical/intel-iaa-enabling-guide.html)
-
-**Intel® QPL Documentation**
-
-Check out the [Intel® Query Processing Library (Intel® QPL)
Documentation](https://intel.github.io/qpl/index.html)
diff --git a/docs/get-started/build-guide.md b/docs/get-started/build-guide.md
index 5706487e87..ffd02ad7ab 100644
--- a/docs/get-started/build-guide.md
+++ b/docs/get-started/build-guide.md
@@ -16,8 +16,6 @@ Please set them via `--`, e.g. `--build_type=Release`.
| build_benchmarks | Build gluten cpp benchmarks.
| OFF |
| enable_jemalloc_stats | Print jemalloc stats for debugging.
| OFF |
| enable_qat | Enable QAT for shuffle data de/compression.
| OFF |
-| enable_iaa | Enable IAA for shuffle data de/compression.
| OFF |
-| enable_hbm | Enable HBM allocator.
| OFF |
| enable_s3 | Build with S3 support.
| OFF |
| enable_gcs | Build with GCS support.
| OFF |
| enable_hdfs | Build with HDFS support.
| OFF |
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
index c5bd12259b..07a862e175 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
@@ -207,14 +207,10 @@ class GlutenConfig(conf: SQLConf) extends
GlutenCoreConfig(conf) {
getConf(COLUMNAR_SHUFFLE_COMPRESSION_MODE)
def columnarShuffleCodecBackend: Option[String] =
getConf(COLUMNAR_SHUFFLE_CODEC_BACKEND)
- .filter(Set(GLUTEN_QAT_BACKEND_NAME, GLUTEN_IAA_BACKEND_NAME).contains(_))
def columnarShuffleEnableQat: Boolean =
columnarShuffleCodecBackend.contains(GlutenConfig.GLUTEN_QAT_BACKEND_NAME)
- def columnarShuffleEnableIaa: Boolean =
- columnarShuffleCodecBackend.contains(GlutenConfig.GLUTEN_IAA_BACKEND_NAME)
-
def columnarShuffleCompressionThreshold: Int =
getConf(COLUMNAR_SHUFFLE_COMPRESSION_THRESHOLD)
@@ -445,9 +441,6 @@ object GlutenConfig {
// QAT config
val GLUTEN_QAT_BACKEND_NAME = "qat"
val GLUTEN_QAT_SUPPORTED_CODEC: Set[String] = Set("gzip", "zstd")
- // IAA config
- val GLUTEN_IAA_BACKEND_NAME = "iaa"
- val GLUTEN_IAA_SUPPORTED_CODEC: Set[String] = Set("gzip")
// Private Spark configs.
val SPARK_OVERHEAD_SIZE_KEY = "spark.executor.memoryOverhead"
@@ -1037,9 +1030,7 @@ object GlutenConfig {
.doc(
"By default, the supported codecs are lz4 and zstd. " +
"When spark.gluten.sql.columnar.shuffle.codecBackend=qat," +
- "the supported codecs are gzip and zstd. " +
- "When spark.gluten.sql.columnar.shuffle.codecBackend=iaa," +
- "the supported codec is gzip.")
+ "the supported codecs are gzip and zstd.")
.stringConf
.transform(_.toLowerCase(Locale.ROOT))
.createOptional
diff --git
a/gluten-substrait/src/main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala
b/gluten-substrait/src/main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala
index b06ccd8b29..ec67b07207 100644
---
a/gluten-substrait/src/main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala
+++
b/gluten-substrait/src/main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala
@@ -60,8 +60,6 @@ object GlutenShuffleUtils {
val glutenCodecKey = GlutenConfig.COLUMNAR_SHUFFLE_CODEC.key
if (glutenConfig.columnarShuffleEnableQat) {
checkCodecValues(glutenCodecKey, codec,
GlutenConfig.GLUTEN_QAT_SUPPORTED_CODEC)
- } else if (glutenConfig.columnarShuffleEnableIaa) {
- checkCodecValues(glutenCodecKey, codec,
GlutenConfig.GLUTEN_IAA_SUPPORTED_CODEC)
} else {
checkCodecValues(
glutenCodecKey,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
