This is an automated email from the ASF dual-hosted git repository.
marong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new efe73aea6c [VL][Minor] Remove unused code for shuffle compression mode
(#10609)
efe73aea6c is described below
commit efe73aea6ce46c9792c9295c0de2283f93612d9d
Author: Rong Ma <[email protected]>
AuthorDate: Tue Sep 2 15:29:24 2025 +0100
[VL][Minor] Remove unused code for shuffle compression mode (#10609)
---
cpp/core/jni/JniCommon.h | 12 --
cpp/core/jni/JniWrapper.cc | 3 +-
cpp/core/shuffle/Utils.cc | 183 +--------------------
cpp/core/shuffle/Utils.h | 17 --
cpp/core/utils/Compression.cc | 2 +-
cpp/core/utils/Compression.h | 6 +-
cpp/velox/benchmarks/GenericBenchmark.cc | 2 +-
cpp/velox/compute/VeloxRuntime.cc | 2 +-
cpp/velox/jni/VeloxJniWrapper.cc | 8 +-
cpp/velox/tests/VeloxShuffleWriterTest.cc | 4 +-
docs/Configuration.md | 1 -
.../org/apache/gluten/config/GlutenConfig.scala | 12 --
12 files changed, 15 insertions(+), 237 deletions(-)
diff --git a/cpp/core/jni/JniCommon.h b/cpp/core/jni/JniCommon.h
index bc59a2a485..fdc084fd2f 100644
--- a/cpp/core/jni/JniCommon.h
+++ b/cpp/core/jni/JniCommon.h
@@ -365,18 +365,6 @@ static inline gluten::CodecBackend getCodecBackend(JNIEnv*
env, jstring codecBac
throw std::invalid_argument("Not support this codec backend " +
codecBackend);
}
-static inline gluten::CompressionMode getCompressionMode(JNIEnv* env, jstring
compressionModeJstr) {
- GLUTEN_DCHECK(compressionModeJstr != nullptr, "CompressionMode cannot be
null");
- auto compressionMode = jStringToCString(env, compressionModeJstr);
- if (compressionMode == "buffer") {
- return gluten::CompressionMode::BUFFER;
- } else if (compressionMode == "rowvector") {
- return gluten::CompressionMode::ROWVECTOR;
- } else {
- throw std::invalid_argument("Not support this compression mode " +
compressionMode);
- }
-}
-
/*
NOTE: the class must be thread safe
*/
diff --git a/cpp/core/jni/JniWrapper.cc b/cpp/core/jni/JniWrapper.cc
index 523c4a7494..d124178c02 100644
--- a/cpp/core/jni/JniWrapper.cc
+++ b/cpp/core/jni/JniWrapper.cc
@@ -827,7 +827,8 @@
Java_org_apache_gluten_vectorized_LocalPartitionWriterJniWrapper_createPartition
auto partitionWriter = std::make_shared<LocalPartitionWriter>(
numPartitions,
- createArrowIpcCodec(getCompressionType(env, codecJstr),
getCodecBackend(env, codecBackendJstr), compressionLevel),
+ createCompressionCodec(
+ getCompressionType(env, codecJstr), getCodecBackend(env,
codecBackendJstr), compressionLevel),
ctx->memoryManager(),
partitionWriterOptions,
dataFile,
diff --git a/cpp/core/shuffle/Utils.cc b/cpp/core/shuffle/Utils.cc
index 4e4dcb5d7e..51667274ed 100644
--- a/cpp/core/shuffle/Utils.cc
+++ b/cpp/core/shuffle/Utils.cc
@@ -32,128 +32,8 @@
#include "utils/Timer.h"
namespace gluten {
-namespace {
-arrow::Result<std::shared_ptr<arrow::Array>> makeNullBinaryArray(
- std::shared_ptr<arrow::DataType> type,
- arrow::MemoryPool* pool) {
- ARROW_ASSIGN_OR_RAISE(auto offsetBuffer,
arrow::AllocateResizableBuffer(kSizeOfIpcOffsetBuffer << 1, pool));
- // set the first offset to 0, and set the value offset
- uint8_t* offsetaddr = offsetBuffer->mutable_data();
- memset(offsetaddr, 0, kSizeOfIpcOffsetBuffer);
- // second value offset 0
- memset(offsetaddr + kSizeOfIpcOffsetBuffer, 0, kSizeOfIpcOffsetBuffer);
- // If it is not compressed array, null valueBuffer
- // worked, but if compress, will core dump at buffer::size(), so replace by
kNullBuffer
- static std::shared_ptr<arrow::Buffer> kNullBuffer =
std::make_shared<arrow::Buffer>(nullptr, 0);
- return arrow::MakeArray(arrow::ArrayData::Make(type, 1, {nullptr,
std::move(offsetBuffer), kNullBuffer}));
-}
-
-arrow::Result<std::shared_ptr<arrow::Array>> makeBinaryArray(
- std::shared_ptr<arrow::DataType> type,
- std::shared_ptr<arrow::Buffer> valueBuffer,
- arrow::MemoryPool* pool) {
- if (valueBuffer == nullptr) {
- return makeNullBinaryArray(type, pool);
- }
-
- ARROW_ASSIGN_OR_RAISE(auto offsetBuffer,
arrow::AllocateResizableBuffer(kSizeOfIpcOffsetBuffer << 1, pool));
- // set the first offset to 0, and set the value offset
- uint8_t* offsetaddr = offsetBuffer->mutable_data();
- memset(offsetaddr, 0, kSizeOfIpcOffsetBuffer);
- int64_t length = valueBuffer->size();
- memcpy(offsetaddr + kSizeOfIpcOffsetBuffer,
reinterpret_cast<uint8_t*>(&length), kSizeOfIpcOffsetBuffer);
- return arrow::MakeArray(arrow::ArrayData::Make(type, 1, {nullptr,
std::move(offsetBuffer), valueBuffer}));
-}
-
-// Length buffer layout |compressionMode|buffers.size()|buffer1
unCompressedLength|buffer1 compressedLength| buffer2...
-arrow::Status getLengthBufferAndValueBufferOneByOne(
- const std::vector<std::shared_ptr<arrow::Buffer>>& buffers,
- arrow::MemoryPool* pool,
- arrow::util::Codec* codec,
- std::shared_ptr<arrow::ResizableBuffer>& lengthBuffer,
- std::shared_ptr<arrow::ResizableBuffer>& valueBuffer) {
- ARROW_ASSIGN_OR_RAISE(
- lengthBuffer, arrow::AllocateResizableBuffer((1 + 1 + buffers.size() *
2) * sizeof(int64_t), pool));
- auto lengthBufferPtr = (int64_t*)(lengthBuffer->mutable_data());
- // Write compression mode.
- *lengthBufferPtr++ = CompressionMode::BUFFER;
- // Write number of buffers.
- *lengthBufferPtr++ = buffers.size();
-
- int64_t compressedBufferMaxSize = getMaxCompressedBufferSize(buffers, codec);
- ARROW_ASSIGN_OR_RAISE(valueBuffer,
arrow::AllocateResizableBuffer(compressedBufferMaxSize, pool));
- int64_t compressValueOffset = 0;
- for (auto& buffer : buffers) {
- if (buffer != nullptr && buffer->size() != 0) {
- int64_t actualLength;
- int64_t maxLength = codec->MaxCompressedLen(buffer->size(), nullptr);
- ARROW_ASSIGN_OR_RAISE(
- actualLength,
- codec->Compress(
- buffer->size(), buffer->data(), maxLength,
valueBuffer->mutable_data() + compressValueOffset));
- compressValueOffset += actualLength;
- *lengthBufferPtr++ = buffer->size();
- *lengthBufferPtr++ = actualLength;
- } else {
- *lengthBufferPtr++ = 0;
- *lengthBufferPtr++ = 0;
- }
- }
- RETURN_NOT_OK(valueBuffer->Resize(compressValueOffset, /*shrink*/ true));
- return arrow::Status::OK();
-}
-
-// Length buffer layout |compressionMode|buffer unCompressedLength|buffer
compressedLength|buffers.size()| buffer1 size
-// | buffer2 size
-arrow::Status getLengthBufferAndValueBufferStream(
- const std::vector<std::shared_ptr<arrow::Buffer>>& buffers,
- arrow::MemoryPool* pool,
- arrow::util::Codec* codec,
- std::shared_ptr<arrow::ResizableBuffer>& lengthBuffer,
- std::shared_ptr<arrow::ResizableBuffer>& compressedBuffer) {
- ARROW_ASSIGN_OR_RAISE(lengthBuffer, arrow::AllocateResizableBuffer((1 + 3 +
buffers.size()) * sizeof(int64_t), pool));
- auto originalBufferSize = getBufferSize(buffers);
-
- // because 64B align, uncompressedBuffer size maybe bigger than
unCompressedBufferSize which is
- // getBuffersSize(buffers), then cannot use this size
- ARROW_ASSIGN_OR_RAISE(auto uncompressedBuffer,
arrow::AllocateResizableBuffer(originalBufferSize, pool));
- int64_t uncompressedSize = uncompressedBuffer->size();
-
- auto lengthBufferPtr = (int64_t*)(lengthBuffer->mutable_data());
- // First write metadata.
- // Write compression mode.
- *lengthBufferPtr++ = CompressionMode::ROWVECTOR;
- // Store uncompressed size.
- *lengthBufferPtr++ = uncompressedSize; // uncompressedLength
- // Skip compressed size and update later.
- auto compressedLengthPtr = lengthBufferPtr++;
- // Store number of buffers.
- *lengthBufferPtr++ = buffers.size();
-
- int64_t compressValueOffset = 0;
- for (auto& buffer : buffers) {
- // Copy all buffers into one big buffer.
- if (buffer != nullptr && buffer->size() != 0) {
- *lengthBufferPtr++ = buffer->size();
- memcpy(uncompressedBuffer->mutable_data() + compressValueOffset,
buffer->data(), buffer->size());
- compressValueOffset += buffer->size();
- } else {
- *lengthBufferPtr++ = 0;
- }
- }
- // Compress the big buffer.
- int64_t maxLength = codec->MaxCompressedLen(uncompressedSize, nullptr);
- ARROW_ASSIGN_OR_RAISE(compressedBuffer,
arrow::AllocateResizableBuffer(maxLength, pool));
- ARROW_ASSIGN_OR_RAISE(
- int64_t actualLength,
- codec->Compress(uncompressedSize, uncompressedBuffer->data(), maxLength,
compressedBuffer->mutable_data()));
- RETURN_NOT_OK(compressedBuffer->Resize(actualLength, /*shrink*/ true));
-
- // Update compressed size.
- *compressedLengthPtr = actualLength;
- return arrow::Status::OK();
-}
+namespace {
uint64_t roundUpToPageSize(uint64_t value) {
static auto pageSize = static_cast<size_t>(arrow::internal::GetPageSize());
@@ -162,67 +42,8 @@ uint64_t roundUpToPageSize(uint64_t value) {
DCHECK_EQ(pageMask & pageSize, pageSize);
return (value + pageSize - 1) & pageMask;
}
-} // namespace
-
-arrow::Result<std::shared_ptr<arrow::RecordBatch>> makeCompressedRecordBatch(
- uint32_t numRows,
- const std::vector<std::shared_ptr<arrow::Buffer>>& buffers,
- const std::shared_ptr<arrow::Schema> compressWriteSchema,
- arrow::MemoryPool* pool,
- arrow::util::Codec* codec,
- int32_t bufferCompressThreshold,
- CompressionMode compressionMode,
- int64_t& compressionTime) {
- ScopedTimer timer{&compressionTime};
- std::vector<std::shared_ptr<arrow::Array>> arrays;
- // header col, numRows, compressionType
- {
- ARROW_ASSIGN_OR_RAISE(auto headerBuffer,
arrow::AllocateResizableBuffer(sizeof(uint32_t) + sizeof(int32_t), pool));
- memcpy(headerBuffer->mutable_data(), &numRows, sizeof(uint32_t));
- int32_t compressType = static_cast<int32_t>(codec->compression_type());
- memcpy(headerBuffer->mutable_data() + sizeof(uint32_t), &compressType,
sizeof(int32_t));
- arrays.emplace_back();
- ARROW_ASSIGN_OR_RAISE(
- arrays.back(), makeBinaryArray(compressWriteSchema->field(0)->type(),
std::move(headerBuffer), pool));
- }
- std::shared_ptr<arrow::ResizableBuffer> lengthBuffer;
- std::shared_ptr<arrow::ResizableBuffer> valueBuffer;
- if (compressionMode == CompressionMode::BUFFER && numRows >
bufferCompressThreshold) {
- RETURN_NOT_OK(getLengthBufferAndValueBufferOneByOne(buffers, pool, codec,
lengthBuffer, valueBuffer));
- } else {
- RETURN_NOT_OK(getLengthBufferAndValueBufferStream(buffers, pool, codec,
lengthBuffer, valueBuffer));
- }
-
- arrays.emplace_back();
- ARROW_ASSIGN_OR_RAISE(arrays.back(),
makeBinaryArray(compressWriteSchema->field(1)->type(), lengthBuffer, pool));
- arrays.emplace_back();
- ARROW_ASSIGN_OR_RAISE(arrays.back(),
makeBinaryArray(compressWriteSchema->field(2)->type(), valueBuffer, pool));
- return arrow::RecordBatch::Make(compressWriteSchema, 1, {arrays});
-}
-arrow::Result<std::shared_ptr<arrow::RecordBatch>> makeUncompressedRecordBatch(
- uint32_t numRows,
- const std::vector<std::shared_ptr<arrow::Buffer>>& buffers,
- const std::shared_ptr<arrow::Schema> writeSchema,
- arrow::MemoryPool* pool) {
- std::vector<std::shared_ptr<arrow::Array>> arrays;
- // header col, numRows, compressionType
- {
- ARROW_ASSIGN_OR_RAISE(auto headerBuffer,
arrow::AllocateResizableBuffer(sizeof(uint32_t) + sizeof(int32_t), pool));
- memcpy(headerBuffer->mutable_data(), &numRows, sizeof(uint32_t));
- int32_t compressType =
static_cast<int32_t>(arrow::Compression::type::UNCOMPRESSED);
- memcpy(headerBuffer->mutable_data() + sizeof(uint32_t), &compressType,
sizeof(int32_t));
- arrays.emplace_back();
- ARROW_ASSIGN_OR_RAISE(arrays.back(),
makeBinaryArray(writeSchema->field(0)->type(), std::move(headerBuffer), pool));
- }
-
- int32_t bufferNum = writeSchema->num_fields() - 1;
- for (int32_t i = 0; i < bufferNum; i++) {
- arrays.emplace_back();
- ARROW_ASSIGN_OR_RAISE(arrays.back(), makeBinaryArray(writeSchema->field(i
+ 1)->type(), buffers[i], pool));
- }
- return arrow::RecordBatch::Make(writeSchema, 1, {arrays});
-}
+} // namespace
MmapFileStream::MmapFileStream(arrow::internal::FileDescriptor fd, uint8_t*
data, int64_t size, uint64_t prefetchSize)
: prefetchSize_(roundUpToPageSize(prefetchSize)), fd_(std::move(fd)),
data_(data), size_(size){};
diff --git a/cpp/core/shuffle/Utils.h b/cpp/core/shuffle/Utils.h
index b08e319c50..540662313e 100644
--- a/cpp/core/shuffle/Utils.h
+++ b/cpp/core/shuffle/Utils.h
@@ -71,23 +71,6 @@ int64_t getMaxCompressedBufferSize(
const std::vector<std::shared_ptr<arrow::Buffer>>& buffers,
arrow::util::Codec* codec);
-arrow::Result<std::shared_ptr<arrow::RecordBatch>> makeCompressedRecordBatch(
- uint32_t numRows,
- const std::vector<std::shared_ptr<arrow::Buffer>>& buffers,
- const std::shared_ptr<arrow::Schema> compressWriteSchema,
- arrow::MemoryPool* pool,
- arrow::util::Codec* codec,
- int32_t bufferCompressThreshold,
- CompressionMode compressionMode,
- int64_t& compressionTime);
-
-// generate the new big one row several columns binary recordbatch
-arrow::Result<std::shared_ptr<arrow::RecordBatch>> makeUncompressedRecordBatch(
- uint32_t numRows,
- const std::vector<std::shared_ptr<arrow::Buffer>>& buffers,
- const std::shared_ptr<arrow::Schema> writeSchema,
- arrow::MemoryPool* pool);
-
std::shared_ptr<arrow::Buffer> zeroLengthNullBuffer();
// MmapFileStream is used to optimize sequential file reading. It uses madvise
diff --git a/cpp/core/utils/Compression.cc b/cpp/core/utils/Compression.cc
index 913ee3b55f..236990106b 100644
--- a/cpp/core/utils/Compression.cc
+++ b/cpp/core/utils/Compression.cc
@@ -26,7 +26,7 @@
namespace gluten {
std::unique_ptr<arrow::util::Codec>
-createArrowIpcCodec(arrow::Compression::type compressedType, CodecBackend
codecBackend, int32_t compressionLevel) {
+createCompressionCodec(arrow::Compression::type compressedType, CodecBackend
codecBackend, int32_t compressionLevel) {
std::unique_ptr<arrow::util::Codec> codec;
switch (compressedType) {
case arrow::Compression::LZ4_FRAME: {
diff --git a/cpp/core/utils/Compression.h b/cpp/core/utils/Compression.h
index 5a3e170fb4..b15d68d92e 100644
--- a/cpp/core/utils/Compression.h
+++ b/cpp/core/utils/Compression.h
@@ -23,11 +23,7 @@ namespace gluten {
enum CodecBackend { NONE, QAT };
-// BUFFER mode will preallocate max compressed buffer, and then compress each
buffer to the max compressed buffer
-// ROWVECTOR mode will copy the buffers to a big buffer and then compress the
big buffer
-enum CompressionMode { BUFFER, ROWVECTOR };
-
-std::unique_ptr<arrow::util::Codec> createArrowIpcCodec(
+std::unique_ptr<arrow::util::Codec> createCompressionCodec(
arrow::Compression::type compressedType,
CodecBackend codecBackend,
int32_t compressionLevel = arrow::util::kUseDefaultCompressionLevel);
diff --git a/cpp/velox/benchmarks/GenericBenchmark.cc
b/cpp/velox/benchmarks/GenericBenchmark.cc
index ee91aa950c..d1e08c67b4 100644
--- a/cpp/velox/benchmarks/GenericBenchmark.cc
+++ b/cpp/velox/benchmarks/GenericBenchmark.cc
@@ -185,7 +185,7 @@ std::unique_ptr<arrow::util::Codec> createCodec() {
setCompressionTypeFromFlag(compressionType, codecBackend);
- return createArrowIpcCodec(compressionType, codecBackend);
+ return createCompressionCodec(compressionType, codecBackend);
}
std::shared_ptr<PartitionWriter>
diff --git a/cpp/velox/compute/VeloxRuntime.cc
b/cpp/velox/compute/VeloxRuntime.cc
index 7470ed9825..65195b7ea6 100644
--- a/cpp/velox/compute/VeloxRuntime.cc
+++ b/cpp/velox/compute/VeloxRuntime.cc
@@ -287,7 +287,7 @@ std::shared_ptr<VeloxDataSource>
VeloxRuntime::createDataSource(
std::shared_ptr<ShuffleReader> VeloxRuntime::createShuffleReader(
std::shared_ptr<arrow::Schema> schema,
ShuffleReaderOptions options) {
- auto codec = gluten::createArrowIpcCodec(options.compressionType,
options.codecBackend);
+ auto codec = gluten::createCompressionCodec(options.compressionType,
options.codecBackend);
const auto veloxCompressionKind =
arrowCompressionTypeToVelox(options.compressionType);
const auto rowType =
facebook::velox::asRowType(gluten::fromArrowSchema(schema));
diff --git a/cpp/velox/jni/VeloxJniWrapper.cc b/cpp/velox/jni/VeloxJniWrapper.cc
index 867d3ac8d3..8249365fbb 100644
--- a/cpp/velox/jni/VeloxJniWrapper.cc
+++ b/cpp/velox/jni/VeloxJniWrapper.cc
@@ -619,7 +619,8 @@
Java_org_apache_gluten_vectorized_CelebornPartitionWriterJniWrapper_createPartit
auto partitionWriter = std::make_shared<RssPartitionWriter>(
numPartitions,
- createArrowIpcCodec(getCompressionType(env, codecJstr),
getCodecBackend(env, codecBackendJstr), compressionLevel),
+ createCompressionCodec(
+ getCompressionType(env, codecJstr), getCodecBackend(env,
codecBackendJstr), compressionLevel),
ctx->memoryManager(),
partitionWriterOptions,
celebornClient);
@@ -662,7 +663,8 @@
Java_org_apache_gluten_vectorized_UnifflePartitionWriterJniWrapper_createPartiti
auto partitionWriter = std::make_shared<RssPartitionWriter>(
numPartitions,
- createArrowIpcCodec(getCompressionType(env, codecJstr),
getCodecBackend(env, codecBackendJstr), compressionLevel),
+ createCompressionCodec(
+ getCompressionType(env, codecJstr), getCodecBackend(env,
codecBackendJstr), compressionLevel),
ctx->memoryManager(),
partitionWriterOptions,
uniffleClient);
@@ -706,7 +708,7 @@ JNIEXPORT jlong JNICALL
Java_org_apache_gluten_execution_IcebergWriteJniWrapper_
jstring directory,
jstring codecJstr,
jbyteArray partition,
- jbyteArray fieldBytes) {
+ jbyteArray fieldBytes) {
JNI_METHOD_START
auto ctx = getRuntime(env, wrapper);
auto runtime = dynamic_cast<VeloxRuntime*>(ctx);
diff --git a/cpp/velox/tests/VeloxShuffleWriterTest.cc
b/cpp/velox/tests/VeloxShuffleWriterTest.cc
index 27fc300372..61cf7210a4 100644
--- a/cpp/velox/tests/VeloxShuffleWriterTest.cc
+++ b/cpp/velox/tests/VeloxShuffleWriterTest.cc
@@ -18,7 +18,6 @@
#include <arrow/c/bridge.h>
#include <arrow/io/api.h>
-#include "../utils/VeloxArrowUtils.h"
#include "config/GlutenConfig.h"
#include "shuffle/VeloxHashShuffleWriter.h"
#include "shuffle/VeloxRssSortShuffleWriter.h"
@@ -26,6 +25,7 @@
#include "tests/VeloxShuffleWriterTestBase.h"
#include "tests/utils/TestUtils.h"
#include "utils/TestAllocationListener.h"
+#include "utils/VeloxArrowUtils.h"
#include "velox/vector/tests/utils/VectorTestBase.h"
@@ -292,7 +292,7 @@ class VeloxShuffleWriterTest : public
::testing::TestWithParam<ShuffleTestParams
const auto veloxCompressionType =
arrowCompressionTypeToVelox(compressionType);
const auto schema = toArrowSchema(rowType,
getDefaultMemoryManager()->getLeafMemoryPool().get());
- auto codec = createArrowIpcCodec(compressionType, CodecBackend::NONE);
+ auto codec = createCompressionCodec(compressionType, CodecBackend::NONE);
// Set batchSize to a large value to make all batches are merged by reader.
auto deserializerFactory =
std::make_unique<gluten::VeloxShuffleReaderDeserializerFactory>(
diff --git a/docs/Configuration.md b/docs/Configuration.md
index c0853ccb01..4a81af5222 100644
--- a/docs/Configuration.md
+++ b/docs/Configuration.md
@@ -113,7 +113,6 @@ nav_order: 15
| spark.gluten.sql.columnar.shuffle.codec |
<undefined> | By default, the supported codecs are lz4 and zstd. When
spark.gluten.sql.columnar.shuffle.codecBackend=qat,the supported codecs are
gzip and zstd.
[...]
| spark.gluten.sql.columnar.shuffle.codecBackend |
<undefined> |
| spark.gluten.sql.columnar.shuffle.compression.threshold | 100
| If number of rows in a batch falls below this threshold, will copy
all buffers into one buffer to compress.
[...]
-| spark.gluten.sql.columnar.shuffle.compressionMode | buffer
| buffer means compress each buffer to pre allocated big
buffer,rowvector means to copy the buffers to a big buffer, and then compress
the buffer
[...]
| spark.gluten.sql.columnar.shuffle.dictionary.enabled | false
| Enable dictionary in hash-based shuffle.
[...]
| spark.gluten.sql.columnar.shuffle.merge.threshold | 0.25
|
| spark.gluten.sql.columnar.shuffle.readerBufferSize | 1MB
| Buffer size in bytes for shuffle reader reading input stream from
local or remote.
[...]
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
index 5547be3223..bf4006ab93 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
@@ -202,9 +202,6 @@ class GlutenConfig(conf: SQLConf) extends
GlutenCoreConfig(conf) {
def columnarShuffleCodec: Option[String] = getConf(COLUMNAR_SHUFFLE_CODEC)
- def columnarShuffleCompressionMode: String =
- getConf(COLUMNAR_SHUFFLE_COMPRESSION_MODE)
-
def columnarShuffleCodecBackend: Option[String] =
getConf(COLUMNAR_SHUFFLE_CODEC_BACKEND)
def columnarShuffleEnableQat: Boolean =
@@ -1037,15 +1034,6 @@ object GlutenConfig {
.transform(_.toLowerCase(Locale.ROOT))
.createOptional
- val COLUMNAR_SHUFFLE_COMPRESSION_MODE =
- buildConf("spark.gluten.sql.columnar.shuffle.compressionMode")
- .internal()
- .doc("buffer means compress each buffer to pre allocated big buffer," +
- "rowvector means to copy the buffers to a big buffer, and then
compress the buffer")
- .stringConf
- .checkValues(Set("buffer", "rowvector"))
- .createWithDefault("buffer")
-
val COLUMNAR_SHUFFLE_COMPRESSION_THRESHOLD =
buildConf("spark.gluten.sql.columnar.shuffle.compression.threshold")
.internal()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
