This is an automated email from the ASF dual-hosted git repository.
changchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 02ca1efcf8 [GLUTEN-1632][CH]Daily Update Clickhouse Version (20250705)
(#10127)
02ca1efcf8 is described below
commit 02ca1efcf855fb7fa7f39d2d3e1c436b6726482c
Author: Kyligence Git <[email protected]>
AuthorDate: Sat Jul 5 21:04:57 2025 -0500
[GLUTEN-1632][CH]Daily Update Clickhouse Version (20250705) (#10127)
* [GLUTEN-1632][CH]Daily Update Clickhouse Version (20250705)
* Fix benchmark build
* Fix Benchmark build due to
https://github.com/ClickHouse/ClickHouse/pull/79417
* Revert "Fix Build due to
https://github.com/ClickHouse/ClickHouse/pull/80931"
This reverts commit 02d12f65b0ee21b9f87a3ee146e7a5acd230cc84.
* Fix Build due to https://github.com/ClickHouse/ClickHouse/pull/81886
* Fix Link issue due to https://github.com/ClickHouse/ClickHouse/pull/83121
* Fix Build due to https://github.com/ClickHouse/ClickHouse/pull/82604
* Fix Build due to https://github.com/ClickHouse/ClickHouse/pull/82945
* Fix Build due to https://github.com/ClickHouse/ClickHouse/pull/83214
---------
Co-authored-by: kyligence-git <[email protected]>
Co-authored-by: Chang chen <[email protected]>
---
cpp-ch/CMakeLists.txt | 7 ++++---
cpp-ch/clickhouse.version | 4 ++--
cpp-ch/local-engine/Common/GlutenSignalHandler.cpp | 8 ++++----
cpp-ch/local-engine/Common/LoggerExtend.cpp | 3 ++-
cpp-ch/local-engine/Common/LoggerExtend.h | 6 +++++-
.../ObjectStorages/registerGlutenDiskObjectStorage.cpp | 3 ++-
cpp-ch/local-engine/Parser/RelParsers/CrossRelParser.cpp | 2 ++
cpp-ch/local-engine/Parser/RelParsers/JoinRelParser.cpp | 4 ++++
.../Storages/MergeTree/SparkStorageMergeTree.h | 4 +---
.../Storages/Parquet/VectorizedParquetRecordReader.h | 3 +++
.../Storages/SubstraitSource/ExcelTextFormatFile.cpp | 3 +--
.../Storages/SubstraitSource/ExcelTextFormatFile.h | 3 +--
.../local-engine/Storages/SubstraitSource/FileReader.cpp | 15 +++++++++------
cpp-ch/local-engine/Storages/SubstraitSource/FileReader.h | 2 +-
cpp-ch/local-engine/Storages/SubstraitSource/FormatFile.h | 6 ++----
.../Storages/SubstraitSource/Iceberg/IcebergReader.h | 6 ------
.../Storages/SubstraitSource/JSONFormatFile.cpp | 3 +--
.../Storages/SubstraitSource/JSONFormatFile.h | 3 +--
.../Storages/SubstraitSource/ORCFormatFile.cpp | 6 ++----
.../local-engine/Storages/SubstraitSource/ORCFormatFile.h | 3 +--
.../Storages/SubstraitSource/ParquetFormatFile.cpp | 13 +++++++++----
.../Storages/SubstraitSource/ParquetFormatFile.h | 3 +--
.../Storages/SubstraitSource/SubstraitFileSource.cpp | 12 ++++++------
.../Storages/SubstraitSource/SubstraitFileSource.h | 13 +++----------
.../Storages/SubstraitSource/SubstraitFileSourceStep.cpp | 12 +++++-------
.../Storages/SubstraitSource/SubstraitFileSourceStep.h | 5 +++--
.../Storages/SubstraitSource/TextFormatFile.cpp | 3 +--
.../Storages/SubstraitSource/TextFormatFile.h | 3 +--
cpp-ch/local-engine/tests/benchmark_local_engine.cpp | 6 +++---
cpp-ch/local-engine/tests/benchmark_parquet_read.cpp | 1 +
cpp-ch/local-engine/tests/gtest_ch_join.cpp | 2 +-
cpp-ch/local-engine/tests/gtest_parquet_read.cpp | 5 +----
32 files changed, 83 insertions(+), 89 deletions(-)
diff --git a/cpp-ch/CMakeLists.txt b/cpp-ch/CMakeLists.txt
index 0ec9c31fa2..d7aadcfc88 100644
--- a/cpp-ch/CMakeLists.txt
+++ b/cpp-ch/CMakeLists.txt
@@ -113,9 +113,10 @@ else()
-DENABLE_SQLITE=OFF -DENABLE_S2_GEOMETRY=OFF -DENABLE_ULID=OFF
-DENABLE_MYSQL=OFF -DENABLE_BCRYPT=OFF -DENABLE_LDAP=OFF
-DENABLE_MSGPACK=OFF -DUSE_REPLXX=OFF -DENABLE_CLICKHOUSE_ALL=OFF
- -DENABLE_GWP_ASAN=OFF -DCOMPILER_FLAGS='-fvisibility=hidden
- -fvisibility-inlines-hidden' -S ${CH_SOURCE_DIR} -G Ninja -B
- ${CH_BINARY_DIR} && cmake --build ${CH_BINARY_DIR} --target libch\"
+ -DENABLE_GWP_ASAN=OFF -DENABLE_NUMACTL=OFF
+ -DCOMPILER_FLAGS='-fvisibility=hidden -fvisibility-inlines-hidden' -S
+ ${CH_SOURCE_DIR} -G Ninja -B ${CH_BINARY_DIR} && cmake --build
+ ${CH_BINARY_DIR} --target libch\"
OUTPUT _build_ch)
endif()
diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version
index b2dd1231a9..2d653373b4 100644
--- a/cpp-ch/clickhouse.version
+++ b/cpp-ch/clickhouse.version
@@ -1,3 +1,3 @@
CH_ORG=Kyligence
-CH_BRANCH=rebase_ch/20250629
-CH_COMMIT=addbf00cfd7
+CH_BRANCH=rebase_ch/20250705
+CH_COMMIT=623e9e91c6e
diff --git a/cpp-ch/local-engine/Common/GlutenSignalHandler.cpp
b/cpp-ch/local-engine/Common/GlutenSignalHandler.cpp
index 2eb8913094..6e2587f687 100644
--- a/cpp-ch/local-engine/Common/GlutenSignalHandler.cpp
+++ b/cpp-ch/local-engine/Common/GlutenSignalHandler.cpp
@@ -162,13 +162,13 @@ collectGlutenCrashLog(Int32 signal, UInt64 thread_id,
const String & query_id, c
}
-class SignalListener : public Poco::Runnable
+class GlutenSignalListener : public Poco::Runnable
{
public:
static constexpr int StdTerminate = -1;
static constexpr int StopThread = -2;
static constexpr int SanitizerTrap = -3;
- SignalListener() : log(&Poco::Logger::get("SignalListener")),
git_hash("getGitHash()") { }
+ GlutenSignalListener() : log(&Poco::Logger::get("SignalListener")),
git_hash("getGitHash()") { }
void run() override
{
@@ -396,13 +396,13 @@ struct SignalHandler::Impl
signal_pipe.setNonBlockingWrite();
signal_pipe.tryIncreaseSize(1 << 20);
- signal_listener = std::make_unique<SignalListener>();
+ signal_listener = std::make_unique<GlutenSignalListener>();
signal_listener_thread.start(*signal_listener);
}
~Impl()
{
- writeSignalIDtoSignalPipe(SignalListener::StopThread);
+ writeSignalIDtoSignalPipe(GlutenSignalListener::StopThread);
signal_listener_thread.join();
/// Reset signals to SIG_DFL to avoid trying to write to the
signal_pipe that will be closed after.
diff --git a/cpp-ch/local-engine/Common/LoggerExtend.cpp
b/cpp-ch/local-engine/Common/LoggerExtend.cpp
index 737113f554..979c28ae08 100644
--- a/cpp-ch/local-engine/Common/LoggerExtend.cpp
+++ b/cpp-ch/local-engine/Common/LoggerExtend.cpp
@@ -15,13 +15,14 @@
* limitations under the License.
*/
#include "LoggerExtend.h"
+#include <Loggers/OwnSplitChannel.h>
+
#include <Loggers/Loggers.h>
#include <Poco/AsyncChannel.h>
#include <Poco/AutoPtr.h>
#include <Poco/ConsoleChannel.h>
#include <Poco/FormattingChannel.h>
#include <Poco/PatternFormatter.h>
-#include <Poco/SimpleFileChannel.h>
using Poco::AsyncChannel;
using Poco::AutoPtr;
diff --git a/cpp-ch/local-engine/Common/LoggerExtend.h
b/cpp-ch/local-engine/Common/LoggerExtend.h
index 5cb49cc18a..7d8ba233c8 100644
--- a/cpp-ch/local-engine/Common/LoggerExtend.h
+++ b/cpp-ch/local-engine/Common/LoggerExtend.h
@@ -16,8 +16,12 @@
*/
#pragma once
-#include <Poco/Util/AbstractConfiguration.h>
+#include <string>
+namespace Poco::Util
+{
+class AbstractConfiguration;
+}
namespace local_engine
{
class LoggerExtend
diff --git
a/cpp-ch/local-engine/Disks/ObjectStorages/registerGlutenDiskObjectStorage.cpp
b/cpp-ch/local-engine/Disks/ObjectStorages/registerGlutenDiskObjectStorage.cpp
index 60a5795bc3..e57c31b45e 100644
---
a/cpp-ch/local-engine/Disks/ObjectStorages/registerGlutenDiskObjectStorage.cpp
+++
b/cpp-ch/local-engine/Disks/ObjectStorages/registerGlutenDiskObjectStorage.cpp
@@ -81,7 +81,8 @@ void registerGlutenS3ObjectStorage(ObjectStorageFactory &
factory)
auto uri = getS3URI(config, config_prefix, context);
auto s3_capabilities = getCapabilitiesFromConfig(config,
config_prefix);
auto endpoint = getEndpoint(config, config_prefix, context);
- auto settings = getSettings(config, config_prefix, context,
endpoint, /* validate_settings */ true);
+ auto settings = std::make_unique<S3Settings>();
+ settings->loadFromConfigForObjectStorage(config, config_prefix,
context->getSettingsRef(), uri.uri.getScheme(), true);
auto client = getClient(endpoint, *settings, context, /*
for_disk_s3 */ true);
auto key_generator =
createObjectStorageKeysGeneratorAsIsWithPrefix(uri.key);
diff --git a/cpp-ch/local-engine/Parser/RelParsers/CrossRelParser.cpp
b/cpp-ch/local-engine/Parser/RelParsers/CrossRelParser.cpp
index 67494555ca..4e00a3882f 100644
--- a/cpp-ch/local-engine/Parser/RelParsers/CrossRelParser.cpp
+++ b/cpp-ch/local-engine/Parser/RelParsers/CrossRelParser.cpp
@@ -41,6 +41,7 @@ namespace DB
namespace Setting
{
extern const SettingsUInt64 max_block_size;
+extern const SettingsUInt64 min_joined_block_size_rows;
extern const SettingsUInt64 min_joined_block_size_bytes;
}
namespace ErrorCodes
@@ -201,6 +202,7 @@ DB::QueryPlanPtr CrossRelParser::parseJoin(const
substrait::CrossRel & join, DB:
right->getCurrentHeader(),
hash_join,
context->getSettingsRef()[Setting::max_block_size],
+ context->getSettingsRef()[Setting::min_joined_block_size_rows],
context->getSettingsRef()[Setting::min_joined_block_size_bytes],
1,
/* required_output_ = */ NameSet{},
diff --git a/cpp-ch/local-engine/Parser/RelParsers/JoinRelParser.cpp
b/cpp-ch/local-engine/Parser/RelParsers/JoinRelParser.cpp
index d0f60a4d37..a339c8c10d 100644
--- a/cpp-ch/local-engine/Parser/RelParsers/JoinRelParser.cpp
+++ b/cpp-ch/local-engine/Parser/RelParsers/JoinRelParser.cpp
@@ -46,6 +46,7 @@ namespace Setting
{
extern const SettingsJoinAlgorithm join_algorithm;
extern const SettingsUInt64 max_block_size;
+extern const SettingsUInt64 min_joined_block_size_rows;
extern const SettingsUInt64 min_joined_block_size_bytes;
extern const SettingsNonZeroUInt64 grace_hash_join_initial_buckets;
extern const SettingsNonZeroUInt64 grace_hash_join_max_buckets;
@@ -319,6 +320,7 @@ DB::QueryPlanPtr JoinRelParser::parseJoin(const
substrait::JoinRel & join, DB::Q
right->getCurrentHeader(),
smj_join,
context->getSettingsRef()[Setting::max_block_size],
+ context->getSettingsRef()[Setting::min_joined_block_size_rows],
context->getSettingsRef()[Setting::min_joined_block_size_bytes],
1,
/* required_output_ = */ NameSet{},
@@ -744,6 +746,7 @@ DB::QueryPlanPtr JoinRelParser::buildMultiOnClauseHashJoin(
right_plan->getCurrentHeader(),
hash_join,
context->getSettingsRef()[Setting::max_block_size],
+ context->getSettingsRef()[Setting::min_joined_block_size_rows],
context->getSettingsRef()[Setting::min_joined_block_size_bytes],
1,
/* required_output_ = */ NameSet{},
@@ -788,6 +791,7 @@ DB::QueryPlanPtr JoinRelParser::buildSingleOnClauseHashJoin(
right_plan->getCurrentHeader(),
hash_join,
context->getSettingsRef()[Setting::max_block_size],
+ context->getSettingsRef()[Setting::min_joined_block_size_rows],
context->getSettingsRef()[Setting::min_joined_block_size_bytes],
1,
/* required_output_ = */ NameSet{},
diff --git a/cpp-ch/local-engine/Storages/MergeTree/SparkStorageMergeTree.h
b/cpp-ch/local-engine/Storages/MergeTree/SparkStorageMergeTree.h
index aeb57cad90..4e9f390c9c 100644
--- a/cpp-ch/local-engine/Storages/MergeTree/SparkStorageMergeTree.h
+++ b/cpp-ch/local-engine/Storages/MergeTree/SparkStorageMergeTree.h
@@ -54,15 +54,13 @@ class SparkStorageMergeTree : public DB::MergeTreeData
{
SparkMutationsSnapshot() = default;
- DB::MutationCommands getAlterMutationCommandsForPart(const
MergeTreeData::DataPartPtr & part) const override { return {}; }
+ DB::MutationCommands getOnFlyMutationCommandsForPart(const
MergeTreeData::DataPartPtr & part) const override { return {}; }
std::shared_ptr<MergeTreeData::IMutationsSnapshot> cloneEmpty() const
override
{
return std::make_shared<SparkMutationsSnapshot>();
}
DB::NameSet getAllUpdatedColumns() const override { return {}; }
-
- bool hasMetadataMutations() const override { return
params.min_part_metadata_version < params.metadata_version; }
};
public:
diff --git
a/cpp-ch/local-engine/Storages/Parquet/VectorizedParquetRecordReader.h
b/cpp-ch/local-engine/Storages/Parquet/VectorizedParquetRecordReader.h
index ce1a139014..512cc91bed 100644
--- a/cpp-ch/local-engine/Storages/Parquet/VectorizedParquetRecordReader.h
+++ b/cpp-ch/local-engine/Storages/Parquet/VectorizedParquetRecordReader.h
@@ -214,6 +214,9 @@ class VectorizedParquetBlockInputFormat final : public
DB::IInputFormat
protected:
void onCancel() noexcept override { is_stopped = 1; }
+ // TODO: create ColumnIndexFilter here, currently disable it now.
+ void setKeyCondition(const std::shared_ptr<const DB::KeyCondition> &
key_condition_) override { }
+
public:
VectorizedParquetBlockInputFormat(
DB::ReadBuffer & in_,
diff --git
a/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.cpp
b/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.cpp
index dd191831c3..f094bf915d 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.cpp
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.cpp
@@ -62,8 +62,7 @@ bool ExcelTextFormatFile::useThis(const DB::ContextPtr &
context)
return settingsEqual(context->getSettingsRef(), USE_EXCEL_PARSER, "true");
}
-FormatFile::InputFormatPtr
-ExcelTextFormatFile::createInputFormat(const DB::Block & header, const
std::shared_ptr<const DB::ActionsDAG> & /*filter_actions_dag*/)
+FormatFile::InputFormatPtr ExcelTextFormatFile::createInputFormat(const
DB::Block & header)
{
auto read_buffer = read_buffer_builder->build(file_info);
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.h
b/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.h
index 0b92f86219..5e70d22eeb 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.h
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.h
@@ -47,8 +47,7 @@ public:
~ExcelTextFormatFile() override = default;
- FormatFile::InputFormatPtr
- createInputFormat(const DB::Block & header, const std::shared_ptr<const
DB::ActionsDAG> & filter_actions_dag = nullptr) override;
+ FormatFile::InputFormatPtr createInputFormat(const DB::Block & header)
override;
bool supportSplit() const override { return true; }
String getFileFormat() const override { return "ExcelText"; }
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/FileReader.cpp
b/cpp-ch/local-engine/Storages/SubstraitSource/FileReader.cpp
index 8de7f83509..e5d1f8d8a0 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/FileReader.cpp
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/FileReader.cpp
@@ -274,12 +274,17 @@ std::unique_ptr<NormalFileReader> createNormalFileReader(
const FormatFilePtr & file,
const DB::Block & to_read_header_,
const DB::Block & output_header_,
- const std::shared_ptr<const DB::ActionsDAG> & filter_actions_dag = nullptr,
+ const std::shared_ptr<const DB::KeyCondition> & key_condition = nullptr,
const ColumnIndexFilterPtr & column_index_filter = nullptr)
{
file->initialize(column_index_filter);
auto createInputFormat = [&](const DB::Block & new_read_header_) ->
FormatFile::InputFormatPtr
- { return file->createInputFormat(new_read_header_, filter_actions_dag); };
+ {
+ auto input_format = file->createInputFormat(new_read_header_);
+ if (key_condition && input_format)
+ input_format->inputFormat().setKeyCondition(key_condition);
+ return input_format;
+ };
if (file->getFileInfo().has_iceberg())
return iceberg::IcebergReader::create(file, to_read_header_,
output_header_, createInputFormat);
@@ -311,13 +316,11 @@ std::unique_ptr<NormalFileReader> createNormalFileReader(
return std::make_unique<NormalFileReader>(file, to_read_header_,
output_header_, input_format);
}
}
-
-/// TODO Remove ColumnIndexFilterPtr
std::unique_ptr<BaseReader> BaseReader::create(
const FormatFilePtr & current_file,
const DB::Block & readHeader,
const DB::Block & outputHeader,
- const std::shared_ptr<const DB::ActionsDAG> & filter_actions_dag,
+ const std::shared_ptr<const DB::KeyCondition> & key_condition,
const ColumnIndexFilterPtr & column_index_filter)
{
if (!readHeader)
@@ -332,7 +335,7 @@ std::unique_ptr<BaseReader> BaseReader::create(
}
}
- return createNormalFileReader(current_file, readHeader, outputHeader,
filter_actions_dag, column_index_filter);
+ return createNormalFileReader(current_file, readHeader, outputHeader,
key_condition, column_index_filter);
}
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/FileReader.h
b/cpp-ch/local-engine/Storages/SubstraitSource/FileReader.h
index 467824610e..bdb6772ab5 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/FileReader.h
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/FileReader.h
@@ -77,7 +77,7 @@ public:
const FormatFilePtr & current_file,
const DB::Block & readHeader,
const DB::Block & outputHeader,
- const std::shared_ptr<const DB::ActionsDAG> & filter_actions_dag,
+ const std::shared_ptr<const DB::KeyCondition> & key_condition,
const ColumnIndexFilterPtr & column_index_filter);
};
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/FormatFile.h
b/cpp-ch/local-engine/Storages/SubstraitSource/FormatFile.h
index 60d28f5bba..262aeff6fa 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/FormatFile.h
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/FormatFile.h
@@ -30,7 +30,6 @@
namespace DB
{
-class ActionsDAG;
namespace ErrorCodes
{
extern const int NOT_IMPLEMENTED;
@@ -116,9 +115,7 @@ public:
virtual ~FormatFile() = default;
/// Create a new input format for reading this file
- virtual InputFormatPtr
- createInputFormat(const DB::Block & header, const std::shared_ptr<const
DB::ActionsDAG> & filter_actions_dag = nullptr)
- = 0;
+ virtual InputFormatPtr createInputFormat(const DB::Block & header) = 0;
/// Spark would split a large file into small segements and read in
different tasks
/// If this file doesn't support the split feacture, only the task with
offset 0 will generate data.
@@ -152,6 +149,7 @@ protected:
std::map<String, String> partition_values;
/// partition keys are normalized to lower cases for partition column
case-insensitive matching
std::map<String, String> normalized_partition_values;
+ std::shared_ptr<const DB::KeyCondition> key_condition;
const FileMetaColumns meta_columns;
/// Currently, it is used to read an iceberg format, and initialized in
the constructor of child class
diff --git
a/cpp-ch/local-engine/Storages/SubstraitSource/Iceberg/IcebergReader.h
b/cpp-ch/local-engine/Storages/SubstraitSource/Iceberg/IcebergReader.h
index ddfd0954db..f8845c1367 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/Iceberg/IcebergReader.h
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/Iceberg/IcebergReader.h
@@ -18,12 +18,6 @@
#include <Storages/SubstraitSource/FileReader.h>
-namespace DB
-{
-class ExpressionActions;
-using ExpressionActionsPtr = std::shared_ptr<ExpressionActions>;
-}
-
namespace local_engine
{
class DeltaDVRoaringBitmapArray;
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/JSONFormatFile.cpp
b/cpp-ch/local-engine/Storages/SubstraitSource/JSONFormatFile.cpp
index b5728abbf9..5cbab2bbda 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/JSONFormatFile.cpp
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/JSONFormatFile.cpp
@@ -31,8 +31,7 @@ JSONFormatFile::JSONFormatFile(
{
}
-FormatFile::InputFormatPtr
-JSONFormatFile::createInputFormat(const DB::Block & header, const
std::shared_ptr<const DB::ActionsDAG> & /*filter_actions_dag*/)
+FormatFile::InputFormatPtr JSONFormatFile::createInputFormat(const DB::Block &
header)
{
auto read_buffer =
read_buffer_builder->buildWithCompressionWrapper(file_info);
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/JSONFormatFile.h
b/cpp-ch/local-engine/Storages/SubstraitSource/JSONFormatFile.h
index a98af115a2..97b3a694de 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/JSONFormatFile.h
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/JSONFormatFile.h
@@ -28,8 +28,7 @@ public:
bool supportSplit() const override { return true; }
- FormatFile::InputFormatPtr
- createInputFormat(const DB::Block & header, const std::shared_ptr<const
DB::ActionsDAG> & filter_actions_dag = nullptr) override;
+ FormatFile::InputFormatPtr createInputFormat(const DB::Block & header)
override;
String getFileFormat() const override { return "JSONEachRow"; }
};
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/ORCFormatFile.cpp
b/cpp-ch/local-engine/Storages/SubstraitSource/ORCFormatFile.cpp
index 9b87c6528e..f78cca5d87 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/ORCFormatFile.cpp
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/ORCFormatFile.cpp
@@ -36,8 +36,7 @@ ORCFormatFile::ORCFormatFile(
{
}
-FormatFile::InputFormatPtr
-ORCFormatFile::createInputFormat(const DB::Block & header, const
std::shared_ptr<const DB::ActionsDAG> & filter_actions_dag)
+FormatFile::InputFormatPtr ORCFormatFile::createInputFormat(const DB::Block &
header)
{
auto read_buffer = read_buffer_builder->build(file_info);
@@ -71,8 +70,7 @@ ORCFormatFile::createInputFormat(const DB::Block & header,
const std::shared_ptr
format_settings.orc.reader_time_zone_name = mapped_timezone;
}
//TODO: support prefetch
- auto parser_group =
std::make_shared<DB::FormatParserGroup>(context->getSettingsRef(), 1,
filter_actions_dag, context);
- auto input_format =
std::make_shared<DB::NativeORCBlockInputFormat>(*read_buffer, header,
format_settings, false, 0, parser_group);
+ auto input_format =
std::make_shared<DB::NativeORCBlockInputFormat>(*read_buffer, header,
format_settings, false, 0);
return std::make_shared<InputFormat>(std::move(read_buffer), input_format);
}
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/ORCFormatFile.h
b/cpp-ch/local-engine/Storages/SubstraitSource/ORCFormatFile.h
index 0d8beb5d3b..b6f6fd2321 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/ORCFormatFile.h
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/ORCFormatFile.h
@@ -44,8 +44,7 @@ public:
DB::ContextPtr context_, const
substrait::ReadRel::LocalFiles::FileOrFiles & file_info_, ReadBufferBuilderPtr
read_buffer_builder_);
~ORCFormatFile() override = default;
- FormatFile::InputFormatPtr
- createInputFormat(const DB::Block & header, const std::shared_ptr<const
DB::ActionsDAG> & filter_actions_dag = nullptr) override;
+ FormatFile::InputFormatPtr createInputFormat(const DB::Block & header)
override;
std::optional<size_t> getTotalRows() override;
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.cpp
b/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.cpp
index 50e84e60f9..a4ccfa93b7 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.cpp
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.cpp
@@ -142,8 +142,7 @@ void ParquetFormatFile::initialize(const
ColumnIndexFilterPtr & filter)
file_schema = ParquetMetaBuilder::collectFileSchema(context,
*read_buffer_);
}
-FormatFile::InputFormatPtr
-ParquetFormatFile::createInputFormat(const Block & header, const
std::shared_ptr<const DB::ActionsDAG> & filter_actions_dag)
+FormatFile::InputFormatPtr ParquetFormatFile::createInputFormat(const Block &
header)
{
assert(read_buffer_);
@@ -206,8 +205,14 @@ ParquetFormatFile::createInputFormat(const Block & header,
const std::shared_ptr
// We need to disable fiter push down and read all row groups, so
that we can get correct row index.
format_settings.parquet.filter_push_down = false;
}
- auto parser_group =
std::make_shared<FormatParserGroup>(context->getSettingsRef(), 1,
filter_actions_dag, context);
- auto input = std::make_shared<ParquetBlockInputFormat>(*read_buffer_,
read_header, format_settings, parser_group, 8192);
+
+ auto input = std::make_shared<ParquetBlockInputFormat>(
+ *read_buffer_,
+ read_header,
+ format_settings,
+ settings[Setting::max_parsing_threads],
+ settings[Setting::max_download_threads],
+ 8192);
return std::make_shared<ParquetInputFormat>(
std::move(read_buffer_), input, std::move(provider),
std::move(read_header), std::move(output_header));
};
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.h
b/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.h
index 587b788f79..2014e1d643 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.h
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.h
@@ -35,8 +35,7 @@ public:
bool use_local_format_);
~ParquetFormatFile() override = default;
- InputFormatPtr
- createInputFormat(const DB::Block & header, const std::shared_ptr<const
DB::ActionsDAG> & filter_actions_dag = nullptr) override;
+ InputFormatPtr createInputFormat(const DB::Block & header) override;
std::optional<size_t> getTotalRows() override;
diff --git
a/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.cpp
b/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.cpp
index 51c041fa3c..a5df941a48 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.cpp
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.cpp
@@ -54,7 +54,7 @@ static DB::Block initReadHeader(const DB::Block & block,
const FormatFiles & fil
SubstraitFileSource::SubstraitFileSource(
const DB::ContextPtr & context_, const DB::Block & outputHeader_, const
substrait::ReadRel::LocalFiles & file_infos)
- : DB::ISource(BaseReader::buildRowCountHeader(outputHeader_), false)
+ :
DB::SourceWithKeyCondition(BaseReader::buildRowCountHeader(outputHeader_),
false)
, files(initializeFiles(file_infos, context_))
, outputHeader(outputHeader_)
, readHeader(initReadHeader(outputHeader, files))
@@ -63,11 +63,11 @@ SubstraitFileSource::SubstraitFileSource(
SubstraitFileSource::~SubstraitFileSource() = default;
-void SubstraitFileSource::setKeyCondition(const std::shared_ptr<const
DB::ActionsDAG> & filter_actions_dag_, DB::ContextPtr context_)
+void SubstraitFileSource::setKeyCondition(const std::optional<DB::ActionsDAG>
& filter_actions_dag, DB::ContextPtr context_)
{
- assert(filter_actions_dag_);
- filter_actions_dag = filter_actions_dag_;
- column_index_filter =
std::make_shared<ColumnIndexFilter>(*filter_actions_dag, context_);
+ setKeyConditionImpl(filter_actions_dag, context_, readHeader);
+ if (filter_actions_dag)
+ column_index_filter =
std::make_shared<ColumnIndexFilter>(filter_actions_dag.value(), context_);
}
DB::Chunk SubstraitFileSource::generate()
@@ -105,7 +105,7 @@ bool SubstraitFileSource::tryPrepareReader()
if (!current_file->supportSplit() && current_file->getStartOffset())
continue;
- file_reader = BaseReader::create(current_file, readHeader,
outputHeader, filter_actions_dag, column_index_filter);
+ file_reader = BaseReader::create(current_file, readHeader,
outputHeader, key_condition, column_index_filter);
if (file_reader)
return true;
}
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.h
b/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.h
index b40dd9a82d..a2280da0d2 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.h
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.h
@@ -17,15 +17,9 @@
#pragma once
#include <memory>
-#include <Interpreters/Context_fwd.h>
-#include <Processors/ISource.h>
+#include <Processors/SourceWithKeyCondition.h>
#include <substrait/algebra.pb.h>
-namespace DB
-{
-class ActionsDAG;
-}
-
namespace local_engine
{
class ColumnIndexFilter;
@@ -35,7 +29,7 @@ class FormatFile;
using FormatFilePtr = std::shared_ptr<FormatFile>;
using FormatFiles = std::vector<FormatFilePtr>;
-class SubstraitFileSource : public DB::ISource
+class SubstraitFileSource : public DB::SourceWithKeyCondition
{
public:
SubstraitFileSource(const DB::ContextPtr & context_, const DB::Block &
header_, const substrait::ReadRel::LocalFiles & file_infos);
@@ -43,7 +37,7 @@ public:
String getName() const override { return "SubstraitFileSource"; }
- void setKeyCondition(const std::shared_ptr<const DB::ActionsDAG> &
filter_actions_dag_, DB::ContextPtr context_);
+ void setKeyCondition(const std::optional<DB::ActionsDAG> &
filter_actions_dag, DB::ContextPtr context_) override;
protected:
DB::Chunk generate() override;
@@ -60,6 +54,5 @@ private:
std::unique_ptr<BaseReader> file_reader;
ColumnIndexFilterPtr column_index_filter;
- std::shared_ptr<const DB::ActionsDAG> filter_actions_dag;
};
}
diff --git
a/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSourceStep.cpp
b/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSourceStep.cpp
index 220d181226..91e50b4275 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSourceStep.cpp
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSourceStep.cpp
@@ -15,6 +15,7 @@
* limitations under the License.
*/
+#include <Interpreters/Context_fwd.h>
#include <Processors/QueryPlan/IQueryPlanStep.h>
#include <QueryPipeline/Pipe.h>
#include <QueryPipeline/QueryPipelineBuilder.h>
@@ -65,12 +66,9 @@ void
SubstraitFileSourceStep::initializePipeline(DB::QueryPipelineBuilder & pipe
void SubstraitFileSourceStep::applyFilters(const DB::ActionDAGNodes
added_filter_nodes)
{
- SourceStepWithFilter::applyFilters(std::move(added_filter_nodes));
- if (filter_actions_dag)
- {
- for (const auto & processor : pipe.getProcessors())
- if (auto * source = dynamic_cast<SubstraitFileSource
*>(processor.get()))
- source->setKeyCondition(filter_actions_dag, context);
- }
+ filter_actions_dag =
DB::ActionsDAG::buildFilterActionsDAG(added_filter_nodes.nodes);
+ for (const auto & processor : pipe.getProcessors())
+ if (auto * source = dynamic_cast<DB::SourceWithKeyCondition
*>(processor.get()))
+ source->setKeyCondition(filter_actions_dag, context);
}
}
diff --git
a/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSourceStep.h
b/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSourceStep.h
index e1175507f5..78fcb767f0 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSourceStep.h
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSourceStep.h
@@ -17,10 +17,11 @@
#pragma once
-#include <Interpreters/Context_fwd.h>
#include <Processors/QueryPlan/ReadFromPreparedSource.h>
#include <Processors/QueryPlan/SourceStepWithFilter.h>
-
+#include <Storages/MergeTree/KeyCondition.h>
+#include <Interpreters/Context_fwd.h>
+#include <Core/NamesAndTypes.h>
namespace local_engine
{
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/TextFormatFile.cpp
b/cpp-ch/local-engine/Storages/SubstraitSource/TextFormatFile.cpp
index 9e40209eea..fcbe650d46 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/TextFormatFile.cpp
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/TextFormatFile.cpp
@@ -34,8 +34,7 @@ TextFormatFile::TextFormatFile(
{
}
-FormatFile::InputFormatPtr
-TextFormatFile::createInputFormat(const DB::Block & header, const
std::shared_ptr<const DB::ActionsDAG> & /*filter_actions_dag*/)
+FormatFile::InputFormatPtr TextFormatFile::createInputFormat(const DB::Block &
header)
{
auto read_buffer =
read_buffer_builder->buildWithCompressionWrapper(file_info);
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/TextFormatFile.h
b/cpp-ch/local-engine/Storages/SubstraitSource/TextFormatFile.h
index 0e6827e653..62e60af4a8 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/TextFormatFile.h
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/TextFormatFile.h
@@ -32,8 +32,7 @@ public:
DB::ContextPtr context_, const
substrait::ReadRel::LocalFiles::FileOrFiles & file_info_, ReadBufferBuilderPtr
read_buffer_builder_);
~TextFormatFile() override = default;
- FormatFile::InputFormatPtr
- createInputFormat(const DB::Block & header, const std::shared_ptr<const
DB::ActionsDAG> & filter_actions_dag = nullptr) override;
+ FormatFile::InputFormatPtr createInputFormat(const DB::Block & header)
override;
DB::NamesAndTypesList getSchema() const
{
diff --git a/cpp-ch/local-engine/tests/benchmark_local_engine.cpp
b/cpp-ch/local-engine/tests/benchmark_local_engine.cpp
index 59acf42cf3..32c213610f 100644
--- a/cpp-ch/local-engine/tests/benchmark_local_engine.cpp
+++ b/cpp-ch/local-engine/tests/benchmark_local_engine.cpp
@@ -803,12 +803,12 @@ QueryPlanPtr readFromMergeTree(MergeTreeWithSnapshot
storage)
auto data_parts = storage.merge_tree->getDataPartsVectorForInternalUsage();
auto query_plan = std::make_unique<QueryPlan>();
auto step = storage.merge_tree->reader.readFromParts(
- data_parts, {}, storage.columns.getNames(), storage.snapshot,
*query_info, global_context, 10000, 1);
+ RangesInDataParts{data_parts}, {}, storage.columns.getNames(),
storage.snapshot, *query_info, global_context, 10000, 1);
query_plan->addStep(std::move(step));
return query_plan;
}
-QueryPlanPtr joinPlan(QueryPlanPtr left, QueryPlanPtr right, String left_key,
String right_key, size_t block_size = 8192)
+QueryPlanPtr joinPlan(QueryPlanPtr left, QueryPlanPtr right, String left_key,
String right_key)
{
auto join = std::make_shared<TableJoin>(
global_context->getSettingsRef(),
global_context->getGlobalTemporaryVolume(),
global_context->getTempDataOnDisk());
@@ -846,7 +846,7 @@ QueryPlanPtr joinPlan(QueryPlanPtr left, QueryPlanPtr
right, String left_key, St
auto hash_join = std::make_shared<HashJoin>(join,
right->getCurrentHeader());
QueryPlanStepPtr join_step = std::make_unique<JoinStep>(
- left->getCurrentHeader(), right->getCurrentHeader(), hash_join,
block_size, 8192, 1, NameSet{}, false, false);
+ left->getCurrentHeader(), right->getCurrentHeader(), hash_join,
DEFAULT_BLOCK_SIZE, DEFAULT_BLOCK_SIZE, 524288, 1, NameSet{}, false, false);
std::vector<QueryPlanPtr> plans;
plans.emplace_back(std::move(left));
diff --git a/cpp-ch/local-engine/tests/benchmark_parquet_read.cpp
b/cpp-ch/local-engine/tests/benchmark_parquet_read.cpp
index 2b56f44622..f9b4693aeb 100644
--- a/cpp-ch/local-engine/tests/benchmark_parquet_read.cpp
+++ b/cpp-ch/local-engine/tests/benchmark_parquet_read.cpp
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+#include <random>
#include <Core/Block.h>
#include <DataTypes/DataTypeDate32.h>
#include <DataTypes/DataTypeString.h>
diff --git a/cpp-ch/local-engine/tests/gtest_ch_join.cpp
b/cpp-ch/local-engine/tests/gtest_ch_join.cpp
index 74b04d5def..6350571c56 100644
--- a/cpp-ch/local-engine/tests/gtest_ch_join.cpp
+++ b/cpp-ch/local-engine/tests/gtest_ch_join.cpp
@@ -127,7 +127,7 @@ TEST(TestJoin, simple)
auto hash_join = std::make_shared<HashJoin>(join,
right_plan.getCurrentHeader());
QueryPlanStepPtr join_step = std::make_unique<JoinStep>(
- left_plan.getCurrentHeader(), right_plan.getCurrentHeader(),
hash_join, 8192, 8192, 1, NameSet{}, false, false);
+ left_plan.getCurrentHeader(), right_plan.getCurrentHeader(),
hash_join, DEFAULT_BLOCK_SIZE, DEFAULT_BLOCK_SIZE, 524288, 1, NameSet{}, false,
false);
std::cerr << "join step:" << join_step->getOutputHeader().dumpStructure()
<< std::endl;
diff --git a/cpp-ch/local-engine/tests/gtest_parquet_read.cpp
b/cpp-ch/local-engine/tests/gtest_parquet_read.cpp
index 57b4eca6cf..57b6b86ae0 100644
--- a/cpp-ch/local-engine/tests/gtest_parquet_read.cpp
+++ b/cpp-ch/local-engine/tests/gtest_parquet_read.cpp
@@ -45,7 +45,6 @@
#include <tests/utils/gluten_test_util.h>
#include <Common/BlockTypeUtils.h>
#include <Common/DebugUtils.h>
-#include <Common/QueryContext.h>
using namespace DB;
using namespace local_engine;
@@ -112,10 +111,8 @@ void readData(const String & path, const std::map<String,
Field> & fields)
ReadBufferFromFile in(full_path);
InputFormatPtr format;
- auto parser_group
- =
std::make_shared<FormatParserGroup>(QueryContext::globalContext()->getSettingsRef(),
1, nullptr, QueryContext::globalContext());
if constexpr (std::is_same_v<InputFormat, DB::ParquetBlockInputFormat>)
- format = std::make_shared<InputFormat>(in, header, settings,
parser_group, 8192);
+ format = std::make_shared<InputFormat>(in, header, settings, 1, 1,
8192);
else
format = std::make_shared<InputFormat>(in, header, settings);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]