This is an automated email from the ASF dual-hosted git repository.

changchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git


The following commit(s) were added to refs/heads/main by this push:
     new 02ca1efcf8 [GLUTEN-1632][CH]Daily Update Clickhouse Version (20250705) 
(#10127)
02ca1efcf8 is described below

commit 02ca1efcf855fb7fa7f39d2d3e1c436b6726482c
Author: Kyligence Git <[email protected]>
AuthorDate: Sat Jul 5 21:04:57 2025 -0500

    [GLUTEN-1632][CH]Daily Update Clickhouse Version (20250705) (#10127)
    
    * [GLUTEN-1632][CH]Daily Update Clickhouse Version (20250705)
    
    * Fix benchmark build
    
    * Fix Benchmark build due to 
https://github.com/ClickHouse/ClickHouse/pull/79417
    
    * Revert "Fix Build due to 
https://github.com/ClickHouse/ClickHouse/pull/80931";
    
    This reverts commit 02d12f65b0ee21b9f87a3ee146e7a5acd230cc84.
    
    * Fix Build due to https://github.com/ClickHouse/ClickHouse/pull/81886
    
    * Fix Link issue due to https://github.com/ClickHouse/ClickHouse/pull/83121
    
    * Fix Build due to https://github.com/ClickHouse/ClickHouse/pull/82604
    
    * Fix Build due to https://github.com/ClickHouse/ClickHouse/pull/82945
    
    * Fix Build due to https://github.com/ClickHouse/ClickHouse/pull/83214
    
    ---------
    
    Co-authored-by: kyligence-git <[email protected]>
    Co-authored-by: Chang chen <[email protected]>
---
 cpp-ch/CMakeLists.txt                                     |  7 ++++---
 cpp-ch/clickhouse.version                                 |  4 ++--
 cpp-ch/local-engine/Common/GlutenSignalHandler.cpp        |  8 ++++----
 cpp-ch/local-engine/Common/LoggerExtend.cpp               |  3 ++-
 cpp-ch/local-engine/Common/LoggerExtend.h                 |  6 +++++-
 .../ObjectStorages/registerGlutenDiskObjectStorage.cpp    |  3 ++-
 cpp-ch/local-engine/Parser/RelParsers/CrossRelParser.cpp  |  2 ++
 cpp-ch/local-engine/Parser/RelParsers/JoinRelParser.cpp   |  4 ++++
 .../Storages/MergeTree/SparkStorageMergeTree.h            |  4 +---
 .../Storages/Parquet/VectorizedParquetRecordReader.h      |  3 +++
 .../Storages/SubstraitSource/ExcelTextFormatFile.cpp      |  3 +--
 .../Storages/SubstraitSource/ExcelTextFormatFile.h        |  3 +--
 .../local-engine/Storages/SubstraitSource/FileReader.cpp  | 15 +++++++++------
 cpp-ch/local-engine/Storages/SubstraitSource/FileReader.h |  2 +-
 cpp-ch/local-engine/Storages/SubstraitSource/FormatFile.h |  6 ++----
 .../Storages/SubstraitSource/Iceberg/IcebergReader.h      |  6 ------
 .../Storages/SubstraitSource/JSONFormatFile.cpp           |  3 +--
 .../Storages/SubstraitSource/JSONFormatFile.h             |  3 +--
 .../Storages/SubstraitSource/ORCFormatFile.cpp            |  6 ++----
 .../local-engine/Storages/SubstraitSource/ORCFormatFile.h |  3 +--
 .../Storages/SubstraitSource/ParquetFormatFile.cpp        | 13 +++++++++----
 .../Storages/SubstraitSource/ParquetFormatFile.h          |  3 +--
 .../Storages/SubstraitSource/SubstraitFileSource.cpp      | 12 ++++++------
 .../Storages/SubstraitSource/SubstraitFileSource.h        | 13 +++----------
 .../Storages/SubstraitSource/SubstraitFileSourceStep.cpp  | 12 +++++-------
 .../Storages/SubstraitSource/SubstraitFileSourceStep.h    |  5 +++--
 .../Storages/SubstraitSource/TextFormatFile.cpp           |  3 +--
 .../Storages/SubstraitSource/TextFormatFile.h             |  3 +--
 cpp-ch/local-engine/tests/benchmark_local_engine.cpp      |  6 +++---
 cpp-ch/local-engine/tests/benchmark_parquet_read.cpp      |  1 +
 cpp-ch/local-engine/tests/gtest_ch_join.cpp               |  2 +-
 cpp-ch/local-engine/tests/gtest_parquet_read.cpp          |  5 +----
 32 files changed, 83 insertions(+), 89 deletions(-)

diff --git a/cpp-ch/CMakeLists.txt b/cpp-ch/CMakeLists.txt
index 0ec9c31fa2..d7aadcfc88 100644
--- a/cpp-ch/CMakeLists.txt
+++ b/cpp-ch/CMakeLists.txt
@@ -113,9 +113,10 @@ else()
       -DENABLE_SQLITE=OFF -DENABLE_S2_GEOMETRY=OFF -DENABLE_ULID=OFF
       -DENABLE_MYSQL=OFF -DENABLE_BCRYPT=OFF -DENABLE_LDAP=OFF
       -DENABLE_MSGPACK=OFF -DUSE_REPLXX=OFF -DENABLE_CLICKHOUSE_ALL=OFF
-      -DENABLE_GWP_ASAN=OFF -DCOMPILER_FLAGS='-fvisibility=hidden
-      -fvisibility-inlines-hidden' -S ${CH_SOURCE_DIR} -G Ninja -B
-      ${CH_BINARY_DIR} && cmake --build ${CH_BINARY_DIR} --target libch\"
+      -DENABLE_GWP_ASAN=OFF -DENABLE_NUMACTL=OFF
+      -DCOMPILER_FLAGS='-fvisibility=hidden -fvisibility-inlines-hidden' -S
+      ${CH_SOURCE_DIR} -G Ninja -B ${CH_BINARY_DIR} && cmake --build
+      ${CH_BINARY_DIR} --target libch\"
     OUTPUT _build_ch)
 endif()
 
diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version
index b2dd1231a9..2d653373b4 100644
--- a/cpp-ch/clickhouse.version
+++ b/cpp-ch/clickhouse.version
@@ -1,3 +1,3 @@
 CH_ORG=Kyligence
-CH_BRANCH=rebase_ch/20250629
-CH_COMMIT=addbf00cfd7
+CH_BRANCH=rebase_ch/20250705
+CH_COMMIT=623e9e91c6e
diff --git a/cpp-ch/local-engine/Common/GlutenSignalHandler.cpp 
b/cpp-ch/local-engine/Common/GlutenSignalHandler.cpp
index 2eb8913094..6e2587f687 100644
--- a/cpp-ch/local-engine/Common/GlutenSignalHandler.cpp
+++ b/cpp-ch/local-engine/Common/GlutenSignalHandler.cpp
@@ -162,13 +162,13 @@ collectGlutenCrashLog(Int32 signal, UInt64 thread_id, 
const String & query_id, c
     
 }
 
-class SignalListener : public Poco::Runnable
+class GlutenSignalListener : public Poco::Runnable
 {
 public:
     static constexpr int StdTerminate = -1;
     static constexpr int StopThread = -2;
     static constexpr int SanitizerTrap = -3;
-    SignalListener() : log(&Poco::Logger::get("SignalListener")), 
git_hash("getGitHash()") { }
+    GlutenSignalListener() : log(&Poco::Logger::get("SignalListener")), 
git_hash("getGitHash()") { }
 
     void run() override
     {
@@ -396,13 +396,13 @@ struct SignalHandler::Impl
 
         signal_pipe.setNonBlockingWrite();
         signal_pipe.tryIncreaseSize(1 << 20);
-        signal_listener = std::make_unique<SignalListener>();
+        signal_listener = std::make_unique<GlutenSignalListener>();
         signal_listener_thread.start(*signal_listener);
     }
 
     ~Impl()
     {
-        writeSignalIDtoSignalPipe(SignalListener::StopThread);
+        writeSignalIDtoSignalPipe(GlutenSignalListener::StopThread);
         signal_listener_thread.join();
 
         /// Reset signals to SIG_DFL to avoid trying to write to the 
signal_pipe that will be closed after.
diff --git a/cpp-ch/local-engine/Common/LoggerExtend.cpp 
b/cpp-ch/local-engine/Common/LoggerExtend.cpp
index 737113f554..979c28ae08 100644
--- a/cpp-ch/local-engine/Common/LoggerExtend.cpp
+++ b/cpp-ch/local-engine/Common/LoggerExtend.cpp
@@ -15,13 +15,14 @@
  * limitations under the License.
  */
 #include "LoggerExtend.h"
+#include <Loggers/OwnSplitChannel.h>
+
 #include <Loggers/Loggers.h>
 #include <Poco/AsyncChannel.h>
 #include <Poco/AutoPtr.h>
 #include <Poco/ConsoleChannel.h>
 #include <Poco/FormattingChannel.h>
 #include <Poco/PatternFormatter.h>
-#include <Poco/SimpleFileChannel.h>
 
 using Poco::AsyncChannel;
 using Poco::AutoPtr;
diff --git a/cpp-ch/local-engine/Common/LoggerExtend.h 
b/cpp-ch/local-engine/Common/LoggerExtend.h
index 5cb49cc18a..7d8ba233c8 100644
--- a/cpp-ch/local-engine/Common/LoggerExtend.h
+++ b/cpp-ch/local-engine/Common/LoggerExtend.h
@@ -16,8 +16,12 @@
  */
 #pragma once
 
-#include <Poco/Util/AbstractConfiguration.h>
+#include <string>
 
+namespace Poco::Util
+{
+class AbstractConfiguration;
+}
 namespace local_engine
 {
 class LoggerExtend
diff --git 
a/cpp-ch/local-engine/Disks/ObjectStorages/registerGlutenDiskObjectStorage.cpp 
b/cpp-ch/local-engine/Disks/ObjectStorages/registerGlutenDiskObjectStorage.cpp
index 60a5795bc3..e57c31b45e 100644
--- 
a/cpp-ch/local-engine/Disks/ObjectStorages/registerGlutenDiskObjectStorage.cpp
+++ 
b/cpp-ch/local-engine/Disks/ObjectStorages/registerGlutenDiskObjectStorage.cpp
@@ -81,7 +81,8 @@ void registerGlutenS3ObjectStorage(ObjectStorageFactory & 
factory)
             auto uri = getS3URI(config, config_prefix, context);
             auto s3_capabilities = getCapabilitiesFromConfig(config, 
config_prefix);
             auto endpoint = getEndpoint(config, config_prefix, context);
-            auto settings = getSettings(config, config_prefix, context, 
endpoint, /* validate_settings */ true);
+            auto settings = std::make_unique<S3Settings>();
+            settings->loadFromConfigForObjectStorage(config, config_prefix, 
context->getSettingsRef(), uri.uri.getScheme(), true);
             auto client = getClient(endpoint, *settings, context, /* 
for_disk_s3 */ true);
             auto key_generator = 
createObjectStorageKeysGeneratorAsIsWithPrefix(uri.key);
 
diff --git a/cpp-ch/local-engine/Parser/RelParsers/CrossRelParser.cpp 
b/cpp-ch/local-engine/Parser/RelParsers/CrossRelParser.cpp
index 67494555ca..4e00a3882f 100644
--- a/cpp-ch/local-engine/Parser/RelParsers/CrossRelParser.cpp
+++ b/cpp-ch/local-engine/Parser/RelParsers/CrossRelParser.cpp
@@ -41,6 +41,7 @@ namespace DB
 namespace Setting
 {
 extern const SettingsUInt64 max_block_size;
+extern const SettingsUInt64 min_joined_block_size_rows;
 extern const SettingsUInt64 min_joined_block_size_bytes;
 }
 namespace ErrorCodes
@@ -201,6 +202,7 @@ DB::QueryPlanPtr CrossRelParser::parseJoin(const 
substrait::CrossRel & join, DB:
             right->getCurrentHeader(),
             hash_join,
             context->getSettingsRef()[Setting::max_block_size],
+            context->getSettingsRef()[Setting::min_joined_block_size_rows],
             context->getSettingsRef()[Setting::min_joined_block_size_bytes],
             1,
             /* required_output_ = */ NameSet{},
diff --git a/cpp-ch/local-engine/Parser/RelParsers/JoinRelParser.cpp 
b/cpp-ch/local-engine/Parser/RelParsers/JoinRelParser.cpp
index d0f60a4d37..a339c8c10d 100644
--- a/cpp-ch/local-engine/Parser/RelParsers/JoinRelParser.cpp
+++ b/cpp-ch/local-engine/Parser/RelParsers/JoinRelParser.cpp
@@ -46,6 +46,7 @@ namespace Setting
 {
 extern const SettingsJoinAlgorithm join_algorithm;
 extern const SettingsUInt64 max_block_size;
+extern const SettingsUInt64 min_joined_block_size_rows;
 extern const SettingsUInt64 min_joined_block_size_bytes;
 extern const SettingsNonZeroUInt64 grace_hash_join_initial_buckets;
 extern const SettingsNonZeroUInt64 grace_hash_join_max_buckets;
@@ -319,6 +320,7 @@ DB::QueryPlanPtr JoinRelParser::parseJoin(const 
substrait::JoinRel & join, DB::Q
             right->getCurrentHeader(),
             smj_join,
             context->getSettingsRef()[Setting::max_block_size],
+            context->getSettingsRef()[Setting::min_joined_block_size_rows],
             context->getSettingsRef()[Setting::min_joined_block_size_bytes],
             1,
             /* required_output_ = */ NameSet{},
@@ -744,6 +746,7 @@ DB::QueryPlanPtr JoinRelParser::buildMultiOnClauseHashJoin(
         right_plan->getCurrentHeader(),
         hash_join,
         context->getSettingsRef()[Setting::max_block_size],
+        context->getSettingsRef()[Setting::min_joined_block_size_rows],
         context->getSettingsRef()[Setting::min_joined_block_size_bytes],
         1,
         /* required_output_ = */ NameSet{},
@@ -788,6 +791,7 @@ DB::QueryPlanPtr JoinRelParser::buildSingleOnClauseHashJoin(
         right_plan->getCurrentHeader(),
         hash_join,
         context->getSettingsRef()[Setting::max_block_size],
+        context->getSettingsRef()[Setting::min_joined_block_size_rows],
         context->getSettingsRef()[Setting::min_joined_block_size_bytes],
         1,
         /* required_output_ = */ NameSet{},
diff --git a/cpp-ch/local-engine/Storages/MergeTree/SparkStorageMergeTree.h 
b/cpp-ch/local-engine/Storages/MergeTree/SparkStorageMergeTree.h
index aeb57cad90..4e9f390c9c 100644
--- a/cpp-ch/local-engine/Storages/MergeTree/SparkStorageMergeTree.h
+++ b/cpp-ch/local-engine/Storages/MergeTree/SparkStorageMergeTree.h
@@ -54,15 +54,13 @@ class SparkStorageMergeTree : public DB::MergeTreeData
     {
         SparkMutationsSnapshot() = default;
 
-        DB::MutationCommands getAlterMutationCommandsForPart(const 
MergeTreeData::DataPartPtr & part) const override { return {}; }
+        DB::MutationCommands getOnFlyMutationCommandsForPart(const 
MergeTreeData::DataPartPtr & part) const override { return {}; }
         std::shared_ptr<MergeTreeData::IMutationsSnapshot> cloneEmpty() const 
override
         {
             return std::make_shared<SparkMutationsSnapshot>();
         }
 
         DB::NameSet getAllUpdatedColumns() const override { return {}; }
-
-        bool hasMetadataMutations() const override { return 
params.min_part_metadata_version < params.metadata_version; }
     };
 
 public:
diff --git 
a/cpp-ch/local-engine/Storages/Parquet/VectorizedParquetRecordReader.h 
b/cpp-ch/local-engine/Storages/Parquet/VectorizedParquetRecordReader.h
index ce1a139014..512cc91bed 100644
--- a/cpp-ch/local-engine/Storages/Parquet/VectorizedParquetRecordReader.h
+++ b/cpp-ch/local-engine/Storages/Parquet/VectorizedParquetRecordReader.h
@@ -214,6 +214,9 @@ class VectorizedParquetBlockInputFormat final : public 
DB::IInputFormat
 protected:
     void onCancel() noexcept override { is_stopped = 1; }
 
+    // TODO: create ColumnIndexFilter here, currently disable it now.
+    void setKeyCondition(const std::shared_ptr<const DB::KeyCondition> & 
key_condition_) override { }
+
 public:
     VectorizedParquetBlockInputFormat(
         DB::ReadBuffer & in_,
diff --git 
a/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.cpp 
b/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.cpp
index dd191831c3..f094bf915d 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.cpp
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.cpp
@@ -62,8 +62,7 @@ bool ExcelTextFormatFile::useThis(const DB::ContextPtr & 
context)
     return settingsEqual(context->getSettingsRef(), USE_EXCEL_PARSER, "true");
 }
 
-FormatFile::InputFormatPtr
-ExcelTextFormatFile::createInputFormat(const DB::Block & header, const 
std::shared_ptr<const DB::ActionsDAG> & /*filter_actions_dag*/)
+FormatFile::InputFormatPtr ExcelTextFormatFile::createInputFormat(const 
DB::Block & header)
 {
     auto read_buffer = read_buffer_builder->build(file_info);
 
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.h 
b/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.h
index 0b92f86219..5e70d22eeb 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.h
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/ExcelTextFormatFile.h
@@ -47,8 +47,7 @@ public:
 
     ~ExcelTextFormatFile() override = default;
 
-    FormatFile::InputFormatPtr
-    createInputFormat(const DB::Block & header, const std::shared_ptr<const 
DB::ActionsDAG> & filter_actions_dag = nullptr) override;
+    FormatFile::InputFormatPtr createInputFormat(const DB::Block & header) 
override;
 
     bool supportSplit() const override { return true; }
     String getFileFormat() const override { return "ExcelText"; }
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/FileReader.cpp 
b/cpp-ch/local-engine/Storages/SubstraitSource/FileReader.cpp
index 8de7f83509..e5d1f8d8a0 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/FileReader.cpp
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/FileReader.cpp
@@ -274,12 +274,17 @@ std::unique_ptr<NormalFileReader> createNormalFileReader(
     const FormatFilePtr & file,
     const DB::Block & to_read_header_,
     const DB::Block & output_header_,
-    const std::shared_ptr<const DB::ActionsDAG> & filter_actions_dag = nullptr,
+    const std::shared_ptr<const DB::KeyCondition> & key_condition = nullptr,
     const ColumnIndexFilterPtr & column_index_filter = nullptr)
 {
     file->initialize(column_index_filter);
     auto createInputFormat = [&](const DB::Block & new_read_header_) -> 
FormatFile::InputFormatPtr
-    { return file->createInputFormat(new_read_header_, filter_actions_dag); };
+    {
+        auto input_format = file->createInputFormat(new_read_header_);
+        if (key_condition && input_format)
+            input_format->inputFormat().setKeyCondition(key_condition);
+        return input_format;
+    };
 
     if (file->getFileInfo().has_iceberg())
         return iceberg::IcebergReader::create(file, to_read_header_, 
output_header_, createInputFormat);
@@ -311,13 +316,11 @@ std::unique_ptr<NormalFileReader> createNormalFileReader(
     return std::make_unique<NormalFileReader>(file, to_read_header_, 
output_header_, input_format);
 }
 }
-
-/// TODO Remove ColumnIndexFilterPtr
 std::unique_ptr<BaseReader> BaseReader::create(
     const FormatFilePtr & current_file,
     const DB::Block & readHeader,
     const DB::Block & outputHeader,
-    const std::shared_ptr<const DB::ActionsDAG> & filter_actions_dag,
+    const std::shared_ptr<const DB::KeyCondition> & key_condition,
     const ColumnIndexFilterPtr & column_index_filter)
 {
     if (!readHeader)
@@ -332,7 +335,7 @@ std::unique_ptr<BaseReader> BaseReader::create(
         }
     }
 
-    return createNormalFileReader(current_file, readHeader, outputHeader, 
filter_actions_dag, column_index_filter);
+    return createNormalFileReader(current_file, readHeader, outputHeader, 
key_condition, column_index_filter);
 }
 
 
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/FileReader.h 
b/cpp-ch/local-engine/Storages/SubstraitSource/FileReader.h
index 467824610e..bdb6772ab5 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/FileReader.h
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/FileReader.h
@@ -77,7 +77,7 @@ public:
         const FormatFilePtr & current_file,
         const DB::Block & readHeader,
         const DB::Block & outputHeader,
-        const std::shared_ptr<const DB::ActionsDAG> & filter_actions_dag,
+        const std::shared_ptr<const DB::KeyCondition> & key_condition,
         const ColumnIndexFilterPtr & column_index_filter);
 };
 
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/FormatFile.h 
b/cpp-ch/local-engine/Storages/SubstraitSource/FormatFile.h
index 60d28f5bba..262aeff6fa 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/FormatFile.h
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/FormatFile.h
@@ -30,7 +30,6 @@
 
 namespace DB
 {
-class ActionsDAG;
 namespace ErrorCodes
 {
 extern const int NOT_IMPLEMENTED;
@@ -116,9 +115,7 @@ public:
     virtual ~FormatFile() = default;
 
     /// Create a new input format for reading this file
-    virtual InputFormatPtr
-    createInputFormat(const DB::Block & header, const std::shared_ptr<const 
DB::ActionsDAG> & filter_actions_dag = nullptr)
-        = 0;
+    virtual InputFormatPtr createInputFormat(const DB::Block & header) = 0;
 
     /// Spark would split a large file into small segements and read in 
different tasks
     /// If this file doesn't support the split feacture, only the task with 
offset 0 will generate data.
@@ -152,6 +149,7 @@ protected:
     std::map<String, String> partition_values;
     /// partition keys are normalized to lower cases for partition column 
case-insensitive matching
     std::map<String, String> normalized_partition_values;
+    std::shared_ptr<const DB::KeyCondition> key_condition;
     const FileMetaColumns meta_columns;
 
     /// Currently, it is used to read an iceberg format, and initialized in 
the constructor of child class
diff --git 
a/cpp-ch/local-engine/Storages/SubstraitSource/Iceberg/IcebergReader.h 
b/cpp-ch/local-engine/Storages/SubstraitSource/Iceberg/IcebergReader.h
index ddfd0954db..f8845c1367 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/Iceberg/IcebergReader.h
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/Iceberg/IcebergReader.h
@@ -18,12 +18,6 @@
 
 #include <Storages/SubstraitSource/FileReader.h>
 
-namespace DB
-{
-class ExpressionActions;
-using ExpressionActionsPtr = std::shared_ptr<ExpressionActions>;
-}
-
 namespace local_engine
 {
 class DeltaDVRoaringBitmapArray;
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/JSONFormatFile.cpp 
b/cpp-ch/local-engine/Storages/SubstraitSource/JSONFormatFile.cpp
index b5728abbf9..5cbab2bbda 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/JSONFormatFile.cpp
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/JSONFormatFile.cpp
@@ -31,8 +31,7 @@ JSONFormatFile::JSONFormatFile(
 {
 }
 
-FormatFile::InputFormatPtr
-JSONFormatFile::createInputFormat(const DB::Block & header, const 
std::shared_ptr<const DB::ActionsDAG> & /*filter_actions_dag*/)
+FormatFile::InputFormatPtr JSONFormatFile::createInputFormat(const DB::Block & 
header)
 {
     auto read_buffer = 
read_buffer_builder->buildWithCompressionWrapper(file_info);
 
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/JSONFormatFile.h 
b/cpp-ch/local-engine/Storages/SubstraitSource/JSONFormatFile.h
index a98af115a2..97b3a694de 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/JSONFormatFile.h
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/JSONFormatFile.h
@@ -28,8 +28,7 @@ public:
 
     bool supportSplit() const override { return true; }
 
-    FormatFile::InputFormatPtr
-    createInputFormat(const DB::Block & header, const std::shared_ptr<const 
DB::ActionsDAG> & filter_actions_dag = nullptr) override;
+    FormatFile::InputFormatPtr createInputFormat(const DB::Block & header) 
override;
 
     String getFileFormat() const override { return "JSONEachRow"; }
 };
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/ORCFormatFile.cpp 
b/cpp-ch/local-engine/Storages/SubstraitSource/ORCFormatFile.cpp
index 9b87c6528e..f78cca5d87 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/ORCFormatFile.cpp
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/ORCFormatFile.cpp
@@ -36,8 +36,7 @@ ORCFormatFile::ORCFormatFile(
 {
 }
 
-FormatFile::InputFormatPtr
-ORCFormatFile::createInputFormat(const DB::Block & header, const 
std::shared_ptr<const DB::ActionsDAG> & filter_actions_dag)
+FormatFile::InputFormatPtr ORCFormatFile::createInputFormat(const DB::Block & 
header)
 {
     auto read_buffer = read_buffer_builder->build(file_info);
 
@@ -71,8 +70,7 @@ ORCFormatFile::createInputFormat(const DB::Block & header, 
const std::shared_ptr
         format_settings.orc.reader_time_zone_name = mapped_timezone;
     }
     //TODO: support prefetch
-    auto parser_group = 
std::make_shared<DB::FormatParserGroup>(context->getSettingsRef(), 1, 
filter_actions_dag, context);
-    auto input_format = 
std::make_shared<DB::NativeORCBlockInputFormat>(*read_buffer, header, 
format_settings, false, 0, parser_group);
+    auto input_format = 
std::make_shared<DB::NativeORCBlockInputFormat>(*read_buffer, header, 
format_settings, false, 0);
     return std::make_shared<InputFormat>(std::move(read_buffer), input_format);
 }
 
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/ORCFormatFile.h 
b/cpp-ch/local-engine/Storages/SubstraitSource/ORCFormatFile.h
index 0d8beb5d3b..b6f6fd2321 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/ORCFormatFile.h
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/ORCFormatFile.h
@@ -44,8 +44,7 @@ public:
         DB::ContextPtr context_, const 
substrait::ReadRel::LocalFiles::FileOrFiles & file_info_, ReadBufferBuilderPtr 
read_buffer_builder_);
     ~ORCFormatFile() override = default;
 
-    FormatFile::InputFormatPtr
-    createInputFormat(const DB::Block & header, const std::shared_ptr<const 
DB::ActionsDAG> & filter_actions_dag = nullptr) override;
+    FormatFile::InputFormatPtr createInputFormat(const DB::Block & header) 
override;
 
     std::optional<size_t> getTotalRows() override;
 
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.cpp 
b/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.cpp
index 50e84e60f9..a4ccfa93b7 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.cpp
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.cpp
@@ -142,8 +142,7 @@ void ParquetFormatFile::initialize(const 
ColumnIndexFilterPtr & filter)
         file_schema = ParquetMetaBuilder::collectFileSchema(context, 
*read_buffer_);
 }
 
-FormatFile::InputFormatPtr
-ParquetFormatFile::createInputFormat(const Block & header, const 
std::shared_ptr<const DB::ActionsDAG> & filter_actions_dag)
+FormatFile::InputFormatPtr ParquetFormatFile::createInputFormat(const Block & 
header)
 {
     assert(read_buffer_);
 
@@ -206,8 +205,14 @@ ParquetFormatFile::createInputFormat(const Block & header, 
const std::shared_ptr
             // We need to disable fiter push down and read all row groups, so 
that we can get correct row index.
             format_settings.parquet.filter_push_down = false;
         }
-        auto parser_group = 
std::make_shared<FormatParserGroup>(context->getSettingsRef(), 1, 
filter_actions_dag, context);
-        auto input = std::make_shared<ParquetBlockInputFormat>(*read_buffer_, 
read_header, format_settings, parser_group, 8192);
+
+        auto input = std::make_shared<ParquetBlockInputFormat>(
+            *read_buffer_,
+            read_header,
+            format_settings,
+            settings[Setting::max_parsing_threads],
+            settings[Setting::max_download_threads],
+            8192);
         return std::make_shared<ParquetInputFormat>(
             std::move(read_buffer_), input, std::move(provider), 
std::move(read_header), std::move(output_header));
     };
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.h 
b/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.h
index 587b788f79..2014e1d643 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.h
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/ParquetFormatFile.h
@@ -35,8 +35,7 @@ public:
         bool use_local_format_);
     ~ParquetFormatFile() override = default;
 
-    InputFormatPtr
-    createInputFormat(const DB::Block & header, const std::shared_ptr<const 
DB::ActionsDAG> & filter_actions_dag = nullptr) override;
+    InputFormatPtr createInputFormat(const DB::Block & header) override;
 
     std::optional<size_t> getTotalRows() override;
 
diff --git 
a/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.cpp 
b/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.cpp
index 51c041fa3c..a5df941a48 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.cpp
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.cpp
@@ -54,7 +54,7 @@ static DB::Block initReadHeader(const DB::Block & block, 
const FormatFiles & fil
 
 SubstraitFileSource::SubstraitFileSource(
     const DB::ContextPtr & context_, const DB::Block & outputHeader_, const 
substrait::ReadRel::LocalFiles & file_infos)
-    : DB::ISource(BaseReader::buildRowCountHeader(outputHeader_), false)
+    : 
DB::SourceWithKeyCondition(BaseReader::buildRowCountHeader(outputHeader_), 
false)
     , files(initializeFiles(file_infos, context_))
     , outputHeader(outputHeader_)
     , readHeader(initReadHeader(outputHeader, files))
@@ -63,11 +63,11 @@ SubstraitFileSource::SubstraitFileSource(
 
 SubstraitFileSource::~SubstraitFileSource() = default;
 
-void SubstraitFileSource::setKeyCondition(const std::shared_ptr<const 
DB::ActionsDAG> & filter_actions_dag_, DB::ContextPtr context_)
+void SubstraitFileSource::setKeyCondition(const std::optional<DB::ActionsDAG> 
& filter_actions_dag, DB::ContextPtr context_)
 {
-    assert(filter_actions_dag_);
-    filter_actions_dag = filter_actions_dag_;
-    column_index_filter = 
std::make_shared<ColumnIndexFilter>(*filter_actions_dag, context_);
+    setKeyConditionImpl(filter_actions_dag, context_, readHeader);
+    if (filter_actions_dag)
+        column_index_filter = 
std::make_shared<ColumnIndexFilter>(filter_actions_dag.value(), context_);
 }
 
 DB::Chunk SubstraitFileSource::generate()
@@ -105,7 +105,7 @@ bool SubstraitFileSource::tryPrepareReader()
         if (!current_file->supportSplit() && current_file->getStartOffset())
             continue;
 
-        file_reader = BaseReader::create(current_file, readHeader, 
outputHeader, filter_actions_dag, column_index_filter);
+        file_reader = BaseReader::create(current_file, readHeader, 
outputHeader, key_condition, column_index_filter);
         if (file_reader)
             return true;
     }
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.h 
b/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.h
index b40dd9a82d..a2280da0d2 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.h
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSource.h
@@ -17,15 +17,9 @@
 #pragma once
 
 #include <memory>
-#include <Interpreters/Context_fwd.h>
-#include <Processors/ISource.h>
+#include <Processors/SourceWithKeyCondition.h>
 #include <substrait/algebra.pb.h>
 
-namespace DB
-{
-class ActionsDAG;
-}
-
 namespace local_engine
 {
 class ColumnIndexFilter;
@@ -35,7 +29,7 @@ class FormatFile;
 using FormatFilePtr = std::shared_ptr<FormatFile>;
 using FormatFiles = std::vector<FormatFilePtr>;
 
-class SubstraitFileSource : public DB::ISource
+class SubstraitFileSource : public DB::SourceWithKeyCondition
 {
 public:
     SubstraitFileSource(const DB::ContextPtr & context_, const DB::Block & 
header_, const substrait::ReadRel::LocalFiles & file_infos);
@@ -43,7 +37,7 @@ public:
 
     String getName() const override { return "SubstraitFileSource"; }
 
-    void setKeyCondition(const std::shared_ptr<const DB::ActionsDAG> & 
filter_actions_dag_, DB::ContextPtr context_);
+    void setKeyCondition(const std::optional<DB::ActionsDAG> & 
filter_actions_dag, DB::ContextPtr context_) override;
 
 protected:
     DB::Chunk generate() override;
@@ -60,6 +54,5 @@ private:
 
     std::unique_ptr<BaseReader> file_reader;
     ColumnIndexFilterPtr column_index_filter;
-    std::shared_ptr<const DB::ActionsDAG> filter_actions_dag;
 };
 }
diff --git 
a/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSourceStep.cpp 
b/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSourceStep.cpp
index 220d181226..91e50b4275 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSourceStep.cpp
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSourceStep.cpp
@@ -15,6 +15,7 @@
  * limitations under the License.
  */
 
+#include <Interpreters/Context_fwd.h>
 #include <Processors/QueryPlan/IQueryPlanStep.h>
 #include <QueryPipeline/Pipe.h>
 #include <QueryPipeline/QueryPipelineBuilder.h>
@@ -65,12 +66,9 @@ void 
SubstraitFileSourceStep::initializePipeline(DB::QueryPipelineBuilder & pipe
 
 void SubstraitFileSourceStep::applyFilters(const DB::ActionDAGNodes 
added_filter_nodes)
 {
-    SourceStepWithFilter::applyFilters(std::move(added_filter_nodes));
-    if (filter_actions_dag)
-    {
-        for (const auto & processor : pipe.getProcessors())
-            if (auto * source = dynamic_cast<SubstraitFileSource 
*>(processor.get()))
-                source->setKeyCondition(filter_actions_dag, context);
-    }
+    filter_actions_dag = 
DB::ActionsDAG::buildFilterActionsDAG(added_filter_nodes.nodes);
+    for (const auto & processor : pipe.getProcessors())
+        if (auto * source = dynamic_cast<DB::SourceWithKeyCondition 
*>(processor.get()))
+            source->setKeyCondition(filter_actions_dag, context);
 }
 }
diff --git 
a/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSourceStep.h 
b/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSourceStep.h
index e1175507f5..78fcb767f0 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSourceStep.h
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/SubstraitFileSourceStep.h
@@ -17,10 +17,11 @@
 
 #pragma once
 
-#include <Interpreters/Context_fwd.h>
 #include <Processors/QueryPlan/ReadFromPreparedSource.h>
 #include <Processors/QueryPlan/SourceStepWithFilter.h>
-
+#include <Storages/MergeTree/KeyCondition.h>
+#include <Interpreters/Context_fwd.h>
+#include <Core/NamesAndTypes.h>
 
 namespace local_engine
 {
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/TextFormatFile.cpp 
b/cpp-ch/local-engine/Storages/SubstraitSource/TextFormatFile.cpp
index 9e40209eea..fcbe650d46 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/TextFormatFile.cpp
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/TextFormatFile.cpp
@@ -34,8 +34,7 @@ TextFormatFile::TextFormatFile(
 {
 }
 
-FormatFile::InputFormatPtr
-TextFormatFile::createInputFormat(const DB::Block & header, const 
std::shared_ptr<const DB::ActionsDAG> & /*filter_actions_dag*/)
+FormatFile::InputFormatPtr TextFormatFile::createInputFormat(const DB::Block & 
header)
 {
     auto read_buffer = 
read_buffer_builder->buildWithCompressionWrapper(file_info);
 
diff --git a/cpp-ch/local-engine/Storages/SubstraitSource/TextFormatFile.h 
b/cpp-ch/local-engine/Storages/SubstraitSource/TextFormatFile.h
index 0e6827e653..62e60af4a8 100644
--- a/cpp-ch/local-engine/Storages/SubstraitSource/TextFormatFile.h
+++ b/cpp-ch/local-engine/Storages/SubstraitSource/TextFormatFile.h
@@ -32,8 +32,7 @@ public:
         DB::ContextPtr context_, const 
substrait::ReadRel::LocalFiles::FileOrFiles & file_info_, ReadBufferBuilderPtr 
read_buffer_builder_);
     ~TextFormatFile() override = default;
 
-    FormatFile::InputFormatPtr
-    createInputFormat(const DB::Block & header, const std::shared_ptr<const 
DB::ActionsDAG> & filter_actions_dag = nullptr) override;
+    FormatFile::InputFormatPtr createInputFormat(const DB::Block & header) 
override;
 
     DB::NamesAndTypesList getSchema() const
     {
diff --git a/cpp-ch/local-engine/tests/benchmark_local_engine.cpp 
b/cpp-ch/local-engine/tests/benchmark_local_engine.cpp
index 59acf42cf3..32c213610f 100644
--- a/cpp-ch/local-engine/tests/benchmark_local_engine.cpp
+++ b/cpp-ch/local-engine/tests/benchmark_local_engine.cpp
@@ -803,12 +803,12 @@ QueryPlanPtr readFromMergeTree(MergeTreeWithSnapshot 
storage)
     auto data_parts = storage.merge_tree->getDataPartsVectorForInternalUsage();
     auto query_plan = std::make_unique<QueryPlan>();
     auto step = storage.merge_tree->reader.readFromParts(
-        data_parts, {}, storage.columns.getNames(), storage.snapshot, 
*query_info, global_context, 10000, 1);
+        RangesInDataParts{data_parts}, {}, storage.columns.getNames(), 
storage.snapshot, *query_info, global_context, 10000, 1);
     query_plan->addStep(std::move(step));
     return query_plan;
 }
 
-QueryPlanPtr joinPlan(QueryPlanPtr left, QueryPlanPtr right, String left_key, 
String right_key, size_t block_size = 8192)
+QueryPlanPtr joinPlan(QueryPlanPtr left, QueryPlanPtr right, String left_key, 
String right_key)
 {
     auto join = std::make_shared<TableJoin>(
         global_context->getSettingsRef(), 
global_context->getGlobalTemporaryVolume(), 
global_context->getTempDataOnDisk());
@@ -846,7 +846,7 @@ QueryPlanPtr joinPlan(QueryPlanPtr left, QueryPlanPtr 
right, String left_key, St
     auto hash_join = std::make_shared<HashJoin>(join, 
right->getCurrentHeader());
 
     QueryPlanStepPtr join_step = std::make_unique<JoinStep>(
-        left->getCurrentHeader(), right->getCurrentHeader(), hash_join, 
block_size, 8192, 1, NameSet{}, false, false);
+        left->getCurrentHeader(), right->getCurrentHeader(), hash_join, 
DEFAULT_BLOCK_SIZE, DEFAULT_BLOCK_SIZE, 524288, 1, NameSet{}, false, false);
 
     std::vector<QueryPlanPtr> plans;
     plans.emplace_back(std::move(left));
diff --git a/cpp-ch/local-engine/tests/benchmark_parquet_read.cpp 
b/cpp-ch/local-engine/tests/benchmark_parquet_read.cpp
index 2b56f44622..f9b4693aeb 100644
--- a/cpp-ch/local-engine/tests/benchmark_parquet_read.cpp
+++ b/cpp-ch/local-engine/tests/benchmark_parquet_read.cpp
@@ -14,6 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+#include <random>
 #include <Core/Block.h>
 #include <DataTypes/DataTypeDate32.h>
 #include <DataTypes/DataTypeString.h>
diff --git a/cpp-ch/local-engine/tests/gtest_ch_join.cpp 
b/cpp-ch/local-engine/tests/gtest_ch_join.cpp
index 74b04d5def..6350571c56 100644
--- a/cpp-ch/local-engine/tests/gtest_ch_join.cpp
+++ b/cpp-ch/local-engine/tests/gtest_ch_join.cpp
@@ -127,7 +127,7 @@ TEST(TestJoin, simple)
     auto hash_join = std::make_shared<HashJoin>(join, 
right_plan.getCurrentHeader());
 
     QueryPlanStepPtr join_step = std::make_unique<JoinStep>(
-        left_plan.getCurrentHeader(), right_plan.getCurrentHeader(), 
hash_join, 8192, 8192, 1, NameSet{}, false, false);
+        left_plan.getCurrentHeader(), right_plan.getCurrentHeader(), 
hash_join, DEFAULT_BLOCK_SIZE, DEFAULT_BLOCK_SIZE, 524288, 1, NameSet{}, false, 
false);
 
     std::cerr << "join step:" << join_step->getOutputHeader().dumpStructure() 
<< std::endl;
 
diff --git a/cpp-ch/local-engine/tests/gtest_parquet_read.cpp 
b/cpp-ch/local-engine/tests/gtest_parquet_read.cpp
index 57b4eca6cf..57b6b86ae0 100644
--- a/cpp-ch/local-engine/tests/gtest_parquet_read.cpp
+++ b/cpp-ch/local-engine/tests/gtest_parquet_read.cpp
@@ -45,7 +45,6 @@
 #include <tests/utils/gluten_test_util.h>
 #include <Common/BlockTypeUtils.h>
 #include <Common/DebugUtils.h>
-#include <Common/QueryContext.h>
 
 using namespace DB;
 using namespace local_engine;
@@ -112,10 +111,8 @@ void readData(const String & path, const std::map<String, 
Field> & fields)
     ReadBufferFromFile in(full_path);
 
     InputFormatPtr format;
-    auto parser_group
-        = 
std::make_shared<FormatParserGroup>(QueryContext::globalContext()->getSettingsRef(),
 1, nullptr, QueryContext::globalContext());
     if constexpr (std::is_same_v<InputFormat, DB::ParquetBlockInputFormat>)
-        format = std::make_shared<InputFormat>(in, header, settings, 
parser_group, 8192);
+        format = std::make_shared<InputFormat>(in, header, settings, 1, 1, 
8192);
     else
         format = std::make_shared<InputFormat>(in, header, settings);
 


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]


Reply via email to