This is an automated email from the ASF dual-hosted git repository.
chengchengjin pushed a change to branch cudf
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
discard 24ea32be9f fix docker
discard e4717a700e update velox branch
discard 59d4f41f8a update CMakeLists.txt
discard 64327b26c1 update Dockerfile
discard cf1462d126 Revert "update the velox branch to gpu branch"
discard 81890293c0 update docker branch
discard ae81a95266 update the velox branch to gpu branch
discard 558f4a4718 support cudf
add 45ca1c38ea [GLUTEN-6887][VL] Daily Update Velox Version (2025_04_09)
(#9263)
add 722bc79539 [GLUTEN-9244][CORE] Change the way of passing default
timezone to native config (#9249)
add 9b9f63f7ca [GLUTEN-8497][VL] Fix columnar batch type mismatch in table
cache (#9230)
add 7d11bb6be7 [VL] Support Spark legacy statistical aggregation function
behavior (#9181)
add 30f301adb2 [CORE] Remove library unloading API from JniLibLoader as
unused (#9277)
add 8ef968cf48 [GLUTEN-9237][CH] Fix the nullability missmatch issue for
the Nothing type (#9238)
add d0175ae8a1 [Minor] Remove duplicate fallback log (#9261)
add f667e81df3 [VL] Disable FlushableHashAggreagte when aggregates contain
sum/avg for floating type (#8986)
add 8b1d018d4b [GLUTEN-6887][VL] Daily Update Velox Version (2025_04_10)
(#9283)
add 118b114996 [CORE] Refine the test APIs for specified spark versions
(#9274)
add 4ad257f0e3 [CH] Add a comment to explain why the endpoint uses a
single thread (#9257)
add 9e33a5f0bb [GLUTEN-8891][VL] Refine local ssd cache feature (#9228)
add cf2a528979 [VL] Fix merge conflict on VeloxLocalCacheSuite (#9297)
add 2614db9f3e fix (#9293)
add d1e2504df4 [TESTS][FOLLOWUP] Disable spark ui in VeloxUdfSuite (#9273)
add 5d933797de [VL] Remove param original of ColumnarPartialProjectExec
(#9290)
add 6329b82b93 [GLUTEN-6887][VL] Daily Update Velox Version (2025_04_11)
(#9299)
add 9687247216 [VL] Fix weekly build job on Centos 7 (#9294)
add cb35706408 [GLUTEN-6887][VL] Daily Update Velox Version (2025_04_12)
(#9303)
add ba3358d815 [GLUTEN-6887][VL] Daily Update Velox Version (2025_04_13)
(#9308)
add d93128ce10 [GLUTEN-9178][CH] Fix cse in aggregate operator not working
(#9301)
add 4dcd2803fa [CORE] Post events until both spark ui and gluten ui are
enable (#9272)
add d78825a59d [CORE] Correctly handle driver configurations when
`spark.sql.extensions` is explicitly set for GlutenSessionExtensions (#9312)
add a66e1a1788 [GLUTEN-8851][VL] feat: Support cudf (#9229)
add e9fa5b6ec8 [GLUTEN-6887][VL] Daily Update Velox Version (2025_04_14)
(#9310)
add dd5a1a9740 [GLUTEN-9288][VL] Enable array_prepend function for spark
3.5+ (#9305)
add 383f7081cd [DOC] Fix broken links in documents (#9320)
add 0b7c9beb1c [GLUTEN-9317][CH]Fix: duplicated column names in shuffle
read (#9318)
add 65ad57add7 [Gluten-9254][CH] Support RDDScanExec (#9270)
add e8b7237910 [GLUTEN-9329][DOC] Fix log path when parsing log in
gen-function-support-docs.py (#9330)
add c2c82715ac [VL] Count total JVM memory as the on-heap portion for the
off-heap sizing feature (#9321)
add c1d12e0c1f [GLUTEN-6887][VL] Daily Update Velox Version (2025_04_15)
(#9322)
add 30df1dc7f0 [GLUTEN-1632][CH]Daily Update Clickhouse Version (20250416)
(#9338)
add 46c4feea76 [GLUTEN-9319][INFRA] Add an option for gluten version in
bug report and enhancement templates (#9324)
add 6e3896f201 [GLUTEN-9300][DOC] Support replacement expression in
gen-function-support-docs (#9331)
add 7e401878a0 [GLUTEN-9239][CH] [PART-1] Support Java-17 Rmove
`JNI_OnUnload`(#9275)
add 79db240513 [GLUTEN-7652][VL] Support binary as string (#9325)
add b10ee0d6b2 [Gluten-9334][CH] Support delta metadata column `file_path`
and `row_index` for mergetree (#9340)
add c275b1ee1f [GLUTEN-6867][CH] Fix Bug that cann't read file on minio
(#9332)
add fb28ad9973 [GLUTEN-6887][VL] Daily Update Velox Version (2025_04_16)
(#9339)
add a35ffe8a45 [VL][CI] Upload core dump error log if test fails (#9347)
add 48bb014d90 [GLUTEN-1632][CH]Daily Update Clickhouse Version (20250417)
(#9345)
add b99fff82a2 [VL] Provide a configuration option to completely turn off
off-heap memory tracking with Spark memory manager (#9341)
add ef011c0ed3 [GLUTEN-9313][VL] ColumnarPartialProject supports built-in
but blacklisted function (#9315)
add 55b2e92f54 [GLUTEN-8772][CORE] refactor: Refactoring the use of
SubstraitContext#functionMap (#8775)
add 083ecfbbe9 [VL] Move pre-configuration code of dynamic off-heap sizing
to its own place (#9336)
add 1b8d42026f [GLUTEN-9163][VL] Use stream de/compressor in sort-based
shuffle (#9278)
add d659c80413 [GLUTEN-9287][VL] Enable array_compact function for Spark
3.4+ (#9349)
add 2b0ceb181c [GLUTEN-6887][VL] Daily Update Velox Version (2025_04_17)
(#9346)
add 78540d6a0f [VL] Fix UT log level to warn to decrease CI output (#9352)
add f0bdaec953 [GLUTEN-9095][UT] Remove Vanilla Spark InternalRow based
checkEvaluation (#9096)
add 675c99e55e [CORE] Make max broadcast table size configurable (#9359)
add d8293a3a20 [CH] Fix build error (#9363)
add a47422fa1b [GLUTEN-6887][VL] Daily Update Velox Version (2025_04_18)
(#9362)
add 6dd3899c34 [GLUTEN-9243][VL] Fix cuda docker image (#9333)
add 4e5125c8ce [GLUTEN-8912][VL] Add Offset support for CollectLimitExec
(#8914)
add f74d31cc50 [GLUTEN-7589][VL] Support date_trunc function (#7611)
add 082fa3f901 [GLUTEN-6887][VL] Daily Update Velox Version (2025_04_19)
(#9371)
add 396940100f [GLUTEN-6887][VL] Daily Update Velox Version (2025_04_20)
(#9376)
add 79dccb8ee5 [GLUTEN][UT] Disable flaky UT from GlutenCSVv1Suite (#9355)
add cce8d24ca9 [GLUTEN-9279] Not pulling out expression from PartialMerge
aggregate function to avoid invalid reference binding in ProjectExecTransformer
(#9280)
add 96bcdfc99f [GLUTEN-1632][CH]Daily Update Clickhouse Version (20250421)
(#9378)
add 77e1d2c14e [GLUTEN-9379] [VL] Disable flaky ArrowCsvScanSuiteV2 (#9380)
add ae3526cedd [GLUTEN-6887][VL] Daily Update Velox Version (2025_04_21)
(#9377)
add ef5e8af42c [Gluten-8792][CH] Support delta project incrementMetric
expr (#9353)
add 231a9acda5 [GLUTEN-9369][DOC] Fix commands in the Gluten C++ debugging
developer doc (#9368)
add 9a7d5fc6b1 [GLUTEN-9034][VL] Add VeloxResizeBatchesExec for Shuffle
(#9035)
add a0f7d1da92 [VL][MINOR] Improve log level in ManagedReservationListener
(#9389)
add 82fa11d41f Fix ColumnarToRowRemovalGuard not able to be copied (#9384)
add 33180440f5 [GLUTEN-8846][CH] [Part 4] Add full-chain UT (#9256)
add 9949ce77ee [GLUTEN-6887][VL] Daily Update Velox Version (2025_04_22)
(#9387)
add fa58cbce05 [GLUTEN-8846][CH] Fix `mvn clean install` (#9401)
new 12bad09781 update velox branch
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (24ea32be9f)
\
N -- N -- N refs/heads/cudf (12bad09781)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/ISSUE_TEMPLATE/bug.yml | 16 +
.github/ISSUE_TEMPLATE/enhancement.yml | 12 +
.github/workflows/docker_image.yml | 27 ++
.github/workflows/velox_backend.yml | 79 ++-
backends-clickhouse/pom.xml | 393 +++++++++++++--
.../merge/MergeIntoMaterializeSource.scala | 499 +++++++++++++++++++
.../GlutenDeltaMergetreeDeletionVectorSuite.scala} | 77 ++-
.../sql/execution/GlutenDeltaExpressionSuite.scala | 92 ++++
.../apache/spark/sql/execution/RDDScanSuite.scala | 86 ++++
.../apache/gluten/component/CHDeltaComponent.scala | 21 +-
...ClickHouseIcebergMOREqualityDeletionSuite.scala | 2 +-
.../execution/iceberg/ClickHouseIcebergSuite.scala | 17 +-
.../apache/gluten/component/CHKafkaComponent.scala | 2 +
.../columnar/KafkaMiscColumnarRules.scala | 47 ++
.../gluten/execution/iceberg/TestFlinkUpsert.java | 538 +++++++++++++++++++++
.../iceberg/TestPositionDeletesTableGluten.java | 91 ++++
.../ClickHouseIcebergHiveTableSupport.scala | 101 ++++
.../iceberg/TestPositionDeletesTableGluten.java | 89 ++++
.../ClickHouseIcebergHiveTableSupport.scala | 101 ++++
.../gluten/vectorized/BlockOutputStream.java | 2 +
.../apache/gluten/vectorized/CHNativeBlock.java | 6 +
.../apache/gluten/vectorized/CHStreamReader.java | 2 +
.../gluten/backendsapi/clickhouse/CHBackend.scala | 2 +
.../backendsapi/clickhouse/CHIteratorApi.scala | 2 -
.../backendsapi/clickhouse/CHListenerApi.scala | 4 +-
.../backendsapi/clickhouse/CHMetricsApi.scala | 4 +-
.../gluten/backendsapi/clickhouse/CHRuleApi.scala | 1 +
.../clickhouse/CHSparkPlanExecApi.scala | 69 ++-
.../backendsapi/clickhouse/CHTransformerApi.scala | 7 +-
.../backendsapi/clickhouse/CHValidatorApi.scala | 2 +-
.../CHAggregateGroupLimitExecTransformer.scala | 5 +-
.../execution/CHHashAggregateExecTransformer.scala | 15 +-
.../execution/CHHashJoinExecTransformer.scala | 5 +-
.../CHWindowGroupLimitExecTransformer.scala | 5 +-
.../expression/CHExpressionTransformer.scala | 60 +--
.../apache/gluten/expression/CHExpressions.scala | 10 +-
.../extension/CHRemoveTopmostColumnarToRow.scala | 71 +++
.../CommonSubexpressionEliminateRule.scala | 16 +-
.../EliminateDeduplicateAggregateWithAnyJoin.scala | 43 +-
.../extension/JoinAggregateToAggregateUnion.scala | 4 +-
.../RewriteSortMergeJoinToHashJoinRule.scala | 3 +-
.../gluten/metrics/FilterMetricsUpdater.scala | 31 +-
.../org/apache/gluten/metrics/MetricsUtil.scala | 24 +-
.../gluten/metrics/ProjectMetricsUpdater.scala | 52 +-
.../org/apache/gluten/utils/PlanNodesUtil.scala | 3 +-
.../apache/spark/rpc/GlutenDriverEndpoint.scala | 3 +-
.../apache/spark/rpc/GlutenExecutorEndpoint.scala | 3 +-
.../utils/RangePartitionerBoundsGenerator.scala | 6 +-
.../spark/sql/execution/CHRDDScanTransformer.scala | 133 +++++
.../utils/MergeTreePartsPartitionsUtil.scala | 2 +-
.../GlutenClickHouseExcelFormatSuite.scala | 9 +-
.../GlutenClickHouseTPCHAbstractSuite.scala | 83 ++--
.../execution/GlutenClickHouseTPCHSuite.scala | 34 ++
...lutenClickHouseWholeStageTransformerSuite.scala | 74 +--
.../execution/GlutenEliminateJoinSuite.scala | 72 +++
.../execution/GlutenFunctionValidateSuite.scala | 40 +-
.../gluten/execution/GlutenNothingValueCheck.scala | 194 ++++++++
.../GlutenClickHouseCacheBaseTestSuite.scala} | 90 +---
.../cache/GlutenClickHouseHDFSSuite.scala | 45 ++
.../cache/GlutenClickHouseMINIOSuite.scala | 52 ++
.../GlutenClickhouseFunctionSuite.scala | 12 +-
.../GlutenClickHouseMergeTreeCacheDataSuite.scala | 44 +-
...GlutenClickHouseMergeTreeWriteOnHDFSSuite.scala | 10 +-
...eMergeTreeWriteOnHDFSWithRocksDBMetaSuite.scala | 10 +-
.../GlutenClickHouseMergeTreeWriteOnS3Suite.scala | 82 +---
.../GlutenClickHouseMergetreeWriteStatsSuite.scala | 8 +-
.../GlutenClickHouseTPCHSaltNullParquetSuite.scala | 51 +-
.../org/apache/gluten/utils/CacheTestHelper.scala | 65 +++
.../org/apache/gluten/utils/HDFSTestHelper.scala | 82 ++++
.../org/apache/gluten/utils/MinioTestHelper.scala | 117 +++++
.../execution/benchmarks/CHSqlBasedBenchmark.scala | 4 -
.../backendsapi/velox/VeloxListenerApi.scala | 38 +-
.../gluten/backendsapi/velox/VeloxRuleApi.scala | 10 +-
.../backendsapi/velox/VeloxSparkPlanExecApi.scala | 45 +-
.../backendsapi/velox/VeloxTransformerApi.scala | 3 +-
.../org/apache/gluten/config/VeloxConfig.scala | 87 +++-
.../execution/ColumnarCollectLimitExec.scala | 142 +++---
.../execution/ColumnarPartialProjectExec.scala | 104 ++--
.../execution/HashAggregateExecTransformer.scala | 52 +-
.../apache/gluten/execution/TopNTransformer.scala | 3 +-
.../gluten/execution/VeloxResizeBatchesExec.scala | 1 +
.../gluten/expression/ExpressionTransformer.scala | 22 +-
...AppendBatchResizeForShuffleInputAndOutput.scala | 58 +++
.../extension/FlushableHashAggregateRule.scala | 26 +-
.../gluten/utils/SharedLibraryLoaderCentos7.scala | 37 +-
.../gluten/utils/SharedLibraryLoaderCentos8.scala | 45 +-
.../gluten/utils/SharedLibraryLoaderCentos9.scala | 45 +-
.../gluten/utils/SharedLibraryLoaderDebian11.scala | 48 +-
.../gluten/utils/SharedLibraryLoaderDebian12.scala | 60 +--
.../utils/SharedLibraryLoaderUbuntu2004.scala | 80 +--
.../utils/SharedLibraryLoaderUbuntu2204.scala | 50 +-
.../tags/{SkipTestTags.java => SkipTest.java} | 2 +-
.../gluten/execution/ArrowCsvScanSuite.scala | 2 +
.../execution/DynamicOffHeapSizingSuite.scala | 6 +-
.../execution/JsonFunctionsValidateSuite.scala | 22 +-
.../gluten/execution/MiscOperatorSuite.scala | 58 ++-
.../execution/ScalarFunctionsValidateSuite.scala | 180 ++++++-
.../execution/VeloxAggregateFunctionsSuite.scala | 62 ++-
.../gluten/execution/VeloxColumnarCacheSuite.scala | 11 +-
.../gluten/execution/VeloxHashJoinSuite.scala | 2 +-
.../apache/gluten/execution/VeloxScanSuite.scala | 20 +
.../execution/VeloxStringFunctionsSuite.scala | 28 +-
.../apache/gluten/expression/VeloxUdfSuite.scala | 7 +-
.../apache/gluten/fuzzer/RowToColumnarFuzzer.scala | 4 +-
.../apache/gluten/fuzzer/ShuffleWriterFuzzer.scala | 4 +-
.../spark/sql/execution/GlutenHiveUDFSuite.scala | 2 +-
...tReadSuite.scala => VeloxLocalCacheSuite.scala} | 8 +-
.../sql/execution/VeloxParquetReadSuite.scala | 2 +-
.../execution/VeloxParquetWriteForHiveSuite.scala | 6 +-
cpp-ch/clickhouse.version | 4 +-
.../AggregateFunctionDVRoaringBitmap.h | 1 -
cpp-ch/local-engine/Common/AggregateUtil.cpp | 2 +-
cpp-ch/local-engine/Common/CHUtil.cpp | 26 -
cpp-ch/local-engine/Common/CHUtil.h | 6 -
cpp-ch/local-engine/Parser/ExpressionParser.cpp | 8 +-
.../Parser/RelParsers/AggregateRelParser.cpp | 1 -
.../Parser/RelParsers/FilterRelParser.cpp | 2 +-
.../Parser/RelParsers/MergeTreeRelParser.cpp | 286 +++++++----
.../Parser/RelParsers/MergeTreeRelParser.h | 37 +-
cpp-ch/local-engine/Parser/SparkRowToCHColumn.cpp | 2 +-
cpp-ch/local-engine/Parser/TypeParser.cpp | 15 +-
cpp-ch/local-engine/Shuffle/ShuffleReader.cpp | 14 +-
cpp-ch/local-engine/Shuffle/ShuffleReader.h | 12 +-
cpp-ch/local-engine/Storages/IO/NativeReader.cpp | 59 ++-
cpp-ch/local-engine/Storages/IO/NativeReader.h | 24 +-
.../Storages/MergeTree/SparkMergeTreeSink.cpp | 2 +-
.../Storages/MergeTree/SparkStorageMergeTree.cpp | 12 +-
.../Storages/MergeTree/SparkStorageMergeTree.h | 2 +-
.../Storages/Parquet/ColumnIndexFilter.cpp | 9 +-
.../Storages/SubstraitSource/Delta/DeltaWriter.cpp | 15 +-
.../Storages/SubstraitSource/Delta/DeltaWriter.h | 2 +-
.../Storages/SubstraitSource/FileReader.cpp | 4 +-
.../Storages/SubstraitSource/FormatFile.h | 9 -
.../Storages/SubstraitSource/ReadBufferBuilder.cpp | 115 +++--
cpp-ch/local-engine/libch-hide-jemalloc.map | 1 -
cpp-ch/local-engine/libch.map | 1 -
cpp-ch/local-engine/local_engine_jni.cpp | 64 ++-
cpp/core/config/GlutenConfig.h | 4 +-
cpp/core/jni/JniWrapper.cc | 8 +-
cpp/core/shuffle/LocalPartitionWriter.cc | 210 +++++---
cpp/core/shuffle/LocalPartitionWriter.h | 14 +-
cpp/core/shuffle/Options.h | 8 +-
cpp/core/shuffle/PartitionWriter.h | 10 +-
cpp/core/shuffle/Payload.cc | 24 +-
cpp/core/shuffle/Payload.h | 20 +-
cpp/core/shuffle/Spill.cc | 6 -
cpp/core/shuffle/Spill.h | 7 -
cpp/core/shuffle/Utils.h | 395 +++++++++++++++
cpp/core/shuffle/rss/RssPartitionWriter.cc | 67 ++-
cpp/core/shuffle/rss/RssPartitionWriter.h | 97 +++-
cpp/velox/benchmarks/GenericBenchmark.cc | 8 +
cpp/velox/compute/VeloxBackend.cc | 65 ++-
cpp/velox/compute/VeloxBackend.h | 26 +-
cpp/velox/compute/WholeStageResultIterator.cc | 26 +-
cpp/velox/config/VeloxConfig.h | 9 +-
cpp/velox/shuffle/VeloxHashShuffleWriter.cc | 10 +-
cpp/velox/shuffle/VeloxShuffleReader.cc | 125 ++---
cpp/velox/shuffle/VeloxShuffleReader.h | 18 +-
cpp/velox/shuffle/VeloxSortShuffleWriter.cc | 76 ++-
cpp/velox/shuffle/VeloxSortShuffleWriter.h | 8 +-
cpp/velox/substrait/SubstraitToVeloxPlan.cc | 16 +-
.../tests/Substrait2VeloxPlanConversionTest.cc | 12 +-
cpp/velox/tests/VeloxShuffleWriterTest.cc | 19 +-
cpp/velox/utils/tests/VeloxShuffleWriterTestBase.h | 8 +-
dev/docker/cudf/Dockerfile | 2 +-
docs/developers/HowTo.md | 37 +-
docs/developers/NewToGluten.md | 2 +-
docs/get-started/Velox.md | 8 +-
docs/velox-backend-scalar-function-support.md | 28 +-
ep/build-velox/src/build_velox.sh | 3 +-
ep/build-velox/src/get_velox.sh | 2 +-
ep/build-velox/src/setup-centos7.sh | 9 +-
.../memory/arrow/alloc/ArrowBufferAllocators.java | 19 +-
.../listener/ManagedReservationListener.java | 4 +-
.../memory/listener/ReservationListeners.java | 13 +-
.../java/org/apache/gluten/jni/JniLibLoader.java | 78 +--
.../DynamicOffHeapSizingMemoryTarget.java | 91 ++--
.../memory/memtarget/MemoryTargetVisitor.java | 4 +
.../gluten/memory/memtarget/MemoryTargets.java | 2 +-
.../scala/org/apache/gluten/GlutenPlugin.scala | 158 +++---
.../gluten/extension/caller/CallerInfo.scala | 22 +-
.../apache/spark/memory/GlobalOffHeapMemory.scala | 53 ++
.../spark/memory/GlobalOffHeapMemoryTarget.scala | 106 ++--
.../org/apache/spark/memory/SparkMemoryUtil.scala | 10 +-
.../gluten/component/ComponentSkippable.scala | 13 +-
.../execution/DeltaFilterExecTransformer.scala | 5 +-
.../execution/DeltaProjectExecTransformer.scala | 38 +-
.../org/apache/gluten/execution/DeltaSuite.scala | 20 +-
.../org/apache/gluten/execution/HudiSuite.scala | 8 +-
.../org/apache/gluten/execution/IcebergSuite.scala | 18 +-
.../substrait/expression/ExpressionBuilder.java | 10 -
.../substrait/expression/WindowFunctionNode.java | 4 +-
.../gluten/backendsapi/SparkPlanExecApi.scala | 55 ++-
.../apache/gluten/backendsapi/TransformerApi.scala | 3 +-
.../BasicPhysicalOperatorTransformer.scala | 5 +-
.../execution/BasicScanExecTransformer.scala | 2 +-
.../CartesianProductExecTransformer.scala | 2 +-
.../execution/ColumnarCollectLimitBaseExec.scala | 22 +-
.../gluten/execution/ExpandExecTransformer.scala | 3 +-
.../execution/GenerateExecTransformerBase.scala | 2 +-
.../gluten/execution/JoinExecTransformer.scala | 13 +-
.../org/apache/gluten/execution/JoinUtils.scala | 12 +-
.../gluten/execution/SampleExecTransformer.scala | 2 +-
.../gluten/execution/SortExecTransformer.scala | 3 +-
.../gluten/execution/WholeStageTransformer.scala | 10 +-
.../gluten/execution/WindowExecTransformer.scala | 7 +-
.../WindowGroupLimitExecTransformer.scala | 5 +-
.../expression/AggregateFunctionsBuilder.scala | 9 +-
.../expression/ArrayExpressionTransformer.scala | 5 +-
.../gluten/expression/ConditionalTransformer.scala | 17 +-
.../DateTimeExpressionsTransformer.scala | 3 +-
.../gluten/expression/ExpressionConverter.scala | 2 +-
.../gluten/expression/ExpressionMappings.scala | 26 +-
.../gluten/expression/ExpressionTransformer.scala | 10 +-
.../apache/gluten/expression/ExpressionUtils.scala | 6 +-
.../JsonTupleExpressionTransformer.scala | 10 +-
.../expression/LambdaFunctionTransformer.scala | 5 +-
.../expression/MapExpressionTransformer.scala | 9 +-
.../expression/NamedExpressionsTransformer.scala | 5 +-
.../PredicateExpressionTransformer.scala | 9 +-
.../expression/ScalarSubqueryTransformer.scala | 3 +-
.../expression/UnaryExpressionTransformer.scala | 29 +-
.../gluten/expression/WindowFunctionsBuilder.scala | 7 +-
.../columnar/CollectLimitTransformerRule.scala | 29 +-
.../extension/columnar/MiscColumnarRules.scala | 50 +-
.../columnar/offload/OffloadSingleNodeRules.scala | 8 +-
.../apache/gluten/utils/PullOutProjectHelper.scala | 24 +-
.../org/apache/gluten/utils/SubstraitUtil.scala | 2 +-
.../sql/execution/GlutenFallbackReporter.scala | 6 +-
.../sql/execution/RDDScanTransformer.scala} | 53 +-
.../FakeRowEnhancement.scala} | 39 +-
.../datasources/GlutenWriterColumnarRules.scala | 2 +-
.../python/EvalPythonExecTransformer.scala | 10 +-
.../columnar/MiscColumnarRulesSuite.scala | 24 +-
.../org/apache/spark/sql/GlutenQueryTest.scala | 76 ++-
gluten-ui/pom.xml | 6 +
...{GlutenEventUtils.scala => GlutenUIUtils.scala} | 20 +-
.../org/apache/spark/sql/GlutenTestsTrait.scala | 20 +-
.../utils/clickhouse/ClickHouseTestSettings.scala | 1 +
.../gluten/utils/velox/VeloxTestSettings.scala | 4 +
.../expressions/GlutenDateExpressionsSuite.scala | 3 +-
.../GlutenSQLAggregateFunctionSuite.scala | 26 +
.../execution/GlutenSQLCollectLimitExecSuite.scala | 34 +-
.../benchmarks/ParquetReadBenchmark.scala | 4 -
.../utils/clickhouse/ClickHouseTestSettings.scala | 2 +
.../gluten/utils/velox/VeloxTestSettings.scala | 3 +
.../expressions/GlutenDateExpressionsSuite.scala | 3 +-
.../GlutenSQLAggregateFunctionSuite.scala | 26 +
.../execution/GlutenSQLCollectLimitExecSuite.scala | 38 +-
.../benchmarks/ParquetReadBenchmark.scala | 4 -
.../spark/sql/gluten/GlutenFallbackSuite.scala | 3 +
.../utils/clickhouse/ClickHouseTestSettings.scala | 1 +
.../gluten/utils/velox/VeloxTestSettings.scala | 6 +-
.../expressions/GlutenDateExpressionsSuite.scala | 3 +-
.../GlutenSQLAggregateFunctionSuite.scala | 26 +
.../execution/GlutenSQLCollectLimitExecSuite.scala | 262 ++++++++++
.../benchmarks/ParquetReadBenchmark.scala | 4 -
.../spark/sql/gluten/GlutenFallbackSuite.scala | 3 +
.../utils/clickhouse/ClickHouseTestSettings.scala | 7 +
.../gluten/utils/velox/VeloxTestSettings.scala | 7 +-
.../expressions/GlutenDateExpressionsSuite.scala | 3 +-
.../GlutenSQLAggregateFunctionSuite.scala | 26 +
.../execution/GlutenSQLCollectLimitExecSuite.scala | 264 ++++++++++
.../benchmarks/ParquetReadBenchmark.scala | 4 -
.../spark/sql/gluten/GlutenFallbackSuite.scala | 3 +
.../hive/execution/GlutenHiveSQLQuerySuite.scala | 1 -
.../expressions/GlutenExpressionMappingSuite.scala | 8 +-
.../GlutenExtensionRewriteRuleSuite.scala | 29 +-
.../org/apache/gluten/sql/SQLQuerySuite.scala | 1 -
.../sql/GlutenExpressionDataTypesValidation.scala | 1 -
.../org/apache/gluten/config/GlutenConfig.scala | 51 +-
.../apache/gluten/expression/ExpressionNames.scala | 12 +
.../org/apache/gluten/sql/shims/SparkShims.scala | 6 +-
.../spark/sql/execution/datasources/FakeRow.scala | 2 +-
.../spark/sql/internal/SparkConfigUtil.scala | 9 +-
.../gluten/sql/shims/spark32/Spark32Shims.scala | 2 +
.../gluten/sql/shims/spark33/Spark33Shims.scala | 8 +
.../gluten/sql/shims/spark34/Spark34Shims.scala | 17 +-
.../gluten/sql/shims/spark35/Spark35Shims.scala | 18 +-
tools/scripts/gen-function-support-docs.py | 8 +-
280 files changed, 7394 insertions(+), 2462 deletions(-)
create mode 100644
backends-clickhouse/src-delta-32/main/scala/org/apache/spark/sql/delta/commands/merge/MergeIntoMaterializeSource.scala
copy
backends-clickhouse/{src/test/scala/org/apache/gluten/execution/mergetree/GlutenClickHouseMergeTreeWriteTaskNotSerializableSuite.scala
=>
src-delta-32/test/scala/org/apache/spark/gluten/delta/GlutenDeltaMergetreeDeletionVectorSuite.scala}
(50%)
create mode 100644
backends-clickhouse/src-delta-32/test/scala/org/apache/spark/sql/execution/GlutenDeltaExpressionSuite.scala
create mode 100644
backends-clickhouse/src-delta-32/test/scala/org/apache/spark/sql/execution/RDDScanSuite.scala
create mode 100644
backends-clickhouse/src-kafka/main/scala/org/apache/gluten/extension/columnar/KafkaMiscColumnarRules.scala
create mode 100644
backends-clickhouse/src-spark33/test/java/org/apache/gluten/execution/iceberg/TestFlinkUpsert.java
create mode 100644
backends-clickhouse/src-spark33/test/java/org/apache/gluten/execution/iceberg/TestPositionDeletesTableGluten.java
create mode 100644
backends-clickhouse/src-spark33/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergHiveTableSupport.scala
create mode 100644
backends-clickhouse/src-spark35/test/java/org/apache/gluten/execution/iceberg/TestPositionDeletesTableGluten.java
create mode 100644
backends-clickhouse/src-spark35/test/scala/org/apache/gluten/execution/iceberg/ClickHouseIcebergHiveTableSupport.scala
create mode 100644
backends-clickhouse/src/main/scala/org/apache/gluten/extension/CHRemoveTopmostColumnarToRow.scala
create mode 100644
backends-clickhouse/src/main/scala/org/apache/spark/sql/execution/CHRDDScanTransformer.scala
create mode 100644
backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenNothingValueCheck.scala
rename
backends-clickhouse/src/test/scala/org/apache/gluten/execution/{tpch/GlutenClickHouseHDFSSuite.scala
=> cache/GlutenClickHouseCacheBaseTestSuite.scala} (67%)
create mode 100644
backends-clickhouse/src/test/scala/org/apache/gluten/execution/cache/GlutenClickHouseHDFSSuite.scala
create mode 100644
backends-clickhouse/src/test/scala/org/apache/gluten/execution/cache/GlutenClickHouseMINIOSuite.scala
create mode 100644
backends-clickhouse/src/test/scala/org/apache/gluten/utils/CacheTestHelper.scala
create mode 100644
backends-clickhouse/src/test/scala/org/apache/gluten/utils/HDFSTestHelper.scala
create mode 100644
backends-clickhouse/src/test/scala/org/apache/gluten/utils/MinioTestHelper.scala
create mode 100644
backends-velox/src/main/scala/org/apache/gluten/extension/AppendBatchResizeForShuffleInputAndOutput.scala
rename backends-velox/src/test/java/org/apache/gluten/tags/{SkipTestTags.java
=> SkipTest.java} (96%)
copy
backends-velox/src/test/scala/org/apache/spark/sql/execution/{VeloxParquetReadSuite.scala
=> VeloxLocalCacheSuite.scala} (84%)
create mode 100644
gluten-core/src/main/scala/org/apache/spark/memory/GlobalOffHeapMemory.scala
rename
backends-velox/src/main/scala/org/apache/spark/memory/GlobalOffHeapMemory.scala
=>
gluten-core/src/main/scala/org/apache/spark/memory/GlobalOffHeapMemoryTarget.scala
(50%)
copy
gluten-ut/spark32/src/test/scala/org/apache/spark/GlutenSortShuffleSuite.scala
=>
gluten-core/src/test/scala/org/apache/gluten/component/ComponentSkippable.scala
(74%)
copy
backends-velox/src/main/scala/org/apache/gluten/extension/PartialProjectRule.scala
=>
gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/CollectLimitTransformerRule.scala
(59%)
copy
gluten-substrait/src/main/scala/org/apache/{gluten/execution/RangeExecBaseTransformer.scala
=> spark/sql/execution/RDDScanTransformer.scala} (56%)
copy
gluten-substrait/src/main/scala/org/apache/spark/sql/execution/{joins/BuildSideRelation.scala
=> datasources/FakeRowEnhancement.scala} (50%)
copy
backends-velox/src/main/scala/org/apache/spark/sql/execution/BaseArrowScanExec.scala
=>
gluten-substrait/src/test/scala/org/apache/gluten/extension/columnar/MiscColumnarRulesSuite.scala
(52%)
rename
gluten-ui/src/main/scala/org/apache/spark/sql/execution/ui/{GlutenEventUtils.scala
=> GlutenUIUtils.scala} (62%)
copy {backends-velox/src/test/scala/org/apache/gluten =>
gluten-ut/spark32/src/test/scala/org/apache/spark/sql}/execution/GlutenSQLCollectLimitExecSuite.scala
(76%)
rename {backends-velox/src/test/scala/org/apache/gluten =>
gluten-ut/spark33/src/test/scala/org/apache/spark/sql}/execution/GlutenSQLCollectLimitExecSuite.scala
(75%)
create mode 100644
gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/GlutenSQLCollectLimitExecSuite.scala
create mode 100644
gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/GlutenSQLCollectLimitExecSuite.scala
copy
gluten-ut/spark32/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala
=>
shims/common/src/main/scala/org/apache/spark/sql/internal/SparkConfigUtil.scala
(79%)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
