This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a change to branch release-0.12.2-shadow
in repository https://gitbox.apache.org/repos/asf/hudi.git
from abb86ca5920 [HUDI-5347] FIxing performance traps in Spark SQL `MERGE
INTO` implementation (#7395)
new 4e2414434c3 [HUDI-5345] Avoid fs.exists calls for metadata table in
HFileBootstrapIndex (#7404)
new 7abc074dac1 [HUDI-5291] Fixing NPE in MOR column stats accounting
(#7349)
new bae596359e8 [HUDI-5346][HUDI-5320] Fixing Create Table as Select
(CTAS) performance gaps (#7370)
new b930002b785 [HUDI-5350] Fix oom cause compaction event lost problem
(#7408)
new ae21564b053 [HUDI-4881] Push down filters if possible when syncing
partitions to Hive (#6725)
new 75c60b0d58d [HUDI-5358] Fix flaky tests in
TestCleanerInsertAndCleanByCommits (#7420)
new e4bdb733bcb [HUDI-5342] Add new bulk insert sort modes repartitioning
data by partition path (#7402)
new 8d83a875711 [HUDI-5356] Call close on SparkRDDWriteClient several
places (#7418)
new ca3dbbcfa57 [HUDI-5338] Adjust coalesce behavior within NONE sort mode
for bulk insert (#7396)
new d39bae6c59a [HUDI-5336] Fixing parsing of log files while building
file groups (#7393)
The 10 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/hudi/cli/commands/SparkMain.java | 13 +-
.../org/apache/hudi/config/HoodieWriteConfig.java | 11 ++
.../execution/bulkinsert/BulkInsertSortMode.java | 8 +-
.../apache/hudi/table/BulkInsertPartitioner.java | 18 +-
.../java/org/apache/hudi/table/HoodieTable.java | 7 +
.../MultipleSparkJobExecutionStrategy.java | 13 +-
.../BulkInsertInternalPartitionerFactory.java | 25 ++-
...lkInsertInternalPartitionerWithRowsFactory.java | 14 +-
.../execution/bulkinsert/NonSortPartitioner.java | 33 +++-
.../bulkinsert/NonSortPartitionerWithRows.java | 34 +++-
.../bulkinsert/PartitionPathRDDPartitioner.java | 52 +++++
...PartitionPathRepartitionAndSortPartitioner.java | 71 +++++++
...nPathRepartitionAndSortPartitionerWithRows.java | 62 ++++++
.../PartitionPathRepartitionPartitioner.java | 70 +++++++
...artitionPathRepartitionPartitionerWithRows.java | 60 ++++++
.../table/action/commit/SparkBulkInsertHelper.java | 2 +-
.../hudi/HoodieDatasetBulkInsertHelper.scala | 68 ++++---
.../org/apache/spark/sql/HoodieUnsafeUtils.scala | 12 +-
.../TestBulkInsertInternalPartitioner.java | 73 +++++--
.../TestBulkInsertInternalPartitionerForRows.java | 71 +++++--
.../clean/TestCleanerInsertAndCleanByCommits.java | 136 +++++++++----
.../java/org/apache/hudi/avro/HoodieAvroUtils.java | 18 +-
.../bootstrap/index/HFileBootstrapIndex.java | 6 +-
.../java/org/apache/hudi/common/fs/FSUtils.java | 18 +-
...ion.java => SerializableFunctionUnchecked.java} | 13 +-
.../apache/hudi/common/model/HoodieLogFile.java | 1 +
.../hudi/common/table/HoodieTableConfig.java | 4 +
.../table/view/AbstractTableFileSystemView.java | 8 +-
.../apache/hudi/common/util/CollectionUtils.java | 9 +
.../table/view/TestHoodieTableFileSystemView.java | 46 +++++
.../examples/spark/HoodieWriteClientExample.java | 2 +-
.../apache/hudi/sink/utils/NonThrownExecutor.java | 8 +-
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 15 +-
.../command/CreateHoodieTableAsSelectCommand.scala | 76 +++-----
.../command/InsertIntoHoodieTableCommand.scala | 3 +-
.../cow-updated2-column-stats-index-table.json | 4 +-
.../mor-updated2-column-stats-index-table.json | 4 +-
...-7e680484-e7e1-48b6-8289-1a7c483b530b-c000.json | 2 +-
.../hudi/functional/TestColumnStatsIndex.scala | 24 +--
.../spark/sql/hudi/HoodieSparkSqlTestBase.scala | 4 +-
.../apache/spark/sql/hudi/TestInsertTable.scala | 22 ++-
.../spark/sql/HoodieSpark2CatalystPlanUtils.scala | 11 +-
.../java/org/apache/hudi/hive/HiveSyncConfig.java | 11 ++
.../java/org/apache/hudi/hive/HiveSyncTool.java | 34 +++-
.../org/apache/hudi/hive/HoodieHiveSyncClient.java | 13 ++
.../expression/AttributeReferenceExpression.java | 23 +--
.../hudi/hive/expression/BinaryOperator.java | 75 ++++++++
.../apache/hudi/hive/expression/Expression.java | 41 ++--
.../hudi/hive/expression/ExpressionVisitor.java | 17 +-
.../hudi/hive/expression/LeafExpression.java | 13 +-
.../org/apache/hudi/hive/expression/Literal.java | 32 ++--
.../apache/hudi/hive/util/FilterGenVisitor.java | 119 ++++++++++++
.../org/apache/hudi/hive/util/HiveSchemaUtil.java | 33 ++--
.../hudi/hive/util/PartitionFilterGenerator.java | 212 +++++++++++++++++++++
.../org/apache/hudi/hive/TestHiveSyncTool.java | 103 +++++++---
.../hive/util/TestPartitionFilterGenerator.java | 134 +++++++++++++
.../hudi/sync/common/HoodieMetaSyncOperations.java | 9 +
57 files changed, 1667 insertions(+), 353 deletions(-)
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/PartitionPathRDDPartitioner.java
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/PartitionPathRepartitionAndSortPartitioner.java
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/PartitionPathRepartitionAndSortPartitionerWithRows.java
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/PartitionPathRepartitionPartitioner.java
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/execution/bulkinsert/PartitionPathRepartitionPartitionerWithRows.java
copy
hudi-common/src/main/java/org/apache/hudi/common/function/{SerializablePairFlatMapFunction.java
=> SerializableFunctionUnchecked.java} (75%)
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/InMemoryMetricsReporter.java
=>
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/expression/AttributeReferenceExpression.java
(69%)
create mode 100644
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/expression/BinaryOperator.java
copy
hudi-common/src/main/java/org/apache/hudi/common/fs/NoOpConsistencyGuard.java
=>
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/expression/Expression.java
(54%)
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metrics/InMemoryMetricsReporter.java
=>
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/expression/ExpressionVisitor.java
(74%)
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/execution/bulkinsert/BulkInsertSortMode.java
=>
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/expression/LeafExpression.java
(80%)
copy hudi-common/src/main/java/org/apache/hudi/common/metrics/Counter.java =>
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/expression/Literal.java
(65%)
create mode 100644
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/FilterGenVisitor.java
create mode 100644
hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/util/PartitionFilterGenerator.java
create mode 100644
hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/util/TestPartitionFilterGenerator.java
