This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a change to branch release-1.2.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
from d42b416c78fd perf: Add dedicated batch size config for LSM timeline
migration on u… (#19052)
new ff1a69c47ddf test(clean): Cover tests executing pending clean before
scheduling a new one (#19051)
new af0fbfea603a test(spark): de-flake TestHoodieClientMultiWriter
early-conflict detection (#19069)
new 3e671a234fa3 fix(test): close TestingServer + write clients on all
exit paths in TestHoodieClientMultiWriter (#19062)
new 1fc97453dcfe [MINOR] Guard detailed metadata size metrics with a
config (#18803)
new 340deedcda60 test(metadata): Add test coverage for deferred RLI init
and bulk_insert (#18865)
new 2396fb08d92f fix(spark): support consistent hashing clustering on
non-partitioned tables (#18968)
new 50236434dd54 chore(integ-test): bump trino-jdbc 390 to 481 (#19073)
new 964d366224fc Fix typo in
PartitionTTLStrategyType#getPartitionTTLStrategyClassName (#19076)
new e2467a4f57e8 fix: Skip missing properties files gracefully in
DFSPropertiesConfiguration (#18805)
new e70f83016016 [HUDI-18827] Fix per-task write token for MOR (table v6)
rollback log files (#18828)
new 796104c079b5 fix(spark): reject INSERT_OVERWRITE when overlapping with
pending clustering (#18829)
new 4570ecdf2db7 fix(streamer): override all deserialize() overloads in
KafkaAvroSchemaDeserializer (#18892)
new fbd320236009 fix(spark): preserve Spark's native unresolved-column
errors in HoodieAnalysis (#18147)
new 4ce7ef834e2d [MINOR] Forward spark.hoodie.* SparkConf to write path
(parity with read path) (#18650)
new a29a48831df9 fix(partition-ttl): Fix the integer overflow issue when
TTL exceeded 24 days (#19075)
new 4a2c490dfdef perf(common): Avoid per-record HoodieSchema rebuilds on
Avro read/merge paths (#18967)
new d35272a7447c test(flink): de-flake testLookupJoin lookup-join IT
(#19093)
new d7c5a22bfe86 fix(partition-ttl): Fix IllegalArgumentException in
KeepByTimeStrategy when no candidate partitions exist (#19092)
new a132f0af254e feat(spark): add repair_orphan_files stored procedure
(#19121)
The 19 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../metadata/HoodieBackedTableMetadataWriter.java | 6 +-
.../action/cluster/strategy/UpdateStrategy.java | 17 +-
.../table/action/index/RunIndexActionExecutor.java | 2 +-
.../table/action/rollback/RollbackHelperV1.java | 35 +-
.../action/ttl/strategy/KeepByTimeStrategy.java | 7 +-
.../ttl/strategy/PartitionTTLStrategyType.java | 3 +-
.../table/action/rollback/TestRollbackHelper.java | 25 +-
.../ttl/strategy/TestKeepByTimeStrategy.java | 62 ++
.../ttl/strategy/TestPartitionTTLStrategy.java | 82 ++
.../ttl/strategy/TestPartitionTTLStrategyType.java | 62 ++
.../FlinkHoodieBackedTableMetadataWriter.java | 2 +-
.../hudi/table/format/FlinkRecordContext.java | 3 +-
.../JavaHoodieBackedTableMetadataWriter.java | 2 +-
.../hudi/testutils/TestHoodieMetadataBase.java | 1 +
...SparkJobConsistentHashingExecutionStrategy.java | 6 +-
.../update/strategy/BaseSparkUpdateStrategy.java | 13 +-
.../update/strategy/SparkAllowUpdateStrategy.java | 11 +-
...arkConsistentBucketDuplicateUpdateStrategy.java | 8 +-
.../update/strategy/SparkRejectUpdateStrategy.java | 17 +-
.../SparkHoodieBackedTableMetadataWriter.java | 2 +-
...ieBackedTableMetadataWriterTableVersionSix.java | 2 +-
.../commit/BaseSparkCommitActionExecutor.java | 53 +-
.../SparkInsertOverwriteCommitActionExecutor.java | 28 +-
...rkInsertOverwriteTableCommitActionExecutor.java | 22 +
.../SparkFileFormatInternalRecordContext.scala | 4 +-
.../client/functional/TestHoodieMetadataBase.java | 1 +
.../fs/TestHoodieSerializableFileStatus.java | 8 +-
.../commit/TestInsertOverwriteWithClustering.java | 834 +++++++++++++++++++++
.../TestMarkerBasedRollbackStrategy.java | 16 +-
.../org/apache/hudi/avro/AvroRecordContext.java | 6 +-
.../java/org/apache/hudi/avro/HoodieAvroUtils.java | 3 +-
.../org/apache/hudi/common/HoodieJsonPayload.java | 5 +-
.../hudi/common/config/HoodieMetadataConfig.java | 18 +
.../hudi/common/schema/HoodieAvroSchemaCache.java | 52 ++
.../apache/hudi/common/schema/HoodieSchema.java | 14 +-
.../table/log/block/HoodieAvroDataBlock.java | 5 +-
.../apache/hudi/metadata/BaseTableMetadata.java | 3 +-
.../hudi/metadata/HoodieMetadataMetrics.java | 8 +-
.../apache/hudi/avro/TestAvroRecordContext.java | 68 ++
.../common/config/TestHoodieMetadataConfig.java | 24 +
.../strategy/ConsistentBucketUpdateStrategy.java | 2 +-
.../FlinkConsistentBucketUpdateStrategy.java | 2 +-
.../apache/hudi/table/ITTestHoodieDataSource.java | 19 +-
.../common/config/DFSPropertiesConfiguration.java | 44 +-
.../util/TestDFSPropertiesConfiguration.java | 83 ++
.../realtime/RealtimeCompactedRecordReader.java | 5 +-
.../BaseDatasetBulkInsertCommitActionExecutor.java | 89 +++
...setBulkInsertOverwriteCommitActionExecutor.java | 44 ++
...lkInsertOverwriteTableCommitActionExecutor.java | 11 +
.../scala/org/apache/hudi/DataSourceOptions.scala | 88 ++-
.../main/scala/org/apache/hudi/DefaultSource.scala | 22 +-
.../scala/org/apache/hudi/HoodieWriterUtils.scala | 8 +-
.../hudi/analysis/HoodieSparkBaseAnalysis.scala | 9 +-
.../hudi/command/payload/ExpressionPayload.scala | 14 +-
.../org/apache/hudi/TestDataSourceOptions.scala | 131 +++-
.../spark/sql/hudi/analysis/HoodieAnalysis.scala | 7 +-
.../hudi/command/procedures/HoodieProcedures.scala | 1 +
.../procedures/RepairOrphanFilesProcedure.scala | 336 +++++++++
.../hudi/client/TestHoodieClientMultiWriter.java | 146 ++--
.../TestHoodieClientOnCopyOnWriteStorage.java | 57 ++
.../TestSparkConsistentBucketClustering.java | 36 +-
.../TestMergeOnReadRollbackActionExecutor.java | 235 ++++++
.../hudi/functional/TestRecordLevelIndex.scala | 116 ++-
.../analysis/TestHoodieAnalysisErrorHandling.scala | 196 +++++
.../procedure/TestRepairOrphanFilesProcedure.scala | 256 +++++++
.../deser/KafkaAvroSchemaDeserializer.java | 16 +
.../deser/TestKafkaAvroSchemaDeserializer.java | 205 +++++
.../test/resources/schema/cdc_envelope_new.avsc | 261 +++++++
.../test/resources/schema/cdc_envelope_old.avsc | 237 ++++++
packaging/hudi-integ-test-bundle/pom.xml | 10 +
pom.xml | 2 +-
71 files changed, 4045 insertions(+), 183 deletions(-)
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/ttl/strategy/TestKeepByTimeStrategy.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/ttl/strategy/TestPartitionTTLStrategy.java
create mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/table/action/ttl/strategy/TestPartitionTTLStrategyType.java
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestInsertOverwriteWithClustering.java
create mode 100644
hudi-common/src/main/java/org/apache/hudi/common/schema/HoodieAvroSchemaCache.java
create mode 100644
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairOrphanFilesProcedure.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/analysis/TestHoodieAnalysisErrorHandling.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestRepairOrphanFilesProcedure.scala
create mode 100644
hudi-utilities/src/test/resources/schema/cdc_envelope_new.avsc
create mode 100644
hudi-utilities/src/test/resources/schema/cdc_envelope_old.avsc
