This is an automated email from the ASF dual-hosted git repository.
satish pushed a change to branch release-0.12.2
in repository https://gitbox.apache.org/repos/asf/hudi.git
from 1d47f95024f [HUDI-5283] Replace deprecated method Schema.parse with
Schema.Parser (#7308)
new e6520bbb365 [HUDI-5241] Optimize HoodieDefaultTimeline API (#7241)
new 31b6acb973f [HUDI-4209] Avoid using HDFS in HoodieClientTestHarness
(#7305)
new 9126df029b8 [HUDI-5286] UnsupportedOperationException throws when
enabling filesystem retry (#7313)
new 854fd37f1b4 [MINOR] Re-enable Cleaner tests (#7317)
new 5b7f18c13aa [HUDI-5242] Do not fail Meta sync in Deltastreamer when
inline table service fails (#7243)
new bf6a7586ecb [HUDI-5007] Prevent Hudi from reading the entire
timeline's when performing a LATEST streaming read (#6920)
new 3b2b98da37c [MINOR] Re-enable Cleaner tests (#7317)
new 6f2a1e9c58e [HUDI-5242] Do not fail Meta sync in Deltastreamer when
inline table service fails (#7243)
new 2f419a503ab [HUDI-5007] Prevent Hudi from reading the entire
timeline's when performing a LATEST streaming read (#6920)
new fab13611a56 [MINOR] Fix IncrementalInputSplits compilation failure
(#7319)
new daa010d46e7 [HUDI-5285] Exclude *-site.xml files from jar packaging
(#7310)
new c0ef19ff0db [MINOR] Remove the log refrence and imports in
HoodieHFileDataBlock (#7311)
new b975295ca26 [HUDI-5253] HoodieMergeOnReadTableInputFormat could have
duplicate records issue if it contains delta files while still splittable
(#7264)
new 0dda39d4c53 [HUDI-5151] Fix bug with broken flink data skipping caused
by ClassNotFoundException of InLineFileSystem (#7124)
new 9913de4ebfa [HUDI-5269] Enhancing spark-sql write tests for some of
the core user flows (#7230)
new a83916c4347 [HUDI-5278] Support more conf to cluster procedure (#7304)
new 1db97a8ae7b [MINOR] Reuse empty InternalSchema instance (#7287)
new 60973dcf56f [HUDI-3981] Flink engine support for comprehensive schema
evolution (#5830)
new 8d0fe3c50ec [HUDI-5279] move logic for deleting active instant to
HoodieActiveTimeline (#7196)
new 11221fda2b3 Rebased MOR iterators onto a `CachingIterator` (to be
idempotent) (#7334)
new cb43615f838 [MINOR] Bumping Azure Ubuntu image to 22.04, as 18.04 will
be deprecated soon (#7347)
new 737811f69c0 [HUDI-5304] Disabling spark-sql core flow tests to unblock
CI (#7346)
The 22 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
azure-pipelines.yml | 2 +-
.../apache/hudi/cli/commands/CommitsCommand.java | 12 +-
.../hudi/cli/commands/CompactionCommand.java | 2 +-
.../org/apache/hudi/cli/commands/DiffCommand.java | 2 +-
.../apache/hudi/cli/commands/ExportCommand.java | 2 +-
.../hudi/cli/commands/FileSystemViewCommand.java | 2 +-
.../cli/commands/HoodieSyncValidateCommand.java | 2 +-
.../apache/hudi/cli/commands/RepairsCommand.java | 2 +-
.../hudi/cli/commands/SavepointsCommand.java | 2 +-
.../org/apache/hudi/cli/commands/StatsCommand.java | 3 +-
.../apache/hudi/cli/commands/TimelineCommand.java | 5 +-
.../hudi/cli/commands/TestArchiveCommand.java | 4 +-
.../hudi/cli/commands/TestCleansCommand.java | 8 +-
.../hudi/cli/commands/TestCommitsCommand.java | 2 +-
.../hudi/cli/commands/TestRepairsCommand.java | 6 +-
.../hudi/cli/commands/TestRollbacksCommand.java | 4 +-
.../hudi/cli/integ/ITTestClusteringCommand.java | 6 +-
.../hudi/cli/integ/ITTestCompactionCommand.java | 4 +-
.../apache/hudi/client/BaseHoodieWriteClient.java | 24 +-
.../apache/hudi/client/HoodieTimelineArchiver.java | 46 +-
...urrentFileWritesConflictResolutionStrategy.java | 4 +-
.../apache/hudi/client/utils/TransactionUtils.java | 4 +-
.../apache/hudi/config/HoodieClusteringConfig.java | 52 +++
.../org/apache/hudi/config/HoodieWriteConfig.java | 15 +
.../metadata/HoodieBackedTableMetadataWriter.java | 6 +-
.../java/org/apache/hudi/table/HoodieTable.java | 2 +-
.../table/action/clean/CleanActionExecutor.java | 2 +-
.../hudi/table/action/clean/CleanPlanner.java | 4 +-
.../compact/ScheduleCompactionActionExecutor.java | 2 +-
.../table/action/index/RunIndexActionExecutor.java | 8 +-
.../rollback/BaseRollbackActionExecutor.java | 4 +-
.../org/apache/hudi/table/repair/RepairUtils.java | 4 +-
.../table/upgrade/ZeroToOneUpgradeHandler.java | 2 +-
.../org/apache/hudi/table/HoodieFlinkTable.java | 13 +
.../index/bloom/TestFlinkHoodieBloomIndex.java | 2 +
.../testutils/HoodieFlinkClientTestHarness.java | 84 +---
.../testutils/HoodieFlinkWriteableTestTable.java | 2 +-
.../scala/org/apache/hudi/HoodieSparkUtils.scala | 2 +-
.../org/apache/hudi/client/TestClientRollback.java | 12 +-
.../hudi/client/TestFileBasedLockProvider.java | 91 ++--
.../hudi/client/TestHoodieClientMultiWriter.java | 4 +-
.../apache/hudi/client/TestHoodieReadClient.java | 37 +-
.../java/org/apache/hudi/client/TestMultiFS.java | 63 ++-
.../hudi/client/TestUpdateSchemaEvolution.java | 3 +-
.../TestHoodieClientOnCopyOnWriteStorage.java | 70 +++-
.../TestBulkInsertInternalPartitioner.java | 16 +-
.../apache/hudi/io/TestHoodieTimelineArchiver.java | 15 +-
.../clean/TestCleanerInsertAndCleanByCommits.java | 4 +-
.../clean/TestCleanerInsertAndCleanByVersions.java | 2 +-
.../commit/TestCopyOnWriteActionExecutor.java | 3 +-
.../table/action/compact/CompactionTestBase.java | 4 +-
.../table/action/compact/TestAsyncCompaction.java | 4 +-
.../table/action/compact/TestInlineCompaction.java | 2 +-
.../TestMergeOnReadRollbackActionExecutor.java | 25 +-
.../hudi/table/marker/TestDirectWriteMarkers.java | 13 +-
.../hudi/testutils/HoodieClientTestHarness.java | 110 +----
.../hudi/testutils/HoodieClientTestUtils.java | 4 +-
.../common/fs/HoodieRetryWrapperFileSystem.java | 5 +
.../hudi/common/table/HoodieTableMetaClient.java | 2 +-
.../table/log/block/HoodieHFileDataBlock.java | 5 +-
.../table/timeline/HoodieActiveTimeline.java | 18 +-
.../table/timeline/HoodieArchivedTimeline.java | 4 +-
.../table/timeline/HoodieDefaultTimeline.java | 83 ++--
.../hudi/common/table/timeline/HoodieTimeline.java | 28 +-
.../common/table/timeline/TimelineDiffHelper.java | 11 +-
.../hudi/common/table/timeline/TimelineUtils.java | 4 +-
.../common/table/timeline/dto/TimelineDTO.java | 2 +-
.../table/view/AbstractTableFileSystemView.java | 4 +-
.../apache/hudi/common/util/ClusteringUtils.java | 4 +-
.../apache/hudi/common/util/CompactionUtils.java | 7 +-
.../hudi/common/util/InternalSchemaCache.java | 2 +-
.../hudi/internal/schema/InternalSchema.java | 3 +-
.../io/FileBasedInternalSchemaStorageManager.java | 2 +-
.../hudi/metadata/HoodieBackedTableMetadata.java | 4 +-
.../org/apache/hudi/common/fs/TestFSUtils.java | 89 ++--
.../fs/TestFSUtilsWithRetryWrapperEnable.java | 15 +-
.../common/functional/TestHoodieLogFormat.java | 62 +--
.../TestHoodieLogFormatAppendFailure.java | 8 +-
.../common/table/TestHoodieTableMetaClient.java | 4 +-
.../hudi/common/table/TestTimelineUtils.java | 14 +-
.../table/timeline/TestHoodieActiveTimeline.java | 16 +-
.../table/view/TestIncrementalFSViewSync.java | 135 +++---
.../common/testutils/HoodieCommonTestHarness.java | 25 +-
.../common/testutils/HoodieTestDataGenerator.java | 181 ++++++--
.../testutils/minicluster/HdfsTestService.java | 33 +-
.../testutils/minicluster/MiniClusterUtil.java | 59 ---
.../hudi/common/util/TestCompactionUtils.java | 6 +-
.../apache/hudi/configuration/OptionsResolver.java | 15 +
.../compact/strategy/CompactionPlanStrategies.java | 7 +-
.../sink/partitioner/profile/WriteProfile.java | 2 +-
.../apache/hudi/source/IncrementalInputSplits.java | 19 +-
.../org/apache/hudi/table/HoodieTableSource.java | 16 +-
.../java/org/apache/hudi/table/format/CastMap.java | 220 ++++++++++
.../org/apache/hudi/table/format/FormatUtils.java | 8 +-
.../format/HoodieParquetEvolvedSplitReader.java} | 38 +-
.../hudi/table/format/HoodieParquetReader.java | 99 +++++
.../table/format/HoodieParquetSplitReader.java} | 34 +-
.../hudi/table/format/InternalSchemaManager.java | 170 ++++++++
.../table/format/cow/CopyOnWriteInputFormat.java | 14 +-
.../table/format/mor/MergeOnReadInputFormat.java | 180 +++++---
.../apache/hudi/util/RowDataCastProjection.java | 33 +-
.../org/apache/hudi/util/RowDataProjection.java | 9 +-
.../sink/TestStreamWriteOperatorCoordinator.java | 16 +-
.../hudi/source/TestIncrementalInputSplits.java | 88 ++++
.../apache/hudi/table/ITTestHoodieDataSource.java | 2 +-
.../apache/hudi/table/ITTestSchemaEvolution.java | 461 +++++++++++++++++++++
.../org/apache/hudi/table/format/TestCastMap.java | 120 ++++++
.../apache/hudi/table/format/TestInputFormat.java | 6 +-
.../org/apache/hudi/utils/TestCompactionUtil.java | 12 +-
.../org/apache/hudi/utils/TestConfigurations.java | 23 +
.../HoodieMergeOnReadTableInputFormat.java | 2 +-
.../hudi/hadoop/realtime/HoodieRealtimePath.java | 2 +-
.../hudi/hadoop/utils/HoodieInputFormatUtils.java | 3 +-
.../hive/TestHoodieCombineHiveInputFormat.java | 32 +-
.../TestHoodieMergeOnReadTableInputFormat.java | 13 +
.../dag/nodes/ValidateAsyncOperations.java | 2 +-
.../testsuite/job/TestHoodieTestSuiteJob.java | 10 +-
.../main/scala/org/apache/hudi/DefaultSource.scala | 2 +-
.../org/apache/hudi/HoodieMergeOnReadRDD.scala | 176 +-------
.../org/apache/hudi/IncrementalRelation.scala | 6 +-
.../src/main/scala/org/apache/hudi/Iterators.scala | 332 +++++++++++++++
.../hudi/MergeOnReadIncrementalRelation.scala | 4 +-
.../org/apache/hudi/util/CachingIterator.scala | 27 +-
.../org/apache/hudi/HoodieDataSourceHelpers.java | 2 +-
.../procedures/CommitsCompareProcedure.scala | 4 +-
.../procedures/RunClusteringProcedure.scala | 91 +++-
.../procedures/RunCompactionProcedure.scala | 2 +-
.../procedures/ValidateHoodieSyncProcedure.scala | 2 +-
.../src/test/java/HoodieJavaStreamingApp.java | 9 +-
.../apache/hudi/functional/TestCOWDataSource.scala | 2 +-
.../TestIncrementalReadWithFullTableScan.scala | 2 +-
.../hudi/functional/TestSparkSqlCoreFlow.scala | 442 ++++++++++++++++++++
.../hudi/procedure/TestClusteringProcedure.scala | 269 +++++++++++-
.../sql/hudi/procedure/TestRepairsProcedure.scala | 4 +-
.../org/apache/hudi/hive/HoodieHiveSyncClient.java | 2 +-
.../org/apache/hudi/hive/TestHiveSyncTool.java | 4 +-
.../hudi/timeline/service/RequestHandler.java | 7 +-
hudi-utilities/pom.xml | 4 +
.../hudi/utilities/HiveIncrementalPuller.java | 4 +-
.../hudi/utilities/HoodieDataTableValidator.java | 2 +-
.../utilities/HoodieMetadataTableValidator.java | 2 +-
.../hudi/utilities/deltastreamer/DeltaSync.java | 4 +-
.../sources/helpers/IncrSourceHelper.java | 2 +-
.../functional/TestHoodieDeltaStreamer.java | 46 +-
.../TestHoodieDeltaStreamerWithMultiWriter.java | 3 +-
145 files changed, 3564 insertions(+), 1217 deletions(-)
delete mode 100644
hudi-common/src/test/java/org/apache/hudi/common/testutils/minicluster/MiniClusterUtil.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/CastMap.java
copy
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/{sink/transform/ChainedTransformer.java
=> table/format/HoodieParquetEvolvedSplitReader.java} (51%)
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/HoodieParquetReader.java
copy
hudi-flink-datasource/{hudi-flink1.15.x/src/main/java/org/apache/hudi/adapter/DataStreamSinkProviderAdapter.java
=>
hudi-flink/src/main/java/org/apache/hudi/table/format/HoodieParquetSplitReader.java}
(56%)
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/InternalSchemaManager.java
copy
hudi-spark-datasource/hudi-spark-common/src/main/java/org/apache/hudi/async/SparkStreamingAsyncCompactService.java
=>
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/RowDataCastProjection.java
(53%)
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/source/TestIncrementalInputSplits.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/ITTestSchemaEvolution.java
create mode 100644
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestCastMap.java
create mode 100644
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/Iterators.scala
copy
hudi-kafka-connect/src/main/java/org/apache/hudi/connect/writers/ConnectWriter.java
=>
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/util/CachingIterator.scala
(50%)
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlCoreFlow.scala
