This is an automated email from the ASF dual-hosted git repository. nagarwal pushed a change to branch hudi_test_suite_refactor in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git.
from a096128 more fixes add c0f96e0 [HUDI-687] Stop incremental reader on RO table when there is a pending compaction (#1396) add a464a29 [HUDI-700]Add unit test for FileSystemViewCommand (#1490) add 5d717a2 [HUDI-782] Add support of Aliyun object storage service. (#1506) add 6d7ca2c [HUDI-727]: Copy default values of fields if not present when rewriting incoming record with new schema (#1427) add 447ba3b [MINOR] Disabling flaky test in InlineFileSystem (#1510) add 17bf930 [HUDI-770] Organize upsert/insert API implementation under a single package (#1495) add 661b0b3 [HUDI-761] Refactoring rollback and restore actions using the ActionExecutor abstraction (#1492) add 644c1cc [HUDI-698]Add unit test for CleansCommand (#1449) add 14d4fea [HUDI-759] Integrate checkpoint provider with delta streamer (#1486) add d65efe6 [HUDI-780] Migrate test cases to Junit 5 (#1504) add 9ca710c [HUDI-777] Updated description for --target-table parameter (#1519) add 1689d01 Merge branch 'master' into hudi_test_suite_refactor No new revisions were added by this update. Summary of changes: hudi-cli/pom.xml | 56 +++ .../apache/hudi/cli/HoodieTableHeaderFields.java | 65 +++ .../apache/hudi/cli/commands/CleansCommand.java | 15 +- .../hudi/cli/commands/FileSystemViewCommand.java | 47 +- .../org/apache/hudi/cli/commands/SparkMain.java | 4 +- .../hudi/cli/commands/TestCleansCommand.java | 183 ++++++++ .../cli/commands/TestFileSystemViewCommand.java | 267 ++++++++++++ .../apache/hudi/cli/integ/ITTestCleansCommand.java | 101 +++++ .../src/test/resources/clean.properties | 5 +- hudi-client/pom.xml | 24 ++ .../hudi/client/AbstractHoodieWriteClient.java | 138 +----- .../org/apache/hudi/client/HoodieWriteClient.java | 475 +++++---------------- ...kException.java => HoodieRestoreException.java} | 8 +- .../apache/hudi/table/HoodieCopyOnWriteTable.java | 428 +++---------------- .../apache/hudi/table/HoodieMergeOnReadTable.java | 376 +++------------- .../java/org/apache/hudi/table/HoodieTable.java | 131 ++++-- .../hudi/table/action/BaseActionExecutor.java | 5 +- .../table/action/clean/CleanActionExecutor.java | 8 +- .../action/commit/BaseCommitActionExecutor.java | 291 +++++++++++++ .../hudi/table/action/commit/BucketInfo.java | 28 +- .../hudi/table/action/commit/BucketType.java | 6 +- .../commit/BulkInsertCommitActionExecutor.java | 60 +++ .../hudi/table/action/commit/BulkInsertHelper.java | 84 ++++ .../BulkInsertPreppedCommitActionExecutor.java | 61 +++ .../table/action/commit/CommitActionExecutor.java | 176 ++++++++ .../action/commit/DeleteCommitActionExecutor.java} | 31 +- .../hudi/table/action/commit/DeleteHelper.java | 96 +++++ .../table/action/commit/HoodieWriteMetadata.java | 104 +++++ .../hudi/table/action/commit/InsertBucket.java | 30 +- .../action/commit/InsertCommitActionExecutor.java} | 33 +- .../commit/InsertPreppedCommitActionExecutor.java} | 31 +- .../apache/hudi/table/action/commit/SmallFile.java | 29 +- .../action/commit/UpsertCommitActionExecutor.java} | 33 +- .../table/action/commit/UpsertPartitioner.java | 316 ++++++++++++++ .../commit/UpsertPreppedCommitActionExecutor.java} | 31 +- .../hudi/table/action/commit/WriteHelper.java | 105 +++++ .../BulkInsertDeltaCommitActionExecutor.java | 62 +++ ...BulkInsertPreppedDeltaCommitActionExecutor.java | 63 +++ .../DeleteDeltaCommitActionExecutor.java} | 33 +- .../deltacommit/DeltaCommitActionExecutor.java | 94 ++++ .../InsertDeltaCommitActionExecutor.java | 49 +++ .../InsertPreppedDeltaCommitActionExecutor.java} | 31 +- .../UpsertDeltaCommitActionExecutor.java | 49 +++ .../deltacommit/UpsertDeltaCommitPartitioner.java | 142 ++++++ .../UpsertPreppedDeltaCommitActionExecutor.java} | 31 +- .../action/restore/BaseRestoreActionExecutor.java | 111 +++++ .../restore/CopyOnWriteRestoreActionExecutor.java | 57 +++ .../restore/MergeOnReadRestoreActionExecutor.java | 64 +++ .../rollback/BaseRollbackActionExecutor.java | 192 +++++++++ .../CopyOnWriteRollbackActionExecutor.java | 94 ++++ .../MergeOnReadRollbackActionExecutor.java | 241 +++++++++++ .../{ => action}/rollback/RollbackHelper.java | 2 +- .../{ => action}/rollback/RollbackRequest.java | 2 +- .../org/apache/hudi/client/TestClientRollback.java | 16 +- .../apache/hudi/client/TestHoodieClientBase.java | 3 +- .../TestHoodieClientOnCopyOnWriteStorage.java | 48 ++- .../hudi/common/HoodieMergeOnReadTestUtils.java | 40 +- .../apache/hudi/metrics/TestHoodieJmxMetrics.java | 4 +- .../org/apache/hudi/metrics/TestHoodieMetrics.java | 15 +- .../java/org/apache/hudi/table/TestCleaner.java | 2 +- .../apache/hudi/table/TestMergeOnReadTable.java | 287 ++++++++++--- .../commit/TestCopyOnWriteActionExecutor.java} | 214 ++++------ .../table/action/commit/TestUpsertPartitioner.java | 148 +++++++ hudi-common/pom.xml | 24 ++ .../java/org/apache/hudi/avro/HoodieAvroUtils.java | 43 +- .../org/apache/hudi/common/fs/StorageSchemes.java | 2 + .../hudi/common/model/WriteOperationType.java | 4 + .../table/timeline/HoodieDefaultTimeline.java | 9 +- .../hudi/common/table/timeline/HoodieTimeline.java | 5 + .../table/timeline/TimelineMetadataUtils.java | 15 +- .../org/apache/hudi/avro/TestHoodieAvroUtils.java | 72 +++- .../hudi/avro/TestHoodieAvroWriteSupport.java | 21 +- .../common/fs/inline/TestInLineFileSystem.java | 2 + .../hudi/common/storage/TestStorageSchemes.java | 1 + .../table/timeline/TestHoodieActiveTimeline.java | 16 +- hudi-hadoop-mr/pom.xml | 24 ++ .../org/apache/hudi/hadoop/HoodieHiveUtil.java | 24 ++ .../hudi/hadoop/HoodieParquetInputFormat.java | 67 ++- .../realtime/AbstractRealtimeRecordReader.java | 4 +- .../realtime/HoodieParquetRealtimeInputFormat.java | 7 + .../apache/hudi/hadoop/InputFormatTestUtil.java | 7 +- .../apache/hudi/hadoop/InputPathHandlerTest.java | 28 +- .../org/apache/hudi/hadoop/TestAnnotation.java | 4 +- .../hudi/hadoop/TestHoodieParquetInputFormat.java | 122 ++++++ .../hudi/hadoop/TestRecordReaderValueIterator.java | 13 +- hudi-hive-sync/pom.xml | 26 +- .../org/apache/hudi/hive/TestHiveSyncTool.java | 297 +++++++------ .../test/java/org/apache/hudi/hive/TestUtil.java | 2 +- hudi-integ-test/pom.xml | 24 +- .../java/org/apache/hudi/integ/ITTestBase.java | 18 +- .../org/apache/hudi/integ/ITTestHoodieDemo.java | 20 +- .../org/apache/hudi/integ/ITTestHoodieSanity.java | 33 +- hudi-spark/pom.xml | 29 +- hudi-spark/src/test/java/DataSourceUtilsTest.java | 6 +- hudi-spark/src/test/scala/TestDataSource.scala | 21 +- .../src/test/scala/TestDataSourceDefaults.scala | 18 +- hudi-timeline-service/pom.xml | 22 +- hudi-utilities/pom.xml | 24 ++ .../org/apache/hudi/utilities/UtilHelpers.java | 28 +- .../checkpointing/InitialCheckPointProvider.java | 33 +- .../checkpointing/KafkaConnectHdfsProvider.java | 27 +- .../deltastreamer/HoodieDeltaStreamer.java | 59 +-- .../HoodieMultiTableDeltaStreamer.java | 2 +- .../hudi/utilities/TestHoodieDeltaStreamer.java | 34 +- .../TestKafkaConnectHdfsProvider.java | 19 +- .../transform/TestChainedTransformer.java | 14 +- pom.xml | 34 +- scripts/run_travis_tests.sh | 7 + 108 files changed, 5229 insertions(+), 2057 deletions(-) create mode 100644 hudi-cli/src/main/java/org/apache/hudi/cli/HoodieTableHeaderFields.java create mode 100644 hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestCleansCommand.java create mode 100644 hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestFileSystemViewCommand.java create mode 100644 hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestCleansCommand.java copy hudi-utilities/src/test/resources/delta-streamer-config/base.properties => hudi-cli/src/test/resources/clean.properties (87%) copy hudi-client/src/main/java/org/apache/hudi/exception/{HoodieRollbackException.java => HoodieRestoreException.java} (81%) create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/commit/BaseCommitActionExecutor.java copy hudi-common/src/main/java/org/apache/hudi/common/model/HoodieArchivedLogFile.java => hudi-client/src/main/java/org/apache/hudi/table/action/commit/BucketInfo.java (60%) copy docker/demo/presto-table-check.commands => hudi-client/src/main/java/org/apache/hudi/table/action/commit/BucketType.java (89%) create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/commit/BulkInsertCommitActionExecutor.java create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/commit/BulkInsertHelper.java create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/commit/BulkInsertPreppedCommitActionExecutor.java create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/commit/CommitActionExecutor.java copy hudi-client/src/main/java/org/apache/hudi/{io/HoodieRangeInfoHandle.java => table/action/commit/DeleteCommitActionExecutor.java} (55%) create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/commit/DeleteHelper.java create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/commit/HoodieWriteMetadata.java copy hudi-common/src/main/java/org/apache/hudi/common/model/HoodieArchivedLogFile.java => hudi-client/src/main/java/org/apache/hudi/table/action/commit/InsertBucket.java (58%) copy hudi-client/src/main/java/org/apache/hudi/{io/HoodieRangeInfoHandle.java => table/action/commit/InsertCommitActionExecutor.java} (50%) copy hudi-client/src/main/java/org/apache/hudi/{io/HoodieRangeInfoHandle.java => table/action/commit/InsertPreppedCommitActionExecutor.java} (54%) copy hudi-common/src/main/java/org/apache/hudi/common/model/HoodieArchivedLogFile.java => hudi-client/src/main/java/org/apache/hudi/table/action/commit/SmallFile.java (61%) copy hudi-client/src/main/java/org/apache/hudi/{io/HoodieRangeInfoHandle.java => table/action/commit/UpsertCommitActionExecutor.java} (50%) create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/commit/UpsertPartitioner.java copy hudi-client/src/main/java/org/apache/hudi/{io/HoodieRangeInfoHandle.java => table/action/commit/UpsertPreppedCommitActionExecutor.java} (54%) create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/commit/WriteHelper.java create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/deltacommit/BulkInsertDeltaCommitActionExecutor.java create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/deltacommit/BulkInsertPreppedDeltaCommitActionExecutor.java copy hudi-client/src/main/java/org/apache/hudi/{io/HoodieRangeInfoHandle.java => table/action/deltacommit/DeleteDeltaCommitActionExecutor.java} (51%) create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/deltacommit/DeltaCommitActionExecutor.java create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/deltacommit/InsertDeltaCommitActionExecutor.java copy hudi-client/src/main/java/org/apache/hudi/{io/HoodieRangeInfoHandle.java => table/action/deltacommit/InsertPreppedDeltaCommitActionExecutor.java} (52%) create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/deltacommit/UpsertDeltaCommitActionExecutor.java create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/deltacommit/UpsertDeltaCommitPartitioner.java copy hudi-client/src/main/java/org/apache/hudi/{io/HoodieRangeInfoHandle.java => table/action/deltacommit/UpsertPreppedDeltaCommitActionExecutor.java} (52%) create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/restore/CopyOnWriteRestoreActionExecutor.java create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/restore/MergeOnReadRestoreActionExecutor.java create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackActionExecutor.java create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/rollback/CopyOnWriteRollbackActionExecutor.java create mode 100644 hudi-client/src/main/java/org/apache/hudi/table/action/rollback/MergeOnReadRollbackActionExecutor.java rename hudi-client/src/main/java/org/apache/hudi/table/{ => action}/rollback/RollbackHelper.java (99%) rename hudi-client/src/main/java/org/apache/hudi/table/{ => action}/rollback/RollbackRequest.java (98%) rename hudi-client/src/test/java/org/apache/hudi/table/{TestCopyOnWriteTable.java => action/commit/TestCopyOnWriteActionExecutor.java} (68%) create mode 100644 hudi-client/src/test/java/org/apache/hudi/table/action/commit/TestUpsertPartitioner.java