This is an automated email from the ASF dual-hosted git repository. leesf pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/hudi.git.
from 195dac9 [MINOR] Disable flaky tests to unlock CI (#4592) add 5ce45c4 [HUDI-3172] Refactor hudi existing modules to make more code reuse in V2 Implementation (#4514) No new revisions were added by this update. Summary of changes: .../org/apache/spark/sql/hudi/SparkAdapter.scala | 4 +- hudi-spark-datasource/README.md | 38 +++ hudi-spark-datasource/hudi-spark-common/pom.xml | 42 +++ .../apache/hudi/HoodieDatasetBulkInsertHelper.java | 0 .../java/org/apache/hudi/SparkRowWriteHelper.java | 0 .../SparkStreamingAsyncClusteringService.java | 0 .../async/SparkStreamingAsyncCompactService.java | 0 .../org/apache/hudi/sql/IExpressionEvaluator.java | 0 .../main/java/org/apache/hudi/sql/InsertMode.java | 0 ...org.apache.spark.sql.sources.DataSourceRegister | 0 .../main/scala/org/apache/hudi/DefaultSource.scala | 2 +- .../scala/org/apache/hudi/HoodieBootstrapRDD.scala | 0 .../org/apache/hudi/HoodieBootstrapRelation.scala | 0 .../org/apache/hudi/HoodieEmptyRelation.scala | 0 .../scala/org/apache/hudi/HoodieFileIndex.scala | 4 +- .../org/apache/hudi/HoodieMergeOnReadRDD.scala | 46 ++-- .../org/apache/hudi/HoodieSparkSqlWriter.scala | 10 +- .../org/apache/hudi/HoodieStreamingSink.scala | 0 .../scala/org/apache/hudi/HoodieWriterUtils.scala | 0 .../org/apache/hudi/IncrementalRelation.scala | 0 .../hudi/MergeOnReadIncrementalRelation.scala | 0 .../apache/hudi/MergeOnReadSnapshotRelation.scala | 4 +- .../src/main/scala/org/apache/hudi/package.scala | 0 .../spark/sql/avro/HoodieAvroDeserializer.scala | 0 .../spark/sql/avro/HoodieAvroSerializer.scala | 0 .../sql/catalyst/catalog/HoodieCatalogTable.scala | 26 +- .../spark/sql/catalyst/trees/HoodieLeafLike.scala | 0 .../apache/spark/sql/hive/HiveClientUtils.scala | 0 .../apache/spark/sql/hudi/DataSkippingUtils.scala | 0 .../apache/spark/sql/hudi/HoodieOptionConfig.scala | 2 +- .../spark/sql/hudi/HoodieSqlCommonUtils.scala} | 64 ++--- .../org/apache/spark/sql/hudi/SerDeUtils.scala | 0 .../AlterHoodieTableAddColumnsCommand.scala | 2 - .../AlterHoodieTableChangeColumnCommand.scala | 2 - .../AlterHoodieTableDropPartitionCommand.scala | 5 +- .../command/AlterHoodieTableRenameCommand.scala | 0 .../hudi/command/CreateHoodieTableCommand.scala | 10 +- .../hudi/command/HoodieLeafRunnableCommand.scala | 0 .../command/ShowHoodieTablePartitionsCommand.scala | 2 +- .../spark/sql/hudi/command/SqlKeyGenerator.scala | 2 +- .../hudi/command/TruncateHoodieTableCommand.scala | 0 .../hudi/command/ValidateDuplicateKeyPayload.scala | 45 ++++ .../hudi/command/payload/ExpressionCodeGen.scala | 0 .../hudi/command/payload/ExpressionPayload.scala | 0 .../sql/hudi/command/payload/SqlTypedRecord.scala | 0 .../sql/hudi/streaming/HoodieSourceOffset.scala | 0 .../sql/hudi/streaming/HoodieStreamSource.scala | 0 hudi-spark-datasource/hudi-spark/pom.xml | 12 + .../sql/hudi/HoodieSparkSessionExtension.scala | 2 +- .../org/apache/spark/sql/hudi/HoodieSqlUtils.scala | 289 +-------------------- .../spark/sql/hudi/analysis/HoodieAnalysis.scala | 28 +- .../hudi/command/CompactionHoodiePathCommand.scala | 11 +- .../command/CompactionHoodieTableCommand.scala | 5 +- .../command/CompactionShowHoodieTableCommand.scala | 5 +- .../command/CreateHoodieTableAsSelectCommand.scala | 5 +- .../hudi/command/DeleteHoodieTableCommand.scala | 7 +- .../sql/hudi/command/DropHoodieTableCommand.scala | 2 +- .../command/InsertIntoHoodieTableCommand.scala | 33 +-- .../hudi/command/MergeIntoHoodieTableCommand.scala | 8 +- .../hudi/command/UpdateHoodieTableCommand.scala | 10 +- .../apache/hudi/functional/TestMORDataSource.scala | 2 - .../org/apache/spark/sql/hudi/TestAlterTable.scala | 2 +- hudi-spark-datasource/hudi-spark2-common/pom.xml | 19 ++ ...org.apache.spark.sql.sources.DataSourceRegister | 2 +- .../Spark2DefaultSource.scala} | 16 +- .../apache/spark/sql/adapter/Spark2Adapter.scala | 8 +- hudi-spark-datasource/hudi-spark3-common/pom.xml | 247 ++++++++++++++++++ .../apache/hudi/spark3/internal/DefaultSource.java | 0 .../HoodieBulkInsertDataInternalWriter.java | 0 .../HoodieBulkInsertDataInternalWriterFactory.java | 0 .../HoodieDataSourceInternalBatchWrite.java | 0 .../HoodieDataSourceInternalBatchWriteBuilder.java | 0 .../internal/HoodieDataSourceInternalTable.java | 0 .../spark3/internal/HoodieWriterCommitMessage.java | 0 .../apache/hudi/spark3/internal/ReflectUtil.java | 0 .../scala/org/apache/hudi/Spark3RowSerDe.scala | 0 .../apache/spark/sql/adapter/Spark3Adapter.scala | 15 +- .../datasources/Spark3ParsePartitionUtil.scala | 2 +- .../{hudi-spark3 => hudi-spark3.1.x}/pom.xml | 11 +- ...org.apache.spark.sql.sources.DataSourceRegister | 2 +- .../org/apache/hudi/Spark3xDefaultSource.scala} | 12 +- hudi-spark-datasource/hudi-spark3/pom.xml | 5 + ...org.apache.spark.sql.sources.DataSourceRegister | 2 +- .../org/apache/hudi/Spark3DefaultSource.scala} | 12 +- hudi-spark-datasource/pom.xml | 4 + hudi-utilities/pom.xml | 6 + packaging/hudi-integ-test-bundle/pom.xml | 7 + packaging/hudi-spark-bundle/pom.xml | 9 + packaging/hudi-utilities-bundle/pom.xml | 6 + pom.xml | 5 +- 90 files changed, 620 insertions(+), 501 deletions(-) create mode 100644 hudi-spark-datasource/README.md rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/java/org/apache/hudi/HoodieDatasetBulkInsertHelper.java (100%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/java/org/apache/hudi/SparkRowWriteHelper.java (100%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/java/org/apache/hudi/async/SparkStreamingAsyncClusteringService.java (100%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/java/org/apache/hudi/async/SparkStreamingAsyncCompactService.java (100%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/java/org/apache/hudi/sql/IExpressionEvaluator.java (100%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/java/org/apache/hudi/sql/InsertMode.java (100%) copy hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister (100%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/hudi/DefaultSource.scala (99%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/hudi/HoodieBootstrapRDD.scala (100%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/hudi/HoodieBootstrapRelation.scala (100%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/hudi/HoodieEmptyRelation.scala (100%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/hudi/HoodieFileIndex.scala (99%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/hudi/HoodieMergeOnReadRDD.scala (92%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala (99%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala (100%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/hudi/HoodieWriterUtils.scala (100%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/hudi/IncrementalRelation.scala (100%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/hudi/MergeOnReadIncrementalRelation.scala (100%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/hudi/MergeOnReadSnapshotRelation.scala (99%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/hudi/package.scala (100%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/avro/HoodieAvroDeserializer.scala (100%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/avro/HoodieAvroSerializer.scala (100%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/catalyst/catalog/HoodieCatalogTable.scala (91%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/catalyst/trees/HoodieLeafLike.scala (100%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hive/HiveClientUtils.scala (100%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/DataSkippingUtils.scala (100%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/HoodieOptionConfig.scala (99%) copy hudi-spark-datasource/{hudi-spark/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlUtils.scala => hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieSqlCommonUtils.scala} (87%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/SerDeUtils.scala (100%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/command/AlterHoodieTableAddColumnsCommand.scala (98%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/command/AlterHoodieTableChangeColumnCommand.scala (98%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/command/AlterHoodieTableDropPartitionCommand.scala (98%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/command/AlterHoodieTableRenameCommand.scala (100%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/command/CreateHoodieTableCommand.scala (96%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/command/HoodieLeafRunnableCommand.scala (100%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/command/ShowHoodieTablePartitionsCommand.scala (98%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/command/SqlKeyGenerator.scala (98%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/command/TruncateHoodieTableCommand.scala (100%) create mode 100644 hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/command/ValidateDuplicateKeyPayload.scala rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/command/payload/ExpressionCodeGen.scala (100%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/command/payload/ExpressionPayload.scala (100%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/command/payload/SqlTypedRecord.scala (100%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieSourceOffset.scala (100%) rename hudi-spark-datasource/{hudi-spark => hudi-spark-common}/src/main/scala/org/apache/spark/sql/hudi/streaming/HoodieStreamSource.scala (100%) create mode 100644 hudi-spark-datasource/hudi-spark2-common/pom.xml copy hudi-spark-datasource/{hudi-spark => hudi-spark2}/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister (95%) copy hudi-spark-datasource/hudi-spark2/src/main/scala/org/apache/{spark/sql/catalyst/plans/logical/DeleteFromTable.scala => hudi/Spark2DefaultSource.scala} (70%) create mode 100644 hudi-spark-datasource/hudi-spark3-common/pom.xml rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3-common}/src/main/java/org/apache/hudi/spark3/internal/DefaultSource.java (100%) rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3-common}/src/main/java/org/apache/hudi/spark3/internal/HoodieBulkInsertDataInternalWriter.java (100%) rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3-common}/src/main/java/org/apache/hudi/spark3/internal/HoodieBulkInsertDataInternalWriterFactory.java (100%) rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3-common}/src/main/java/org/apache/hudi/spark3/internal/HoodieDataSourceInternalBatchWrite.java (100%) rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3-common}/src/main/java/org/apache/hudi/spark3/internal/HoodieDataSourceInternalBatchWriteBuilder.java (100%) rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3-common}/src/main/java/org/apache/hudi/spark3/internal/HoodieDataSourceInternalTable.java (100%) rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3-common}/src/main/java/org/apache/hudi/spark3/internal/HoodieWriterCommitMessage.java (100%) rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3-common}/src/main/java/org/apache/hudi/spark3/internal/ReflectUtil.java (100%) rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3-common}/src/main/scala/org/apache/hudi/Spark3RowSerDe.scala (100%) rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3-common}/src/main/scala/org/apache/spark/sql/adapter/Spark3Adapter.scala (85%) rename hudi-spark-datasource/{hudi-spark3 => hudi-spark3-common}/src/main/scala/org/apache/spark/sql/execution/datasources/Spark3ParsePartitionUtil.scala (99%) copy hudi-spark-datasource/{hudi-spark3 => hudi-spark3.1.x}/pom.xml (96%) copy hudi-spark-datasource/{hudi-spark => hudi-spark3.1.x}/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister (95%) copy hudi-spark-datasource/{hudi-spark2/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DeleteFromTable.scala => hudi-spark3.1.x/src/main/scala/org/apache/hudi/Spark3xDefaultSource.scala} (70%) rename hudi-spark-datasource/{hudi-spark => hudi-spark3}/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister (95%) copy hudi-spark-datasource/{hudi-spark2/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DeleteFromTable.scala => hudi-spark3/src/main/scala/org/apache/hudi/Spark3DefaultSource.scala} (70%)