codope commented on code in PR #12342:
URL: https://github.com/apache/hudi/pull/12342#discussion_r1865696203
##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DefaultSource.scala:
##########
@@ -301,11 +302,29 @@ object DefaultSource {
resolveBaseFileOnlyRelation(sqlContext, globPaths, userSchema,
metaClient, parameters)
}
case (COPY_ON_WRITE, QUERY_TYPE_INCREMENTAL_OPT_VAL, _) =>
- if (useNewParquetFileFormat) {
- new HoodieCopyOnWriteIncrementalHadoopFsRelationFactory(
- sqlContext, metaClient, parameters, userSchema,
isBootstrappedTable).build()
+ if (SparkConfigUtils.containsConfigProperty(parameters,
INCREMENTAL_READ_VERSION)) {
+ val writeTableVersion =
Integer.parseInt(parameters(INCREMENTAL_READ_VERSION.key))
+ if (writeTableVersion >= 8) {
+ if (useNewParquetFileFormat) {
+ new HoodieCopyOnWriteIncrementalHadoopFsRelationFactory(
+ sqlContext, metaClient, parameters, userSchema,
isBootstrappedTable).build()
+ } else {
+ new IncrementalRelationV2(sqlContext, parameters,
userSchema, metaClient, RangeType.CLOSED_CLOSED)
+ }
+ } else {
+ new IncrementalRelationV1(sqlContext, parameters, userSchema,
metaClient)
+ }
} else {
- new IncrementalRelation(sqlContext, parameters, userSchema,
metaClient)
+ if (metaClient.getTableConfig.getTableVersion.versionCode() >=
8) {
+ if (useNewParquetFileFormat) {
+ new HoodieCopyOnWriteIncrementalHadoopFsRelationFactory(
+ sqlContext, metaClient, parameters, userSchema,
isBootstrappedTable).build()
+ } else {
+ new IncrementalRelationV2(sqlContext, parameters,
userSchema, metaClient, RangeType.CLOSED_CLOSED)
Review Comment:
strongly agree.. HUDI-8627 to track all refactoring and code reuse related
stuff
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
