linliu-code commented on code in PR #9888:
URL: https://github.com/apache/hudi/pull/9888#discussion_r1371808164
##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/HoodieFileGroupReaderBasedParquetFileFormat.scala:
##########
@@ -141,12 +145,37 @@ class
HoodieFileGroupReaderBasedParquetFileFormat(tableState: HoodieTableState,
case _ => baseFileReader(file)
}
}
+ // CDC queries.
+ case hoodiePartitionCDCFileGroupSliceMapping:
HoodiePartitionCDCFileGroupMapping =>
+ val filePath: Path =
sparkAdapter.getSparkPartitionedFileUtils.getPathFromPartitionedFile(file)
+ val fileGroupId: HoodieFileGroupId = new
HoodieFileGroupId(filePath.getParent.toString, filePath.getName)
+ val fileSplits =
hoodiePartitionCDCFileGroupSliceMapping.getFileSplitsFor(fileGroupId).get.toArray
+ val fileGroupSplit: HoodieCDCFileGroupSplit =
HoodieCDCFileGroupSplit(fileSplits)
+ buildCDCRecordIterator(fileGroupSplit, preMergeBaseFileReader,
hadoopConf, requiredSchema, props)
// TODO: Use FileGroupReader here: HUDI-6942.
case _ => baseFileReader(file)
}
}
}
+ protected def buildCDCRecordIterator(cdcFileGroupSplit:
HoodieCDCFileGroupSplit,
+ preMergeBaseFileReader: PartitionedFile
=> Iterator[InternalRow],
+ hadoopConf: Configuration,
+ requiredSchema: StructType,
+ props: TypedProperties):
Iterator[InternalRow] = {
+ val metaClient =
HoodieTableMetaClient.initTableAndGetMetaClient(hadoopConf,
tableState.tablePath, props)
+ val cdcSchema = CDCRelation.FULL_CDC_SPARK_SCHEMA
+ new CDCFileGroupIterator(
Review Comment:
Right, it can not right now due to its complex logic and our tight schedule.
We should definitely do that later.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@hudi.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
