beyond1920 commented on code in PR #8072:
URL: https://github.com/apache/hudi/pull/8072#discussion_r1124145638
##########
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkInsertOverwriteCommitActionExecutor.java:
##########
@@ -85,4 +102,19 @@ protected List<String> getAllExistingFileIds(String
partitionPath) {
// because new commit is not complete. it is safe to mark all existing
file Ids as old files
return
table.getSliceView().getLatestFileSlices(partitionPath).map(FileSlice::getFileId).distinct().collect(Collectors.toList());
}
+
+ @Override
+ protected Iterator<List<WriteStatus>> handleInsertPartition(String
instantTime, Integer partition, Iterator recordItr, Partitioner partitioner) {
+ SparkHoodiePartitioner upsertPartitioner = (SparkHoodiePartitioner)
partitioner;
+ BucketInfo binfo = upsertPartitioner.getBucketInfo(partition);
+ BucketType btype = binfo.bucketType;
+ if (btype.equals(BucketType.INSERT)) {
+ return handleInsert(binfo.fileIdPrefix, recordItr);
+ } else if (btype.equals(BucketType.UPDATE)) {
+ throw new HoodieInsertOverwriteException(
+ "Insert overwrite should always use INSERT bucketType, please
correct the logical of " + partitioner.getClass().getName());
Review Comment:
This is a protected code to prevent hit this bug again when introduce new
partitioner class in the future.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@hudi.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
