bhat-vinay commented on code in PR #10876: URL: https://github.com/apache/hudi/pull/10876#discussion_r1536560898
########## hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/BaseSparkCommitActionExecutor.java: ########## @@ -411,4 +427,90 @@ public Partitioner getLayoutPartitioner(WorkloadProfile profile, String layoutPa protected void runPrecommitValidators(HoodieWriteMetadata<HoodieData<WriteStatus>> writeMetadata) { SparkValidatorUtils.runValidators(config, writeMetadata, context, table, instantTime); } + + private HoodieData<WriteStatus> sortAndMapPartitionsAsRDD(HoodieData<HoodieRecord<T>> dedupedRecords, Partitioner partitioner) { + JavaPairRDD<Tuple2<HoodieKey, Long>, HoodieRecord<T>> mappedRDD = getSortedIndexedRecords(dedupedRecords); + JavaPairRDD<Tuple2<HoodieKey, Long>, HoodieRecord<T>> partitionedRDD; + if (table.requireSortedRecords()) { + // Partition and sort within each partition as a single step. This is faster than partitioning first and then + // applying a sort. + Comparator<Tuple2<HoodieKey, Long>> comparator = (Comparator<Tuple2<HoodieKey, Long>> & Serializable) (t1, t2) -> { + HoodieKey key1 = t1._1(); + HoodieKey key2 = t2._1(); + return key1.getRecordKey().compareTo(key2.getRecordKey()); + }; + partitionedRDD = mappedRDD.repartitionAndSortWithinPartitions(partitioner, comparator); + } else { + // Partition only + partitionedRDD = mappedRDD.partitionBy(partitioner); + } + + return HoodieJavaRDD.of(partitionedRDD.map(Tuple2::_2).mapPartitionsWithIndex((partition, recordItr) -> { + if (WriteOperationType.isChangingRecords(operationType)) { + return handleUpsertPartition(instantTime, partition, recordItr, partitioner); + } else { + return handleInsertPartition(instantTime, partition, recordItr, partitioner); + } + }, true).flatMap(List::iterator)); + } + + private boolean operationRequiresSorting() { + return operationType == WriteOperationType.INSERT && config.getBoolean(INSERT_SORT); Review Comment: The current implementation (in this PR) does not support sorting for UPSERT operation -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@hudi.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org