linliu-code commented on code in PR #12122:
URL: https://github.com/apache/hudi/pull/12122#discussion_r1807204447
##########
hudi-common/src/main/java/org/apache/hudi/common/table/read/HoodieBaseFileGroupRecordBuffer.java:
##########
@@ -278,14 +289,19 @@ protected Option<DeleteRecord>
doProcessNextDeletedRecord(DeleteRecord deleteRec
if
(isDeleteRecordWithNaturalOrder(existingRecordMetadataPair.getLeft(),
existingOrderingVal)) {
return Option.empty();
}
- Comparable deleteOrderingVal = deleteRecord.getOrderingValue();
- // Checks the ordering value does not equal to 0
- // because we use 0 as the default value which means natural order
- boolean chooseExisting = !deleteOrderingVal.equals(0)
- && ReflectionUtils.isSameClass(existingOrderingVal,
deleteOrderingVal)
- && existingOrderingVal.compareTo(deleteOrderingVal) > 0;
+ Comparable deleteOrderingVal = readerContext.getOrderingValue(
Review Comment:
Yeah, we can do that.
##########
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/Iterators.scala:
##########
@@ -302,33 +300,49 @@ class RecordMergingFileIterator(logFiles:
List[HoodieLogFile],
private def merge(curRow: InternalRow, newRecord: HoodieRecord[_]):
Option[InternalRow] = {
// NOTE: We have to pass in Avro Schema used to read from Delta Log file
since we invoke combining API
// on the record from the Delta Log
- recordMerger.getRecordType match {
- case HoodieRecordType.SPARK =>
- val curRecord = new HoodieSparkRecord(curRow, readerSchema)
- val result = recordMerger.merge(curRecord, baseFileReaderAvroSchema,
newRecord, logFileReaderAvroSchema, payloadProps)
- toScalaOption(result)
- .flatMap { r =>
- val data = r.getLeft.getData.asInstanceOf[InternalRow]
- if (isDeleteOperation(data)) {
- None
- } else {
- val schema = HoodieInternalRowUtils.getCachedSchema(r.getRight)
- val projection =
HoodieInternalRowUtils.getCachedUnsafeProjection(schema, structTypeSchema)
- Some(projection.apply(data))
+
+ // Delete records are in-between; no merge is needed.
+ if
(newRecord.getMetaDataInfo(HoodieReaderContext.INTERNAL_META_OPERATION).isPresent)
{
Review Comment:
Will do.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
