nsivabalan commented on a change in pull request #2421: URL: https://github.com/apache/hudi/pull/2421#discussion_r554629618
########## File path: hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java ########## @@ -262,18 +264,31 @@ private static void processRollbackMetadata(HoodieRollbackMetadata rollbackMetad partitionToDeletedFiles.get(partition).addAll(deletedFiles); } - if (!pm.getAppendFiles().isEmpty()) { + if (pm.getRollbackLogFiles() != null && !pm.getRollbackLogFiles().isEmpty()) { if (!partitionToAppendedFiles.containsKey(partition)) { partitionToAppendedFiles.put(partition, new HashMap<>()); } // Extract appended file name from the absolute paths saved in getAppendFiles() - pm.getAppendFiles().forEach((path, size) -> { + pm.getRollbackLogFiles().forEach((path, size) -> { partitionToAppendedFiles.get(partition).merge(new Path(path).getName(), size, (oldSize, newSizeCopy) -> { return size + oldSize; }); }); } + + if (pm.getWrittenLogFiles() != null && !pm.getWrittenLogFiles().isEmpty()) { + if (!partitionToAppendedFiles.containsKey(partition)) { + partitionToAppendedFiles.put(partition, new HashMap<>()); + } + + // Extract appended file name from the absolute paths saved in getWrittenLogFiles() + pm.getWrittenLogFiles().forEach((path, size) -> { + partitionToAppendedFiles.get(partition).merge(new Path(path).getName(), size, (oldSize, newSizeCopy) -> { + return oldSize; Review comment: if rolledback file was also in written log file list, we have to take the rolledback file size since it is the snapshot after the log file was written. So, leaving it to old value itself. ########## File path: hudi-common/src/main/java/org/apache/hudi/metadata/HoodieTableMetadataUtil.java ########## @@ -262,13 +251,26 @@ private static void processRollbackMetadata(HoodieRollbackMetadata rollbackMetad partitionToDeletedFiles.get(partition).addAll(deletedFiles); } - if (!pm.getAppendFiles().isEmpty()) { + if (pm.getRollbackLogFiles() != null && !pm.getRollbackLogFiles().isEmpty()) { if (!partitionToAppendedFiles.containsKey(partition)) { partitionToAppendedFiles.put(partition, new HashMap<>()); } // Extract appended file name from the absolute paths saved in getAppendFiles() - pm.getAppendFiles().forEach((path, size) -> { + pm.getRollbackLogFiles().forEach((path, size) -> { + partitionToAppendedFiles.get(partition).merge(new Path(path).getName(), size, (oldSize, newSizeCopy) -> { + return size + oldSize; Review comment: this is merging entries within rollback and delete files only. Ideally we shouldn't see any merge in this section but only in line 286. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org