deniskuzZ commented on code in PR #5328:
URL: https://github.com/apache/hive/pull/5328#discussion_r1700025774
##########
iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java:
##########
@@ -393,28 +393,47 @@ public static PartitionData toPartitionData(StructLike
sourceKey, Types.StructTy
return data;
}
- public static List<DataFile> getDataFiles(Table table, int specId,
- String partitionPath) {
+ /**
+ * Returns list of data files filtered by specId and partitionPath as
following:
+ * 1. If matchBySpecId is true, then filters files by specId == file's
specId, else by specId != file's specId
+ * 2. If partitionPath is not null, then also filters files where
partitionPath == file's partition path
+ * @param table the iceberg table
+ * @param specId partition spec id
+ * @param partitionPath partition path
+ * @param matchBySpecId If true, searches for files matching the specId,
else searches for ones that don't match
+ */
+ public static List<DataFile> getDataFiles(Table table, int specId, String
partitionPath, boolean matchBySpecId) {
CloseableIterable<FileScanTask> fileScanTasks =
table.newScan().useSnapshot(table.currentSnapshot().snapshotId()).ignoreResiduals().planFiles();
CloseableIterable<FileScanTask> filteredFileScanTasks =
CloseableIterable.filter(fileScanTasks, t -> {
DataFile file = t.asFileScanTask().file();
- return file.specId() == specId && table.specs()
-
.get(specId).partitionToPath(file.partition()).equals(partitionPath);
+ return ((matchBySpecId && file.specId() == specId) ||
(!matchBySpecId && file.specId() != specId)) &&
+ (partitionPath == null || (partitionPath != null &&
+
table.specs().get(specId).partitionToPath(file.partition()).equals(partitionPath)));
});
return
Lists.newArrayList(CloseableIterable.transform(filteredFileScanTasks, t ->
t.file()));
}
- public static List<DeleteFile> getDeleteFiles(Table table, int specId,
String partitionPath) {
+ /**
+ * Returns list of delete files filtered by specId and partitionPath as
following:
+ * 1. If matchBySpecId is true, then filters files by specId == file's
specId, else by specId != file's specId
+ * 2. If partitionPath is not null, then also filters files where
partitionPath == file's partition path
+ * @param table the iceberg table
+ * @param specId partition spec id
+ * @param partitionPath partition path
+ * @param matchBySpecId If true, searches for files matching the specId,
else searches for ones that don't match
+ */
+ public static List<DeleteFile> getDeleteFiles(Table table, int specId,
String partitionPath, boolean matchBySpecId) {
Review Comment:
same as above
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: gitbox-unsubscr...@hive.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscr...@hive.apache.org
For additional commands, e-mail: gitbox-h...@hive.apache.org
