deniskuzZ commented on code in PR #5389:
URL: https://github.com/apache/hive/pull/5389#discussion_r1709797978
##########
iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergTableUtil.java:
##########
@@ -396,45 +396,49 @@ public static PartitionData toPartitionData(StructLike
sourceKey, Types.StructTy
/**
* Returns list of data files filtered by specId and partitionPath as
following:
- * 1. If matchBySpecId is true, then filters files by specId == file's
specId, else by specId != file's specId
- * 2. If partitionPath is not null, then also filters files where
partitionPath == file's partition path
+ * 1. If table is unpartitioned, returns all data files without filtering.
+ * 2. If matchBySpecId is true, then filters files by specId == file's
specId, else by specId != file's specId
+ * 3. If partitionPath is not null, then also filters files where
partitionPath == file's partition path
* @param table the iceberg table
* @param specId partition spec id
* @param partitionPath partition path
* @param matchBySpecId filter that's applied on data files' spec ids
*/
- public static List<DataFile> getDataFiles(Table table, int specId, String
partitionPath,
+ public static List<DataFile> getDataFiles(Table table, Integer specId,
String partitionPath,
Predicate<Object> matchBySpecId) {
CloseableIterable<FileScanTask> fileScanTasks =
table.newScan().useSnapshot(table.currentSnapshot().snapshotId()).ignoreResiduals().planFiles();
CloseableIterable<FileScanTask> filteredFileScanTasks =
CloseableIterable.filter(fileScanTasks, t -> {
DataFile file = t.asFileScanTask().file();
- return matchBySpecId.test(file.specId()) && (partitionPath == null
|| (partitionPath != null &&
-
table.specs().get(specId).partitionToPath(file.partition()).equals(partitionPath)));
+ return !table.spec().isPartitioned() ||
+ (matchBySpecId.test(file.specId()) && (partitionPath == null ||
(partitionPath != null &&
+
table.specs().get(specId).partitionToPath(file.partition()).equals(partitionPath))));
});
return
Lists.newArrayList(CloseableIterable.transform(filteredFileScanTasks, t ->
t.file()));
}
/**
* Returns list of delete files filtered by specId and partitionPath as
following:
- * 1. If matchBySpecId is true, then filters files by specId == file's
specId, else by specId != file's specId
- * 2. If partitionPath is not null, then also filters files where
partitionPath == file's partition path
+ * 1. If table is unpartitioned, returns all delete files without filtering.
+ * 2. If matchBySpecId is true, then filters files by specId == file's
specId, else by specId != file's specId
+ * 3. If partitionPath is not null, then also filters files where
partitionPath == file's partition path
* @param table the iceberg table
* @param specId partition spec id
* @param partitionPath partition path
* @param matchBySpecId filter that's applied on delete files' spec ids
*/
- public static List<DeleteFile> getDeleteFiles(Table table, int specId,
String partitionPath,
+ public static List<DeleteFile> getDeleteFiles(Table table, Integer specId,
String partitionPath,
Predicate<Object> matchBySpecId) {
Table deletesTable =
MetadataTableUtils.createMetadataTableInstance(table,
MetadataTableType.POSITION_DELETES);
CloseableIterable<ScanTask> deletesScanTasks =
deletesTable.newBatchScan().planFiles();
CloseableIterable<ScanTask> filteredDeletesScanTasks =
CloseableIterable.filter(deletesScanTasks, t -> {
DeleteFile file = ((PositionDeletesScanTask) t).file();
- return matchBySpecId.test(file.specId()) && (partitionPath == null
|| (partitionPath != null &&
-
table.specs().get(specId).partitionToPath(file.partition()).equals(partitionPath)));
+ return !table.spec().isPartitioned() ||
+ (matchBySpecId.test(file.specId()) && (partitionPath == null ||
(partitionPath != null &&
Review Comment:
same here
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
