vvysotskyi commented on a change in pull request #1810: DRILL-7271: Refactor
Metadata interfaces and classes to contain all needed information for the File
based Metastore
URL: https://github.com/apache/drill/pull/1810#discussion_r296612824
##########
File path:
exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/AbstractParquetGroupScan.java
##########
@@ -281,31 +265,50 @@ public AbstractGroupScanWithMetadata
applyFilter(LogicalExpression filterExpr, U
logger.debug("All row groups have been filtered out. Add back one to get
schema from scanner");
+ Map<Path, SegmentMetadata> segmentsMap =
getNextOrEmpty(getSegmentsMetadata().values()).stream()
+ .collect(Collectors.toMap(SegmentMetadata::getPath,
Function.identity()));
+
Map<Path, FileMetadata> filesMap =
getNextOrEmpty(getFilesMetadata().values()).stream()
- .collect(Collectors.toMap(FileMetadata::getLocation,
Function.identity()));
+ .collect(Collectors.toMap(FileMetadata::getPath,
Function.identity()));
Multimap<Path, RowGroupMetadata> rowGroupsMap =
LinkedListMultimap.create();
- getNextOrEmpty(getRowGroupsMetadata().values()).forEach(entry ->
rowGroupsMap.put(entry.getLocation(), entry));
+ getNextOrEmpty(getRowGroupsMetadata().values()).forEach(entry ->
rowGroupsMap.put(entry.getPath(), entry));
- builder.withRowGroups(rowGroupsMap)
+ filteredMetadata.withRowGroups(rowGroupsMap)
.withTable(getTableMetadata())
+ .withSegments(segmentsMap)
.withPartitions(getNextOrEmpty(getPartitionsMetadata()))
.withNonInterestingColumns(getNonInterestingColumnsMetadata())
.withFiles(filesMap)
.withMatching(false);
}
- if (builder.getOverflowLevel() != MetadataLevel.NONE) {
+ if (filteredMetadata.getOverflowLevel() != MetadataType.NONE) {
logger.warn("applyFilter {} wasn't able to do pruning for all metadata
levels filter condition, since metadata count for " +
"{} level exceeds
`planner.store.parquet.rowgroup.filter.pushdown.threshold` value.\n" +
"But underlying metadata was pruned without filter expression
according to the metadata with above level.",
- ExpressionStringBuilder.toString(filterExpr),
builder.getOverflowLevel());
+ ExpressionStringBuilder.toString(filterExpr),
filteredMetadata.getOverflowLevel());
}
logger.debug("applyFilter {} reduce row groups # from {} to {}",
- ExpressionStringBuilder.toString(filterExpr),
getRowGroupsMetadata().size(), builder.getRowGroups().size());
+ ExpressionStringBuilder.toString(filterExpr),
getRowGroupsMetadata().size(), filteredMetadata.getRowGroups().size());
Review comment:
Thanks, done.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
