[ https://issues.apache.org/jira/browse/DRILL-7199?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16831172#comment-16831172 ]
ASF GitHub Bot commented on DRILL-7199: --------------------------------------- dvjyothsna commented on pull request #1771: DRILL-7199: Optimize population of metadata for non-interesting columns URL: https://github.com/apache/drill/pull/1771#discussion_r280170622 ########## File path: exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetTableMetadataUtils.java ########## @@ -286,25 +287,27 @@ public static PartitionMetadata getPartitionMetadata(SchemaPath partitionColumn, statistics.put(ColumnStatisticsKind.NULLS_COUNT, nulls); columnsStatistics.put(colPath, new ColumnStatisticsImpl(statistics, comparator)); } - columnsStatistics.putAll(populateNonInterestingColumnsStats(columnsStatistics.keySet(), tableMetadata)); return columnsStatistics; } /** - * Populates the non-interesting column's statistics - * @param schemaPaths columns paths which should be ignored + * Returns the non-interesting column's metadata * @param parquetTableMetadata the source of column metadata for non-interesting column's statistics - * @return returns non-interesting column statistics map + * @return returns non-interesting columns metadata */ - @SuppressWarnings("unchecked") - public static Map<SchemaPath, ColumnStatistics> populateNonInterestingColumnsStats( - Set<SchemaPath> schemaPaths, MetadataBase.ParquetTableMetadataBase parquetTableMetadata) { + public static NonInterestingColumnsMetadata getNonInterestingColumnsMeta(MetadataBase.ParquetTableMetadataBase parquetTableMetadata) { Map<SchemaPath, ColumnStatistics> columnsStatistics = new HashMap<>(); if (parquetTableMetadata instanceof Metadata_V4.ParquetTableMetadata_v4) { - for (Metadata_V4.ColumnTypeMetadata_v4 columnTypeMetadata : - ((Metadata_V4.ParquetTableMetadata_v4) parquetTableMetadata).getColumnTypeInfoMap().values()) { - SchemaPath schemaPath = SchemaPath.getCompoundPath(columnTypeMetadata.name); - if (!schemaPaths.contains(schemaPath)) { + ConcurrentHashMap<Metadata_V4.ColumnTypeMetadata_v4.Key, Metadata_V4.ColumnTypeMetadata_v4> columnTypeInfoMap = + ((Metadata_V4.ParquetTableMetadata_v4) parquetTableMetadata).getColumnTypeInfoMap(); + + if (columnTypeInfoMap == null) { + return new NonInterestingColumnsMetadata(columnsStatistics); + } // in some cases for runtime pruning Review comment: @Ben-Zvi Added this comment. I'm assuming columnTypeInfoMap will be null while populating metadata during run-time pruning and non-interesting columns don't make sense. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org > Optimize the time taken to populate column statistics for non-interesting > columns > --------------------------------------------------------------------------------- > > Key: DRILL-7199 > URL: https://issues.apache.org/jira/browse/DRILL-7199 > Project: Apache Drill > Issue Type: Bug > Reporter: Venkata Jyothsna Donapati > Assignee: Venkata Jyothsna Donapati > Priority: Minor > Fix For: 1.17.0 > > Original Estimate: 48h > Remaining Estimate: 48h > > Currently populating column statistics for non-interesting columns very long > since it is populated for every row group. Since non-interesting column > statistics are common for the table, it can be populated once and can be > reused. -- This message was sent by Atlassian JIRA (v7.6.3#76005)