[
https://issues.apache.org/jira/browse/PARQUET-2261?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17707655#comment-17707655
]
ASF GitHub Bot commented on PARQUET-2261:
-----------------------------------------
gszadovszky commented on code in PR #197:
URL: https://github.com/apache/parquet-format/pull/197#discussion_r1155304277
##########
src/main/thrift/parquet.thrift:
##########
@@ -190,6 +190,60 @@ enum FieldRepetitionType {
/** The field is repeated and can contain 0 or more values */
REPEATED = 2;
}
+ /**
+ * Tracks a histogram of repetition and definition levels for either a page
or column chunk.
+ *
+ * This is useful for:
+ * 1. Estimating the size of the data when materialized in memory
+ * 2. For filter push-down on nulls at various levels of nested structures
and
+ * list lengths.
+ */
+ struct RepetitionDefinitionLevelHistogram {
+ /**
+ * When present there is expected to be one element corresponding to each
repetition (i.e. size=max repetition_level+1)
+ * where each element represents the number of time the repetition level
was observed in the data.
+ *
+ * This value is should not be written if max_repetition_level is 0.
Review Comment:
```suggestion
* This value should not be written if max_repetition_level is 0.
```
##########
src/main/thrift/parquet.thrift:
##########
@@ -966,6 +1039,13 @@ struct ColumnIndex {
/** A list containing the number of null values for each page **/
5: optional list<i64> null_counts
+ /**
+ * Repetition and definition level histograms for the page.
Review Comment:
```suggestion
* Repetition and definition level histograms for the pages.
```
##########
src/main/thrift/parquet.thrift:
##########
@@ -190,6 +190,60 @@ enum FieldRepetitionType {
/** The field is repeated and can contain 0 or more values */
REPEATED = 2;
}
+ /**
+ * Tracks a histogram of repetition and definition levels for either a page
or column chunk.
+ *
+ * This is useful for:
+ * 1. Estimating the size of the data when materialized in memory
+ * 2. For filter push-down on nulls at various levels of nested structures
and
+ * list lengths.
+ */
+ struct RepetitionDefinitionLevelHistogram {
+ /**
+ * When present there is expected to be one element corresponding to each
repetition (i.e. size=max repetition_level+1)
+ * where each element represents the number of time the repetition level
was observed in the data.
+ *
+ * This value is should not be written if max_repetition_level is 0.
+ **/
+ 1: optional list<i64> repetition_level_histogram;
+ /**
+ * Same as repetition_level_histogram except for definition levels.
+ *
+ * This value is should not be written when max_definition_level is 0.
Review Comment:
```suggestion
* This value should not be written when max_definition_level is 0.
```
##########
src/main/thrift/parquet.thrift:
##########
@@ -966,6 +1039,13 @@ struct ColumnIndex {
/** A list containing the number of null values for each page **/
5: optional list<i64> null_counts
+ /**
+ * Repetition and definition level histograms for the page.
+ *
+ * This contains has some redundancy with null_counts, however, to accomdate
+ * the widest range of readers both should be populated.
Review Comment:
```suggestion
* This contains some redundancy with null_counts, however, to accomodate
the
* widest range of readers both should be populated.
```
##########
src/main/thrift/parquet.thrift:
##########
@@ -580,7 +648,12 @@ struct DataPageHeaderV2 {
If missing it is considered compressed */
7: optional bool is_compressed = 1;
- /** optional statistics for the data in this page **/
+ /**
+ * optional statistics for the data in this page
+ *
+ * See notes on DataPage struct for use-cases and recommendations on a
discussion on
+ * recommendations around populating this field.
Review Comment:
```suggestion
* See notes on DataPage struct for use-cases and recommendations around
populating
* this field.
```
> [Format] Add statistics that reflect decoded size to metadata
> -------------------------------------------------------------
>
> Key: PARQUET-2261
> URL: https://issues.apache.org/jira/browse/PARQUET-2261
> Project: Parquet
> Issue Type: Improvement
> Components: parquet-format
> Reporter: Micah Kornfield
> Assignee: Micah Kornfield
> Priority: Major
>
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
