This is an automated email from the ASF dual-hosted git repository. gangwu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/parquet-format.git
The following commit(s) were added to refs/heads/master by this push: new be0478a PARQUET-2480: Clarify what "page index" means in Parquet.thrift (#245) be0478a is described below commit be0478a7f7d6b0cc12503983233c85f8ae0a5fa7 Author: Andrew Lamb <and...@nerdnetworks.org> AuthorDate: Wed May 22 02:03:32 2024 -0400 PARQUET-2480: Clarify what "page index" means in Parquet.thrift (#245) --- PageIndex.md | 10 ++++++---- src/main/thrift/parquet.thrift | 22 +++++++++++++++++----- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/PageIndex.md b/PageIndex.md index f4a8f64..a371c42 100644 --- a/PageIndex.md +++ b/PageIndex.md @@ -17,11 +17,13 @@ - under the License. --> -# ColumnIndex Layout to Support Page Skipping +# Parquet page index: Layout to Support Page Skipping -This document describes the format for column index pages in the Parquet -footer. These pages contain statistics for DataPages and can be used to skip -pages when scanning data in ordered and unordered columns. +In Parquet, a *page index* is optional metadata for a +ColumnChunk, containing statistics for DataPages that can be used +to skip those pages when scanning in ordered and unordered columns. +The page index is stored using the OffsetIndex and ColumnIndex structures, +defined in [`parquet.thrift`](src/main/thrift/parquet.thrift) ## Problem Statement In previous versions of the format, Statistics are stored for ColumnChunks in diff --git a/src/main/thrift/parquet.thrift b/src/main/thrift/parquet.thrift index 27d4043..c928ad6 100644 --- a/src/main/thrift/parquet.thrift +++ b/src/main/thrift/parquet.thrift @@ -738,10 +738,10 @@ struct PageHeader { } /** - * Wrapper struct to specify sort order + * Sort order within a RowGroup of a leaf column */ struct SortingColumn { - /** The column index (in this row group) **/ + /** The ordinal position of the column (in this row group) **/ 1: required i32 column_idx /** If true, indicates this column is sorted in descending order. **/ @@ -1001,6 +1001,13 @@ struct PageLocation { 3: required i64 first_row_index } +/** + * Optional offsets for each data page in a ColumnChunk. + * + * Forms part of the page index, along with ColumnIndex. + * + * OffsetIndex may be present even if ColumnIndex is not. + */ struct OffsetIndex { /** * PageLocations, ordered by increasing PageLocation.offset. It is required @@ -1017,8 +1024,14 @@ struct OffsetIndex { } /** - * Description for ColumnIndex. - * Each <array-field>[i] refers to the page at OffsetIndex.page_locations[i] + * Optional statistics for each data page in a ColumnChunk. + * + * Forms part the page index, along with OffsetIndex. + * + * If this structure is present, OffsetIndex must also be present. + * + * For each field in this structure, <field>[i] refers to the page at + * OffsetIndex.page_locations[i] */ struct ColumnIndex { /** @@ -1071,7 +1084,6 @@ struct ColumnIndex { * Same as repetition_level_histograms except for definitions levels. **/ 7: optional list<i64> definition_level_histograms; - } struct AesGcmV1 {