This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-12003-6014b2dce20f1dc3fac660641cf5041ad1b933c7 in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
commit 5649fe151886aa91b925e900438bdf5f2cfd520f Author: Jörn Horstmann <[email protected]> AuthorDate: Mon Nov 2 19:36:09 2026 +0000 fix(parquet): do not write a dictionary page in empty row group (#12003) # Which issue does this PR close? <!-- It is not necessary to raise an issue, but it is encouraged if it is a non-trivial enhancement/bug. If this PR relates to an existing issue, but does not fully resolve it, please change the wording below from "Closes #NNN" to "Relates to #NNN" or "Part of #NNN" to avoid false closing an issue. --> - Closes #7356 # Rationale for this change I ran into this issue again, where an empty parquet file had a non-zero `total_uncompressed_size` and `total_compressed_size`, leading to my code trying to parse a data page at offset 0 and stumbling over that "PAR1" magic. <!-- Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed. Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes. Please try to explain the problem you are trying to solve in terms of the user-visible behavior, rather than the implementation. --> # What changes are included in this PR? Empty row groups no longer write a dictionary page, and their size is 0. This also skips the encodings, if no actual data using that encoding is written. # Are these changes tested? One new unit test specifically for the empty file, other tests still pass. # Are there any significant user-facing changes? The changes to the encodings metadata could be unexpected, it would be fine to leave these out if wanted. --- parquet/src/arrow/mod.rs | 2 +- parquet/src/column/writer/mod.rs | 188 +++++++++++++++++++++------ parquet/src/file/writer.rs | 40 ++++++ parquet/tests/arrow_reader/io/sync_reader.rs | 18 +-- 4 files changed, 195 insertions(+), 53 deletions(-) diff --git a/parquet/src/arrow/mod.rs b/parquet/src/arrow/mod.rs index d25126cbd7..1ebb2321a0 100644 --- a/parquet/src/arrow/mod.rs +++ b/parquet/src/arrow/mod.rs @@ -534,7 +534,7 @@ mod test { .unwrap(); assert_eq!( err.to_string(), - "EOF: Parquet file too small. Page index range 82..115 overlaps with file metadata 0..357" + "EOF: Parquet file too small. Page index range 82..115 overlaps with file metadata 0..356" ); } diff --git a/parquet/src/column/writer/mod.rs b/parquet/src/column/writer/mod.rs index bff91faf28..179153894d 100644 --- a/parquet/src/column/writer/mod.rs +++ b/parquet/src/column/writer/mod.rs @@ -512,9 +512,7 @@ impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E> { let statistics_enabled = column_props.statistics_enabled; - let mut encodings = BTreeSet::new(); - // Used for level information - encodings.insert(Encoding::RLE); + let encodings = BTreeSet::new(); let mut page_metrics = PageMetrics::new(); let mut column_metrics = ColumnMetrics::<E::T>::new(); @@ -819,7 +817,7 @@ impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E> { if self.page_metrics.num_buffered_values > 0 { self.add_data_page()?; } - if self.encoder.has_dictionary() { + if self.encoder.has_dictionary() && self.column_metrics.total_rows_written > 0 { self.write_dictionary_page()?; } self.flush_data_pages()?; @@ -1739,6 +1737,9 @@ impl<'a, E: ColumnValueEncoder> GenericColumnWriter<'a, E> { #[inline] fn write_data_page(&mut self, page: CompressedPage) -> Result<()> { self.encodings.insert(page.encoding()); + if self.descr.max_def_level() > 0 || self.descr.max_rep_level() > 0 { + self.encodings.insert(Encoding::RLE); + } match self.encoding_stats.last_mut() { Some(encoding_stats) if encoding_stats.page_type == page.page_type() @@ -2259,7 +2260,7 @@ mod tests { let metadata = r.metadata; assert_eq!( metadata.encodings().collect::<Vec<_>>(), - vec![Encoding::PLAIN, Encoding::RLE] + vec![Encoding::PLAIN] ); assert_eq!(metadata.num_values(), 4); // just values assert_eq!(metadata.dictionary_page_offset(), None); @@ -2294,7 +2295,7 @@ mod tests { true, &[true, false], None, - &[Encoding::PLAIN, Encoding::RLE], + &[Encoding::PLAIN], &[encoding_stats(PageType::DATA_PAGE, Encoding::PLAIN, 1)], ); check_encoding_write_support::<BoolType>( @@ -2302,7 +2303,7 @@ mod tests { false, &[true, false], None, - &[Encoding::PLAIN, Encoding::RLE], + &[Encoding::PLAIN], &[encoding_stats(PageType::DATA_PAGE, Encoding::PLAIN, 1)], ); check_encoding_write_support::<BoolType>( @@ -2330,7 +2331,7 @@ mod tests { true, &[1, 2], Some(0), - &[Encoding::PLAIN, Encoding::RLE, Encoding::RLE_DICTIONARY], + &[Encoding::PLAIN, Encoding::RLE_DICTIONARY], &[ encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1), encoding_stats(PageType::DATA_PAGE, Encoding::RLE_DICTIONARY, 1), @@ -2341,7 +2342,7 @@ mod tests { false, &[1, 2], None, - &[Encoding::PLAIN, Encoding::RLE], + &[Encoding::PLAIN], &[encoding_stats(PageType::DATA_PAGE, Encoding::PLAIN, 1)], ); check_encoding_write_support::<Int32Type>( @@ -2349,7 +2350,7 @@ mod tests { true, &[1, 2], Some(0), - &[Encoding::PLAIN, Encoding::RLE, Encoding::RLE_DICTIONARY], + &[Encoding::PLAIN, Encoding::RLE_DICTIONARY], &[ encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1), encoding_stats(PageType::DATA_PAGE_V2, Encoding::RLE_DICTIONARY, 1), @@ -2360,7 +2361,7 @@ mod tests { false, &[1, 2], None, - &[Encoding::RLE, Encoding::DELTA_BINARY_PACKED], + &[Encoding::DELTA_BINARY_PACKED], &[encoding_stats( PageType::DATA_PAGE_V2, Encoding::DELTA_BINARY_PACKED, @@ -2376,7 +2377,7 @@ mod tests { true, &[1, 2], Some(0), - &[Encoding::PLAIN, Encoding::RLE, Encoding::RLE_DICTIONARY], + &[Encoding::PLAIN, Encoding::RLE_DICTIONARY], &[ encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1), encoding_stats(PageType::DATA_PAGE, Encoding::RLE_DICTIONARY, 1), @@ -2387,7 +2388,7 @@ mod tests { false, &[1, 2], None, - &[Encoding::PLAIN, Encoding::RLE], + &[Encoding::PLAIN], &[encoding_stats(PageType::DATA_PAGE, Encoding::PLAIN, 1)], ); check_encoding_write_support::<Int64Type>( @@ -2395,7 +2396,7 @@ mod tests { true, &[1, 2], Some(0), - &[Encoding::PLAIN, Encoding::RLE, Encoding::RLE_DICTIONARY], + &[Encoding::PLAIN, Encoding::RLE_DICTIONARY], &[ encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1), encoding_stats(PageType::DATA_PAGE_V2, Encoding::RLE_DICTIONARY, 1), @@ -2406,7 +2407,7 @@ mod tests { false, &[1, 2], None, - &[Encoding::RLE, Encoding::DELTA_BINARY_PACKED], + &[Encoding::DELTA_BINARY_PACKED], &[encoding_stats( PageType::DATA_PAGE_V2, Encoding::DELTA_BINARY_PACKED, @@ -2422,7 +2423,7 @@ mod tests { true, &[Int96::from(vec![1, 2, 3])], Some(0), - &[Encoding::PLAIN, Encoding::RLE, Encoding::RLE_DICTIONARY], + &[Encoding::PLAIN, Encoding::RLE_DICTIONARY], &[ encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1), encoding_stats(PageType::DATA_PAGE, Encoding::RLE_DICTIONARY, 1), @@ -2433,7 +2434,7 @@ mod tests { false, &[Int96::from(vec![1, 2, 3])], None, - &[Encoding::PLAIN, Encoding::RLE], + &[Encoding::PLAIN], &[encoding_stats(PageType::DATA_PAGE, Encoding::PLAIN, 1)], ); check_encoding_write_support::<Int96Type>( @@ -2441,7 +2442,7 @@ mod tests { true, &[Int96::from(vec![1, 2, 3])], Some(0), - &[Encoding::PLAIN, Encoding::RLE, Encoding::RLE_DICTIONARY], + &[Encoding::PLAIN, Encoding::RLE_DICTIONARY], &[ encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1), encoding_stats(PageType::DATA_PAGE_V2, Encoding::RLE_DICTIONARY, 1), @@ -2452,7 +2453,7 @@ mod tests { false, &[Int96::from(vec![1, 2, 3])], None, - &[Encoding::PLAIN, Encoding::RLE], + &[Encoding::PLAIN], &[encoding_stats(PageType::DATA_PAGE_V2, Encoding::PLAIN, 1)], ); } @@ -2464,7 +2465,7 @@ mod tests { true, &[1.0, 2.0], Some(0), - &[Encoding::PLAIN, Encoding::RLE, Encoding::RLE_DICTIONARY], + &[Encoding::PLAIN, Encoding::RLE_DICTIONARY], &[ encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1), encoding_stats(PageType::DATA_PAGE, Encoding::RLE_DICTIONARY, 1), @@ -2475,7 +2476,7 @@ mod tests { false, &[1.0, 2.0], None, - &[Encoding::PLAIN, Encoding::RLE], + &[Encoding::PLAIN], &[encoding_stats(PageType::DATA_PAGE, Encoding::PLAIN, 1)], ); check_encoding_write_support::<FloatType>( @@ -2483,7 +2484,7 @@ mod tests { true, &[1.0, 2.0], Some(0), - &[Encoding::PLAIN, Encoding::RLE, Encoding::RLE_DICTIONARY], + &[Encoding::PLAIN, Encoding::RLE_DICTIONARY], &[ encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1), encoding_stats(PageType::DATA_PAGE_V2, Encoding::RLE_DICTIONARY, 1), @@ -2494,7 +2495,7 @@ mod tests { false, &[1.0, 2.0], None, - &[Encoding::PLAIN, Encoding::RLE], + &[Encoding::PLAIN], &[encoding_stats(PageType::DATA_PAGE_V2, Encoding::PLAIN, 1)], ); } @@ -2506,7 +2507,7 @@ mod tests { true, &[1.0, 2.0], Some(0), - &[Encoding::PLAIN, Encoding::RLE, Encoding::RLE_DICTIONARY], + &[Encoding::PLAIN, Encoding::RLE_DICTIONARY], &[ encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1), encoding_stats(PageType::DATA_PAGE, Encoding::RLE_DICTIONARY, 1), @@ -2517,7 +2518,7 @@ mod tests { false, &[1.0, 2.0], None, - &[Encoding::PLAIN, Encoding::RLE], + &[Encoding::PLAIN], &[encoding_stats(PageType::DATA_PAGE, Encoding::PLAIN, 1)], ); check_encoding_write_support::<DoubleType>( @@ -2525,7 +2526,7 @@ mod tests { true, &[1.0, 2.0], Some(0), - &[Encoding::PLAIN, Encoding::RLE, Encoding::RLE_DICTIONARY], + &[Encoding::PLAIN, Encoding::RLE_DICTIONARY], &[ encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1), encoding_stats(PageType::DATA_PAGE_V2, Encoding::RLE_DICTIONARY, 1), @@ -2536,7 +2537,7 @@ mod tests { false, &[1.0, 2.0], None, - &[Encoding::PLAIN, Encoding::RLE], + &[Encoding::PLAIN], &[encoding_stats(PageType::DATA_PAGE_V2, Encoding::PLAIN, 1)], ); } @@ -2548,7 +2549,7 @@ mod tests { true, &[ByteArray::from(vec![1u8])], Some(0), - &[Encoding::PLAIN, Encoding::RLE, Encoding::RLE_DICTIONARY], + &[Encoding::PLAIN, Encoding::RLE_DICTIONARY], &[ encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1), encoding_stats(PageType::DATA_PAGE, Encoding::RLE_DICTIONARY, 1), @@ -2559,7 +2560,7 @@ mod tests { false, &[ByteArray::from(vec![1u8])], None, - &[Encoding::PLAIN, Encoding::RLE], + &[Encoding::PLAIN], &[encoding_stats(PageType::DATA_PAGE, Encoding::PLAIN, 1)], ); check_encoding_write_support::<ByteArrayType>( @@ -2567,7 +2568,7 @@ mod tests { true, &[ByteArray::from(vec![1u8])], Some(0), - &[Encoding::PLAIN, Encoding::RLE, Encoding::RLE_DICTIONARY], + &[Encoding::PLAIN, Encoding::RLE_DICTIONARY], &[ encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1), encoding_stats(PageType::DATA_PAGE_V2, Encoding::RLE_DICTIONARY, 1), @@ -2578,7 +2579,7 @@ mod tests { false, &[ByteArray::from(vec![1u8])], None, - &[Encoding::RLE, Encoding::DELTA_BYTE_ARRAY], + &[Encoding::DELTA_BYTE_ARRAY], &[encoding_stats( PageType::DATA_PAGE_V2, Encoding::DELTA_BYTE_ARRAY, @@ -2605,7 +2606,7 @@ mod tests { true, &[ByteArray::from(vec![1u8]).into()], Some(0), - &[Encoding::PLAIN, Encoding::RLE, Encoding::RLE_DICTIONARY], + &[Encoding::PLAIN, Encoding::RLE_DICTIONARY], &[ encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1), encoding_stats(PageType::DATA_PAGE, Encoding::RLE_DICTIONARY, 1), @@ -2626,7 +2627,7 @@ mod tests { false, &[ByteArray::from(vec![1u8]).into()], None, - &[Encoding::PLAIN, Encoding::RLE], + &[Encoding::PLAIN], &[encoding_stats(PageType::DATA_PAGE, Encoding::PLAIN, 1)], ); check_encoding_write_support::<FixedLenByteArrayType>( @@ -2634,7 +2635,7 @@ mod tests { true, &[ByteArray::from(vec![1u8]).into()], Some(0), - &[Encoding::PLAIN, Encoding::RLE, Encoding::RLE_DICTIONARY], + &[Encoding::PLAIN, Encoding::RLE_DICTIONARY], &[ encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1), encoding_stats(PageType::DATA_PAGE_V2, Encoding::RLE_DICTIONARY, 1), @@ -2645,7 +2646,7 @@ mod tests { false, &[ByteArray::from(vec![1u8]).into()], None, - &[Encoding::RLE, Encoding::DELTA_BYTE_ARRAY], + &[Encoding::DELTA_BYTE_ARRAY], &[encoding_stats( PageType::DATA_PAGE_V2, Encoding::DELTA_BYTE_ARRAY, @@ -2654,6 +2655,90 @@ mod tests { ); } + #[test] + fn test_column_writer_no_dictionary_page_for_empty_required() { + let props = WriterProperties::builder() + .set_writer_version(WriterVersion::PARQUET_1_0) + .set_dictionary_enabled(true) + .build(); + let meta = column_write_and_get_metadata_with_def_levels::<Int64Type>(props, &[], &[], 0); + assert_eq!(meta.dictionary_page_offset(), None); + assert_eq!(meta.data_page_offset(), 0); + assert_eq!(meta.uncompressed_size(), 0); + assert_eq!(meta.compressed_size(), 0); + assert_eq!(meta.encodings().collect::<Vec<_>>(), &[]); + assert_eq!(meta.page_encoding_stats().unwrap(), &[]); + } + + #[test] + fn test_column_writer_no_dictionary_page_for_empty_optional() { + let props = WriterProperties::builder() + .set_writer_version(WriterVersion::PARQUET_1_0) + .set_dictionary_enabled(true) + .build(); + let meta = column_write_and_get_metadata_with_def_levels::<Int64Type>(props, &[], &[], 1); + assert_eq!(meta.dictionary_page_offset(), None); + assert_eq!(meta.data_page_offset(), 0); + assert_eq!(meta.uncompressed_size(), 0); + assert_eq!(meta.compressed_size(), 0); + assert_eq!(meta.encodings().collect::<Vec<_>>(), &[]); + assert_eq!(meta.page_encoding_stats().unwrap(), &[]); + } + + #[test] + fn test_column_writer_encoding_includes_rle_for_all_null_data() { + let props = WriterProperties::builder() + .set_writer_version(WriterVersion::PARQUET_1_0) + .set_dictionary_enabled(true) + .build(); + let meta = column_write_and_get_metadata_with_def_levels::<Int64Type>(props, &[], &[0], 1); + // this writes an empty dictionary page because we write the all-null data page with RLE_DICTIONARY encoding + assert_eq!(meta.dictionary_page_offset(), Some(0)); + assert_eq!(meta.data_page_offset(), 0); + assert_eq!(meta.uncompressed_size(), 7); + assert_eq!(meta.compressed_size(), 7); + assert_eq!( + meta.encodings().collect::<Vec<_>>(), + &[Encoding::PLAIN, Encoding::RLE, Encoding::RLE_DICTIONARY] + ); + assert_eq!( + meta.page_encoding_stats().unwrap(), + &[ + encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1), + encoding_stats(PageType::DATA_PAGE, Encoding::RLE_DICTIONARY, 1) + ] + ); + } + + #[test] + fn test_column_writer_encoding_includes_rle_for_mixed_null_data() { + let props = WriterProperties::builder() + .set_writer_version(WriterVersion::PARQUET_1_0) + .set_dictionary_enabled(true) + .build(); + let meta = column_write_and_get_metadata_with_def_levels::<Int64Type>( + props, + &[42_i64], + &[1, 0], + 1, + ); + assert_eq!(meta.dictionary_page_offset(), Some(0)); + assert_eq!(meta.data_page_offset(), 8); + assert_eq!(meta.uncompressed_size(), 16); + assert_eq!(meta.compressed_size(), 16); + assert_eq!( + meta.encodings().collect::<Vec<_>>(), + &[Encoding::PLAIN, Encoding::RLE, Encoding::RLE_DICTIONARY] + ); + assert_eq!( + meta.page_encoding_stats().unwrap(), + &[ + encoding_stats(PageType::DICTIONARY_PAGE, Encoding::PLAIN, 1), + encoding_stats(PageType::DATA_PAGE, Encoding::RLE_DICTIONARY, 1) + ] + ); + } + #[test] fn test_column_writer_check_metadata() { let page_writer = get_test_page_writer(); @@ -2668,12 +2753,12 @@ mod tests { let metadata = r.metadata; assert_eq!( metadata.encodings().collect::<Vec<_>>(), - vec![Encoding::PLAIN, Encoding::RLE, Encoding::RLE_DICTIONARY] + vec![Encoding::PLAIN, Encoding::RLE_DICTIONARY] ); assert_eq!(metadata.num_values(), 4); assert_eq!(metadata.compressed_size(), 20); assert_eq!(metadata.uncompressed_size(), 20); - assert_eq!(metadata.data_page_offset(), 0); + assert_eq!(metadata.data_page_offset(), 16); assert_eq!(metadata.dictionary_page_offset(), Some(0)); if let Some(stats) = metadata.statistics() { assert_eq!(stats.null_count_opt(), Some(0)); @@ -2830,12 +2915,12 @@ mod tests { let metadata = r.metadata; assert_eq!( metadata.encodings().collect::<Vec<_>>(), - vec![Encoding::PLAIN, Encoding::RLE, Encoding::RLE_DICTIONARY] + vec![Encoding::PLAIN, Encoding::RLE_DICTIONARY] ); assert_eq!(metadata.num_values(), 4); assert_eq!(metadata.compressed_size(), 20); assert_eq!(metadata.uncompressed_size(), 20); - assert_eq!(metadata.data_page_offset(), 0); + assert_eq!(metadata.data_page_offset(), 16); assert_eq!(metadata.dictionary_page_offset(), Some(0)); if let Some(stats) = metadata.statistics() { assert_eq!(stats.null_count_opt(), Some(0)); @@ -5541,6 +5626,19 @@ mod tests { writer.close().unwrap().metadata } + fn column_write_and_get_metadata_with_def_levels<T: DataType>( + props: WriterProperties, + values: &[T::T], + def_levels: &[i16], + max_def_level: i16, + ) -> ColumnChunkMetaData { + let page_writer = get_test_page_writer(); + let props = Arc::new(props); + let mut writer = get_test_column_writer::<T>(page_writer, max_def_level, 0, props); + writer.write_batch(values, Some(def_levels), None).unwrap(); + writer.close().unwrap().metadata + } + // Helper function to more compactly create a PageEncodingStats struct. fn encoding_stats(page_type: PageType, encoding: Encoding, count: i32) -> PageEncodingStats { PageEncodingStats { @@ -5743,10 +5841,13 @@ mod tests { /// Returns page writer that collects pages without serializing them. fn get_test_page_writer() -> Box<dyn PageWriter> { - Box::new(TestPageWriter {}) + Box::new(TestPageWriter::default()) } - struct TestPageWriter {} + #[derive(Default)] + struct TestPageWriter { + bytes_written: u64, + } impl PageWriter for TestPageWriter { fn write_page(&mut self, page: CompressedPage) -> Result<PageWriteSpec> { @@ -5755,8 +5856,9 @@ mod tests { res.uncompressed_size = page.uncompressed_size(); res.compressed_size = page.compressed_size(); res.num_values = page.num_values(); - res.offset = 0; + res.offset = self.bytes_written; res.bytes_written = page.data().len() as u64; + self.bytes_written += res.bytes_written; Ok(res) } @@ -5896,7 +5998,7 @@ mod tests { #[test] fn test_page_v2_snappy_compression_fallback() { // Test that PageV2 sets is_compressed to false when Snappy compression increases data size - let page_writer = TestPageWriter {}; + let page_writer = TestPageWriter::default(); // Create WriterProperties with PageV2 and Snappy compression let props = WriterProperties::builder() diff --git a/parquet/src/file/writer.rs b/parquet/src/file/writer.rs index 7030cb28c6..7654cb8ce9 100644 --- a/parquet/src/file/writer.rs +++ b/parquet/src/file/writer.rs @@ -2832,4 +2832,44 @@ mod tests { assert_eq!(new_col_min, exp_col_min); assert_eq!(new_col_max, exp_col_max); } + + #[test] + fn test_empty_row_group_data_page_offset() { + // Write a parquet file with one row group and one column but no data written. + // With no rows written no pages are flushed, so sizes should be zero. + let schema = Arc::new( + types::Type::group_type_builder("schema") + .with_fields(vec![Arc::new( + types::Type::primitive_type_builder("col1", Type::INT32) + .build() + .unwrap(), + )]) + .build() + .unwrap(), + ); + let props = Default::default(); + let mut buf = Vec::new(); + let mut writer = + SerializedFileWriter::new(std::io::Cursor::new(&mut buf), schema, props).unwrap(); + + let mut row_group = writer.next_row_group().unwrap(); + let col = row_group.next_column().unwrap().unwrap(); + col.close().unwrap(); + row_group.close().unwrap(); + let metadata = writer.close().unwrap(); + + assert_eq!(metadata.num_row_groups(), 1); + let col_meta = metadata.row_group(0).column(0); + // No pages written: sizes are zero. + assert_eq!(col_meta.compressed_size(), 0); + assert_eq!(col_meta.uncompressed_size(), 0); + assert_eq!(metadata.row_group(0).compressed_size(), 0); + // No dictionary page written. + assert_eq!(col_meta.dictionary_page_offset(), None); + // data_page_offset is 0. The field is required, + // but 0 would be invalid since it points to the PAR1 magic. + assert_eq!(col_meta.data_page_offset(), 0); + // No pages written: no encodings recorded. + assert_eq!(col_meta.encodings().collect::<Vec<_>>(), vec![]); + } } diff --git a/parquet/tests/arrow_reader/io/sync_reader.rs b/parquet/tests/arrow_reader/io/sync_reader.rs index 1c1c1f8dd1..039cfc9a13 100644 --- a/parquet/tests/arrow_reader/io/sync_reader.rs +++ b/parquet/tests/arrow_reader/io/sync_reader.rs @@ -42,7 +42,7 @@ fn test_read_entire_file() { @r#" [ "Footer: 8 bytes", - "Metadata: 1162", + "Metadata: 1156", "UNKNOWN: 22230..22877 (maybe Page Index)", "Event: Builder Configured", "Event: Reader Built", @@ -78,7 +78,7 @@ fn test_read_single_group() { @r#" [ "Footer: 8 bytes", - "Metadata: 1162", + "Metadata: 1156", "UNKNOWN: 22230..22877 (maybe Page Index)", "Event: Builder Configured", "Event: Reader Built", @@ -106,7 +106,7 @@ fn test_read_single_column() { @r#" [ "Footer: 8 bytes", - "Metadata: 1162", + "Metadata: 1156", "UNKNOWN: 22230..22877 (maybe Page Index)", "Event: Builder Configured", "Event: Reader Built", @@ -135,7 +135,7 @@ fn test_read_single_column_no_page_index() { @r#" [ "Footer: 8 bytes", - "Metadata: 1162", + "Metadata: 1156", "Event: Builder Configured", "Event: Reader Built", "Row Group 0, column 'b': DictionaryPage (17 bytes , 17 requests) [header]", @@ -182,7 +182,7 @@ fn test_read_row_selection() { @r#" [ "Footer: 8 bytes", - "Metadata: 1162", + "Metadata: 1156", "UNKNOWN: 22230..22877 (maybe Page Index)", "Event: Builder Configured", "Event: Reader Built", @@ -213,7 +213,7 @@ fn test_read_limit() { @r#" [ "Footer: 8 bytes", - "Metadata: 1162", + "Metadata: 1156", "UNKNOWN: 22230..22877 (maybe Page Index)", "Event: Builder Configured", "Event: Reader Built", @@ -255,7 +255,7 @@ fn test_read_single_row_filter() { @r#" [ "Footer: 8 bytes", - "Metadata: 1162", + "Metadata: 1156", "UNKNOWN: 22230..22877 (maybe Page Index)", "Event: Builder Configured", "Row Group 0, column 'b': DictionaryPage (1617 bytes, 1 requests) [data]", @@ -311,7 +311,7 @@ fn test_read_multiple_row_filter() { @r#" [ "Footer: 8 bytes", - "Metadata: 1162", + "Metadata: 1156", "UNKNOWN: 22230..22877 (maybe Page Index)", "Event: Builder Configured", "Row Group 0, column 'a': DictionaryPage (1617 bytes, 1 requests) [data]", @@ -357,7 +357,7 @@ fn test_read_single_row_filter_all() { @r#" [ "Footer: 8 bytes", - "Metadata: 1162", + "Metadata: 1156", "UNKNOWN: 22230..22877 (maybe Page Index)", "Event: Builder Configured", "Row Group 0, column 'b': DictionaryPage (1617 bytes, 1 requests) [data]",
