yordan-pavlov commented on a change in pull request #692:
URL: https://github.com/apache/arrow-rs/pull/692#discussion_r696960150
##########
File path: parquet/src/arrow/arrow_array_reader.rs
##########
@@ -754,20 +786,40 @@ impl<I: Iterator<Item = Box<dyn ValueDecoder>>>
ValueDecoder
}
}
- Ok(num_values - values_to_read)
+ Ok((num_values - values_to_read, child_values_to_read))
}
}
+/// The level type. This is used to determine whether
+/// to read levels ahead and determine the number of
+/// values to read.
+#[derive(PartialEq)]
+enum LevelType {
+ Definition,
+ Repetition,
+}
+
struct LevelValueDecoder {
level_decoder: crate::encodings::levels::LevelDecoder,
level_value_buffer: Vec<i16>,
+ level_type: LevelType,
+ total_consumed_levels: usize,
+ current_unconsumed_levels: usize,
+ total_consumed_values: usize,
}
impl LevelValueDecoder {
- fn new(level_decoder: crate::encodings::levels::LevelDecoder) -> Self {
+ fn new(
+ level_decoder: crate::encodings::levels::LevelDecoder,
+ level_type: LevelType,
+ ) -> Self {
Self {
level_decoder,
- level_value_buffer: vec![0i16; 2048],
+ level_value_buffer: vec![0i16; BUFFER_LEN],
+ level_type,
+ total_consumed_levels: 0,
Review comment:
another reason to split `LevelValueDecoder` into separate decoders for
repetition and definition levels is because they use different state - these 3
new fields are only used for repetition levels
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@arrow.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
