itsjunetime commented on code in PR #6690:
URL: https://github.com/apache/arrow-rs/pull/6690#discussion_r1838630127
##########
arrow-data/src/data.rs:
##########
@@ -423,72 +423,97 @@ impl ArrayData {
size
}
- /// Returns the total number of the bytes of memory occupied by
- /// the buffers by this slice of [`ArrayData`] (See also diagram on
[`ArrayData`]).
+ /// Returns the total number of the bytes of memory occupied by the
buffers by this slice of
+ /// [`ArrayData`] (See also diagram on [`ArrayData`]).
///
- /// This is approximately the number of bytes if a new
- /// [`ArrayData`] was formed by creating new [`Buffer`]s with
- /// exactly the data needed.
+ /// This is approximately the number of bytes if a new [`ArrayData`] was
formed by creating new
+ /// [`Buffer`]s with exactly the data needed.
///
- /// For example, a [`DataType::Int64`] with `100` elements,
- /// [`Self::get_slice_memory_size`] would return `100 * 8 = 800`. If
- /// the [`ArrayData`] was then [`Self::slice`]ed to refer to its
- /// first `20` elements, then [`Self::get_slice_memory_size`] on the
- /// sliced [`ArrayData`] would return `20 * 8 = 160`.
- pub fn get_slice_memory_size(&self) -> Result<usize, ArrowError> {
- let mut result: usize = 0;
+ /// For example, a [`DataType::Int64`] with `100` elements,
[`Self::get_slice_memory_size`]
+ /// would return `100 * 8 = 800`. If the [`ArrayData`] was then
[`Self::slice`]d to refer to
+ /// its first `20` elements, then [`Self::get_slice_memory_size`] on the
sliced [`ArrayData`]
+ /// would return `20 * 8 = 160`.
+ ///
+ /// The `alignment` parameter is used to add padding to each buffer being
counted, to ensure
+ /// the size for each one is aligned to `alignment` bytes (if it is `Some`)
+ pub fn get_slice_memory_size_with_alignment(
+ &self,
+ alignment: Option<u8>,
+ ) -> Result<usize, ArrowError> {
+ // Note: This accounts for data used by the Dictionary DataType that
isn't actually encoded
+ // as a part of `write_array_data` in arrow-ipc - specifically, the
`values` part of
+ // each Dictionary are encoded in the `child_data` of the `ArrayData`
it produces, but (for
+ // some reason that I don't fully understand) it doesn't encode those
values. hmm.
let layout = layout(&self.data_type);
- for spec in layout.buffers.iter() {
- match spec {
+ // Just pulled from arrow-ipc
+ #[inline]
+ fn pad_to_alignment(alignment: u8, len: usize) -> usize {
Review Comment:
Thanks for catching that - I think the best move would just be documenting
this in the function doc-comment so people can be aware of that.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
