JakeDern commented on code in PR #10128:
URL: https://github.com/apache/arrow-rs/pull/10128#discussion_r3670823110
##########
arrow-ipc/src/writer.rs:
##########
@@ -857,34 +898,34 @@ impl IpcDataGenerator {
let batch_compression_level = write_options.batch_compression_level;
let compression_codec: Option<CompressionCodec> =
batch_compression_type
- .map(|batch_compression_type| match batch_compression_level {
+ .map(|compression_type| match batch_compression_level {
Some(level) => {
-
CompressionCodec::try_new_with_compression_level(batch_compression_type, level)
+
CompressionCodec::try_new_with_compression_level(compression_type, level)
}
- None => batch_compression_type.try_into(),
+ None => compression_type.try_into(),
})
.transpose()?;
- let alignment = write_options.alignment;
let mut meta = IpcMetadataBuilder::default();
- let mut sink = IpcBodySink::Write(&mut arrow_data);
- let offset = write_array_data(
- array_data,
- &mut meta,
- &mut sink,
- 0,
- compression_codec,
- ipc_write_context,
- write_options,
- )?;
+ let mut variadic_buffer_counts: Vec<i64> = vec![];
+ let mut offset = 0i64;
- let mut variadic_buffer_counts = vec![];
- append_variadic_buffer_counts(&mut variadic_buffer_counts, array_data);
+ for array_data in columns {
+ offset = write_array_data(
+ &array_data,
+ &mut meta,
+ sink,
+ offset,
+ compression_codec,
+ ipc_write_context,
+ write_options,
+ )?;
+ append_variadic_buffer_counts(&mut variadic_buffer_counts,
&array_data);
+ }
- // pad the tail of body data
- let tail_pad = pad_to_alignment(alignment, offset as usize);
- let body_len = offset as usize + tail_pad;
- arrow_data.extend_from_slice(&PADDING[..tail_pad]);
+ // Each buffer is padded to the alignment as it is written, so
`offset` is
+ // already a multiple of the alignment -- the body needs no trailing
padding.
+ let body_len = offset as usize;
Review Comment:
Sure, done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
