This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-erlang.git
The following commit(s) were added to refs/heads/main by this push:
new ff5969a fix: arrow_buffer:size/1 use 64 byte padding not 8 (#114)
ff5969a is described below
commit ff5969a77f947f47e044b9d39834512576513063
Author: Benjamin Philip <[email protected]>
AuthorDate: Mon Aug 3 07:49:31 2026 +0530
fix: arrow_buffer:size/1 use 64 byte padding not 8 (#114)
## What issue does this PR close?
Closes #113.
Previously, arrow_buffer:size/1 took 8 byte padding to calculate size
when the buffers actually used 64 bytes. This resulted in various gross
errors in offsets and other metadata, rendering any IPC binaries
produced unusable.
## What's Changed
`arrow_buffer:size/1` now correctly uses 64 byte padding. Record Batch
test data
has been updated with the correct offsets.
---
native/arrow_format_nif/src/lib.rs | 18 +++++++++---------
native/arrow_format_nif/src/utils.rs | 18 +++++++++---------
src/arrow_buffer.erl | 6 +++---
test/arrow_buffer_SUITE.erl | 2 +-
test/arrow_ipc_record_batch_SUITE.erl | 15 ++++++++++-----
5 files changed, 32 insertions(+), 27 deletions(-)
diff --git a/native/arrow_format_nif/src/lib.rs
b/native/arrow_format_nif/src/lib.rs
index 1117440..02c3b0f 100644
--- a/native/arrow_format_nif/src/lib.rs
+++ b/native/arrow_format_nif/src/lib.rs
@@ -168,39 +168,39 @@ pub mod test {
length: 1,
},
ipc::Buffer {
- offset: 8,
+ offset: 64,
length: 4,
},
ipc::Buffer {
- offset: 16,
+ offset: 128,
length: 1,
},
ipc::Buffer {
- offset: 24,
+ offset: 192,
length: 20,
},
ipc::Buffer {
- offset: 48,
+ offset: 256,
length: 15,
},
ipc::Buffer {
- offset: 64,
+ offset: 320,
length: 1,
},
ipc::Buffer {
- offset: 72,
+ offset: 384,
length: 4,
},
ipc::Buffer {
- offset: 80,
+ offset: 448,
length: 1,
},
ipc::Buffer {
- offset: 88,
+ offset: 512,
length: 2,
},
ipc::Buffer {
- offset: 96,
+ offset: 576,
length: 10,
},
]),
diff --git a/native/arrow_format_nif/src/utils.rs
b/native/arrow_format_nif/src/utils.rs
index b1920f6..233be19 100644
--- a/native/arrow_format_nif/src/utils.rs
+++ b/native/arrow_format_nif/src/utils.rs
@@ -120,39 +120,39 @@ pub fn record_batch() -> Message {
length: 1,
},
Buffer {
- offset: 8,
+ offset: 64,
length: 4,
},
Buffer {
- offset: 16,
+ offset: 128,
length: 1,
},
Buffer {
- offset: 24,
+ offset: 192,
length: 20,
},
Buffer {
- offset: 48,
+ offset: 256,
length: 15,
},
Buffer {
- offset: 64,
+ offset: 320,
length: 1,
},
Buffer {
- offset: 72,
+ offset: 384,
length: 4,
},
Buffer {
- offset: 80,
+ offset: 448,
length: 1,
},
Buffer {
- offset: 88,
+ offset: 512,
length: 2,
},
Buffer {
- offset: 96,
+ offset: 576,
length: 10,
},
],
diff --git a/src/arrow_buffer.erl b/src/arrow_buffer.erl
index ecec6d5..d64af67 100644
--- a/src/arrow_buffer.erl
+++ b/src/arrow_buffer.erl
@@ -117,10 +117,10 @@ to_erlang(_Buffer) ->
erlang:error(badarg).
-doc "Returns the size of the buffer inclusive of padding in bytes.".
--spec size(Buffer :: arrow_buffer:buffer()) -> pos_integer().
+-spec size(Buffer :: arrow_buffer:buffer()) -> non_neg_integer().
size(Buffer) ->
- Len = Buffer#buffer.length * 8,
- round((Len + arrow_utils:pad_len(Len)) / 8).
+ Len = Buffer#buffer.length,
+ Len + arrow_utils:pad_len(Len).
-spec slot(
Value :: arrow_type:native_type(),
diff --git a/test/arrow_buffer_SUITE.erl b/test/arrow_buffer_SUITE.erl
index d91a07f..ca5cdac 100644
--- a/test/arrow_buffer_SUITE.erl
+++ b/test/arrow_buffer_SUITE.erl
@@ -157,7 +157,7 @@ to_erlang(_Config) ->
size(_Config) ->
Buffer = arrow_buffer:from_erlang([1, 2, 3], {s, 8}),
- ?assertEqual(arrow_buffer:size(Buffer), 8).
+ ?assertEqual(arrow_buffer:size(Buffer),
byte_size(arrow_buffer:to_arrow(Buffer))).
%%%%%%%%%%%
%% Utils %%
diff --git a/test/arrow_ipc_record_batch_SUITE.erl
b/test/arrow_ipc_record_batch_SUITE.erl
index 157d5b1..2195e4f 100644
--- a/test/arrow_ipc_record_batch_SUITE.erl
+++ b/test/arrow_ipc_record_batch_SUITE.erl
@@ -43,14 +43,19 @@ valid_nodes_on_from_erlang(_Config) ->
?assertEqual((?RecordBatch)#record_batch.nodes, FieldNodes).
valid_buffers_on_from_erlang(_Config) ->
- ID = [#{offset => 0, length => 1}, #{offset => 8, length => 4}],
+ ID = [#{offset => 0, length => 1}, #{offset => 64, length => 4}],
Name = [
- #{offset => 16, length => 1}, #{offset => 24, length => 20}, #{offset
=> 48, length => 15}
+ #{offset => 128, length => 1},
+ #{offset => 192, length => 20},
+ #{offset => 256, length => 15}
],
- Age = [#{offset => 64, length => 1}, #{offset => 72, length => 4}],
+ Age = [#{offset => 320, length => 1}, #{offset => 384, length => 4}],
Marks =
- [#{offset => 80, length => 1}] ++
- [#{offset => 88, length => 2}, #{offset => 96, length => 10}],
+ [
+ #{offset => 448, length => 1},
+ #{offset => 512, length => 2},
+ #{offset => 576, length => 10}
+ ],
Buffers = ID ++ Name ++ Age ++ Marks,
?assertEqual((?RecordBatch)#record_batch.buffers, Buffers).
