[ https://issues.apache.org/jira/browse/ARROW-15971?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Arrow User updated ARROW-15971: ------------------------------- Description: h2. Description When using C++ (or Python) to construct a null or empty outer array of type `array_1: list<item: struct<array_sub_col: list<item: string>>>`, either: - `array_1:null` or - `array_1:[]` an out of bounds exceptions (see stack trace below) follows when later retrieving the field reader for the inner list (`array_sub_col`) in Java. h2. Reproduction Java: 7.0.0 C++: 4.0.0 Python: 7.0.0 Creating a stream on C++ of type `array_1: list<item: struct<array_sub_col: list<item: string>>> ` with an empty (or null) outer list: {code:c++}arrow::MemoryPool* pool = arrow::default_memory_pool(); arrow::Result<std::shared_ptr<arrow::io::BufferOutputStream>> stream_buffer = arrow::io::BufferOutputStream::Create(1, pool); std::vector<std::shared_ptr<arrow::Field>> inner_list_field\{std::make_shared<arrow::Field>("array_sub_col",arrow::list(arrow::utf8()))}; // Datatype for the builder: list<struct<list<string>>> std::shared_ptr<DataType> data_type = list(struct_(inner_list_field)); std::unique_ptr<arrow::ArrayBuilder> builder; arrow::MakeBuilder(pool, data_type, &builder); auto* list_builder = dynamic_cast<arrow::ListBuilder*>(builder.get()); // Append a null or an empty list to the outer list list_builder->AppendNull(); // or list_builder->AppendEmptyValue() std::vector<std::shared_ptr<arrow::Array>> value_batch; value_batch.resize(1); list_builder->Finish(&value_batch[0]); std::vector<std::shared_ptr<arrow::Field>> outer_list_field\{std::make_shared<arrow::Field>("array_1",data_type)}; auto schema = std::make_shared<arrow::Schema>(outer_list_field); // Build a single row record batch std::shared_ptr<arrow::RecordBatch> batch = RecordBatch::Make(schema, 1, value_batch); ASSERT_OK(batch->Validate()); // Stream the batch to a file arrow::Result<std::shared_ptr<ipc::RecordBatchWriter>> stream_writer = arrow::ipc::MakeStreamWriter(stream_buffer.ValueOrDie().get(), schema, arrow::ipc::IpcWriteOptions::Defaults()); stream_writer.ValueOrDie()->WriteRecordBatch(*batch); arrow::Result<std::shared_ptr<arrow::Buffer>> buffer_result = stream_buffer.ValueOrDie()->Finish(); std::shared_ptr<arrow::Buffer> buffer = buffer_result.ValueOrDie(); auto file_output = arrow::io::FileOutputStream::Open("/tmp/batch_stream.out").ValueOrDie(); file_output->Write(buffer->data(), buffer->size()); file_output->Close(); {code} As expected, Python holds the same memory layout for the field vectors as the code above: {code:python}array = pa.array([None], type=pa.list_(pa.struct([pa.field("array_sub_col", pa.list_(pa.utf8()))]))) batch = pa.record_batch([struct_array], names=["array_1"]) sink = pa.BufferOutputStream() with pa.ipc.new_stream(sink, batch.schema) as writer: writer.write_batch(batch) buf = sink.getvalue() with open('/tmp/batch_stream.out', 'wb') as f: f.write(buf) {code} *Java fails when then trying to access the inner list's field reader:* {code:java}File file = new File("/tmp/batch_stream.out"); byte[] bytes = FileUtils.readFileToByteArray(file); try (ArrowStreamReader reader = new ArrowStreamReader(new ByteArrayInputStream(bytes), allocator)) { Schema schema = reader.getVectorSchemaRoot().getSchema(); reader.loadNextBatch(); readBatch.getVector("array_1").getReader().reader().reader("array_sub_col"); // <- fails: reader("array_sub_col") fails with OOB // Concrete readers: // FieldVector array_1 = readBatch.getVector("array_1"); // UnionListReader array_1_reader = (UnionListReader) array_1.getReader(); // NullableStructReaderImpl struct_reader = (NullableStructReaderImpl) array_1_reader.reader(); // FieldReader union_list_reader = struct_reader.reader("array_sub_col"); // <- fails: OOB {code} h3. Stack trace: {quote}java.lang.reflect.InvocationTargetException at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0 (Native Method) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke (NativeMethodAccessorImpl.java:62) at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke (DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke (Method.java:566) at org.codehaus.mojo.exec.ExecJavaMojo$1.run (ExecJavaMojo.java:297) at java.lang.Thread.run (Thread.java:829) Caused by: java.lang.IndexOutOfBoundsException: index: 4, length: 4 (expected: range(0, 4)) at org.apache.arrow.memory.ArrowBuf.checkIndexD (ArrowBuf.java:318) at org.apache.arrow.memory.ArrowBuf.chk (ArrowBuf.java:305) at org.apache.arrow.memory.ArrowBuf.getInt (ArrowBuf.java:424) at com.test.arrow.ValidateArrow.testArrow (ValidateArrow.java:433) at com.test.arrow.ValidateArrow.main (ValidateArrow.java:440) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0 (Native Method) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke (NativeMethodAccessorImpl.java:62) at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke (DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke (Method.java:566) at org.codehaus.mojo.exec.ExecJavaMojo$1.run (ExecJavaMojo.java:297) at java.lang.Thread.run (Thread.java:829) {quote} was: h2. Description When using C++ (or Python) to construct a null or empty outer array of type `array_1: list<item: struct<array_sub_col: list<item: string>>>`, either: - `array_1:null` or - `array_1:[]` an out of bounds exceptions (see stack trace below) follows when later retrieving the field reader for the inner list (`array_sub_col`) in Java. h2. Reproduction Java: 7.0.0 C++: 4.0.0 Python: 7.0.0 Creating a stream on C++ of type `array_1: list<item: struct<array_sub_col: list<item: string>>> ` with an empty (or null) outer list: {quote}arrow::MemoryPool* pool = arrow::default_memory_pool(); arrow::Result<std::shared_ptr<arrow::io::BufferOutputStream>> stream_buffer = arrow::io::BufferOutputStream::Create(1, pool); std::vector<std::shared_ptr<arrow::Field>> inner_list_field\{std::make_shared<arrow::Field>("array_sub_col",arrow::list(arrow::utf8()))}; // Datatype for the builder: list<struct<list<string>>> std::shared_ptr<DataType> data_type = list(struct_(inner_list_field)); std::unique_ptr<arrow::ArrayBuilder> builder; arrow::MakeBuilder(pool, data_type, &builder); auto* list_builder = dynamic_cast<arrow::ListBuilder*>(builder.get()); // Append a null or an empty list to the outer list list_builder->AppendNull(); // or list_builder->AppendEmptyValue() std::vector<std::shared_ptr<arrow::Array>> value_batch; value_batch.resize(1); list_builder->Finish(&value_batch[0]); std::vector<std::shared_ptr<arrow::Field>> outer_list_field\{std::make_shared<arrow::Field>("array_1",data_type)}; auto schema = std::make_shared<arrow::Schema>(outer_list_field); // Build a single row record batch std::shared_ptr<arrow::RecordBatch> batch = RecordBatch::Make(schema, 1, value_batch); ASSERT_OK(batch->Validate()); // Stream the batch to a file arrow::Result<std::shared_ptr<ipc::RecordBatchWriter>> stream_writer = arrow::ipc::MakeStreamWriter(stream_buffer.ValueOrDie().get(), schema, arrow::ipc::IpcWriteOptions::Defaults()); stream_writer.ValueOrDie()->WriteRecordBatch(*batch); arrow::Result<std::shared_ptr<arrow::Buffer>> buffer_result = stream_buffer.ValueOrDie()->Finish(); std::shared_ptr<arrow::Buffer> buffer = buffer_result.ValueOrDie(); auto file_output = arrow::io::FileOutputStream::Open("/tmp/batch_stream.out").ValueOrDie(); file_output->Write(buffer->data(), buffer->size()); file_output->Close(); {quote} As expected, Python holds the same memory layout for the field vectors as the code above: {quote}array = pa.array([None], type=pa.list_(pa.struct([pa.field("array_sub_col", pa.list_(pa.utf8()))]))) batch = pa.record_batch([struct_array], names=["array_1"]) sink = pa.BufferOutputStream() with pa.ipc.new_stream(sink, batch.schema) as writer: writer.write_batch(batch) buf = sink.getvalue() with open('/tmp/batch_stream.out', 'wb') as f: f.write(buf) {quote} *Java fails when then trying to access the inner list's field reader:* {quote}File file = new File("/tmp/batch_stream.out"); byte[] bytes = FileUtils.readFileToByteArray(file); try (ArrowStreamReader reader = new ArrowStreamReader(new ByteArrayInputStream(bytes), allocator)) { Schema schema = reader.getVectorSchemaRoot().getSchema(); reader.loadNextBatch(); readBatch.getVector("array_1").getReader().reader().reader("array_sub_col"); // <- fails: reader("array_sub_col") fails with OOB // Concrete readers: // FieldVector array_1 = readBatch.getVector("array_1"); // UnionListReader array_1_reader = (UnionListReader) array_1.getReader(); // NullableStructReaderImpl struct_reader = (NullableStructReaderImpl) array_1_reader.reader(); // FieldReader union_list_reader = struct_reader.reader("array_sub_col"); // <- fails: OOB {quote} h3. Stack trace: {quote}java.lang.reflect.InvocationTargetException at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0 (Native Method) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke (NativeMethodAccessorImpl.java:62) at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke (DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke (Method.java:566) at org.codehaus.mojo.exec.ExecJavaMojo$1.run (ExecJavaMojo.java:297) at java.lang.Thread.run (Thread.java:829) Caused by: java.lang.IndexOutOfBoundsException: index: 4, length: 4 (expected: range(0, 4)) at org.apache.arrow.memory.ArrowBuf.checkIndexD (ArrowBuf.java:318) at org.apache.arrow.memory.ArrowBuf.chk (ArrowBuf.java:305) at org.apache.arrow.memory.ArrowBuf.getInt (ArrowBuf.java:424) at com.test.arrow.ValidateArrow.testArrow (ValidateArrow.java:433) at com.test.arrow.ValidateArrow.main (ValidateArrow.java:440) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0 (Native Method) at jdk.internal.reflect.NativeMethodAccessorImpl.invoke (NativeMethodAccessorImpl.java:62) at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke (DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke (Method.java:566) at org.codehaus.mojo.exec.ExecJavaMojo$1.run (ExecJavaMojo.java:297) at java.lang.Thread.run (Thread.java:829) {quote} > Error when reading inner lists within a struct in empty outer lists from > C++/Python in Java > ------------------------------------------------------------------------------------------- > > Key: ARROW-15971 > URL: https://issues.apache.org/jira/browse/ARROW-15971 > Project: Apache Arrow > Issue Type: Bug > Components: C++, Java, Python > Affects Versions: 7.0.0 > Reporter: Arrow User > Priority: Major > Labels: bug > > h2. Description > When using C++ (or Python) to construct a null or empty outer array of type > `array_1: list<item: struct<array_sub_col: list<item: string>>>`, either: > - `array_1:null` or > - `array_1:[]` > an out of bounds exceptions (see stack trace below) follows when later > retrieving the field reader for the inner list (`array_sub_col`) in Java. > h2. Reproduction > Java: 7.0.0 > C++: 4.0.0 > Python: 7.0.0 > Creating a stream on C++ of type `array_1: list<item: struct<array_sub_col: > list<item: string>>> ` with an empty (or null) outer list: > {code:c++}arrow::MemoryPool* pool = arrow::default_memory_pool(); > arrow::Result<std::shared_ptr<arrow::io::BufferOutputStream>> stream_buffer = > arrow::io::BufferOutputStream::Create(1, pool); > std::vector<std::shared_ptr<arrow::Field>> > inner_list_field\{std::make_shared<arrow::Field>("array_sub_col",arrow::list(arrow::utf8()))}; > // Datatype for the builder: list<struct<list<string>>> > std::shared_ptr<DataType> data_type = list(struct_(inner_list_field)); > std::unique_ptr<arrow::ArrayBuilder> builder; > arrow::MakeBuilder(pool, data_type, &builder); > auto* list_builder = dynamic_cast<arrow::ListBuilder*>(builder.get()); > // Append a null or an empty list to the outer list > list_builder->AppendNull(); // or list_builder->AppendEmptyValue() > std::vector<std::shared_ptr<arrow::Array>> value_batch; > value_batch.resize(1); > list_builder->Finish(&value_batch[0]); > std::vector<std::shared_ptr<arrow::Field>> > outer_list_field\{std::make_shared<arrow::Field>("array_1",data_type)}; > auto schema = std::make_shared<arrow::Schema>(outer_list_field); > // Build a single row record batch > std::shared_ptr<arrow::RecordBatch> batch = RecordBatch::Make(schema, 1, > value_batch); > ASSERT_OK(batch->Validate()); > // Stream the batch to a file > arrow::Result<std::shared_ptr<ipc::RecordBatchWriter>> stream_writer = > arrow::ipc::MakeStreamWriter(stream_buffer.ValueOrDie().get(), schema, > arrow::ipc::IpcWriteOptions::Defaults()); > stream_writer.ValueOrDie()->WriteRecordBatch(*batch); > arrow::Result<std::shared_ptr<arrow::Buffer>> buffer_result = > stream_buffer.ValueOrDie()->Finish(); > std::shared_ptr<arrow::Buffer> buffer = buffer_result.ValueOrDie(); > auto file_output = > arrow::io::FileOutputStream::Open("/tmp/batch_stream.out").ValueOrDie(); > file_output->Write(buffer->data(), buffer->size()); > file_output->Close(); > {code} > As expected, Python holds the same memory layout for the field vectors as the > code above: > {code:python}array = pa.array([None], > type=pa.list_(pa.struct([pa.field("array_sub_col", pa.list_(pa.utf8()))]))) > batch = pa.record_batch([struct_array], names=["array_1"]) > sink = pa.BufferOutputStream() > with pa.ipc.new_stream(sink, batch.schema) as writer: > writer.write_batch(batch) > buf = sink.getvalue() > with open('/tmp/batch_stream.out', 'wb') as f: > f.write(buf) > {code} > *Java fails when then trying to access the inner list's field reader:* > {code:java}File file = new File("/tmp/batch_stream.out"); > byte[] bytes = FileUtils.readFileToByteArray(file); > try (ArrowStreamReader reader = new ArrowStreamReader(new > ByteArrayInputStream(bytes), allocator)) { > Schema schema = reader.getVectorSchemaRoot().getSchema(); > reader.loadNextBatch(); > > readBatch.getVector("array_1").getReader().reader().reader("array_sub_col"); > // <- fails: reader("array_sub_col") fails with OOB > // Concrete readers: > // FieldVector array_1 = readBatch.getVector("array_1"); > // UnionListReader array_1_reader = (UnionListReader) > array_1.getReader(); > // NullableStructReaderImpl struct_reader = (NullableStructReaderImpl) > array_1_reader.reader(); > // FieldReader union_list_reader = > struct_reader.reader("array_sub_col"); // <- fails: OOB > {code} > h3. Stack trace: > {quote}java.lang.reflect.InvocationTargetException > at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0 (Native Method) > at jdk.internal.reflect.NativeMethodAccessorImpl.invoke > (NativeMethodAccessorImpl.java:62) > at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke > (DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke (Method.java:566) > at org.codehaus.mojo.exec.ExecJavaMojo$1.run (ExecJavaMojo.java:297) > at java.lang.Thread.run (Thread.java:829) > Caused by: java.lang.IndexOutOfBoundsException: index: 4, length: 4 > (expected: range(0, 4)) > at org.apache.arrow.memory.ArrowBuf.checkIndexD (ArrowBuf.java:318) > at org.apache.arrow.memory.ArrowBuf.chk (ArrowBuf.java:305) > at org.apache.arrow.memory.ArrowBuf.getInt (ArrowBuf.java:424) > at com.test.arrow.ValidateArrow.testArrow (ValidateArrow.java:433) > at com.test.arrow.ValidateArrow.main (ValidateArrow.java:440) > at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0 (Native Method) > at jdk.internal.reflect.NativeMethodAccessorImpl.invoke > (NativeMethodAccessorImpl.java:62) > at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke > (DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke (Method.java:566) > at org.codehaus.mojo.exec.ExecJavaMojo$1.run (ExecJavaMojo.java:297) > at java.lang.Thread.run (Thread.java:829) > {quote} -- This message was sent by Atlassian Jira (v8.20.1#820001)