[
https://issues.apache.org/jira/browse/ARROW-560?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17046821#comment-17046821
]
Antoine Pitrou commented on ARROW-560:
--------------------------------------
In any case, I've built a prototype, but it crashes when releasing a buffer in
the Python tests.
{code}
#0 raise (sig=<optimized out>) at ../sysdeps/unix/sysv/linux/raise.c:51
#1 0x00007efbfaaa4298 in os::Linux::chained_handler(int, siginfo*, void*) ()
from /opt/conda/envs/arrow//jre/lib/amd64/server/libjvm.so
#2 0x00007efbfaaab585 in JVM_handle_linux_signal () from
/opt/conda/envs/arrow//jre/lib/amd64/server/libjvm.so
#3 0x00007efbfaaa0c93 in signalHandler(int, siginfo*, void*) () from
/opt/conda/envs/arrow//jre/lib/amd64/server/libjvm.so
#4 <signal handler called>
#5 0x00007efbfa86c230 in jni_invoke_nonstatic(JNIEnv_*, JavaValue*, _jobject*,
JNICallType, _jmethodID*, JNI_ArgumentPusher*, Thread*) ()
from /opt/conda/envs/arrow//jre/lib/amd64/server/libjvm.so
#6 0x00007efbfa86ff7c in jni_CallVoidMethodV () from
/opt/conda/envs/arrow//jre/lib/amd64/server/libjvm.so
#7 0x00007efbf9fe7f90 in invokeMethod (env=env@entry=0x5655584ff9e0,
retval=retval@entry=0x7ffbfffef420, methType=<optimized out>,
methType@entry=INSTANCE,
instObj=0x565559963960, className=className@entry=0x7efbf9ff05f8
"org/apache/hadoop/fs/FSDataInputStream",
methName=methName@entry=0x7efbf9fefc74 "releaseBuffer",
methSignature=0x7efbf9fefc98 ")V", methSignature@entry=0x7efbf9fefc82
"(Ljava/nio/ByteBuffer;)V")
at
/build/source/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/jni_helper.c:150
#8 0x00007efbf9fed97d in hadoopRzBufferFree (file=0x565559964ab0,
buffer=0x565559325270)
at
/build/source/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfs/hdfs.c:2712
#9 0x00007efbbfdd70cd in arrow::io::internal::LibHdfsShim::RzBufferFree
(this=0x7efbc0b7cc40 <arrow::io::internal::(anonymous
namespace)::libhdfs_shim>, file=0x565559964ab0,
buffer=0x565559325270) at /arrow/cpp/src/arrow/io/hdfs_internal.cc:579
#10 0x00007efbbfdc9a18 in arrow::io::HdfsBuffer::~HdfsBuffer
(this=0x5655599638b0, __in_chrg=<optimized out>) at
/arrow/cpp/src/arrow/io/hdfs.cc:134
#11 0x00007efbbfdd2e37 in
__gnu_cxx::new_allocator<arrow::io::HdfsBuffer>::destroy<arrow::io::HdfsBuffer>
(this=0x5655599638b0, __p=0x5655599638b0)
at
/opt/conda/envs/arrow/x86_64-conda_cos6-linux-gnu/include/c++/7.3.0/ext/new_allocator.h:140
#12 0x00007efbbfdd2e03 in
std::allocator_traits<std::allocator<arrow::io::HdfsBuffer>
>::destroy<arrow::io::HdfsBuffer> (__a=..., __p=0x5655599638b0)
at
/opt/conda/envs/arrow/x86_64-conda_cos6-linux-gnu/include/c++/7.3.0/bits/alloc_traits.h:487
#13 0x00007efbbfdd2b53 in std::_Sp_counted_ptr_inplace<arrow::io::HdfsBuffer,
std::allocator<arrow::io::HdfsBuffer>, (__gnu_cxx::_Lock_policy)2>::_M_dispose
(this=0x5655599638a0)
at
/opt/conda/envs/arrow/x86_64-conda_cos6-linux-gnu/include/c++/7.3.0/bits/shared_ptr_base.h:535
#14 0x00007efbbf92f508 in
std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release
(this=0x5655599638a0)
at
/opt/conda/envs/arrow/x86_64-conda_cos6-linux-gnu/include/c++/7.3.0/bits/shared_ptr_base.h:154
#15 0x00007efbbf9272ff in
std::__shared_count<(__gnu_cxx::_Lock_policy)2>::~__shared_count
(this=0x5655599631c8, __in_chrg=<optimized out>)
at
/opt/conda/envs/arrow/x86_64-conda_cos6-linux-gnu/include/c++/7.3.0/bits/shared_ptr_base.h:684
#16 0x00007efbbf9231b2 in std::__shared_ptr<arrow::Buffer,
(__gnu_cxx::_Lock_policy)2>::~__shared_ptr (this=0x5655599631c0,
__in_chrg=<optimized out>)
at
/opt/conda/envs/arrow/x86_64-conda_cos6-linux-gnu/include/c++/7.3.0/bits/shared_ptr_base.h:1123
#17 0x00007efbbf9231ce in std::shared_ptr<arrow::Buffer>::~shared_ptr
(this=0x5655599631c0, __in_chrg=<optimized out>)
at
/opt/conda/envs/arrow/x86_64-conda_cos6-linux-gnu/include/c++/7.3.0/bits/shared_ptr.h:93
#18 0x00007efbbfdb8306 in arrow::io::CompressedInputStream::Impl::~Impl
(this=0x565559963190, __in_chrg=<optimized out>) at
/arrow/cpp/src/arrow/io/compressed.cc:242
#19 0x00007efbbfdb834c in
std::default_delete<arrow::io::CompressedInputStream::Impl>::operator()
(this=0x565558a161b8, __ptr=0x565559963190)
at
/opt/conda/envs/arrow/x86_64-conda_cos6-linux-gnu/include/c++/7.3.0/bits/unique_ptr.h:78
#20 0x00007efbbfdb7485 in
std::unique_ptr<arrow::io::CompressedInputStream::Impl,
std::default_delete<arrow::io::CompressedInputStream::Impl> >::~unique_ptr
(this=0x565558a161b8,
__in_chrg=<optimized out>) at
/opt/conda/envs/arrow/x86_64-conda_cos6-linux-gnu/include/c++/7.3.0/bits/unique_ptr.h:268
#21 0x00007efbbfdb355d in
arrow::io::CompressedInputStream::~CompressedInputStream (this=0x565558a161a0,
__in_chrg=<optimized out>, __vtt_parm=<optimized out>)
at /arrow/cpp/src/arrow/io/compressed.cc:443
#22 0x00007efbbfdb35ba in
arrow::io::CompressedInputStream::~CompressedInputStream (this=0x565558a161a0,
__in_chrg=<optimized out>, __vtt_parm=<optimized out>)
at /arrow/cpp/src/arrow/io/compressed.cc:443
#23 0x00007efbbfdb91cc in
std::_Sp_counted_ptr<arrow::io::CompressedInputStream*,
(__gnu_cxx::_Lock_policy)2>::_M_dispose (this=0x565558ad9d40)
at
/opt/conda/envs/arrow/x86_64-conda_cos6-linux-gnu/include/c++/7.3.0/bits/shared_ptr_base.h:376
#24 0x00007efbc1092ad6 in
std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release
(this=0x565558ad9d40)
at
/opt/conda/envs/arrow/x86_64-conda_cos6-linux-gnu/include/c++/7.3.0/bits/shared_ptr_base.h:154
#25 0x00007efbc1089248 in
std::__shared_count<(__gnu_cxx::_Lock_policy)2>::~__shared_count
(this=0x7efbb64b3890, __in_chrg=<optimized out>)
at
/opt/conda/envs/arrow/x86_64-conda_cos6-linux-gnu/include/c++/7.3.0/bits/shared_ptr_base.h:684
#26 0x00007efbc0f7c955 in std::__shared_ptr<arrow::io::InputStream,
(__gnu_cxx::_Lock_policy)2>::~__shared_ptr (this=0x7efbb64b3888,
__in_chrg=<optimized out>)
at
/opt/conda/envs/arrow/x86_64-conda_cos6-linux-gnu/include/c++/7.3.0/bits/shared_ptr_base.h:1123
#27 0x00007efbc0f7c970 in std::shared_ptr<arrow::io::InputStream>::~shared_ptr
(this=0x7efbb64b3888, __in_chrg=<optimized out>)
at
/opt/conda/envs/arrow/x86_64-conda_cos6-linux-gnu/include/c++/7.3.0/bits/shared_ptr.h:93
#28 0x00007efbc1092493 in
__Pyx_call_destructor<std::shared_ptr<arrow::io::InputStream> > (x=...) at
lib.cpp:299
#29 0x00007efbc1020d50 in __pyx_tp_dealloc_7pyarrow_3lib_NativeFile
(o=<pyarrow.lib.CompressedInputStream at remote 0x7efbb64b3870>) at
lib.cpp:165579
{code}
> [C++] Add support for zero-copy libhdfs reads
> ---------------------------------------------
>
> Key: ARROW-560
> URL: https://issues.apache.org/jira/browse/ARROW-560
> Project: Apache Arrow
> Issue Type: New Feature
> Components: C++
> Reporter: Wes McKinney
> Priority: Major
> Labels: filesystem
>
> See *Rz* functions in
> https://github.com/apache/arrow/blob/master/cpp/thirdparty/hadoop/include/hdfs.h
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
