This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
The following commit(s) were added to refs/heads/asf-site by this push:
new b76291e37 deploy: 6f2231353f92dbffea7d5fd9c93db2641c7ffc41
b76291e37 is described below
commit b76291e3797b3854b29378bd9861c9a86c39a685
Author: tustvold <[email protected]>
AuthorDate: Mon Jul 10 15:14:21 2023 +0000
deploy: 6f2231353f92dbffea7d5fd9c93db2641c7ffc41
---
parquet/arrow/async_writer/index.html | 2 +-
.../async_writer/struct.AsyncArrowWriter.html | 19 ++++++++------
.../arrow/async_writer/struct.SharedBuffer.html | 4 +--
search-index.js | 2 +-
src/parquet/arrow/async_writer/mod.rs.html | 30 ++++++++++++++++++----
5 files changed, 40 insertions(+), 17 deletions(-)
diff --git a/parquet/arrow/async_writer/index.html
b/parquet/arrow/async_writer/index.html
index b9b48a761..7ecc6a46d 100644
--- a/parquet/arrow/async_writer/index.html
+++ b/parquet/arrow/async_writer/index.html
@@ -1,4 +1,4 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta
name="viewport" content="width=device-width, initial-scale=1.0"><meta
name="generator" content="rustdoc"><meta name="description" content="Contains
async writer which writes arrow data into parquet
data."><title>parquet::arrow::async_writer - Rust</title><link rel="preload"
as="font" type="font/woff2" crossorigin
href="../../../static.files/SourceSerif4-Regular-46f98efaafac5295.ttf.woff2"><link
rel="preload" as="font" type= [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta
name="viewport" content="width=device-width, initial-scale=1.0"><meta
name="generator" content="rustdoc"><meta name="description" content="Contains
async writer which writes arrow data into parquet
data."><title>parquet::arrow::async_writer - Rust</title><link rel="preload"
as="font" type="font/woff2" crossorigin
href="../../../static.files/SourceSerif4-Regular-46f98efaafac5295.ttf.woff2"><link
rel="preload" as="font" type= [...]
<p>Provides <code>async</code> API for writing <a
href="../../../arrow_array/record_batch/struct.RecordBatch.html" title="struct
arrow_array::record_batch::RecordBatch"><code>RecordBatch</code></a>es as
parquet files. The API is
similar to the <a href="../arrow_writer/struct.ArrowWriter.html" title="struct
parquet::arrow::arrow_writer::ArrowWriter"><code>sync</code> API</a>, so please
read the documentation there before using this API.</p>
diff --git a/parquet/arrow/async_writer/struct.AsyncArrowWriter.html
b/parquet/arrow/async_writer/struct.AsyncArrowWriter.html
index 7dc202ddb..b0b74d94a 100644
--- a/parquet/arrow/async_writer/struct.AsyncArrowWriter.html
+++ b/parquet/arrow/async_writer/struct.AsyncArrowWriter.html
@@ -1,7 +1,8 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta
name="viewport" content="width=device-width, initial-scale=1.0"><meta
name="generator" content="rustdoc"><meta name="description" content="Async
arrow writer."><title>AsyncArrowWriter in parquet::arrow::async_writer -
Rust</title><link rel="preload" as="font" type="font/woff2" crossorigin
href="../../../static.files/SourceSerif4-Regular-46f98efaafac5295.ttf.woff2"><link
rel="preload" as="font" type="font/woff2" crossorigin [...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta
name="viewport" content="width=device-width, initial-scale=1.0"><meta
name="generator" content="rustdoc"><meta name="description" content="Async
arrow writer."><title>AsyncArrowWriter in parquet::arrow::async_writer -
Rust</title><link rel="preload" as="font" type="font/woff2" crossorigin
href="../../../static.files/SourceSerif4-Regular-46f98efaafac5295.ttf.woff2"><link
rel="preload" as="font" type="font/woff2" crossorigin [...]
sync_writer: <a class="struct"
href="../arrow_writer/struct.ArrowWriter.html" title="struct
parquet::arrow::arrow_writer::ArrowWriter">ArrowWriter</a><<a class="struct"
href="struct.SharedBuffer.html" title="struct
parquet::arrow::async_writer::SharedBuffer">SharedBuffer</a>>,
async_writer: W,
shared_buffer: <a class="struct" href="struct.SharedBuffer.html"
title="struct parquet::arrow::async_writer::SharedBuffer">SharedBuffer</a>,
+ buffer_size: <a class="primitive"
href="https://doc.rust-lang.org/nightly/std/primitive.usize.html";>usize</a>,
}</code></pre><details class="toggle top-doc" open><summary
class="hideme"><span>Expand description</span></summary><div
class="docblock"><p>Async arrow writer.</p>
<p>It is implemented based on the sync writer <a
href="../arrow_writer/struct.ArrowWriter.html" title="struct
parquet::arrow::arrow_writer::ArrowWriter"><code>ArrowWriter</code></a> with an
inner buffer.
The buffered data will be flushed to the writer provided by caller when the
@@ -9,27 +10,29 @@ buffer’s threshold is exceeded.</p>
</div></details><h2 id="fields" class="fields small-section-header">Fields<a
href="#fields" class="anchor">§</a></h2><span id="structfield.sync_writer"
class="structfield small-section-header"><a href="#structfield.sync_writer"
class="anchor field">§</a><code>sync_writer: <a class="struct"
href="../arrow_writer/struct.ArrowWriter.html" title="struct
parquet::arrow::arrow_writer::ArrowWriter">ArrowWriter</a><<a class="struct"
href="struct.SharedBuffer.html" title="struct parquet::arrow [...]
</div><span id="structfield.async_writer" class="structfield
small-section-header"><a href="#structfield.async_writer" class="anchor
field">§</a><code>async_writer: W</code></span><div class="docblock"><p>Async
writer provided by caller</p>
</div><span id="structfield.shared_buffer" class="structfield
small-section-header"><a href="#structfield.shared_buffer" class="anchor
field">§</a><code>shared_buffer: <a class="struct"
href="struct.SharedBuffer.html" title="struct
parquet::arrow::async_writer::SharedBuffer">SharedBuffer</a></code></span><div
class="docblock"><p>The inner buffer shared by the <code>sync_writer</code> and
the <code>async_writer</code></p>
-</div><h2 id="implementations" class="small-section-header">Implementations<a
href="#implementations" class="anchor">§</a></h2><div
id="implementations-list"><details class="toggle implementors-toggle"
open><summary><section id="impl-AsyncArrowWriter%3CW%3E" class="impl"><a
class="srclink rightside"
href="../../../src/parquet/arrow/async_writer/mod.rs.html#82-165">source</a><a
href="#impl-AsyncArrowWriter%3CW%3E" class="anchor">§</a><h3
class="code-header">impl<W: AsyncWrite + <a clas [...]
+</div><span id="structfield.buffer_size" class="structfield
small-section-header"><a href="#structfield.buffer_size" class="anchor
field">§</a><code>buffer_size: <a class="primitive"
href="https://doc.rust-lang.org/nightly/std/primitive.usize.html";>usize</a></code></span><div
class="docblock"><p>Trigger forced flushing once buffer size reaches this
value</p>
+</div><h2 id="implementations" class="small-section-header">Implementations<a
href="#implementations" class="anchor">§</a></h2><div
id="implementations-list"><details class="toggle implementors-toggle"
open><summary><section id="impl-AsyncArrowWriter%3CW%3E" class="impl"><a
class="srclink rightside"
href="../../../src/parquet/arrow/async_writer/mod.rs.html#85-175">source</a><a
href="#impl-AsyncArrowWriter%3CW%3E" class="anchor">§</a><h3
class="code-header">impl<W: AsyncWrite + <a clas [...]
writer: W,
arrow_schema: <a class="type"
href="../../../arrow_schema/schema/type.SchemaRef.html" title="type
arrow_schema::schema::SchemaRef">SchemaRef</a>,
buffer_size: <a class="primitive"
href="https://doc.rust-lang.org/nightly/std/primitive.usize.html";>usize</a>,
props: <a class="enum"
href="https://doc.rust-lang.org/nightly/core/option/enum.Option.html";
title="enum core::option::Option">Option</a><<a class="struct"
href="../../file/properties/struct.WriterProperties.html" title="struct
parquet::file::properties::WriterProperties">WriterProperties</a>>
) -> <a class="type" href="../../errors/type.Result.html" title="type
parquet::errors::Result">Result</a><Self></h4></section></summary><div
class="docblock"><p>Try to create a new Async Arrow Writer.</p>
-<p><code>buffer_size</code> determines the initial size of the intermediate
buffer.</p>
+<p><code>buffer_size</code> determines the number of bytes to buffer before
flushing
+to the underlying [<code>AsyncWrite</code>]</p>
<p>The intermediate buffer will automatically be resized if necessary</p>
<p><a href="struct.AsyncArrowWriter.html#method.write" title="method
parquet::arrow::async_writer::AsyncArrowWriter::write"><code>Self::write</code></a>
will flush this intermediate buffer if it is at least
half full</p>
-</div></details><details class="toggle method-toggle" open><summary><section
id="method.write" class="method"><a class="srclink rightside"
href="../../../src/parquet/arrow/async_writer/mod.rs.html#112-115">source</a><h4
class="code-header">pub async fn <a href="#method.write"
class="fn">write</a>(&mut self, batch: &<a class="struct"
href="../../../arrow_array/record_batch/struct.RecordBatch.html" title="struct
arrow_array::record_batch::RecordBatch">RecordBatch</a>) -> <a clas [...]
+</div></details><details class="toggle method-toggle" open><summary><section
id="method.write" class="method"><a class="srclink rightside"
href="../../../src/parquet/arrow/async_writer/mod.rs.html#117-125">source</a><h4
class="code-header">pub async fn <a href="#method.write"
class="fn">write</a>(&mut self, batch: &<a class="struct"
href="../../../arrow_array/record_batch/struct.RecordBatch.html" title="struct
arrow_array::record_batch::RecordBatch">RecordBatch</a>) -> <a clas [...]
<p>After every sync write by the inner <a
href="../arrow_writer/struct.ArrowWriter.html" title="struct
parquet::arrow::arrow_writer::ArrowWriter">ArrowWriter</a>, the inner buffer
will be
checked and flush if at least half full</p>
-</div></details><details class="toggle method-toggle" open><summary><section
id="method.append_key_value_metadata" class="method"><a class="srclink
rightside"
href="../../../src/parquet/arrow/async_writer/mod.rs.html#120-122">source</a><h4
class="code-header">pub fn <a href="#method.append_key_value_metadata"
class="fn">append_key_value_metadata</a>(&mut self, kv_metadata: <a
class="struct" href="../../format/struct.KeyValue.html" title="struct
parquet::format::KeyValue">KeyValue</a> [...]
+</div></details><details class="toggle method-toggle" open><summary><section
id="method.append_key_value_metadata" class="method"><a class="srclink
rightside"
href="../../../src/parquet/arrow/async_writer/mod.rs.html#130-132">source</a><h4
class="code-header">pub fn <a href="#method.append_key_value_metadata"
class="fn">append_key_value_metadata</a>(&mut self, kv_metadata: <a
class="struct" href="../../format/struct.KeyValue.html" title="struct
parquet::format::KeyValue">KeyValue</a> [...]
<p>This method allows to append metadata after <a
href="../../../arrow_array/record_batch/struct.RecordBatch.html" title="struct
arrow_array::record_batch::RecordBatch"><code>RecordBatch</code></a>es are
written.</p>
-</div></details><details class="toggle method-toggle" open><summary><section
id="method.close" class="method"><a class="srclink rightside"
href="../../../src/parquet/arrow/async_writer/mod.rs.html#127-135">source</a><h4
class="code-header">pub async fn <a href="#method.close"
class="fn">close</a>(self) -> <a class="type"
href="../../errors/type.Result.html" title="type
parquet::errors::Result">Result</a><<a class="struct"
href="../../format/struct.FileMetaData.html" title="struct p [...]
+</div></details><details class="toggle method-toggle" open><summary><section
id="method.close" class="method"><a class="srclink rightside"
href="../../../src/parquet/arrow/async_writer/mod.rs.html#137-145">source</a><h4
class="code-header">pub async fn <a href="#method.close"
class="fn">close</a>(self) -> <a class="type"
href="../../errors/type.Result.html" title="type
parquet::errors::Result">Result</a><<a class="struct"
href="../../format/struct.FileMetaData.html" title="struct p [...]
<p>All the data in the inner buffer will be force flushed.</p>
-</div></details><details class="toggle method-toggle" open><summary><section
id="method.try_flush" class="method"><a class="srclink rightside"
href="../../../src/parquet/arrow/async_writer/mod.rs.html#139-164">source</a><h4
class="code-header">async fn <a href="#method.try_flush"
class="fn">try_flush</a>(
+</div></details><details class="toggle method-toggle" open><summary><section
id="method.try_flush" class="method"><a class="srclink rightside"
href="../../../src/parquet/arrow/async_writer/mod.rs.html#149-174">source</a><h4
class="code-header">async fn <a href="#method.try_flush"
class="fn">try_flush</a>(
shared_buffer: &mut <a class="struct" href="struct.SharedBuffer.html"
title="struct parquet::arrow::async_writer::SharedBuffer">SharedBuffer</a>,
async_writer: <a class="primitive"
href="https://doc.rust-lang.org/nightly/std/primitive.reference.html";>&mut
W</a>,
- force: <a class="primitive"
href="https://doc.rust-lang.org/nightly/std/primitive.bool.html";>bool</a>
+ buffer_size: <a class="primitive"
href="https://doc.rust-lang.org/nightly/std/primitive.usize.html";>usize</a>
) -> <a class="type" href="../../errors/type.Result.html" title="type
parquet::errors::Result">Result</a><<a class="primitive"
href="https://doc.rust-lang.org/nightly/std/primitive.unit.html";>()</a>></h4></section></summary><div
class="docblock"><p>Flush the data in the <a href="struct.SharedBuffer.html"
title="struct
parquet::arrow::async_writer::SharedBuffer"><code>SharedBuffer</code></a> into
the <code>async_writer</code> if its size
exceeds the threshold.</p>
</div></details></div></details></div><h2 id="synthetic-implementations"
class="small-section-header">Auto Trait Implementations<a
href="#synthetic-implementations" class="anchor">§</a></h2><div
id="synthetic-implementations-list"><section
id="impl-RefUnwindSafe-for-AsyncArrowWriter%3CW%3E" class="impl"><a
href="#impl-RefUnwindSafe-for-AsyncArrowWriter%3CW%3E" class="anchor">§</a><h3
class="code-header">impl<W> !<a class="trait"
href="https://doc.rust-lang.org/nightly/core/panic/un [...]
diff --git a/parquet/arrow/async_writer/struct.SharedBuffer.html
b/parquet/arrow/async_writer/struct.SharedBuffer.html
index 8503877bf..2adbb8980 100644
--- a/parquet/arrow/async_writer/struct.SharedBuffer.html
+++ b/parquet/arrow/async_writer/struct.SharedBuffer.html
@@ -1,11 +1,11 @@
-<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta
name="viewport" content="width=device-width, initial-scale=1.0"><meta
name="generator" content="rustdoc"><meta name="description" content="A buffer
with interior mutability shared by the `ArrowWriter` and
`AsyncArrowWriter`."><title>SharedBuffer in parquet::arrow::async_writer -
Rust</title><link rel="preload" as="font" type="font/woff2" crossorigin
href="../../../static.files/SourceSerif4-Regular-46f98efaafac5295.ttf.woff2
[...]
+<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta
name="viewport" content="width=device-width, initial-scale=1.0"><meta
name="generator" content="rustdoc"><meta name="description" content="A buffer
with interior mutability shared by the `ArrowWriter` and
`AsyncArrowWriter`."><title>SharedBuffer in parquet::arrow::async_writer -
Rust</title><link rel="preload" as="font" type="font/woff2" crossorigin
href="../../../static.files/SourceSerif4-Regular-46f98efaafac5295.ttf.woff2
[...]
buffer: <a class="struct"
href="https://doc.rust-lang.org/nightly/alloc/sync/struct.Arc.html";
title="struct alloc::sync::Arc">Arc</a><Mutex<<a class="struct"
href="https://doc.rust-lang.org/nightly/alloc/vec/struct.Vec.html";
title="struct alloc::vec::Vec">Vec</a><<a class="primitive"
href="https://doc.rust-lang.org/nightly/std/primitive.u8.html";>u8</a>>>>,
}</code></pre><details class="toggle top-doc" open><summary
class="hideme"><span>Expand description</span></summary><div
class="docblock"><p>A buffer with interior mutability shared by the <a
href="../arrow_writer/struct.ArrowWriter.html" title="struct
parquet::arrow::arrow_writer::ArrowWriter"><code>ArrowWriter</code></a> and
<a href="struct.AsyncArrowWriter.html" title="struct
parquet::arrow::async_writer::AsyncArrowWriter"><code>AsyncArrowWriter</code></a>.</p>
</div></details><h2 id="fields" class="fields small-section-header">Fields<a
href="#fields" class="anchor">§</a></h2><span id="structfield.buffer"
class="structfield small-section-header"><a href="#structfield.buffer"
class="anchor field">§</a><code>buffer: <a class="struct"
href="https://doc.rust-lang.org/nightly/alloc/sync/struct.Arc.html";
title="struct alloc::sync::Arc">Arc</a><Mutex<<a class="struct"
href="https://doc.rust-lang.org/nightly/alloc/vec/struct.Vec.html"; title="stru
[...]
<p>The lock is used to obtain internal mutability, so no worry about the
lock contention.</p>
-</div><h2 id="implementations" class="small-section-header">Implementations<a
href="#implementations" class="anchor">§</a></h2><div
id="implementations-list"><details class="toggle implementors-toggle"
open><summary><section id="impl-SharedBuffer" class="impl"><a class="srclink
rightside"
href="../../../src/parquet/arrow/async_writer/mod.rs.html#178-184">source</a><a
href="#impl-SharedBuffer" class="anchor">§</a><h3 class="code-header">impl <a
class="struct" href="struct.SharedBuffer.htm [...]
+</div><h2 id="implementations" class="small-section-header">Implementations<a
href="#implementations" class="anchor">§</a></h2><div
id="implementations-list"><details class="toggle implementors-toggle"
open><summary><section id="impl-SharedBuffer" class="impl"><a class="srclink
rightside"
href="../../../src/parquet/arrow/async_writer/mod.rs.html#188-194">source</a><a
href="#impl-SharedBuffer" class="anchor">§</a><h3 class="code-header">impl <a
class="struct" href="struct.SharedBuffer.htm [...]
contents reach their destination. <a
href="https://doc.rust-lang.org/nightly/std/io/trait.Write.html#tymethod.flush";>Read
more</a></div></details><details class="toggle method-toggle"
open><summary><section id="method.write_vectored" class="method
trait-impl"><span class="rightside"><span class="since" title="Stable since
Rust version 1.36.0">1.36.0</span> · <a class="srclink"
href="https://doc.rust-lang.org/nightly/src/std/io/mod.rs.html#1495";>source</a></span><a
href="#method.write_vec [...]
implementation. <a
href="https://doc.rust-lang.org/nightly/std/io/trait.Write.html#method.is_write_vectored";>Read
more</a></div></details><details class="toggle method-toggle"
open><summary><section id="method.write_all" class="method trait-impl"><span
class="rightside"><span class="since" title="Stable since Rust version
1.0.0">1.0.0</span> · <a class="srclink"
href="https://doc.rust-lang.org/nightly/src/std/io/mod.rs.html#1572";>source</a></span><a
href="#method.write_all" class="anchor [...]
encountered. <a
href="https://doc.rust-lang.org/nightly/std/io/trait.Write.html#method.write_fmt";>Read
more</a></div></details><details class="toggle method-toggle"
open><summary><section id="method.by_ref" class="method trait-impl"><span
class="rightside"><span class="since" title="Stable since Rust version
1.0.0">1.0.0</span> · <a class="srclink"
href="https://doc.rust-lang.org/nightly/src/std/io/mod.rs.html#1749-1751";>source</a></span><a
href="#method.by_ref" class="anchor">§</a><h4 c [...]
diff --git a/search-index.js b/search-index.js
index ad322064c..b723b5cee 100644
--- a/search-index.js
+++ b/search-index.js
@@ -23,7 +23,7 @@ var searchIndex = JSON.parse('{\
"flight_test_integration_client":{"doc":"","t":"DNGNGELLLLLLLLLLLLLLLLLLLLLLMLLLLFMMMLLLLLLLLLLLLL","n":["Args","AuthBasicProto","Error","Middleware","Result","Scenario","augment_args","augment_args_for_update","borrow","borrow","borrow_mut","borrow_mut","cast","cast","clone","clone_into","command","command_for_update","fmt","fmt","from","from","from_arg_matches","from_arg_matches_mut","from_cast","from_cast","from_ref","group_id","host","into","into","into_request","into_request","main"
[...]
"flight_test_integration_server":{"doc":"","t":"DNGNGELLLLLLLLLLLLLLLLLLLLLLLLLLFMMLLLLLLLLLLLLL","n":["Args","AuthBasicProto","Error","Middleware","Result","Scenario","augment_args","augment_args_for_update","borrow","borrow","borrow_mut","borrow_mut","cast","cast","clone","clone_into","command","command_for_update","fmt","fmt","from","from","from_arg_matches","from_arg_matches_mut","from_cast","from_cast","from_ref","group_id","into","into","into_request","into_request","main","port","
[...]
"gen":{"doc":"","t":"FF","n":["main","prost_config"],"q":[[0,"gen"]],"d":["",""],"i":[0,0],"f":[[[],[[3,[[2,[1]]]]]],[[],4]],"c":[],"p":[[8,"Error"],[3,"Box"],[4,"Result"],[3,"Config"]]},\
-"parquet":{"doc":"This crate contains the official Native Rust
…","t":"AAAAAAOAAAARCCDCDLAFAAALLLLALLLLLLAMLLLLLLLLLLMMFFFALLLLLLLLLIDDDDGDIDDFMMMLLLLLLLLLLLLLKLKFMMAMLLLLLLLMLLLLMLMLLLLLLKMMLMKMMMMLLMMAMMMFMMLLLLLLLLLLLLLLLLLLLLLLLIDDLLLLLLKLMLLLLLLLLMKLMLLLLLLDDFLLLLLLLLLLLLLLLLLLLLLLLLLLLLFLLLLLLLMLLMLLMLLLLLLLLLLDEDDDNDNGLMLLLLLLLLLLLLMMALLLLLLLLLLMLLLLLLLLLLLLLLLLLFFLFFLFFFLMLLLLLLLLLLMAMLLLLMLLLLLLLLLLLLLLLLLLLLLLFFLLFMMDDNNDDENLMLLLLLLLLLLLLLLLFLLMOOOFLLMLLLLLMLLLLLLLLLLLLLLLLLLLM
[...]
+"parquet":{"doc":"This crate contains the official Native Rust
…","t":"AAAAAAOAAAARCCDCDLAFAAALLLLALLLLLLAMLLLLLLLLLLMMFFFALLLLLLLLLIDDDDGDIDDFMMMLLLLLLLLLLLLLKLKFMMAMLLLLLLLMLLLLMLMLLLLLLKMMLMKMMMMLLMMAMMMFMMLLLLLLLLLLLLLLLLLLLLLLLIDDLLLLLLKLMLLLLLLLLMKLMLLLLLLDDFLLLLLLLLLLLLLLLLLLLLLLLLLLLLFLLLLLLLMLLMLLMLLLLLLLLLLDEDDDNDNGLMLLLLLLLLLLLLMMALLLLLLLLLLMLLLLLLLLLLLLLLLLLFFLFFLFFFLMLLLLLLLLLLMAMLLLLMLLLLLLLLLLLLLLLLLLLLLLFFLLFMMDDNNDDENLMLLLLLLLLLLLLLLLFLLMOOOFLLMLLLLLMLLLLLLLLLLLLLLLLLLLM
[...]
"parquet_concat":{"doc":"Binary that concatenates the column data of one or
more
…","t":"DLLLLLLLLLLLLLMLFMLLLLLL","n":["Args","augment_args","augment_args_for_update","borrow","borrow_mut","cast","command","command_for_update","fmt","from","from_arg_matches","from_arg_matches_mut","from_cast","group_id","input","into","main","output","run","try_from","try_into","type_id","update_from_arg_matches","update_from_arg_matches_mut"],"q":[[0,"parquet_concat"]],"d":["Concatenates
one or more pa [...]
"parquet_derive":{"doc":"","t":"YANNNDNNNEENNNLLLLLLLLLLLLLLLLLLLLLLLLLMLLLLMLLLLLLLLLMLLLLLLMLLLL","n":["ParquetRecordWriter","parquet_field","Array","ChronoNaiveDate","ChronoNaiveDateTime","Field","Option","Reference","Slice","ThirdPartyType","Type","TypePath","Uuid","Vec","borrow","borrow","borrow","borrow_mut","borrow_mut","borrow_mut","column_writer","converted_type","copied_direct_vals","eq","eq","eq","fmt","fmt","fmt","from","from","from","from","from","from_type","from_type_array
[...]
"parquet_derive_test":{"doc":"","t":"DMMMMLLMMMMMMLMMLLMMLMMMMMMMMMMMLLLM","n":["ACompleteRecord","a_bool","a_borrowed_string","a_str","a_string","borrow","borrow_mut","borrowed_byte_vec","borrowed_maybe_a_str","borrowed_maybe_a_string","borrowed_maybe_borrowed_byte_vec","borrowed_maybe_byte_vec","byte_vec","cast","double","float","from","from_cast","i16","i32","into","isize","maybe_a_str","maybe_a_string","maybe_byte_vec","maybe_double","maybe_float","maybe_i16","maybe_u32","maybe_u8","
[...]
diff --git a/src/parquet/arrow/async_writer/mod.rs.html
b/src/parquet/arrow/async_writer/mod.rs.html
index db4084cb9..0226237c1 100644
--- a/src/parquet/arrow/async_writer/mod.rs.html
+++ b/src/parquet/arrow/async_writer/mod.rs.html
@@ -365,6 +365,16 @@
<a href="#365" id="365">365</a>
<a href="#366" id="366">366</a>
<a href="#367" id="367">367</a>
+<a href="#368" id="368">368</a>
+<a href="#369" id="369">369</a>
+<a href="#370" id="370">370</a>
+<a href="#371" id="371">371</a>
+<a href="#372" id="372">372</a>
+<a href="#373" id="373">373</a>
+<a href="#374" id="374">374</a>
+<a href="#375" id="375">375</a>
+<a href="#376" id="376">376</a>
+<a href="#377" id="377">377</a>
</pre></div><pre class="rust"><code><span class="comment">// Licensed to the
Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
@@ -444,12 +454,16 @@
<span class="doccomment">/// The inner buffer shared by the `sync_writer`
and the `async_writer`
</span>shared_buffer: SharedBuffer,
+
+ <span class="doccomment">/// Trigger forced flushing once buffer size
reaches this value
+ </span>buffer_size: usize,
}
<span class="kw">impl</span><W: AsyncWrite + Unpin + Send>
AsyncArrowWriter<W> {
<span class="doccomment">/// Try to create a new Async Arrow Writer.
///
- /// `buffer_size` determines the initial size of the intermediate buffer.
+ /// `buffer_size` determines the number of bytes to buffer before flushing
+ /// to the underlying [`AsyncWrite`]
///
/// The intermediate buffer will automatically be resized if necessary
///
@@ -469,6 +483,7 @@
sync_writer,
async_writer: writer,
shared_buffer,
+ buffer_size,
})
}
@@ -478,7 +493,12 @@
/// checked and flush if at least half full
</span><span class="kw">pub async fn </span>write(<span
class="kw-2">&mut </span><span class="self">self</span>, batch: <span
class="kw-2">&</span>RecordBatch) -> <span
class="prelude-ty">Result</span><()> {
<span class="self">self</span>.sync_writer.write(batch)<span
class="question-mark">?</span>;
- <span class="self">Self</span>::try_flush(<span class="kw-2">&mut
</span><span class="self">self</span>.shared_buffer, <span
class="kw-2">&mut </span><span class="self">self</span>.async_writer, <span
class="bool-val">false</span>).<span class="kw">await
+ <span class="self">Self</span>::try_flush(
+ <span class="kw-2">&mut </span><span
class="self">self</span>.shared_buffer,
+ <span class="kw-2">&mut </span><span
class="self">self</span>.async_writer,
+ <span class="self">self</span>.buffer_size,
+ )
+ .<span class="kw">await
</span>}
<span class="doccomment">/// Append [`KeyValue`] metadata in addition to
those in [`WriterProperties`]
@@ -495,7 +515,7 @@
<span class="kw">let </span>metadata = <span
class="self">self</span>.sync_writer.close()<span
class="question-mark">?</span>;
<span class="comment">// Force to flush the remaining data.
- </span><span class="self">Self</span>::try_flush(<span
class="kw-2">&mut </span><span class="self">self</span>.shared_buffer,
<span class="kw-2">&mut </span><span class="self">self</span>.async_writer,
<span class="bool-val">true</span>).<span class="kw">await</span><span
class="question-mark">?</span>;
+ </span><span class="self">Self</span>::try_flush(<span
class="kw-2">&mut </span><span class="self">self</span>.shared_buffer,
<span class="kw-2">&mut </span><span class="self">self</span>.async_writer,
<span class="number">0</span>).<span class="kw">await</span><span
class="question-mark">?</span>;
<span class="self">self</span>.async_writer.shutdown().<span
class="kw">await</span><span class="question-mark">?</span>;
<span class="prelude-val">Ok</span>(metadata)
@@ -506,10 +526,10 @@
</span><span class="kw">async fn </span>try_flush(
shared_buffer: <span class="kw-2">&mut </span>SharedBuffer,
async_writer: <span class="kw-2">&mut </span>W,
- force: bool,
+ buffer_size: usize,
) -> <span class="prelude-ty">Result</span><()> {
<span class="kw">let </span><span class="kw-2">mut </span>buffer =
shared_buffer.buffer.try_lock().unwrap();
- <span class="kw">if </span>!force && buffer.len() <
buffer.capacity() / <span class="number">2 </span>{
+ <span class="kw">if </span>buffer.is_empty() || buffer.len() <
buffer_size {
<span class="comment">// no need to flush
</span><span class="kw">return </span><span
class="prelude-val">Ok</span>(());
}
