amoeba commented on code in PR #14452:
URL: https://github.com/apache/arrow/pull/14452#discussion_r1005075568
##########
docs/source/cpp/csv.rst:
##########
@@ -56,19 +67,84 @@ A CSV file is read from a :class:`~arrow::io::InputStream`.
parse_options,
convert_options);
if (!maybe_reader.ok()) {
- // Handle TableReader instantiation error...
+ // Handle TableReader instantiation error...
}
std::shared_ptr<arrow::csv::TableReader> reader = *maybe_reader;
// Read table from CSV file
auto maybe_table = reader->Read();
if (!maybe_table.ok()) {
- // Handle CSV read error
- // (for example a CSV syntax error or failed type conversion)
+ // Handle CSV read error
+ // (for example a CSV syntax error or failed type conversion)
}
std::shared_ptr<arrow::Table> table = *maybe_table;
}
+StreamingReader
+---------------
+
+.. code-block:: cpp
+
+ #include "arrow/csv/api.h"
+
+ {
+ // ...
+ arrow::io::IOContext io_context = arrow::io::default_io_context();
+ std::shared_ptr<arrow::io::InputStream> input = ...;
+
+ auto read_options = arrow::csv::ReadOptions::Defaults();
+ auto parse_options = arrow::csv::ParseOptions::Defaults();
+ auto convert_options = arrow::csv::ConvertOptions::Defaults();
+
+ // Instantiate StreamingReader from input stream and options
+ auto maybe_reader =
+ arrow::csv::StreamingReader::Make(io_context,
+ input,
+ read_options,
+ parse_options,
+ convert_options);
+ if (!maybe_reader.ok()) {
+ // Handle StreamingReader instantiation error...
+ }
+ std::shared_ptr<arrow::csv::StreamingReader> reader = *maybe_reader;
+
+ // Set aside a RecordBatch pointer for re-use while streaming
+ std::shared_ptr<RecordBatch> batch;
+
+ // Attempt to read the first RecordBatch
+ arrow::Status status = reader->ReadNext(&batch);
+
+ if (!status.ok()) {
+ // Handle read error
+ }
+
+ if (batch == NULL) {
+ // Handle end of file
+ }
+ }
+
+.. _cpp-csv-tradeoffs:
+
+Tradeoffs
+---------
+
+The choice between using :class:`~arrow::csv::TableReader` or
+:class:`~arrow::csv::StreamingReader` will depend on your use case but two
+caveats are worth pointing out:
+
+1. :class:`~arrow::csv::TableReader` is capable of using multiple threads (See
+ :ref:`Performance <cpp-csv-performance>`) whereas
+ :class:`~arrow::csv::StreamingReader` is always single-threaded and will
+ ignore :member:`ReadOptions::use_threads`.
+2. :class:`~arrow::csv::StreamingReader` performs type inference off the first
+ block that's read in, after which point the types are frozen. Either set
+ :member:`ReadOptions::block_size` to a large enough value or use
+ :member:`ConvertOptions::column_types` to set the desired data types
+ explicitly.
Review Comment:
Updated in
https://github.com/apache/arrow/commit/3d54059caa2f6dc888be670f98da33d526a12ba4
and 039d23f8aa093017e72446d405a5f45b849d2689. I didn't include specific error
text but said,
> after which point the types
are frozen and any data in subsequent blocks that cannot be converted to
those types will cause an error.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
