pitrou commented on code in PR #14452:
URL: https://github.com/apache/arrow/pull/14452#discussion_r1005514775
##########
docs/source/cpp/csv.rst:
##########
@@ -56,19 +67,97 @@ A CSV file is read from a :class:`~arrow::io::InputStream`.
parse_options,
convert_options);
if (!maybe_reader.ok()) {
- // Handle TableReader instantiation error...
+ // Handle TableReader instantiation error...
}
std::shared_ptr<arrow::csv::TableReader> reader = *maybe_reader;
// Read table from CSV file
auto maybe_table = reader->Read();
if (!maybe_table.ok()) {
- // Handle CSV read error
- // (for example a CSV syntax error or failed type conversion)
+ // Handle CSV read error
+ // (for example a CSV syntax error or failed type conversion)
}
std::shared_ptr<arrow::Table> table = *maybe_table;
}
+StreamingReader
+---------------
+
+.. code-block:: cpp
+
+ #include "arrow/csv/api.h"
+
+ {
+ // ...
+ arrow::io::IOContext io_context = arrow::io::default_io_context();
+ std::shared_ptr<arrow::io::InputStream> input = ...;
+
+ auto read_options = arrow::csv::ReadOptions::Defaults();
+ auto parse_options = arrow::csv::ParseOptions::Defaults();
+ auto convert_options = arrow::csv::ConvertOptions::Defaults();
+
+ // Instantiate StreamingReader from input stream and options
+ auto maybe_reader =
+ arrow::csv::StreamingReader::Make(io_context,
+ input,
+ read_options,
+ parse_options,
+ convert_options);
+ if (!maybe_reader.ok()) {
+ // Handle StreamingReader instantiation error...
+ }
+ std::shared_ptr<arrow::csv::StreamingReader> reader = *maybe_reader;
+
+ // Set aside a RecordBatch pointer for re-use while streaming
+ std::shared_ptr<RecordBatch> batch;
+
+ while (true) {
+ // Attempt to read the first RecordBatch
+ arrow::Status status = reader->ReadNext(&batch);
+
+ if (!status.ok()) {
+ // Handle read error
+ }
+
+ if (batch == NULL) {
+ // Handle end of file
+ break;
+ }
+
+ // Do something with the batch
+ }
+ }
+
+.. _cpp-csv-tradeoffs:
+
+Tradeoffs
+---------
+
+The choice between using :class:`~arrow::csv::TableReader` or
+:class:`~arrow::csv::StreamingReader` will ultimately depend on the use case
+but there are a few tradeoffs to be aware of:
+
+1. **Memory usage:** :class:`~arrow::csv::TableReader` loads all of the data
+ into memory at once and, depending on the amount of data, may require
+ considerably more memory than :class:`~arrow::csv::StreamingReader` which
+ only loads one :class:`~arrow::RecordBatch` at a time. This is likely to be
+ the most significant tradeoff for users.
+3. **Speed:** When reading the entire contents of a CSV,
+ :class:`~arrow::csv::TableReader` will tend to be faster than
+ :class:`~arrow::csv::StreamingReader` because it makes better use of
+ available cores.
Review Comment:
Link to the `cpp-csv-performance` reference here somehow?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
