http://git-wip-us.apache.org/repos/asf/arrow-site/blob/62ef7145/docs/latest/cpp/examples.html ---------------------------------------------------------------------- diff --git a/docs/latest/cpp/examples.html b/docs/latest/cpp/examples.html new file mode 100644 index 0000000..bc96115 --- /dev/null +++ b/docs/latest/cpp/examples.html @@ -0,0 +1,440 @@ + + + +<!DOCTYPE html> +<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> +<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> +<head> + <meta charset="utf-8"> + + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + <title>Examples — Apache Arrow v0.11.1.dev473+g6ed02454</title> + + + + + + + + + + + + + + + + <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" /> + <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> + <link rel="index" title="Index" href="../genindex.html" /> + <link rel="search" title="Search" href="../search.html" /> + <link rel="next" title="API Reference" href="api.html" /> + <link rel="prev" title="Two-dimensional Datasets" href="tables.html" /> + + + <script src="../_static/js/modernizr.min.js"></script> + +</head> + +<body class="wy-body-for-nav"> + + + <div class="wy-grid-for-nav"> + + + <nav data-toggle="wy-nav-shift" class="wy-nav-side"> + <div class="wy-side-scroll"> + <div class="wy-side-nav-search"> + + + + <a href="../index.html" class="icon icon-home"> Apache Arrow + + + + </a> + + + + + <div class="version"> + 0.11.1.dev473+g6ed02454 + </div> + + + + +<div role="search"> + <form id="rtd-search-form" class="wy-form" action="../search.html" method="get"> + <input type="text" name="q" placeholder="Search docs" /> + <input type="hidden" name="check_keywords" value="yes" /> + <input type="hidden" name="area" value="default" /> + </form> +</div> + + + </div> + + <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> + + + + + + + <p class="caption"><span class="caption-text">Memory Format</span></p> +<ul> +<li class="toctree-l1"><a class="reference internal" href="../format/README.html">Arrow specification documents</a></li> +<li class="toctree-l1"><a class="reference internal" href="../format/Guidelines.html">Implementation guidelines</a></li> +<li class="toctree-l1"><a class="reference internal" href="../format/Layout.html">Physical memory layout</a></li> +<li class="toctree-l1"><a class="reference internal" href="../format/Metadata.html">Metadata: Logical types, schemas, data headers</a></li> +<li class="toctree-l1"><a class="reference internal" href="../format/IPC.html">Interprocess messaging / communication (IPC)</a></li> +</ul> +<p class="caption"><span class="caption-text">Languages</span></p> +<ul class="current"> +<li class="toctree-l1 current"><a class="reference internal" href="index.html">C++ Implementation</a><ul class="current"> +<li class="toctree-l2"><a class="reference internal" href="getting_started.html">Getting Started</a></li> +<li class="toctree-l2 current"><a class="current reference internal" href="#">Examples</a><ul> +<li class="toctree-l3"><a class="reference internal" href="#row-to-columnar-conversion">Row to columnar conversion</a></li> +</ul> +</li> +<li class="toctree-l2"><a class="reference internal" href="api.html">API Reference</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../python/index.html">Python bindings</a></li> +</ul> + + + + </div> + </div> + </nav> + + <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> + + + <nav class="wy-nav-top" aria-label="top navigation"> + + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="../index.html">Apache Arrow</a> + + </nav> + + + <div class="wy-nav-content"> + + <div class="rst-content"> + + + + + + + + + + + + + + + + + +<div role="navigation" aria-label="breadcrumbs navigation"> + + <ul class="wy-breadcrumbs"> + + <li><a href="../index.html">Docs</a> »</li> + + <li><a href="index.html">C++ Implementation</a> »</li> + + <li>Examples</li> + + + <li class="wy-breadcrumbs-aside"> + + + <a href="../_sources/cpp/examples.rst.txt" rel="nofollow"> View page source</a> + + + </li> + + </ul> + + + <hr/> +</div> + <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article"> + <div itemprop="articleBody"> + + <div class="section" id="examples"> +<h1>Examples<a class="headerlink" href="#examples" title="Permalink to this headline">¶</a></h1> +<div class="section" id="row-to-columnar-conversion"> +<h2>Row to columnar conversion<a class="headerlink" href="#row-to-columnar-conversion" title="Permalink to this headline">¶</a></h2> +<p>The following example converts an array of structs to a <a class="reference internal" href="api/table.html#_CPPv3N5arrow5TableE" title="arrow::Table"><code class="xref cpp cpp-class docutils literal notranslate"><span class="pre">arrow::Table</span></code></a> +instance, and then converts it back to the original array of structs.</p> +<div class="highlight-cpp notranslate"><div class="highlight"><pre><span></span><span class="c1">// Licensed to the Apache Software Foundation (ASF) under one</span> +<span class="c1">// or more contributor license agreements. See the NOTICE file</span> +<span class="c1">// distributed with this work for additional information</span> +<span class="c1">// regarding copyright ownership. The ASF licenses this file</span> +<span class="c1">// to you under the Apache License, Version 2.0 (the</span> +<span class="c1">// "License"); you may not use this file except in compliance</span> +<span class="c1">// with the License. You may obtain a copy of the License at</span> +<span class="c1">//</span> +<span class="c1">// http://www.apache.org/licenses/LICENSE-2.0</span> +<span class="c1">//</span> +<span class="c1">// Unless required by applicable law or agreed to in writing,</span> +<span class="c1">// software distributed under the License is distributed on an</span> +<span class="c1">// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY</span> +<span class="c1">// KIND, either express or implied. See the License for the</span> +<span class="c1">// specific language governing permissions and limitations</span> +<span class="c1">// under the License.</span> + +<span class="cp">#include</span> <span class="cpf"><cstdint></span><span class="cp"></span> +<span class="cp">#include</span> <span class="cpf"><iostream></span><span class="cp"></span> +<span class="cp">#include</span> <span class="cpf"><vector></span><span class="cp"></span> + +<span class="cp">#include</span> <span class="cpf"><arrow/api.h></span><span class="cp"></span> + +<span class="k">using</span> <span class="n">arrow</span><span class="o">::</span><span class="n">DoubleBuilder</span><span class="p">;</span> +<span class="k">using</span> <span class="n">arrow</span><span class="o">::</span><span class="n">Int64Builder</span><span class="p">;</span> +<span class="k">using</span> <span class="n">arrow</span><span class="o">::</span><span class="n">ListBuilder</span><span class="p">;</span> + +<span class="c1">// While we want to use columnar data structures to build efficient operations, we</span> +<span class="c1">// often receive data in a row-wise fashion from other systems. In the following,</span> +<span class="c1">// we want give a brief introduction into the classes provided by Apache Arrow by</span> +<span class="c1">// showing how to transform row-wise data into a columnar table.</span> +<span class="c1">//</span> +<span class="c1">// The data in this example is stored in the following struct:</span> +<span class="k">struct</span> <span class="n">data_row</span> <span class="p">{</span> + <span class="kt">int64_t</span> <span class="n">id</span><span class="p">;</span> + <span class="kt">double</span> <span class="n">cost</span><span class="p">;</span> + <span class="n">std</span><span class="o">::</span><span class="n">vector</span><span class="o"><</span><span class="kt">double</span><span class="o">></span> <span class="n">cost_components</span><span class="p">;</span> +<span class="p">};</span> + +<span class="c1">// Transforming a vector of structs into a columnar Table.</span> +<span class="c1">//</span> +<span class="c1">// The final representation should be an `arrow::Table` which in turn is made up of</span> +<span class="c1">// an `arrow::Schema` and a list of `arrow::Column`. An `arrow::Column` is again a</span> +<span class="c1">// named collection of one or more `arrow::Array` instances. As the first step, we</span> +<span class="c1">// will iterate over the data and build up the arrays incrementally. For this task,</span> +<span class="c1">// we provide `arrow::ArrayBuilder` classes that help in the construction of the</span> +<span class="c1">// final `arrow::Array` instances.</span> +<span class="c1">//</span> +<span class="c1">// For each type, Arrow has a specially typed builder class. For the primitive</span> +<span class="c1">// values `id` and `cost` we can use the respective `arrow::Int64Builder` and</span> +<span class="c1">// `arrow::DoubleBuilder`. For the `cost_components` vector, we need to have two</span> +<span class="c1">// builders, a top-level `arrow::ListBuilder` that builds the array of offsets and</span> +<span class="c1">// a nested `arrow::DoubleBuilder` that constructs the underlying values array that</span> +<span class="c1">// is referenced by the offsets in the former array.</span> +<span class="n">arrow</span><span class="o">::</span><span class="n">Status</span> <span class="n">VectorToColumnarTable</span><span class="p">(</span><span class="k">const</span> <span class="n">std</span><span class="o">::</span><span class="n">vector</span><span class="o"><</span><span class="k">struct</span> <span class="n">data_row</span><span class="o">>&</span> <span class="n">rows</span><span class="p">,</span> + <span class="n">std</span><span class="o">::</span><span class="n">shared_ptr</span><span class="o"><</span><span class="n">arrow</span><span class="o">::</span><span class="n">Table</span><span class="o">>*</span> <span class="n">table</span><span class="p">)</span> <span class="p">{</span> + <span class="c1">// The builders are more efficient using</span> + <span class="c1">// arrow::jemalloc::MemoryPool::default_pool() as this can increase the size of</span> + <span class="c1">// the underlying memory regions in-place. At the moment, arrow::jemalloc is only</span> + <span class="c1">// supported on Unix systems, not Windows.</span> + <span class="n">arrow</span><span class="o">::</span><span class="n">MemoryPool</span><span class="o">*</span> <span class="n">pool</span> <span class="o">=</span> <span class="n">arrow</span><span class="o">::</span><span class="n">default_memory_pool</span><span class="p">();</span> + + <span class="n">Int64Builder</span> <span class="nf">id_builder</span><span class="p">(</span><span class="n">pool</span><span class="p">);</span> + <span class="n">DoubleBuilder</span> <span class="nf">cost_builder</span><span class="p">(</span><span class="n">pool</span><span class="p">);</span> + <span class="n">ListBuilder</span> <span class="nf">components_builder</span><span class="p">(</span><span class="n">pool</span><span class="p">,</span> <span class="n">std</span><span class="o">::</span><span class="n">make_shared</span><span class="o"><</span><span class="n">DoubleBuilder</span><span class="o">></span><span class="p">(</span><span class="n">pool</span><span class="p">));</span> + <span class="c1">// The following builder is owned by components_builder.</span> + <span class="n">DoubleBuilder</span><span class="o">&</span> <span class="n">cost_components_builder</span> <span class="o">=</span> + <span class="o">*</span><span class="p">(</span><span class="k">static_cast</span><span class="o"><</span><span class="n">DoubleBuilder</span><span class="o">*></span><span class="p">(</span><span class="n">components_builder</span><span class="p">.</span><span class="n">value_builder</span><span class="p">()));</span> + + <span class="c1">// Now we can loop over our existing data and insert it into the builders. The</span> + <span class="c1">// `Append` calls here may fail (e.g. we cannot allocate enough additional memory).</span> + <span class="c1">// Thus we need to check their return values. For more information on these values,</span> + <span class="c1">// check the documentation about `arrow::Status`.</span> + <span class="k">for</span> <span class="p">(</span><span class="k">const</span> <span class="n">data_row</span><span class="o">&</span> <span class="nl">row</span> <span class="p">:</span> <span class="n">rows</span><span class="p">)</span> <span class="p">{</span> + <span class="n">ARROW_RETURN_NOT_OK</span><span class="p">(</span><span class="n">id_builder</span><span class="p">.</span><span class="n">Append</span><span class="p">(</span><span class="n">row</span><span class="p">.</span><span class="n">id</span><span class="p">));</span> + <span class="n">ARROW_RETURN_NOT_OK</span><span class="p">(</span><span class="n">cost_builder</span><span class="p">.</span><span class="n">Append</span><span class="p">(</span><span class="n">row</span><span class="p">.</span><span class="n">cost</span><span class="p">));</span> + + <span class="c1">// Indicate the start of a new list row. This will memorise the current</span> + <span class="c1">// offset in the values builder.</span> + <span class="n">ARROW_RETURN_NOT_OK</span><span class="p">(</span><span class="n">components_builder</span><span class="p">.</span><span class="n">Append</span><span class="p">());</span> + <span class="c1">// Store the actual values. The final nullptr argument tells the underyling</span> + <span class="c1">// builder that all added values are valid, i.e. non-null.</span> + <span class="n">ARROW_RETURN_NOT_OK</span><span class="p">(</span><span class="n">cost_components_builder</span><span class="p">.</span><span class="n">AppendValues</span><span class="p">(</span><span class="n">row</span><span class="p">.</span><span class="n">cost_components</span><span class="p">.</span><span class="n">data</span><span class="p">(),</span> + <span class="n">row</span><span class="p">.</span><span class="n">cost_components</span><span class="p">.</span><span class="n">size</span><span class="p">()));</span> + <span class="p">}</span> + + <span class="c1">// At the end, we finalise the arrays, declare the (type) schema and combine them</span> + <span class="c1">// into a single `arrow::Table`:</span> + <span class="n">std</span><span class="o">::</span><span class="n">shared_ptr</span><span class="o"><</span><span class="n">arrow</span><span class="o">::</span><span class="n">Array</span><span class="o">></span> <span class="n">id_array</span><span class="p">;</span> + <span class="n">ARROW_RETURN_NOT_OK</span><span class="p">(</span><span class="n">id_builder</span><span class="p">.</span><span class="n">Finish</span><span class="p">(</span><span class="o">&</span><span class="n">id_array</span><span class="p">));</span> + <span class="n">std</span><span class="o">::</span><span class="n">shared_ptr</span><span class="o"><</span><span class="n">arrow</span><span class="o">::</span><span class="n">Array</span><span class="o">></span> <span class="n">cost_array</span><span class="p">;</span> + <span class="n">ARROW_RETURN_NOT_OK</span><span class="p">(</span><span class="n">cost_builder</span><span class="p">.</span><span class="n">Finish</span><span class="p">(</span><span class="o">&</span><span class="n">cost_array</span><span class="p">));</span> + <span class="c1">// No need to invoke cost_components_builder.Finish because it is implied by</span> + <span class="c1">// the parent builder's Finish invocation.</span> + <span class="n">std</span><span class="o">::</span><span class="n">shared_ptr</span><span class="o"><</span><span class="n">arrow</span><span class="o">::</span><span class="n">Array</span><span class="o">></span> <span class="n">cost_components_array</span><span class="p">;</span> + <span class="n">ARROW_RETURN_NOT_OK</span><span class="p">(</span><span class="n">components_builder</span><span class="p">.</span><span class="n">Finish</span><span class="p">(</span><span class="o">&</span><span class="n">cost_components_array</span><span class="p">));</span> + + <span class="n">std</span><span class="o">::</span><span class="n">vector</span><span class="o"><</span><span class="n">std</span><span class="o">::</span><span class="n">shared_ptr</span><span class="o"><</span><span class="n">arrow</span><span class="o">::</span><span class="n">Field</span><span class="o">>></span> <span class="n">schema_vector</span> <span class="o">=</span> <span class="p">{</span> + <span class="n">arrow</span><span class="o">::</span><span class="n">field</span><span class="p">(</span><span class="s">"id"</span><span class="p">,</span> <span class="n">arrow</span><span class="o">::</span><span class="n">int64</span><span class="p">()),</span> <span class="n">arrow</span><span class="o">::</span><span class="n">field</span><span class="p">(</span><span class="s">"cost"</span><span class="p">,</span> <span class="n">arrow</span><span class="o">::</span><span class="n">float64</span><span class="p">()),</span> + <span class="n">arrow</span><span class="o">::</span><span class="n">field</span><span class="p">(</span><span class="s">"cost_components"</span><span class="p">,</span> <span class="n">arrow</span><span class="o">::</span><span class="n">list</span><span class="p">(</span><span class="n">arrow</span><span class="o">::</span><span class="n">float64</span><span class="p">()))};</span> + + <span class="k">auto</span> <span class="n">schema</span> <span class="o">=</span> <span class="n">std</span><span class="o">::</span><span class="n">make_shared</span><span class="o"><</span><span class="n">arrow</span><span class="o">::</span><span class="n">Schema</span><span class="o">></span><span class="p">(</span><span class="n">schema_vector</span><span class="p">);</span> + + <span class="c1">// The final `table` variable is the one we then can pass on to other functions</span> + <span class="c1">// that can consume Apache Arrow memory structures. This object has ownership of</span> + <span class="c1">// all referenced data, thus we don't have to care about undefined references once</span> + <span class="c1">// we leave the scope of the function building the table and its underlying arrays.</span> + <span class="o">*</span><span class="n">table</span> <span class="o">=</span> <span class="n">arrow</span><span class="o">::</span><span class="n">Table</span><span class="o">::</span><span class="n">Make</span><span class="p">(</span><span class="n">schema</span><span class="p">,</span> <span class="p">{</span><span class="n">id_array</span><span class="p">,</span> <span class="n">cost_array</span><span class="p">,</span> <span class="n">cost_components_array</span><span class="p">});</span> + + <span class="k">return</span> <span class="n">arrow</span><span class="o">::</span><span class="n">Status</span><span class="o">::</span><span class="n">OK</span><span class="p">();</span> +<span class="p">}</span> + +<span class="n">arrow</span><span class="o">::</span><span class="n">Status</span> <span class="n">ColumnarTableToVector</span><span class="p">(</span><span class="k">const</span> <span class="n">std</span><span class="o">::</span><span class="n">shared_ptr</span><span class="o"><</span><span class="n">arrow</span><span class="o">::</span><span class="n">Table</span><span class="o">>&</span> <span class="n">table</span><span class="p">,</span> + <span class="n">std</span><span class="o">::</span><span class="n">vector</span><span class="o"><</span><span class="k">struct</span> <span class="n">data_row</span><span class="o">>*</span> <span class="n">rows</span><span class="p">)</span> <span class="p">{</span> + <span class="c1">// To convert an Arrow table back into the same row-wise representation as in the</span> + <span class="c1">// above section, we first will check that the table conforms to our expected</span> + <span class="c1">// schema and then will build up the vector of rows incrementally.</span> + <span class="c1">//</span> + <span class="c1">// For the check if the table is as expected, we can utilise solely its schema.</span> + <span class="n">std</span><span class="o">::</span><span class="n">vector</span><span class="o"><</span><span class="n">std</span><span class="o">::</span><span class="n">shared_ptr</span><span class="o"><</span><span class="n">arrow</span><span class="o">::</span><span class="n">Field</span><span class="o">>></span> <span class="n">schema_vector</span> <span class="o">=</span> <span class="p">{</span> + <span class="n">arrow</span><span class="o">::</span><span class="n">field</span><span class="p">(</span><span class="s">"id"</span><span class="p">,</span> <span class="n">arrow</span><span class="o">::</span><span class="n">int64</span><span class="p">()),</span> <span class="n">arrow</span><span class="o">::</span><span class="n">field</span><span class="p">(</span><span class="s">"cost"</span><span class="p">,</span> <span class="n">arrow</span><span class="o">::</span><span class="n">float64</span><span class="p">()),</span> + <span class="n">arrow</span><span class="o">::</span><span class="n">field</span><span class="p">(</span><span class="s">"cost_components"</span><span class="p">,</span> <span class="n">arrow</span><span class="o">::</span><span class="n">list</span><span class="p">(</span><span class="n">arrow</span><span class="o">::</span><span class="n">float64</span><span class="p">()))};</span> + <span class="k">auto</span> <span class="n">expected_schema</span> <span class="o">=</span> <span class="n">std</span><span class="o">::</span><span class="n">make_shared</span><span class="o"><</span><span class="n">arrow</span><span class="o">::</span><span class="n">Schema</span><span class="o">></span><span class="p">(</span><span class="n">schema_vector</span><span class="p">);</span> + + <span class="k">if</span> <span class="p">(</span><span class="o">!</span><span class="n">expected_schema</span><span class="o">-></span><span class="n">Equals</span><span class="p">(</span><span class="o">*</span><span class="n">table</span><span class="o">-></span><span class="n">schema</span><span class="p">()))</span> <span class="p">{</span> + <span class="c1">// The table doesn't have the expected schema thus we cannot directly</span> + <span class="c1">// convert it to our target representation.</span> + <span class="k">return</span> <span class="n">arrow</span><span class="o">::</span><span class="n">Status</span><span class="o">::</span><span class="n">Invalid</span><span class="p">(</span><span class="s">"Schemas are not matching!"</span><span class="p">);</span> + <span class="p">}</span> + + <span class="c1">// As we have ensured that the table has the expected structure, we can unpack the</span> + <span class="c1">// underlying arrays. For the primitive columns `id` and `cost` we can use the high</span> + <span class="c1">// level functions to get the values whereas for the nested column</span> + <span class="c1">// `cost_components` we need to access the C-pointer to the data to copy its</span> + <span class="c1">// contents into the resulting `std::vector<double>`. Here we need to be care to</span> + <span class="c1">// also add the offset to the pointer. This offset is needed to enable zero-copy</span> + <span class="c1">// slicing operations. While this could be adjusted automatically for double</span> + <span class="c1">// arrays, this cannot be done for the accompanying bitmap as often the slicing</span> + <span class="c1">// border would be inside a byte.</span> + + <span class="k">auto</span> <span class="n">ids</span> <span class="o">=</span> + <span class="n">std</span><span class="o">::</span><span class="n">static_pointer_cast</span><span class="o"><</span><span class="n">arrow</span><span class="o">::</span><span class="n">Int64Array</span><span class="o">></span><span class="p">(</span><span class="n">table</span><span class="o">-></span><span class="n">column</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span><span class="o">-></span><span class="n">data</span><span class="p">()</span><span class="o">-></span><span class="n">chunk</span><span class="p">(</span><span class="mi">0</span><span class="p">));</span> + <span class="k">auto</span> <span class="n">costs</span> <span class="o">=</span> + <span class="n">std</span><span class="o">::</span><span class="n">static_pointer_cast</span><span class="o"><</span><span class="n">arrow</span><span class="o">::</span><span class="n">DoubleArray</span><span class="o">></span><span class="p">(</span><span class="n">table</span><span class="o">-></span><span class="n">column</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span><span class="o">-></span><span class="n">data</span><span class="p">()</span><span class="o">-></span><span class="n">chunk</span><span class="p">(</span><span class="mi">0</span><span class="p">));</span> + <span class="k">auto</span> <span class="n">cost_components</span> <span class="o">=</span> + <span class="n">std</span><span class="o">::</span><span class="n">static_pointer_cast</span><span class="o"><</span><span class="n">arrow</span><span class="o">::</span><span class="n">ListArray</span><span class="o">></span><span class="p">(</span><span class="n">table</span><span class="o">-></span><span class="n">column</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span><span class="o">-></span><span class="n">data</span><span class="p">()</span><span class="o">-></span><span class="n">chunk</span><span class="p">(</span><span class="mi">0</span><span class="p">));</span> + <span class="k">auto</span> <span class="n">cost_components_values</span> <span class="o">=</span> + <span class="n">std</span><span class="o">::</span><span class="n">static_pointer_cast</span><span class="o"><</span><span class="n">arrow</span><span class="o">::</span><span class="n">DoubleArray</span><span class="o">></span><span class="p">(</span><span class="n">cost_components</span><span class="o">-></span><span class="n">values</span><span class="p">());</span> + <span class="c1">// To enable zero-copy slices, the native values pointer might need to account</span> + <span class="c1">// for this slicing offset. This is not needed for the higher level functions</span> + <span class="c1">// like Value(â¦) that already account for this offset internally.</span> + <span class="k">const</span> <span class="kt">double</span><span class="o">*</span> <span class="n">ccv_ptr</span> <span class="o">=</span> <span class="n">cost_components_values</span><span class="o">-></span><span class="n">data</span><span class="p">()</span><span class="o">-></span><span class="n">GetValues</span><span class="o"><</span><span class="kt">double</span><span class="o">></span><span class="p">(</span><span class="mi">1</span><span class="p">);</span> + + <span class="k">for</span> <span class="p">(</span><span class="kt">int64_t</span> <span class="n">i</span> <span class="o">=</span> <span class="mi">0</span><span class="p">;</span> <span class="n">i</span> <span class="o"><</span> <span class="n">table</span><span class="o">-></span><span class="n">num_rows</span><span class="p">();</span> <span class="n">i</span><span class="o">++</span><span class="p">)</span> <span class="p">{</span> + <span class="c1">// Another simplification in this example is that we assume that there are</span> + <span class="c1">// no null entries, e.g. each row is fill with valid values.</span> + <span class="kt">int64_t</span> <span class="n">id</span> <span class="o">=</span> <span class="n">ids</span><span class="o">-></span><span class="n">Value</span><span class="p">(</span><span class="n">i</span><span class="p">);</span> + <span class="kt">double</span> <span class="n">cost</span> <span class="o">=</span> <span class="n">costs</span><span class="o">-></span><span class="n">Value</span><span class="p">(</span><span class="n">i</span><span class="p">);</span> + <span class="k">const</span> <span class="kt">double</span><span class="o">*</span> <span class="n">first</span> <span class="o">=</span> <span class="n">ccv_ptr</span> <span class="o">+</span> <span class="n">cost_components</span><span class="o">-></span><span class="n">value_offset</span><span class="p">(</span><span class="n">i</span><span class="p">);</span> + <span class="k">const</span> <span class="kt">double</span><span class="o">*</span> <span class="n">last</span> <span class="o">=</span> <span class="n">ccv_ptr</span> <span class="o">+</span> <span class="n">cost_components</span><span class="o">-></span><span class="n">value_offset</span><span class="p">(</span><span class="n">i</span> <span class="o">+</span> <span class="mi">1</span><span class="p">);</span> + <span class="n">std</span><span class="o">::</span><span class="n">vector</span><span class="o"><</span><span class="kt">double</span><span class="o">></span> <span class="n">components_vec</span><span class="p">(</span><span class="n">first</span><span class="p">,</span> <span class="n">last</span><span class="p">);</span> + <span class="n">rows</span><span class="o">-></span><span class="n">push_back</span><span class="p">({</span><span class="n">id</span><span class="p">,</span> <span class="n">cost</span><span class="p">,</span> <span class="n">components_vec</span><span class="p">});</span> + <span class="p">}</span> + + <span class="k">return</span> <span class="n">arrow</span><span class="o">::</span><span class="n">Status</span><span class="o">::</span><span class="n">OK</span><span class="p">();</span> +<span class="p">}</span> + +<span class="cp">#define EXIT_ON_FAILURE(expr) \</span> +<span class="cp"> do { \</span> +<span class="cp"> arrow::Status status_ = (expr); \</span> +<span class="cp"> if (!status_.ok()) { \</span> +<span class="cp"> std::cerr << status_.message() << std::endl; \</span> +<span class="cp"> return EXIT_FAILURE; \</span> +<span class="cp"> } \</span> +<span class="cp"> } while (0);</span> + +<span class="kt">int</span> <span class="n">main</span><span class="p">(</span><span class="kt">int</span> <span class="n">argc</span><span class="p">,</span> <span class="kt">char</span><span class="o">**</span> <span class="n">argv</span><span class="p">)</span> <span class="p">{</span> + <span class="n">std</span><span class="o">::</span><span class="n">vector</span><span class="o"><</span><span class="n">data_row</span><span class="o">></span> <span class="n">rows</span> <span class="o">=</span> <span class="p">{</span> + <span class="p">{</span><span class="mi">1</span><span class="p">,</span> <span class="mf">1.0</span><span class="p">,</span> <span class="p">{</span><span class="mf">1.0</span><span class="p">}},</span> <span class="p">{</span><span class="mi">2</span><span class="p">,</span> <span class="mf">2.0</span><span class="p">,</span> <span class="p">{</span><span class="mf">1.0</span><span class="p">,</span> <span class="mf">2.0</span><span class="p">}},</span> <span class="p">{</span><span class="mi">3</span><span class="p">,</span> <span class="mf">3.0</span><span class="p">,</span> <span class="p">{</span><span class="mf">1.0</span><span class="p">,</span> <span class="mf">2.0</span><span class="p">,</span> <span class="mf">3.0</span><span class="p">}}};</span> + + <span class="n">std</span><span class="o">::</span><span class="n">shared_ptr</span><span class="o"><</span><span class="n">arrow</span><span class="o">::</span><span class="n">Table</span><span class="o">></span> <span class="n">table</span><span class="p">;</span> + <span class="n">EXIT_ON_FAILURE</span><span class="p">(</span><span class="n">VectorToColumnarTable</span><span class="p">(</span><span class="n">rows</span><span class="p">,</span> <span class="o">&</span><span class="n">table</span><span class="p">));</span> + + <span class="n">std</span><span class="o">::</span><span class="n">vector</span><span class="o"><</span><span class="n">data_row</span><span class="o">></span> <span class="n">expected_rows</span><span class="p">;</span> + <span class="n">EXIT_ON_FAILURE</span><span class="p">(</span><span class="n">ColumnarTableToVector</span><span class="p">(</span><span class="n">table</span><span class="p">,</span> <span class="o">&</span><span class="n">expected_rows</span><span class="p">));</span> + + <span class="n">assert</span><span class="p">(</span><span class="n">rows</span><span class="p">.</span><span class="n">size</span><span class="p">()</span> <span class="o">==</span> <span class="n">expected_rows</span><span class="p">.</span><span class="n">size</span><span class="p">());</span> + + <span class="k">return</span> <span class="n">EXIT_SUCCESS</span><span class="p">;</span> +<span class="p">}</span> +</pre></div> +</div> +</div> +</div> + + + </div> + + </div> + <footer> + + <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation"> + + <a href="api.html" class="btn btn-neutral float-right" title="API Reference" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a> + + + <a href="tables.html" class="btn btn-neutral" title="Two-dimensional Datasets" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a> + + </div> + + + <hr/> + + <div role="contentinfo"> + <p> + © Copyright 2016-2018 Apache Software Foundation + + </p> + </div> + Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. + +</footer> + + </div> + </div> + + </section> + + </div> + + + + + + + + <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script> + <script type="text/javascript" src="../_static/jquery.js"></script> + <script type="text/javascript" src="../_static/underscore.js"></script> + <script type="text/javascript" src="../_static/doctools.js"></script> + + + + + <script type="text/javascript" src="../_static/js/theme.js"></script> + + <script type="text/javascript"> + jQuery(function () { + SphinxRtdTheme.Navigation.enable(true); + }); + </script> +<script async src="https://www.googletagmanager.com/gtag/js?id=UA-107500873-1"></script> +<script> + window.dataLayer = window.dataLayer || []; + function gtag(){dataLayer.push(arguments);} + gtag('js', new Date()); + + gtag('config', 'UA-107500873-1'); +</script> + + +</body> +</html> \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/arrow-site/blob/62ef7145/docs/latest/cpp/getting_started.html ---------------------------------------------------------------------- diff --git a/docs/latest/cpp/getting_started.html b/docs/latest/cpp/getting_started.html new file mode 100644 index 0000000..742e6b3 --- /dev/null +++ b/docs/latest/cpp/getting_started.html @@ -0,0 +1,308 @@ + + + +<!DOCTYPE html> +<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> +<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> +<head> + <meta charset="utf-8"> + + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + <title>Getting Started — Apache Arrow v0.11.1.dev473+g6ed02454</title> + + + + + + + + + + + + + + + + <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" /> + <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> + <link rel="index" title="Index" href="../genindex.html" /> + <link rel="search" title="Search" href="../search.html" /> + <link rel="next" title="High-Level Overview" href="overview.html" /> + <link rel="prev" title="C++ Implementation" href="index.html" /> + + + <script src="../_static/js/modernizr.min.js"></script> + +</head> + +<body class="wy-body-for-nav"> + + + <div class="wy-grid-for-nav"> + + + <nav data-toggle="wy-nav-shift" class="wy-nav-side"> + <div class="wy-side-scroll"> + <div class="wy-side-nav-search"> + + + + <a href="../index.html" class="icon icon-home"> Apache Arrow + + + + </a> + + + + + <div class="version"> + 0.11.1.dev473+g6ed02454 + </div> + + + + +<div role="search"> + <form id="rtd-search-form" class="wy-form" action="../search.html" method="get"> + <input type="text" name="q" placeholder="Search docs" /> + <input type="hidden" name="check_keywords" value="yes" /> + <input type="hidden" name="area" value="default" /> + </form> +</div> + + + </div> + + <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> + + + + + + + <p class="caption"><span class="caption-text">Memory Format</span></p> +<ul> +<li class="toctree-l1"><a class="reference internal" href="../format/README.html">Arrow specification documents</a></li> +<li class="toctree-l1"><a class="reference internal" href="../format/Guidelines.html">Implementation guidelines</a></li> +<li class="toctree-l1"><a class="reference internal" href="../format/Layout.html">Physical memory layout</a></li> +<li class="toctree-l1"><a class="reference internal" href="../format/Metadata.html">Metadata: Logical types, schemas, data headers</a></li> +<li class="toctree-l1"><a class="reference internal" href="../format/IPC.html">Interprocess messaging / communication (IPC)</a></li> +</ul> +<p class="caption"><span class="caption-text">Languages</span></p> +<ul class="current"> +<li class="toctree-l1 current"><a class="reference internal" href="index.html">C++ Implementation</a><ul class="current"> +<li class="toctree-l2 current"><a class="current reference internal" href="#">Getting Started</a><ul> +<li class="toctree-l3"><a class="reference internal" href="overview.html">High-Level Overview</a></li> +<li class="toctree-l3"><a class="reference internal" href="conventions.html">Conventions</a></li> +<li class="toctree-l3"><a class="reference internal" href="memory.html">Memory Management</a></li> +<li class="toctree-l3"><a class="reference internal" href="arrays.html">Arrays</a></li> +<li class="toctree-l3"><a class="reference internal" href="datatypes.html">Data Types</a></li> +<li class="toctree-l3"><a class="reference internal" href="tables.html">Two-dimensional Datasets</a></li> +</ul> +</li> +<li class="toctree-l2"><a class="reference internal" href="examples.html">Examples</a></li> +<li class="toctree-l2"><a class="reference internal" href="api.html">API Reference</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../python/index.html">Python bindings</a></li> +</ul> + + + + </div> + </div> + </nav> + + <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> + + + <nav class="wy-nav-top" aria-label="top navigation"> + + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="../index.html">Apache Arrow</a> + + </nav> + + + <div class="wy-nav-content"> + + <div class="rst-content"> + + + + + + + + + + + + + + + + + +<div role="navigation" aria-label="breadcrumbs navigation"> + + <ul class="wy-breadcrumbs"> + + <li><a href="../index.html">Docs</a> »</li> + + <li><a href="index.html">C++ Implementation</a> »</li> + + <li>Getting Started</li> + + + <li class="wy-breadcrumbs-aside"> + + + <a href="../_sources/cpp/getting_started.rst.txt" rel="nofollow"> View page source</a> + + + </li> + + </ul> + + + <hr/> +</div> + <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article"> + <div itemprop="articleBody"> + + <div class="section" id="getting-started"> +<h1>Getting Started<a class="headerlink" href="#getting-started" title="Permalink to this headline">¶</a></h1> +<div class="toctree-wrapper compound"> +<ul> +<li class="toctree-l1"><a class="reference internal" href="overview.html">High-Level Overview</a><ul> +<li class="toctree-l2"><a class="reference internal" href="overview.html#the-physical-layer">The physical layer</a></li> +<li class="toctree-l2"><a class="reference internal" href="overview.html#the-one-dimensional-layer">The one-dimensional layer</a></li> +<li class="toctree-l2"><a class="reference internal" href="overview.html#the-two-dimensional-layer">The two-dimensional layer</a></li> +<li class="toctree-l2"><a class="reference internal" href="overview.html#the-compute-layer">The compute layer</a></li> +<li class="toctree-l2"><a class="reference internal" href="overview.html#the-io-layer">The IO layer</a></li> +<li class="toctree-l2"><a class="reference internal" href="overview.html#the-inter-process-communication-ipc-layer">The Inter-Process Communication (IPC) layer</a></li> +<li class="toctree-l2"><a class="reference internal" href="overview.html#the-file-formats-layer">The file formats layer</a></li> +<li class="toctree-l2"><a class="reference internal" href="overview.html#the-devices-layer">The devices layer</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="conventions.html">Conventions</a><ul> +<li class="toctree-l2"><a class="reference internal" href="conventions.html#language-version">Language version</a></li> +<li class="toctree-l2"><a class="reference internal" href="conventions.html#namespacing">Namespacing</a></li> +<li class="toctree-l2"><a class="reference internal" href="conventions.html#safe-pointers">Safe pointers</a></li> +<li class="toctree-l2"><a class="reference internal" href="conventions.html#immutability">Immutability</a></li> +<li class="toctree-l2"><a class="reference internal" href="conventions.html#error-reporting">Error reporting</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="memory.html">Memory Management</a><ul> +<li class="toctree-l2"><a class="reference internal" href="memory.html#buffers">Buffers</a><ul> +<li class="toctree-l3"><a class="reference internal" href="memory.html#accessing-buffer-memory">Accessing Buffer Memory</a></li> +<li class="toctree-l3"><a class="reference internal" href="memory.html#slicing">Slicing</a></li> +<li class="toctree-l3"><a class="reference internal" href="memory.html#allocating-a-buffer">Allocating a Buffer</a></li> +<li class="toctree-l3"><a class="reference internal" href="memory.html#building-a-buffer">Building a Buffer</a></li> +</ul> +</li> +<li class="toctree-l2"><a class="reference internal" href="memory.html#memory-pools">Memory Pools</a><ul> +<li class="toctree-l3"><a class="reference internal" href="memory.html#default-memory-pool">Default Memory Pool</a></li> +<li class="toctree-l3"><a class="reference internal" href="memory.html#stl-integration">STL Integration</a></li> +</ul> +</li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="arrays.html">Arrays</a><ul> +<li class="toctree-l2"><a class="reference internal" href="arrays.html#building-an-array">Building an array</a><ul> +<li class="toctree-l3"><a class="reference internal" href="arrays.html#performance">Performance</a></li> +</ul> +</li> +<li class="toctree-l2"><a class="reference internal" href="arrays.html#size-limitations-and-recommendations">Size Limitations and Recommendations</a></li> +<li class="toctree-l2"><a class="reference internal" href="arrays.html#chunked-arrays">Chunked Arrays</a></li> +<li class="toctree-l2"><a class="reference internal" href="arrays.html#slicing">Slicing</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="datatypes.html">Data Types</a><ul> +<li class="toctree-l2"><a class="reference internal" href="datatypes.html#creating-data-types">Creating data types</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="tables.html">Two-dimensional Datasets</a><ul> +<li class="toctree-l2"><a class="reference internal" href="tables.html#fields">Fields</a></li> +<li class="toctree-l2"><a class="reference internal" href="tables.html#schemas">Schemas</a></li> +<li class="toctree-l2"><a class="reference internal" href="tables.html#columns">Columns</a></li> +<li class="toctree-l2"><a class="reference internal" href="tables.html#tables">Tables</a></li> +<li class="toctree-l2"><a class="reference internal" href="tables.html#record-batches">Record Batches</a></li> +</ul> +</li> +</ul> +</div> +</div> + + + </div> + + </div> + <footer> + + <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation"> + + <a href="overview.html" class="btn btn-neutral float-right" title="High-Level Overview" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a> + + + <a href="index.html" class="btn btn-neutral" title="C++ Implementation" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a> + + </div> + + + <hr/> + + <div role="contentinfo"> + <p> + © Copyright 2016-2018 Apache Software Foundation + + </p> + </div> + Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. + +</footer> + + </div> + </div> + + </section> + + </div> + + + + + + + + <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script> + <script type="text/javascript" src="../_static/jquery.js"></script> + <script type="text/javascript" src="../_static/underscore.js"></script> + <script type="text/javascript" src="../_static/doctools.js"></script> + + + + + <script type="text/javascript" src="../_static/js/theme.js"></script> + + <script type="text/javascript"> + jQuery(function () { + SphinxRtdTheme.Navigation.enable(true); + }); + </script> +<script async src="https://www.googletagmanager.com/gtag/js?id=UA-107500873-1"></script> +<script> + window.dataLayer = window.dataLayer || []; + function gtag(){dataLayer.push(arguments);} + gtag('js', new Date()); + + gtag('config', 'UA-107500873-1'); +</script> + + +</body> +</html> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/arrow-site/blob/62ef7145/docs/latest/cpp/index.html ---------------------------------------------------------------------- diff --git a/docs/latest/cpp/index.html b/docs/latest/cpp/index.html new file mode 100644 index 0000000..89fd13a --- /dev/null +++ b/docs/latest/cpp/index.html @@ -0,0 +1,264 @@ + + + +<!DOCTYPE html> +<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> +<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> +<head> + <meta charset="utf-8"> + + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + <title>C++ Implementation — Apache Arrow v0.11.1.dev473+g6ed02454</title> + + + + + + + + + + + + + + + + <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" /> + <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> + <link rel="index" title="Index" href="../genindex.html" /> + <link rel="search" title="Search" href="../search.html" /> + <link rel="next" title="Getting Started" href="getting_started.html" /> + <link rel="prev" title="Interprocess messaging / communication (IPC)" href="../format/IPC.html" /> + + + <script src="../_static/js/modernizr.min.js"></script> + +</head> + +<body class="wy-body-for-nav"> + + + <div class="wy-grid-for-nav"> + + + <nav data-toggle="wy-nav-shift" class="wy-nav-side"> + <div class="wy-side-scroll"> + <div class="wy-side-nav-search"> + + + + <a href="../index.html" class="icon icon-home"> Apache Arrow + + + + </a> + + + + + <div class="version"> + 0.11.1.dev473+g6ed02454 + </div> + + + + +<div role="search"> + <form id="rtd-search-form" class="wy-form" action="../search.html" method="get"> + <input type="text" name="q" placeholder="Search docs" /> + <input type="hidden" name="check_keywords" value="yes" /> + <input type="hidden" name="area" value="default" /> + </form> +</div> + + + </div> + + <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> + + + + + + + <p class="caption"><span class="caption-text">Memory Format</span></p> +<ul> +<li class="toctree-l1"><a class="reference internal" href="../format/README.html">Arrow specification documents</a></li> +<li class="toctree-l1"><a class="reference internal" href="../format/Guidelines.html">Implementation guidelines</a></li> +<li class="toctree-l1"><a class="reference internal" href="../format/Layout.html">Physical memory layout</a></li> +<li class="toctree-l1"><a class="reference internal" href="../format/Metadata.html">Metadata: Logical types, schemas, data headers</a></li> +<li class="toctree-l1"><a class="reference internal" href="../format/IPC.html">Interprocess messaging / communication (IPC)</a></li> +</ul> +<p class="caption"><span class="caption-text">Languages</span></p> +<ul class="current"> +<li class="toctree-l1 current"><a class="current reference internal" href="#">C++ Implementation</a><ul> +<li class="toctree-l2"><a class="reference internal" href="getting_started.html">Getting Started</a></li> +<li class="toctree-l2"><a class="reference internal" href="examples.html">Examples</a></li> +<li class="toctree-l2"><a class="reference internal" href="api.html">API Reference</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../python/index.html">Python bindings</a></li> +</ul> + + + + </div> + </div> + </nav> + + <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> + + + <nav class="wy-nav-top" aria-label="top navigation"> + + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="../index.html">Apache Arrow</a> + + </nav> + + + <div class="wy-nav-content"> + + <div class="rst-content"> + + + + + + + + + + + + + + + + + +<div role="navigation" aria-label="breadcrumbs navigation"> + + <ul class="wy-breadcrumbs"> + + <li><a href="../index.html">Docs</a> »</li> + + <li>C++ Implementation</li> + + + <li class="wy-breadcrumbs-aside"> + + + <a href="../_sources/cpp/index.rst.txt" rel="nofollow"> View page source</a> + + + </li> + + </ul> + + + <hr/> +</div> + <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article"> + <div itemprop="articleBody"> + + <div class="section" id="c-implementation"> +<h1>C++ Implementation<a class="headerlink" href="#c-implementation" title="Permalink to this headline">¶</a></h1> +<div class="toctree-wrapper compound"> +<ul> +<li class="toctree-l1"><a class="reference internal" href="getting_started.html">Getting Started</a><ul> +<li class="toctree-l2"><a class="reference internal" href="overview.html">High-Level Overview</a></li> +<li class="toctree-l2"><a class="reference internal" href="conventions.html">Conventions</a></li> +<li class="toctree-l2"><a class="reference internal" href="memory.html">Memory Management</a></li> +<li class="toctree-l2"><a class="reference internal" href="arrays.html">Arrays</a></li> +<li class="toctree-l2"><a class="reference internal" href="datatypes.html">Data Types</a></li> +<li class="toctree-l2"><a class="reference internal" href="tables.html">Two-dimensional Datasets</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="examples.html">Examples</a><ul> +<li class="toctree-l2"><a class="reference internal" href="examples.html#row-to-columnar-conversion">Row to columnar conversion</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="api.html">API Reference</a><ul> +<li class="toctree-l2"><a class="reference internal" href="api/support.html">Programming Support</a></li> +<li class="toctree-l2"><a class="reference internal" href="api/memory.html">Memory (management)</a></li> +<li class="toctree-l2"><a class="reference internal" href="api/datatype.html">Data Types</a></li> +<li class="toctree-l2"><a class="reference internal" href="api/array.html">Arrays</a></li> +<li class="toctree-l2"><a class="reference internal" href="api/builder.html">Array Builders</a></li> +<li class="toctree-l2"><a class="reference internal" href="api/table.html">Two-dimensional Datasets</a></li> +</ul> +</li> +</ul> +</div> +</div> + + + </div> + + </div> + <footer> + + <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation"> + + <a href="getting_started.html" class="btn btn-neutral float-right" title="Getting Started" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a> + + + <a href="../format/IPC.html" class="btn btn-neutral" title="Interprocess messaging / communication (IPC)" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a> + + </div> + + + <hr/> + + <div role="contentinfo"> + <p> + © Copyright 2016-2018 Apache Software Foundation + + </p> + </div> + Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. + +</footer> + + </div> + </div> + + </section> + + </div> + + + + + + + + <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script> + <script type="text/javascript" src="../_static/jquery.js"></script> + <script type="text/javascript" src="../_static/underscore.js"></script> + <script type="text/javascript" src="../_static/doctools.js"></script> + + + + + <script type="text/javascript" src="../_static/js/theme.js"></script> + + <script type="text/javascript"> + jQuery(function () { + SphinxRtdTheme.Navigation.enable(true); + }); + </script> +<script async src="https://www.googletagmanager.com/gtag/js?id=UA-107500873-1"></script> +<script> + window.dataLayer = window.dataLayer || []; + function gtag(){dataLayer.push(arguments);} + gtag('js', new Date()); + + gtag('config', 'UA-107500873-1'); +</script> + + +</body> +</html> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/arrow-site/blob/62ef7145/docs/latest/cpp/memory.html ---------------------------------------------------------------------- diff --git a/docs/latest/cpp/memory.html b/docs/latest/cpp/memory.html new file mode 100644 index 0000000..451365b --- /dev/null +++ b/docs/latest/cpp/memory.html @@ -0,0 +1,348 @@ + + + +<!DOCTYPE html> +<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> +<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> +<head> + <meta charset="utf-8"> + + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + <title>Memory Management — Apache Arrow v0.11.1.dev473+g6ed02454</title> + + + + + + + + + + + + + + + + <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" /> + <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> + <link rel="index" title="Index" href="../genindex.html" /> + <link rel="search" title="Search" href="../search.html" /> + <link rel="next" title="Arrays" href="arrays.html" /> + <link rel="prev" title="Conventions" href="conventions.html" /> + + + <script src="../_static/js/modernizr.min.js"></script> + +</head> + +<body class="wy-body-for-nav"> + + + <div class="wy-grid-for-nav"> + + + <nav data-toggle="wy-nav-shift" class="wy-nav-side"> + <div class="wy-side-scroll"> + <div class="wy-side-nav-search"> + + + + <a href="../index.html" class="icon icon-home"> Apache Arrow + + + + </a> + + + + + <div class="version"> + 0.11.1.dev473+g6ed02454 + </div> + + + + +<div role="search"> + <form id="rtd-search-form" class="wy-form" action="../search.html" method="get"> + <input type="text" name="q" placeholder="Search docs" /> + <input type="hidden" name="check_keywords" value="yes" /> + <input type="hidden" name="area" value="default" /> + </form> +</div> + + + </div> + + <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> + + + + + + + <p class="caption"><span class="caption-text">Memory Format</span></p> +<ul> +<li class="toctree-l1"><a class="reference internal" href="../format/README.html">Arrow specification documents</a></li> +<li class="toctree-l1"><a class="reference internal" href="../format/Guidelines.html">Implementation guidelines</a></li> +<li class="toctree-l1"><a class="reference internal" href="../format/Layout.html">Physical memory layout</a></li> +<li class="toctree-l1"><a class="reference internal" href="../format/Metadata.html">Metadata: Logical types, schemas, data headers</a></li> +<li class="toctree-l1"><a class="reference internal" href="../format/IPC.html">Interprocess messaging / communication (IPC)</a></li> +</ul> +<p class="caption"><span class="caption-text">Languages</span></p> +<ul class="current"> +<li class="toctree-l1 current"><a class="reference internal" href="index.html">C++ Implementation</a><ul class="current"> +<li class="toctree-l2 current"><a class="reference internal" href="getting_started.html">Getting Started</a><ul class="current"> +<li class="toctree-l3"><a class="reference internal" href="overview.html">High-Level Overview</a></li> +<li class="toctree-l3"><a class="reference internal" href="conventions.html">Conventions</a></li> +<li class="toctree-l3 current"><a class="current reference internal" href="#">Memory Management</a><ul> +<li class="toctree-l4"><a class="reference internal" href="#buffers">Buffers</a></li> +<li class="toctree-l4"><a class="reference internal" href="#memory-pools">Memory Pools</a></li> +</ul> +</li> +<li class="toctree-l3"><a class="reference internal" href="arrays.html">Arrays</a></li> +<li class="toctree-l3"><a class="reference internal" href="datatypes.html">Data Types</a></li> +<li class="toctree-l3"><a class="reference internal" href="tables.html">Two-dimensional Datasets</a></li> +</ul> +</li> +<li class="toctree-l2"><a class="reference internal" href="examples.html">Examples</a></li> +<li class="toctree-l2"><a class="reference internal" href="api.html">API Reference</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../python/index.html">Python bindings</a></li> +</ul> + + + + </div> + </div> + </nav> + + <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> + + + <nav class="wy-nav-top" aria-label="top navigation"> + + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="../index.html">Apache Arrow</a> + + </nav> + + + <div class="wy-nav-content"> + + <div class="rst-content"> + + + + + + + + + + + + + + + + + +<div role="navigation" aria-label="breadcrumbs navigation"> + + <ul class="wy-breadcrumbs"> + + <li><a href="../index.html">Docs</a> »</li> + + <li><a href="index.html">C++ Implementation</a> »</li> + + <li><a href="getting_started.html">Getting Started</a> »</li> + + <li>Memory Management</li> + + + <li class="wy-breadcrumbs-aside"> + + + <a href="../_sources/cpp/memory.rst.txt" rel="nofollow"> View page source</a> + + + </li> + + </ul> + + + <hr/> +</div> + <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article"> + <div itemprop="articleBody"> + + <div class="section" id="memory-management"> +<h1>Memory Management<a class="headerlink" href="#memory-management" title="Permalink to this headline">¶</a></h1> +<div class="section" id="buffers"> +<h2>Buffers<a class="headerlink" href="#buffers" title="Permalink to this headline">¶</a></h2> +<p>To avoid passing around raw data pointers with varying and non-obvious +lifetime rules, Arrow provides a generic abstraction called <a class="reference internal" href="api/memory.html#_CPPv3N5arrow6BufferE" title="arrow::Buffer"><code class="xref cpp cpp-class docutils literal notranslate"><span class="pre">arrow::Buffer</span></code></a>. +A Buffer encapsulates a pointer and data size, and generally also ties its +lifetime to that of an underlying provider (in other words, a Buffer should +<em>always</em> point to valid memory till its destruction). Buffers are untyped: +they simply denote a physical memory area regardless of its intended meaning +or interpretation.</p> +<p>Buffers may be allocated by Arrow itself , or by third-party routines. +For example, it is possible to pass the data of a Python bytestring as a Arrow +buffer, keeping the Python object alive as necessary.</p> +<p>In addition, buffers come in various flavours: mutable or not, resizable or +not. Generally, you will hold a mutable buffer when building up a piece +of data, then it will be frozen as an immutable container such as an +<a class="reference internal" href="arrays.html"><span class="doc">array</span></a>.</p> +<div class="admonition note"> +<p class="first admonition-title">Note</p> +<p class="last">Some buffers may point to non-CPU memory, such as GPU-backed memory +provided by a CUDA context. If youâre writing a GPU-aware application, +you will need to be careful not to interpret a GPU memory pointer as +a CPU-reachable pointer, or vice-versa.</p> +</div> +<div class="section" id="accessing-buffer-memory"> +<h3>Accessing Buffer Memory<a class="headerlink" href="#accessing-buffer-memory" title="Permalink to this headline">¶</a></h3> +<p>Buffers provide fast access to the underlying memory using the +<a class="reference internal" href="api/memory.html#_CPPv3NK5arrow6Buffer4sizeEv" title="arrow::Buffer::size"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">size()</span></code></a> and <a class="reference internal" href="api/memory.html#_CPPv3NK5arrow6Buffer4dataEv" title="arrow::Buffer::data"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">data()</span></code></a> accessors +(or <a class="reference internal" href="api/memory.html#_CPPv3N5arrow6Buffer12mutable_dataEv" title="arrow::Buffer::mutable_data"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">mutable_data()</span></code></a> for writable access to a mutable +buffer).</p> +</div> +<div class="section" id="slicing"> +<h3>Slicing<a class="headerlink" href="#slicing" title="Permalink to this headline">¶</a></h3> +<p>It is possible to make zero-copy slices of buffers, to obtain a buffer +referring to some contiguous subset of the underlying data. This is done +by calling the <a class="reference internal" href="api/memory.html#_CPPv3N5arrow11SliceBufferERKNSt10shared_ptrI6BufferEEK7int64_tK7int64_t" title="arrow::SliceBuffer"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">arrow::SliceBuffer()</span></code></a> and <a class="reference internal" href="api/memory.html#_CPPv3N5arrow18SliceMutableBufferERKNSt10shared_ptrI6BufferEEK7int64_tK7int64_t" title="arrow::SliceMutableBuffer"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">arrow::SliceMutableBuffer()</span></code></a> +functions.</p> +</div> +<div class="section" id="allocating-a-buffer"> +<h3>Allocating a Buffer<a class="headerlink" href="#allocating-a-buffer" title="Permalink to this headline">¶</a></h3> +<p>You can allocate a buffer yourself by calling one of the +<a class="reference internal" href="api/memory.html#_CPPv3N5arrow14AllocateBufferEP10MemoryPoolK7int64_tPNSt10shared_ptrI6BufferEE" title="arrow::AllocateBuffer"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">arrow::AllocateBuffer()</span></code></a> or <a class="reference internal" href="api/memory.html#_CPPv3N5arrow23AllocateResizableBufferEP10MemoryPoolK7int64_tPNSt10shared_ptrI15ResizableBufferEE" title="arrow::AllocateResizableBuffer"><code class="xref cpp cpp-func docutils literal notranslate"><span class="pre">arrow::AllocateResizableBuffer()</span></code></a> +overloads:</p> +<div class="highlight-cpp notranslate"><div class="highlight"><pre><span></span><span class="n">std</span><span class="o">::</span><span class="n">shared_ptr</span><span class="o"><</span><span class="n">arrow</span><span class="o">::</span><span class="n">Buffer</span><span class="o">></span> <span class="n">buffer</span><span class="p">;</span> + +<span class="k">if</span> <span class="p">(</span><span class="o">!</span><span class="n">arrow</span><span class="o">::</span><span class="n">AllocateBuffer</span><span class="p">(</span><span class="mi">4096</span><span class="p">,</span> <span class="o">&</span><span class="n">buffer</span><span class="p">).</span><span class="n">ok</span><span class="p">())</span> <span class="p">{</span> + <span class="c1">// ... handle allocation error</span> +<span class="p">}</span> +<span class="kt">uint8_t</span><span class="o">*</span> <span class="n">buffer_data</span> <span class="o">=</span> <span class="n">buffer</span><span class="o">-></span><span class="n">mutable_data</span><span class="p">();</span> +<span class="n">memcpy</span><span class="p">(</span><span class="n">buffer_data</span><span class="p">,</span> <span class="s">"hello world"</span><span class="p">,</span> <span class="mi">11</span><span class="p">);</span> +</pre></div> +</div> +<p>Allocating a buffer this way ensures it is 64-bytes aligned and padded +as recommended by the <a class="reference internal" href="../format/Layout.html"><span class="doc">Arrow memory specification</span></a>.</p> +</div> +<div class="section" id="building-a-buffer"> +<h3>Building a Buffer<a class="headerlink" href="#building-a-buffer" title="Permalink to this headline">¶</a></h3> +<p>You can also allocate <em>and</em> build a Buffer incrementally, using the +<a class="reference internal" href="api/memory.html#_CPPv3N5arrow13BufferBuilderE" title="arrow::BufferBuilder"><code class="xref cpp cpp-class docutils literal notranslate"><span class="pre">arrow::BufferBuilder</span></code></a> API:</p> +<div class="highlight-cpp notranslate"><div class="highlight"><pre><span></span><span class="n">BufferBuilder</span> <span class="n">builder</span><span class="p">;</span> +<span class="n">builder</span><span class="p">.</span><span class="n">Resize</span><span class="p">(</span><span class="mi">11</span><span class="p">);</span> +<span class="n">builder</span><span class="p">.</span><span class="n">Append</span><span class="p">(</span><span class="s">"hello "</span><span class="p">,</span> <span class="mi">6</span><span class="p">);</span> +<span class="n">builder</span><span class="p">.</span><span class="n">Append</span><span class="p">(</span><span class="s">"world"</span><span class="p">,</span> <span class="mi">5</span><span class="p">);</span> + +<span class="n">std</span><span class="o">::</span><span class="n">shared_ptr</span><span class="o"><</span><span class="n">arrow</span><span class="o">::</span><span class="n">Buffer</span><span class="o">></span> <span class="n">buffer</span><span class="p">;</span> +<span class="k">if</span> <span class="p">(</span><span class="o">!</span><span class="n">builder</span><span class="p">.</span><span class="n">Finish</span><span class="p">(</span><span class="o">&</span><span class="n">buffer</span><span class="p">).</span><span class="n">ok</span><span class="p">())</span> <span class="p">{</span> + <span class="c1">// ... handle buffer allocation error</span> +<span class="p">}</span> +</pre></div> +</div> +</div> +</div> +<div class="section" id="memory-pools"> +<h2>Memory Pools<a class="headerlink" href="#memory-pools" title="Permalink to this headline">¶</a></h2> +<p>When allocating a Buffer using the Arrow C++ API, the bufferâs underlying +memory is allocated by a <a class="reference internal" href="api/memory.html#_CPPv3N5arrow10MemoryPoolE" title="arrow::MemoryPool"><code class="xref cpp cpp-class docutils literal notranslate"><span class="pre">arrow::MemoryPool</span></code></a> instance. Usually this +will be the process-wide <em>default memory pool</em>, but many Arrow APIs allow +you to pass another MemoryPool instance for their internal allocations.</p> +<p>Memory pools are used for large long-lived data such as array buffers. +Other data, such as small C++ objects and temporary workspaces, usually +goes through the regular C++ allocators.</p> +<div class="section" id="default-memory-pool"> +<h3>Default Memory Pool<a class="headerlink" href="#default-memory-pool" title="Permalink to this headline">¶</a></h3> +<p>Depending on how Arrow was compiled, the default memory pool may use the +standard C <code class="docutils literal notranslate"><span class="pre">malloc</span></code> allocator, or a <a class="reference external" href="http://jemalloc.net/">jemalloc</a> heap.</p> +</div> +<div class="section" id="stl-integration"> +<h3>STL Integration<a class="headerlink" href="#stl-integration" title="Permalink to this headline">¶</a></h3> +<p>If you wish to use a Arrow memory pool to allocate the data of STL containers, +you can do so using the <a class="reference internal" href="api/memory.html#_CPPv3N5arrow13stl_allocatorE" title="arrow::stl_allocator"><code class="xref cpp cpp-class docutils literal notranslate"><span class="pre">arrow::stl_allocator</span></code></a> wrapper.</p> +<p>Conversely, you can also use a STL allocator to allocate Arrow memory, +using the <a class="reference internal" href="api/memory.html#_CPPv3N5arrow13STLMemoryPoolE" title="arrow::STLMemoryPool"><code class="xref cpp cpp-class docutils literal notranslate"><span class="pre">arrow::STLMemoryPool</span></code></a> class. However, this may be less +performant, as STL allocators donât provide a resizing operation.</p> +</div> +</div> +</div> + + + </div> + + </div> + <footer> + + <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation"> + + <a href="arrays.html" class="btn btn-neutral float-right" title="Arrays" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a> + + + <a href="conventions.html" class="btn btn-neutral" title="Conventions" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a> + + </div> + + + <hr/> + + <div role="contentinfo"> + <p> + © Copyright 2016-2018 Apache Software Foundation + + </p> + </div> + Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. + +</footer> + + </div> + </div> + + </section> + + </div> + + + + + + + + <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script> + <script type="text/javascript" src="../_static/jquery.js"></script> + <script type="text/javascript" src="../_static/underscore.js"></script> + <script type="text/javascript" src="../_static/doctools.js"></script> + + + + + <script type="text/javascript" src="../_static/js/theme.js"></script> + + <script type="text/javascript"> + jQuery(function () { + SphinxRtdTheme.Navigation.enable(true); + }); + </script> +<script async src="https://www.googletagmanager.com/gtag/js?id=UA-107500873-1"></script> +<script> + window.dataLayer = window.dataLayer || []; + function gtag(){dataLayer.push(arguments);} + gtag('js', new Date()); + + gtag('config', 'UA-107500873-1'); +</script> + + +</body> +</html> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/arrow-site/blob/62ef7145/docs/latest/cpp/overview.html ---------------------------------------------------------------------- diff --git a/docs/latest/cpp/overview.html b/docs/latest/cpp/overview.html new file mode 100644 index 0000000..1b4da24 --- /dev/null +++ b/docs/latest/cpp/overview.html @@ -0,0 +1,316 @@ + + + +<!DOCTYPE html> +<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]--> +<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]--> +<head> + <meta charset="utf-8"> + + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + + <title>High-Level Overview — Apache Arrow v0.11.1.dev473+g6ed02454</title> + + + + + + + + + + + + + + + + <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" /> + <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> + <link rel="index" title="Index" href="../genindex.html" /> + <link rel="search" title="Search" href="../search.html" /> + <link rel="next" title="Conventions" href="conventions.html" /> + <link rel="prev" title="Getting Started" href="getting_started.html" /> + + + <script src="../_static/js/modernizr.min.js"></script> + +</head> + +<body class="wy-body-for-nav"> + + + <div class="wy-grid-for-nav"> + + + <nav data-toggle="wy-nav-shift" class="wy-nav-side"> + <div class="wy-side-scroll"> + <div class="wy-side-nav-search"> + + + + <a href="../index.html" class="icon icon-home"> Apache Arrow + + + + </a> + + + + + <div class="version"> + 0.11.1.dev473+g6ed02454 + </div> + + + + +<div role="search"> + <form id="rtd-search-form" class="wy-form" action="../search.html" method="get"> + <input type="text" name="q" placeholder="Search docs" /> + <input type="hidden" name="check_keywords" value="yes" /> + <input type="hidden" name="area" value="default" /> + </form> +</div> + + + </div> + + <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation"> + + + + + + + <p class="caption"><span class="caption-text">Memory Format</span></p> +<ul> +<li class="toctree-l1"><a class="reference internal" href="../format/README.html">Arrow specification documents</a></li> +<li class="toctree-l1"><a class="reference internal" href="../format/Guidelines.html">Implementation guidelines</a></li> +<li class="toctree-l1"><a class="reference internal" href="../format/Layout.html">Physical memory layout</a></li> +<li class="toctree-l1"><a class="reference internal" href="../format/Metadata.html">Metadata: Logical types, schemas, data headers</a></li> +<li class="toctree-l1"><a class="reference internal" href="../format/IPC.html">Interprocess messaging / communication (IPC)</a></li> +</ul> +<p class="caption"><span class="caption-text">Languages</span></p> +<ul class="current"> +<li class="toctree-l1 current"><a class="reference internal" href="index.html">C++ Implementation</a><ul class="current"> +<li class="toctree-l2 current"><a class="reference internal" href="getting_started.html">Getting Started</a><ul class="current"> +<li class="toctree-l3 current"><a class="current reference internal" href="#">High-Level Overview</a><ul> +<li class="toctree-l4"><a class="reference internal" href="#the-physical-layer">The physical layer</a></li> +<li class="toctree-l4"><a class="reference internal" href="#the-one-dimensional-layer">The one-dimensional layer</a></li> +<li class="toctree-l4"><a class="reference internal" href="#the-two-dimensional-layer">The two-dimensional layer</a></li> +<li class="toctree-l4"><a class="reference internal" href="#the-compute-layer">The compute layer</a></li> +<li class="toctree-l4"><a class="reference internal" href="#the-io-layer">The IO layer</a></li> +<li class="toctree-l4"><a class="reference internal" href="#the-inter-process-communication-ipc-layer">The Inter-Process Communication (IPC) layer</a></li> +<li class="toctree-l4"><a class="reference internal" href="#the-file-formats-layer">The file formats layer</a></li> +<li class="toctree-l4"><a class="reference internal" href="#the-devices-layer">The devices layer</a></li> +</ul> +</li> +<li class="toctree-l3"><a class="reference internal" href="conventions.html">Conventions</a></li> +<li class="toctree-l3"><a class="reference internal" href="memory.html">Memory Management</a></li> +<li class="toctree-l3"><a class="reference internal" href="arrays.html">Arrays</a></li> +<li class="toctree-l3"><a class="reference internal" href="datatypes.html">Data Types</a></li> +<li class="toctree-l3"><a class="reference internal" href="tables.html">Two-dimensional Datasets</a></li> +</ul> +</li> +<li class="toctree-l2"><a class="reference internal" href="examples.html">Examples</a></li> +<li class="toctree-l2"><a class="reference internal" href="api.html">API Reference</a></li> +</ul> +</li> +<li class="toctree-l1"><a class="reference internal" href="../python/index.html">Python bindings</a></li> +</ul> + + + + </div> + </div> + </nav> + + <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap"> + + + <nav class="wy-nav-top" aria-label="top navigation"> + + <i data-toggle="wy-nav-top" class="fa fa-bars"></i> + <a href="../index.html">Apache Arrow</a> + + </nav> + + + <div class="wy-nav-content"> + + <div class="rst-content"> + + + + + + + + + + + + + + + + + +<div role="navigation" aria-label="breadcrumbs navigation"> + + <ul class="wy-breadcrumbs"> + + <li><a href="../index.html">Docs</a> »</li> + + <li><a href="index.html">C++ Implementation</a> »</li> + + <li><a href="getting_started.html">Getting Started</a> »</li> + + <li>High-Level Overview</li> + + + <li class="wy-breadcrumbs-aside"> + + + <a href="../_sources/cpp/overview.rst.txt" rel="nofollow"> View page source</a> + + + </li> + + </ul> + + + <hr/> +</div> + <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article"> + <div itemprop="articleBody"> + + <div class="section" id="high-level-overview"> +<h1>High-Level Overview<a class="headerlink" href="#high-level-overview" title="Permalink to this headline">¶</a></h1> +<p>The Arrow C++ library is comprised of different parts, each of which serves +a specific purpose.</p> +<div class="section" id="the-physical-layer"> +<h2>The physical layer<a class="headerlink" href="#the-physical-layer" title="Permalink to this headline">¶</a></h2> +<p><strong>Memory management</strong> abstractions provide a uniform API over memory that +may be allocated through various means, such as heap allocation, the memory +mapping of a file or a static memory area. In particular, the <strong>buffer</strong> +abstraction represents a contiguous area of physical data.</p> +</div> +<div class="section" id="the-one-dimensional-layer"> +<h2>The one-dimensional layer<a class="headerlink" href="#the-one-dimensional-layer" title="Permalink to this headline">¶</a></h2> +<p><strong>Data types</strong> govern the <em>logical</em> interpretation of <em>physical</em> data. +Many operations in Arrow are parametered, at compile-time or at runtime, +by a data type.</p> +<p><strong>Arrays</strong> assemble one or several buffers with a data type, allowing to +view them as a logical contiguous sequence of values (possibly nested).</p> +<p><strong>Chunked arrays</strong> are a generalization of arrays, comprising several same-type +arrays into a longer logical sequence of values.</p> +</div> +<div class="section" id="the-two-dimensional-layer"> +<h2>The two-dimensional layer<a class="headerlink" href="#the-two-dimensional-layer" title="Permalink to this headline">¶</a></h2> +<p><strong>Schemas</strong> describe a logical collection of several pieces of data, +each with a distinct name and type, and optional metadata.</p> +<p><strong>Columns</strong> are like chunked arrays, but with optional metadata.</p> +<p><strong>Tables</strong> are collections of columns in accordance to a schema. They are +the most capable dataset-providing abstraction in Arrow.</p> +<p><strong>Record batches</strong> are collections of contiguous arrays, described +by a schema. They allow incremental construction or serialization of tables.</p> +</div> +<div class="section" id="the-compute-layer"> +<h2>The compute layer<a class="headerlink" href="#the-compute-layer" title="Permalink to this headline">¶</a></h2> +<p><strong>Datums</strong> are flexible dataset references, able to hold for example an array or table +reference.</p> +<p><strong>Kernels</strong> are specialized computation functions running in a loop over a +given set of datums representing input and output parameters to the functions.</p> +</div> +<div class="section" id="the-io-layer"> +<h2>The IO layer<a class="headerlink" href="#the-io-layer" title="Permalink to this headline">¶</a></h2> +<p><strong>Streams</strong> allow untyped sequential or seekable access over external data +of various kinds (for example compressed or memory-mapped).</p> +</div> +<div class="section" id="the-inter-process-communication-ipc-layer"> +<h2>The Inter-Process Communication (IPC) layer<a class="headerlink" href="#the-inter-process-communication-ipc-layer" title="Permalink to this headline">¶</a></h2> +<p>A <strong>messaging format</strong> allows interchange of Arrow data between processes, using +as few copies as possible.</p> +</div> +<div class="section" id="the-file-formats-layer"> +<h2>The file formats layer<a class="headerlink" href="#the-file-formats-layer" title="Permalink to this headline">¶</a></h2> +<p>Reading and writing Arrow data from/to various file formats is possible, for +example <strong>Parquet</strong>, <strong>CSV</strong>, <strong>Orc</strong> or the Arrow-specific <strong>Feather</strong> format.</p> +</div> +<div class="section" id="the-devices-layer"> +<h2>The devices layer<a class="headerlink" href="#the-devices-layer" title="Permalink to this headline">¶</a></h2> +<p>Basic <strong>CUDA</strong> integration is provided, allowing to describe Arrow data backed +by GPU-allocated memory.</p> +</div> +</div> + + + </div> + + </div> + <footer> + + <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation"> + + <a href="conventions.html" class="btn btn-neutral float-right" title="Conventions" accesskey="n" rel="next">Next <span class="fa fa-arrow-circle-right"></span></a> + + + <a href="getting_started.html" class="btn btn-neutral" title="Getting Started" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> Previous</a> + + </div> + + + <hr/> + + <div role="contentinfo"> + <p> + © Copyright 2016-2018 Apache Software Foundation + + </p> + </div> + Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>. + +</footer> + + </div> + </div> + + </section> + + </div> + + + + + + + + <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script> + <script type="text/javascript" src="../_static/jquery.js"></script> + <script type="text/javascript" src="../_static/underscore.js"></script> + <script type="text/javascript" src="../_static/doctools.js"></script> + + + + + <script type="text/javascript" src="../_static/js/theme.js"></script> + + <script type="text/javascript"> + jQuery(function () { + SphinxRtdTheme.Navigation.enable(true); + }); + </script> +<script async src="https://www.googletagmanager.com/gtag/js?id=UA-107500873-1"></script> +<script> + window.dataLayer = window.dataLayer || []; + function gtag(){dataLayer.push(arguments);} + gtag('js', new Date()); + + gtag('config', 'UA-107500873-1'); +</script> + + +</body> +</html> \ No newline at end of file