http://git-wip-us.apache.org/repos/asf/arrow-site/blob/4d4a3202/docs/python/api.html ---------------------------------------------------------------------- diff --git a/docs/python/api.html b/docs/python/api.html index a35a846..35d537e 100644 --- a/docs/python/api.html +++ b/docs/python/api.html @@ -73,7 +73,8 @@ <li class="toctree-l1"><a class="reference internal" href="memory.html">Memory and IO Interfaces</a></li> <li class="toctree-l1"><a class="reference internal" href="data.html">In-Memory Data Model</a></li> <li class="toctree-l1"><a class="reference internal" href="ipc.html">IPC: Fast Streaming and Serialization</a></li> -<li class="toctree-l1"><a class="reference internal" href="filesystems.html">Filesystem Interfaces</a></li> +<li class="toctree-l1"><a class="reference internal" href="filesystems.html">File System Interfaces</a></li> +<li class="toctree-l1"><a class="reference internal" href="plasma.html">The Plasma In-Memory Object Store</a></li> <li class="toctree-l1"><a class="reference internal" href="pandas.html">Using PyArrow with pandas</a></li> <li class="toctree-l1"><a class="reference internal" href="parquet.html">Reading and Writing the Apache Parquet Format</a></li> <li class="toctree-l1 current"><a class="current reference internal" href="#">API Reference</a></li> @@ -94,13 +95,15 @@ <li><a class="reference internal" href="#">API Reference</a><ul> <li><a class="reference internal" href="#type-and-schema-factory-functions">Type and Schema Factory Functions</a></li> <li><a class="reference internal" href="#scalar-value-types">Scalar Value Types</a></li> -<li><a class="reference internal" href="#array-types-and-constructors">Array Types and Constructors</a></li> +<li><a class="reference internal" href="#array-types">Array Types</a></li> <li><a class="reference internal" href="#tables-and-record-batches">Tables and Record Batches</a></li> <li><a class="reference internal" href="#tensor-type-and-functions">Tensor type and Functions</a></li> <li><a class="reference internal" href="#input-output-and-shared-memory">Input / Output and Shared Memory</a></li> +<li><a class="reference internal" href="#file-systems">File Systems</a></li> <li><a class="reference internal" href="#interprocess-communication-and-messaging">Interprocess Communication and Messaging</a></li> <li><a class="reference internal" href="#memory-pools">Memory Pools</a></li> <li><a class="reference internal" href="#type-classes">Type Classes</a></li> +<li><a class="reference internal" href="#in-memory-object-store">In-Memory Object Store</a></li> <li><a class="reference internal" href="#apache-parquet">Apache Parquet</a></li> </ul> </li> @@ -322,90 +325,87 @@ </tbody> </table> </div> -<div class="section" id="array-types-and-constructors"> -<span id="api-array"></span><h2>Array Types and Constructors<a class="headerlink" href="#array-types-and-constructors" title="Permalink to this headline">¶</a></h2> +<div class="section" id="array-types"> +<span id="api-array"></span><h2>Array Types<a class="headerlink" href="#array-types" title="Permalink to this headline">¶</a></h2> <table border="1" class="longtable docutils"> <colgroup> <col width="10%" /> <col width="90%" /> </colgroup> <tbody valign="top"> -<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.array.html#pyarrow.array" title="pyarrow.array"><code class="xref py py-obj docutils literal"><span class="pre">array</span></code></a>(sequence, DataType type=None, ...[, size])</td> -<td>Create pyarrow.Array instance from a Python sequence</td> -</tr> -<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.Array.html#pyarrow.Array" title="pyarrow.Array"><code class="xref py py-obj docutils literal"><span class="pre">Array</span></code></a></td> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.lib.Array.html#pyarrow.lib.Array" title="pyarrow.lib.Array"><code class="xref py py-obj docutils literal"><span class="pre">Array</span></code></a></td> <td></td> </tr> -<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.BooleanArray.html#pyarrow.BooleanArray" title="pyarrow.BooleanArray"><code class="xref py py-obj docutils literal"><span class="pre">BooleanArray</span></code></a></td> +<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.lib.BooleanArray.html#pyarrow.lib.BooleanArray" title="pyarrow.lib.BooleanArray"><code class="xref py py-obj docutils literal"><span class="pre">BooleanArray</span></code></a></td> <td></td> </tr> -<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.DictionaryArray.html#pyarrow.DictionaryArray" title="pyarrow.DictionaryArray"><code class="xref py py-obj docutils literal"><span class="pre">DictionaryArray</span></code></a></td> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.lib.DictionaryArray.html#pyarrow.lib.DictionaryArray" title="pyarrow.lib.DictionaryArray"><code class="xref py py-obj docutils literal"><span class="pre">DictionaryArray</span></code></a></td> <td></td> </tr> -<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.FloatingPointArray.html#pyarrow.FloatingPointArray" title="pyarrow.FloatingPointArray"><code class="xref py py-obj docutils literal"><span class="pre">FloatingPointArray</span></code></a></td> +<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.lib.FloatingPointArray.html#pyarrow.lib.FloatingPointArray" title="pyarrow.lib.FloatingPointArray"><code class="xref py py-obj docutils literal"><span class="pre">FloatingPointArray</span></code></a></td> <td></td> </tr> -<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.IntegerArray.html#pyarrow.IntegerArray" title="pyarrow.IntegerArray"><code class="xref py py-obj docutils literal"><span class="pre">IntegerArray</span></code></a></td> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.lib.IntegerArray.html#pyarrow.lib.IntegerArray" title="pyarrow.lib.IntegerArray"><code class="xref py py-obj docutils literal"><span class="pre">IntegerArray</span></code></a></td> <td></td> </tr> -<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.Int8Array.html#pyarrow.Int8Array" title="pyarrow.Int8Array"><code class="xref py py-obj docutils literal"><span class="pre">Int8Array</span></code></a></td> +<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.lib.Int8Array.html#pyarrow.lib.Int8Array" title="pyarrow.lib.Int8Array"><code class="xref py py-obj docutils literal"><span class="pre">Int8Array</span></code></a></td> <td></td> </tr> -<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.Int16Array.html#pyarrow.Int16Array" title="pyarrow.Int16Array"><code class="xref py py-obj docutils literal"><span class="pre">Int16Array</span></code></a></td> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.lib.Int16Array.html#pyarrow.lib.Int16Array" title="pyarrow.lib.Int16Array"><code class="xref py py-obj docutils literal"><span class="pre">Int16Array</span></code></a></td> <td></td> </tr> -<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.Int32Array.html#pyarrow.Int32Array" title="pyarrow.Int32Array"><code class="xref py py-obj docutils literal"><span class="pre">Int32Array</span></code></a></td> +<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.lib.Int32Array.html#pyarrow.lib.Int32Array" title="pyarrow.lib.Int32Array"><code class="xref py py-obj docutils literal"><span class="pre">Int32Array</span></code></a></td> <td></td> </tr> -<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.Int64Array.html#pyarrow.Int64Array" title="pyarrow.Int64Array"><code class="xref py py-obj docutils literal"><span class="pre">Int64Array</span></code></a></td> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.lib.Int64Array.html#pyarrow.lib.Int64Array" title="pyarrow.lib.Int64Array"><code class="xref py py-obj docutils literal"><span class="pre">Int64Array</span></code></a></td> <td></td> </tr> -<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.NullArray.html#pyarrow.NullArray" title="pyarrow.NullArray"><code class="xref py py-obj docutils literal"><span class="pre">NullArray</span></code></a></td> +<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.lib.NullArray.html#pyarrow.lib.NullArray" title="pyarrow.lib.NullArray"><code class="xref py py-obj docutils literal"><span class="pre">NullArray</span></code></a></td> <td></td> </tr> -<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.NumericArray.html#pyarrow.NumericArray" title="pyarrow.NumericArray"><code class="xref py py-obj docutils literal"><span class="pre">NumericArray</span></code></a></td> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.lib.NumericArray.html#pyarrow.lib.NumericArray" title="pyarrow.lib.NumericArray"><code class="xref py py-obj docutils literal"><span class="pre">NumericArray</span></code></a></td> <td></td> </tr> -<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.UInt8Array.html#pyarrow.UInt8Array" title="pyarrow.UInt8Array"><code class="xref py py-obj docutils literal"><span class="pre">UInt8Array</span></code></a></td> +<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.lib.UInt8Array.html#pyarrow.lib.UInt8Array" title="pyarrow.lib.UInt8Array"><code class="xref py py-obj docutils literal"><span class="pre">UInt8Array</span></code></a></td> <td></td> </tr> -<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.UInt16Array.html#pyarrow.UInt16Array" title="pyarrow.UInt16Array"><code class="xref py py-obj docutils literal"><span class="pre">UInt16Array</span></code></a></td> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.lib.UInt16Array.html#pyarrow.lib.UInt16Array" title="pyarrow.lib.UInt16Array"><code class="xref py py-obj docutils literal"><span class="pre">UInt16Array</span></code></a></td> <td></td> </tr> -<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.UInt32Array.html#pyarrow.UInt32Array" title="pyarrow.UInt32Array"><code class="xref py py-obj docutils literal"><span class="pre">UInt32Array</span></code></a></td> +<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.lib.UInt32Array.html#pyarrow.lib.UInt32Array" title="pyarrow.lib.UInt32Array"><code class="xref py py-obj docutils literal"><span class="pre">UInt32Array</span></code></a></td> <td></td> </tr> -<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.UInt64Array.html#pyarrow.UInt64Array" title="pyarrow.UInt64Array"><code class="xref py py-obj docutils literal"><span class="pre">UInt64Array</span></code></a></td> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.lib.UInt64Array.html#pyarrow.lib.UInt64Array" title="pyarrow.lib.UInt64Array"><code class="xref py py-obj docutils literal"><span class="pre">UInt64Array</span></code></a></td> <td></td> </tr> -<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.BinaryArray.html#pyarrow.BinaryArray" title="pyarrow.BinaryArray"><code class="xref py py-obj docutils literal"><span class="pre">BinaryArray</span></code></a></td> +<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.lib.BinaryArray.html#pyarrow.lib.BinaryArray" title="pyarrow.lib.BinaryArray"><code class="xref py py-obj docutils literal"><span class="pre">BinaryArray</span></code></a></td> <td></td> </tr> -<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.FixedSizeBinaryArray.html#pyarrow.FixedSizeBinaryArray" title="pyarrow.FixedSizeBinaryArray"><code class="xref py py-obj docutils literal"><span class="pre">FixedSizeBinaryArray</span></code></a></td> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.lib.FixedSizeBinaryArray.html#pyarrow.lib.FixedSizeBinaryArray" title="pyarrow.lib.FixedSizeBinaryArray"><code class="xref py py-obj docutils literal"><span class="pre">FixedSizeBinaryArray</span></code></a></td> <td></td> </tr> -<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.StringArray.html#pyarrow.StringArray" title="pyarrow.StringArray"><code class="xref py py-obj docutils literal"><span class="pre">StringArray</span></code></a></td> +<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.lib.StringArray.html#pyarrow.lib.StringArray" title="pyarrow.lib.StringArray"><code class="xref py py-obj docutils literal"><span class="pre">StringArray</span></code></a></td> <td></td> </tr> -<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.Time32Array.html#pyarrow.Time32Array" title="pyarrow.Time32Array"><code class="xref py py-obj docutils literal"><span class="pre">Time32Array</span></code></a></td> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.lib.Time32Array.html#pyarrow.lib.Time32Array" title="pyarrow.lib.Time32Array"><code class="xref py py-obj docutils literal"><span class="pre">Time32Array</span></code></a></td> <td></td> </tr> -<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.Time64Array.html#pyarrow.Time64Array" title="pyarrow.Time64Array"><code class="xref py py-obj docutils literal"><span class="pre">Time64Array</span></code></a></td> +<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.lib.Time64Array.html#pyarrow.lib.Time64Array" title="pyarrow.lib.Time64Array"><code class="xref py py-obj docutils literal"><span class="pre">Time64Array</span></code></a></td> <td></td> </tr> -<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.Date32Array.html#pyarrow.Date32Array" title="pyarrow.Date32Array"><code class="xref py py-obj docutils literal"><span class="pre">Date32Array</span></code></a></td> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.lib.Date32Array.html#pyarrow.lib.Date32Array" title="pyarrow.lib.Date32Array"><code class="xref py py-obj docutils literal"><span class="pre">Date32Array</span></code></a></td> <td></td> </tr> -<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.Date64Array.html#pyarrow.Date64Array" title="pyarrow.Date64Array"><code class="xref py py-obj docutils literal"><span class="pre">Date64Array</span></code></a></td> +<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.lib.Date64Array.html#pyarrow.lib.Date64Array" title="pyarrow.lib.Date64Array"><code class="xref py py-obj docutils literal"><span class="pre">Date64Array</span></code></a></td> <td></td> </tr> -<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.TimestampArray.html#pyarrow.TimestampArray" title="pyarrow.TimestampArray"><code class="xref py py-obj docutils literal"><span class="pre">TimestampArray</span></code></a></td> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.lib.TimestampArray.html#pyarrow.lib.TimestampArray" title="pyarrow.lib.TimestampArray"><code class="xref py py-obj docutils literal"><span class="pre">TimestampArray</span></code></a></td> <td></td> </tr> -<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.DecimalArray.html#pyarrow.DecimalArray" title="pyarrow.DecimalArray"><code class="xref py py-obj docutils literal"><span class="pre">DecimalArray</span></code></a></td> +<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.lib.DecimalArray.html#pyarrow.lib.DecimalArray" title="pyarrow.lib.DecimalArray"><code class="xref py py-obj docutils literal"><span class="pre">DecimalArray</span></code></a></td> <td></td> </tr> -<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.ListArray.html#pyarrow.ListArray" title="pyarrow.ListArray"><code class="xref py py-obj docutils literal"><span class="pre">ListArray</span></code></a></td> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.lib.ListArray.html#pyarrow.lib.ListArray" title="pyarrow.lib.ListArray"><code class="xref py py-obj docutils literal"><span class="pre">ListArray</span></code></a></td> <td></td> </tr> </tbody> @@ -483,6 +483,28 @@ </tbody> </table> </div> +<div class="section" id="file-systems"> +<h2>File Systems<a class="headerlink" href="#file-systems" title="Permalink to this headline">¶</a></h2> +<table border="1" class="longtable docutils"> +<colgroup> +<col width="10%" /> +<col width="90%" /> +</colgroup> +<tbody valign="top"> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.hdfs.connect.html#pyarrow.hdfs.connect" title="pyarrow.hdfs.connect"><code class="xref py py-obj docutils literal"><span class="pre">hdfs.connect</span></code></a>([host, port, user, ...])</td> +<td>Connect to an HDFS cluster.</td> +</tr> +<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.LocalFileSystem.html#pyarrow.LocalFileSystem" title="pyarrow.LocalFileSystem"><code class="xref py py-obj docutils literal"><span class="pre">LocalFileSystem</span></code></a></td> +<td></td> +</tr> +</tbody> +</table> +<dl class="class"> +<dt> +<em class="property">class </em><code class="descclassname">pyarrow.</code><code class="descname">HadoopFileSystem</code><a class="reference internal" href="_modules/pyarrow/hdfs.html#HadoopFileSystem"><span class="viewcode-link">[source]</span></a></dt> +<dd></dd></dl> + +</div> <div class="section" id="interprocess-communication-and-messaging"> <span id="api-ipc"></span><h2>Interprocess Communication and Messaging<a class="headerlink" href="#interprocess-communication-and-messaging" title="Permalink to this headline">¶</a></h2> <table border="1" class="longtable docutils"> @@ -567,18 +589,38 @@ <col width="90%" /> </colgroup> <tbody valign="top"> -<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.DataType.html#pyarrow.DataType" title="pyarrow.DataType"><code class="xref py py-obj docutils literal"><span class="pre">DataType</span></code></a></td> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.lib.DataType.html#pyarrow.lib.DataType" title="pyarrow.lib.DataType"><code class="xref py py-obj docutils literal"><span class="pre">DataType</span></code></a></td> <td>Base type for Apache Arrow data type instances.</td> </tr> -<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.Field.html#pyarrow.Field" title="pyarrow.Field"><code class="xref py py-obj docutils literal"><span class="pre">Field</span></code></a></td> +<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.lib.Field.html#pyarrow.lib.Field" title="pyarrow.lib.Field"><code class="xref py py-obj docutils literal"><span class="pre">Field</span></code></a></td> <td>Represents a named field, with a data type, nullability, and optional</td> </tr> -<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.Schema.html#pyarrow.Schema" title="pyarrow.Schema"><code class="xref py py-obj docutils literal"><span class="pre">Schema</span></code></a></td> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.lib.Schema.html#pyarrow.lib.Schema" title="pyarrow.lib.Schema"><code class="xref py py-obj docutils literal"><span class="pre">Schema</span></code></a></td> <td></td> </tr> </tbody> </table> </div> +<div class="section" id="in-memory-object-store"> +<span id="api-plasma"></span><h2>In-Memory Object Store<a class="headerlink" href="#in-memory-object-store" title="Permalink to this headline">¶</a></h2> +<table border="1" class="longtable docutils"> +<colgroup> +<col width="10%" /> +<col width="90%" /> +</colgroup> +<tbody valign="top"> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.plasma.ObjectID.html#pyarrow.plasma.ObjectID" title="pyarrow.plasma.ObjectID"><code class="xref py py-obj docutils literal"><span class="pre">ObjectID</span></code></a></td> +<td>An ObjectID represents a string of bytes used to identify Plasma objects.</td> +</tr> +<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.plasma.PlasmaClient.html#pyarrow.plasma.PlasmaClient" title="pyarrow.plasma.PlasmaClient"><code class="xref py py-obj docutils literal"><span class="pre">PlasmaClient</span></code></a></td> +<td>The PlasmaClient is used to interface with a plasma store and manager.</td> +</tr> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.plasma.PlasmaBuffer.html#pyarrow.plasma.PlasmaBuffer" title="pyarrow.plasma.PlasmaBuffer"><code class="xref py py-obj docutils literal"><span class="pre">PlasmaBuffer</span></code></a></td> +<td>This is the type returned by calls to get with a PlasmaClient.</td> +</tr> +</tbody> +</table> +</div> <div class="section" id="apache-parquet"> <span id="api-parquet"></span><h2>Apache Parquet<a class="headerlink" href="#apache-parquet" title="Permalink to this headline">¶</a></h2> <table border="1" class="longtable docutils"> @@ -590,16 +632,25 @@ <tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.parquet.ParquetDataset.html#pyarrow.parquet.ParquetDataset" title="pyarrow.parquet.ParquetDataset"><code class="xref py py-obj docutils literal"><span class="pre">ParquetDataset</span></code></a>(path_or_paths[, filesystem, ...])</td> <td>Encapsulates details of reading a complete Parquet dataset possibly</td> </tr> -<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.parquet.ParquetFile.html#pyarrow.parquet.ParquetFile" title="pyarrow.parquet.ParquetFile"><code class="xref py py-obj docutils literal"><span class="pre">ParquetFile</span></code></a>(source[, metadata])</td> +<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.parquet.ParquetFile.html#pyarrow.parquet.ParquetFile" title="pyarrow.parquet.ParquetFile"><code class="xref py py-obj docutils literal"><span class="pre">ParquetFile</span></code></a>(source[, metadata, common_metadata])</td> <td>Reader interface for a single Parquet file</td> </tr> -<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.parquet.read_table.html#pyarrow.parquet.read_table" title="pyarrow.parquet.read_table"><code class="xref py py-obj docutils literal"><span class="pre">read_table</span></code></a>(source[, columns, nthreads, metadata])</td> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.parquet.read_table.html#pyarrow.parquet.read_table" title="pyarrow.parquet.read_table"><code class="xref py py-obj docutils literal"><span class="pre">read_table</span></code></a>(source[, columns, nthreads, ...])</td> <td>Read a Table from Parquet format</td> </tr> -<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.parquet.write_metadata.html#pyarrow.parquet.write_metadata" title="pyarrow.parquet.write_metadata"><code class="xref py py-obj docutils literal"><span class="pre">write_metadata</span></code></a>(schema, where[, version])</td> +<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.parquet.read_metadata.html#pyarrow.parquet.read_metadata" title="pyarrow.parquet.read_metadata"><code class="xref py py-obj docutils literal"><span class="pre">read_metadata</span></code></a>(where)</td> +<td>Read FileMetadata from footer of a single Parquet file</td> +</tr> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.parquet.read_pandas.html#pyarrow.parquet.read_pandas" title="pyarrow.parquet.read_pandas"><code class="xref py py-obj docutils literal"><span class="pre">read_pandas</span></code></a>(source[, columns, nthreads, ...])</td> +<td>Read a Table from Parquet format, also reading DataFrame index values if</td> +</tr> +<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.parquet.read_schema.html#pyarrow.parquet.read_schema" title="pyarrow.parquet.read_schema"><code class="xref py py-obj docutils literal"><span class="pre">read_schema</span></code></a>(where)</td> +<td>Read effective Arrow schema from Parquet file metadata</td> +</tr> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.parquet.write_metadata.html#pyarrow.parquet.write_metadata" title="pyarrow.parquet.write_metadata"><code class="xref py py-obj docutils literal"><span class="pre">write_metadata</span></code></a>(schema, where[, version, ...])</td> <td>Write metadata-only Parquet file from schema</td> </tr> -<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.parquet.write_table.html#pyarrow.parquet.write_table" title="pyarrow.parquet.write_table"><code class="xref py py-obj docutils literal"><span class="pre">write_table</span></code></a>(table, where[, row_group_size, ...])</td> +<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.parquet.write_table.html#pyarrow.parquet.write_table" title="pyarrow.parquet.write_table"><code class="xref py py-obj docutils literal"><span class="pre">write_table</span></code></a>(table, where[, row_group_size, ...])</td> <td>Write a Table to Parquet format</td> </tr> </tbody>
http://git-wip-us.apache.org/repos/asf/arrow-site/blob/4d4a3202/docs/python/data.html ---------------------------------------------------------------------- diff --git a/docs/python/data.html b/docs/python/data.html index fe35ea3..f60fe3c 100644 --- a/docs/python/data.html +++ b/docs/python/data.html @@ -73,7 +73,8 @@ <li class="toctree-l1"><a class="reference internal" href="memory.html">Memory and IO Interfaces</a></li> <li class="toctree-l1 current"><a class="current reference internal" href="#">In-Memory Data Model</a></li> <li class="toctree-l1"><a class="reference internal" href="ipc.html">IPC: Fast Streaming and Serialization</a></li> -<li class="toctree-l1"><a class="reference internal" href="filesystems.html">Filesystem Interfaces</a></li> +<li class="toctree-l1"><a class="reference internal" href="filesystems.html">File System Interfaces</a></li> +<li class="toctree-l1"><a class="reference internal" href="plasma.html">The Plasma In-Memory Object Store</a></li> <li class="toctree-l1"><a class="reference internal" href="pandas.html">Using PyArrow with pandas</a></li> <li class="toctree-l1"><a class="reference internal" href="parquet.html">Reading and Writing the Apache Parquet Format</a></li> <li class="toctree-l1"><a class="reference internal" href="api.html">API Reference</a></li> @@ -215,7 +216,7 @@ same for one or more types. For example, <code class="docutils literal"><span cl <p>These objects are <cite>metadata</cite>; they are used for describing the data in arrays, schemas, and record batches. In Python, they can be used in functions where the input data (e.g. Python objects) may be coerced to more than one Arrow type.</p> -<p>The <a class="reference internal" href="generated/pyarrow.Field.html#pyarrow.Field" title="pyarrow.Field"><code class="xref py py-class docutils literal"><span class="pre">Field</span></code></a> type is a type plus a name and optional +<p>The <code class="xref py py-class docutils literal"><span class="pre">Field</span></code> type is a type plus a name and optional user-defined metadata:</p> <div class="highlight-ipython"><div class="highlight"><pre><span></span><span class="gp">In [11]: </span><span class="n">f0</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">field</span><span class="p">(</span><span class="s1">'int32_field'</span><span class="p">,</span> <span class="n">t1</span><span class="p">)</span> @@ -258,7 +259,7 @@ functions.</p> </div> <div class="section" id="schemas"> <span id="data-schema"></span><h2>Schemas<a class="headerlink" href="#schemas" title="Permalink to this headline">¶</a></h2> -<p>The <a class="reference internal" href="generated/pyarrow.Schema.html#pyarrow.Schema" title="pyarrow.Schema"><code class="xref py py-class docutils literal"><span class="pre">Schema</span></code></a> type is similar to the <code class="docutils literal"><span class="pre">struct</span></code> array type; it +<p>The <code class="xref py py-class docutils literal"><span class="pre">Schema</span></code> type is similar to the <code class="docutils literal"><span class="pre">struct</span></code> array type; it defines the column names and types in a record batch or table data structure. The <code class="docutils literal"><span class="pre">pyarrow.schema</span></code> factory function makes new Schema objects in Python:</p> @@ -296,7 +297,7 @@ the <code class="docutils literal"><span class="pre">numpy.array</span></code> f <span class="gp">In [24]: </span><span class="n">arr</span> <span class="gh">Out[24]: </span><span class="go"></span> -<span class="go"><pyarrow.lib.Int64Array object at 0x2b964178bb88></span> +<span class="go"><pyarrow.lib.Int64Array object at 0x7fdbf15f60e8></span> <span class="go">[</span> <span class="go"> 1,</span> <span class="go"> 2,</span> @@ -351,7 +352,7 @@ in a file or stream to transmit integer <em>indices</em> referencing a shared particularly often used with strings to save memory and improve performance.</p> <p>The way that dictionaries are handled in the Apache Arrow format and the way they appear in C++ and Python is slightly different. We define a special -<a class="reference internal" href="generated/pyarrow.DictionaryArray.html#pyarrow.DictionaryArray" title="pyarrow.DictionaryArray"><code class="xref py py-class docutils literal"><span class="pre">DictionaryArray</span></code></a> type with a corresponding dictionary type. Let’s +<a class="reference internal" href="generated/pyarrow.lib.DictionaryArray.html#pyarrow.lib.DictionaryArray" title="pyarrow.lib.DictionaryArray"><code class="xref py py-class docutils literal"><span class="pre">DictionaryArray</span></code></a> type with a corresponding dictionary type. Let’s consider an example:</p> <div class="highlight-ipython"><div class="highlight"><pre><span></span><span class="gp">In [33]: </span><span class="n">indices</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="bp">None</span><span class="p">,</span> <span class="mi">2</span><span class="p">])</span> @@ -361,7 +362,7 @@ consider an example:</p> <span class="gp">In [36]: </span><span class="n">dict_array</span> <span class="gh">Out[36]: </span><span class="go"></span> -<span class="go"><pyarrow.lib.DictionaryArray object at 0x2b96417ab6a8></span> +<span class="go"><pyarrow.lib.DictionaryArray object at 0x7fdbf1601588></span> <span class="go">[</span> <span class="go"> 'foo',</span> <span class="go"> 'bar',</span> @@ -376,11 +377,11 @@ consider an example:</p> </div> <p>Here we have:</p> <div class="highlight-ipython"><div class="highlight"><pre><span></span><span class="gp">In [37]: </span><span class="k">print</span><span class="p">(</span><span class="n">dict_array</span><span class="o">.</span><span class="n">type</span><span class="p">)</span> -<span class="go">dictionary<values=string, indices=int64></span> +<span class="go">dictionary<values=string, indices=int64, ordered=0></span> <span class="gp">In [38]: </span><span class="n">dict_array</span><span class="o">.</span><span class="n">indices</span> -<span class="go"> Out[38]: </span> -<span class="go"><pyarrow.lib.Int64Array object at 0x2b96417b67c8></span> +<span class="go"> Out[38]: </span> +<span class="go"><pyarrow.lib.Int64Array object at 0x7fdbf1607908></span> <span class="go">[</span> <span class="go"> 0,</span> <span class="go"> 1,</span> @@ -393,8 +394,8 @@ consider an example:</p> <span class="go">]</span> <span class="gp">In [39]: </span><span class="n">dict_array</span><span class="o">.</span><span class="n">dictionary</span> -<span class="go"> Out[39]: </span> -<span class="go"><pyarrow.lib.StringArray object at 0x2b96417b6ae8></span> +<span class="go"> Out[39]: </span> +<span class="go"><pyarrow.lib.StringArray object at 0x7fdbf15fabd8></span> <span class="go">[</span> <span class="go"> 'foo',</span> <span class="go"> 'bar',</span> @@ -402,7 +403,7 @@ consider an example:</p> <span class="go">]</span> </pre></div> </div> -<p>When using <a class="reference internal" href="generated/pyarrow.DictionaryArray.html#pyarrow.DictionaryArray" title="pyarrow.DictionaryArray"><code class="xref py py-class docutils literal"><span class="pre">DictionaryArray</span></code></a> with pandas, the analogue is +<p>When using <a class="reference internal" href="generated/pyarrow.lib.DictionaryArray.html#pyarrow.lib.DictionaryArray" title="pyarrow.lib.DictionaryArray"><code class="xref py py-class docutils literal"><span class="pre">DictionaryArray</span></code></a> with pandas, the analogue is <code class="docutils literal"><span class="pre">pandas.Categorical</span></code> (more on this later):</p> <div class="highlight-ipython"><div class="highlight"><pre><span></span><span class="gp">In [40]: </span><span class="n">dict_array</span><span class="o">.</span><span class="n">to_pandas</span><span class="p">()</span> <span class="gh">Out[40]: </span><span class="go"></span> @@ -442,7 +443,7 @@ instances. Let’s consider a collection of arrays:</p> <span class="gp">In [46]: </span><span class="n">batch</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="go"> Out[46]: </span> -<span class="go"><pyarrow.lib.StringArray object at 0x2b96417c5138></span> +<span class="go"><pyarrow.lib.StringArray object at 0x7fdbf161a868></span> <span class="go">[</span> <span class="go"> 'foo',</span> <span class="go"> 'bar',</span> @@ -456,7 +457,7 @@ instances. Let’s consider a collection of arrays:</p> <span class="gp">In [48]: </span><span class="n">batch2</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="gh">Out[48]: </span><span class="go"></span> -<span class="go"><pyarrow.lib.StringArray object at 0x2b96417c5958></span> +<span class="go"><pyarrow.lib.StringArray object at 0x7fdbf1600d18></span> <span class="go">[</span> <span class="go"> 'bar',</span> <span class="go"> 'baz',</span> @@ -495,17 +496,158 @@ for one or more arrays of the same type.</p> <div class="highlight-ipython"><div class="highlight"><pre><span></span><span class="gp">In [53]: </span><span class="n">c</span> <span class="o">=</span> <span class="n">table</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="gp">In [54]: </span><span class="n">c</span> -<span class="gh">Out[54]: </span><span class="go"><pyarrow.lib.Column at 0x2b964178e540></span> +<span class="gh">Out[54]: </span><span class="go"></span> +<span class="go"><pyarrow.lib.Column object at 0x7fdbf18b45a0></span> +<span class="go">chunk 0: <pyarrow.lib.Int64Array object at 0x7fdbf16228b8></span> +<span class="go">[</span> +<span class="go"> 1,</span> +<span class="go"> 2,</span> +<span class="go"> 3,</span> +<span class="go"> 4</span> +<span class="go">]</span> +<span class="go">chunk 1: <pyarrow.lib.Int64Array object at 0x7fdbf16228b8></span> +<span class="go">[</span> +<span class="go"> 1,</span> +<span class="go"> 2,</span> +<span class="go"> 3,</span> +<span class="go"> 4</span> +<span class="go">]</span> +<span class="go">chunk 2: <pyarrow.lib.Int64Array object at 0x7fdbf16228b8></span> +<span class="go">[</span> +<span class="go"> 1,</span> +<span class="go"> 2,</span> +<span class="go"> 3,</span> +<span class="go"> 4</span> +<span class="go">]</span> +<span class="go">chunk 3: <pyarrow.lib.Int64Array object at 0x7fdbf16228b8></span> +<span class="go">[</span> +<span class="go"> 1,</span> +<span class="go"> 2,</span> +<span class="go"> 3,</span> +<span class="go"> 4</span> +<span class="go">]</span> +<span class="go">chunk 4: <pyarrow.lib.Int64Array object at 0x7fdbf16228b8></span> +<span class="go">[</span> +<span class="go"> 1,</span> +<span class="go"> 2,</span> +<span class="go"> 3,</span> +<span class="go"> 4</span> +<span class="go">]</span> +<span class="go">chunk 5: <pyarrow.lib.Int64Array object at 0x7fdbf16228b8></span> +<span class="go">[</span> +<span class="go"> 1,</span> +<span class="go"> 2,</span> +<span class="go"> 3,</span> +<span class="go"> 4</span> +<span class="go">]</span> +<span class="go">chunk 6: <pyarrow.lib.Int64Array object at 0x7fdbf16228b8></span> +<span class="go">[</span> +<span class="go"> 1,</span> +<span class="go"> 2,</span> +<span class="go"> 3,</span> +<span class="go"> 4</span> +<span class="go">]</span> +<span class="go">chunk 7: <pyarrow.lib.Int64Array object at 0x7fdbf16228b8></span> +<span class="go">[</span> +<span class="go"> 1,</span> +<span class="go"> 2,</span> +<span class="go"> 3,</span> +<span class="go"> 4</span> +<span class="go">]</span> +<span class="go">chunk 8: <pyarrow.lib.Int64Array object at 0x7fdbf16228b8></span> +<span class="go">[</span> +<span class="go"> 1,</span> +<span class="go"> 2,</span> +<span class="go"> 3,</span> +<span class="go"> 4</span> +<span class="go">]</span> +<span class="go">chunk 9: <pyarrow.lib.Int64Array object at 0x7fdbf16228b8></span> +<span class="go">[</span> +<span class="go"> 1,</span> +<span class="go"> 2,</span> +<span class="go"> 3,</span> +<span class="go"> 4</span> +<span class="go">]</span> +<span class="go">chunk 10: <pyarrow.lib.Int64Array object at 0x7fdbf16228b8></span> +<span class="go">[</span> +<span class="go"> 1,</span> +<span class="go"> 2,</span> +<span class="go"> 3,</span> +<span class="go"> 4</span> +<span class="go">]</span> +<span class="go">chunk 11: <pyarrow.lib.Int64Array object at 0x7fdbf16228b8></span> +<span class="go">[</span> +<span class="go"> 1,</span> +<span class="go"> 2,</span> +<span class="go"> 3,</span> +<span class="go"> 4</span> +<span class="go">]</span> +<span class="go">chunk 12: <pyarrow.lib.Int64Array object at 0x7fdbf16228b8></span> +<span class="go">[</span> +<span class="go"> 1,</span> +<span class="go"> 2,</span> +<span class="go"> 3,</span> +<span class="go"> 4</span> +<span class="go">]</span> +<span class="go">chunk 13: <pyarrow.lib.Int64Array object at 0x7fdbf16228b8></span> +<span class="go">[</span> +<span class="go"> 1,</span> +<span class="go"> 2,</span> +<span class="go"> 3,</span> +<span class="go"> 4</span> +<span class="go">]</span> +<span class="go">chunk 14: <pyarrow.lib.Int64Array object at 0x7fdbf16228b8></span> +<span class="go">[</span> +<span class="go"> 1,</span> +<span class="go"> 2,</span> +<span class="go"> 3,</span> +<span class="go"> 4</span> +<span class="go">]</span> +<span class="go">chunk 15: <pyarrow.lib.Int64Array object at 0x7fdbf16228b8></span> +<span class="go">[</span> +<span class="go"> 1,</span> +<span class="go"> 2,</span> +<span class="go"> 3,</span> +<span class="go"> 4</span> +<span class="go">]</span> +<span class="go">chunk 16: <pyarrow.lib.Int64Array object at 0x7fdbf16228b8></span> +<span class="go">[</span> +<span class="go"> 1,</span> +<span class="go"> 2,</span> +<span class="go"> 3,</span> +<span class="go"> 4</span> +<span class="go">]</span> +<span class="go">chunk 17: <pyarrow.lib.Int64Array object at 0x7fdbf16228b8></span> +<span class="go">[</span> +<span class="go"> 1,</span> +<span class="go"> 2,</span> +<span class="go"> 3,</span> +<span class="go"> 4</span> +<span class="go">]</span> +<span class="go">chunk 18: <pyarrow.lib.Int64Array object at 0x7fdbf16228b8></span> +<span class="go">[</span> +<span class="go"> 1,</span> +<span class="go"> 2,</span> +<span class="go"> 3,</span> +<span class="go"> 4</span> +<span class="go">]</span> +<span class="go">chunk 19: <pyarrow.lib.Int64Array object at 0x7fdbf16228b8></span> +<span class="go">[</span> +<span class="go"> 1,</span> +<span class="go"> 2,</span> +<span class="go"> 3,</span> +<span class="go"> 4</span> +<span class="go">]</span> <span class="gp">In [55]: </span><span class="n">c</span><span class="o">.</span><span class="n">data</span> -<span class="go"> Out[55]: <pyarrow.lib.ChunkedArray at 0x2b964178e360></span> +<span class="go"> Out[55]: <pyarrow.lib.ChunkedArray at 0x7fdbf18b46f0></span> <span class="gp">In [56]: </span><span class="n">c</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">num_chunks</span> -<span class="go"> Out[56]: 5</span> +<span class="go"> Out[56]: 5</span> <span class="gp">In [57]: </span><span class="n">c</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">chunk</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span> -<span class="go"> Out[57]: </span> -<span class="go"><pyarrow.lib.Int64Array object at 0x2b96417d03b8></span> +<span class="go"> Out[57]: </span> +<span class="go"><pyarrow.lib.Int64Array object at 0x7fdbf1622ae8></span> <span class="go">[</span> <span class="go"> 1,</span> <span class="go"> 2,</span> http://git-wip-us.apache.org/repos/asf/arrow-site/blob/4d4a3202/docs/python/development.html ---------------------------------------------------------------------- diff --git a/docs/python/development.html b/docs/python/development.html index a3ab920..7a6a12f 100644 --- a/docs/python/development.html +++ b/docs/python/development.html @@ -73,7 +73,8 @@ <li class="toctree-l1"><a class="reference internal" href="memory.html">Memory and IO Interfaces</a></li> <li class="toctree-l1"><a class="reference internal" href="data.html">In-Memory Data Model</a></li> <li class="toctree-l1"><a class="reference internal" href="ipc.html">IPC: Fast Streaming and Serialization</a></li> -<li class="toctree-l1"><a class="reference internal" href="filesystems.html">Filesystem Interfaces</a></li> +<li class="toctree-l1"><a class="reference internal" href="filesystems.html">File System Interfaces</a></li> +<li class="toctree-l1"><a class="reference internal" href="plasma.html">The Plasma In-Memory Object Store</a></li> <li class="toctree-l1"><a class="reference internal" href="pandas.html">Using PyArrow with pandas</a></li> <li class="toctree-l1"><a class="reference internal" href="parquet.html">Reading and Writing the Apache Parquet Format</a></li> <li class="toctree-l1"><a class="reference internal" href="api.html">API Reference</a></li> @@ -189,9 +190,9 @@ git clone https://github.com/apache/parquet-cpp.git </div> <p>You should now see</p> <div class="highlight-shell"><div class="highlight"><pre><span></span>$ ls -l -total 8 -drwxrwxr-x <span class="m">12</span> wesm wesm <span class="m">4096</span> Apr <span class="m">15</span> 19:19 arrow/ -drwxrwxr-x <span class="m">12</span> wesm wesm <span class="m">4096</span> Apr <span class="m">15</span> 19:19 parquet-cpp/ +total <span class="m">8</span> +drwxrwxr-x <span class="m">12</span> wesm wesm <span class="m">4096</span> Apr <span class="m">15</span> <span class="m">19</span>:19 arrow/ +drwxrwxr-x <span class="m">12</span> wesm wesm <span class="m">4096</span> Apr <span class="m">15</span> <span class="m">19</span>:19 parquet-cpp/ </pre></div> </div> <div class="section" id="using-conda"> @@ -199,9 +200,9 @@ drwxrwxr-x <span class="m">12</span> wesm wesm <span class="m">4096</span> Apr < <p>Let’s create a conda environment with all the C++ build and Python dependencies from conda-forge:</p> <div class="highlight-shell"><div class="highlight"><pre><span></span>conda create -y -q -n pyarrow-dev <span class="se">\</span> - <span class="nv">python</span><span class="o">=</span>3.6 numpy six setuptools cython pandas pytest <span class="se">\</span> + <span class="nv">python</span><span class="o">=</span><span class="m">3</span>.6 numpy six setuptools cython pandas pytest <span class="se">\</span> cmake flatbuffers rapidjson boost-cpp thrift-cpp snappy zlib <span class="se">\</span> - brotli jemalloc -c conda-forge + brotli jemalloc lz4-c zstd -c conda-forge <span class="nb">source</span> activate pyarrow-dev </pre></div> </div> @@ -263,6 +264,7 @@ about our build toolchain:</p> cmake -DCMAKE_BUILD_TYPE<span class="o">=</span><span class="nv">$ARROW_BUILD_TYPE</span> <span class="se">\</span> -DCMAKE_INSTALL_PREFIX<span class="o">=</span><span class="nv">$ARROW_HOME</span> <span class="se">\</span> -DARROW_PYTHON<span class="o">=</span>on <span class="se">\</span> + -DARROW_PLASMA<span class="o">=</span>on <span class="se">\</span> -DARROW_BUILD_TESTS<span class="o">=</span>OFF <span class="se">\</span> .. make -j4 @@ -270,6 +272,8 @@ make install <span class="nb">popd</span> </pre></div> </div> +<p>If you don’t want to build and install the Plasma in-memory object store, +you can omit the <code class="docutils literal"><span class="pre">-DARROW_PLASMA=on</span></code> flag.</p> <p>Now, optionally build and install the Apache Parquet libraries in your toolchain:</p> <div class="highlight-shell"><div class="highlight"><pre><span></span>mkdir parquet-cpp/build @@ -290,14 +294,15 @@ make install <p>Now, build pyarrow:</p> <div class="highlight-shell"><div class="highlight"><pre><span></span><span class="nb">cd</span> arrow/python python setup.py build_ext --build-type<span class="o">=</span><span class="nv">$ARROW_BUILD_TYPE</span> <span class="se">\</span> - --with-parquet --inplace + --with-parquet --with-plasma --inplace </pre></div> </div> -<p>If you did not build parquet-cpp, you can omit <code class="docutils literal"><span class="pre">--with-parquet</span></code>.</p> +<p>If you did not build parquet-cpp, you can omit <code class="docutils literal"><span class="pre">--with-parquet</span></code> and if +you did not build with plasma, you can omit <code class="docutils literal"><span class="pre">--with-plasma</span></code>.</p> <p>You should be able to run the unit tests with:</p> <div class="highlight-shell"><div class="highlight"><pre><span></span>$ py.test <span class="nv">pyarrow</span> <span class="o">================================</span> <span class="nb">test</span> session <span class="nv">starts</span> <span class="o">====================</span> -platform linux -- Python 3.6.1, pytest-3.0.7, py-1.4.33, pluggy-0.4.0 +platform linux -- Python <span class="m">3</span>.6.1, pytest-3.0.7, py-1.4.33, pluggy-0.4.0 rootdir: /home/wesm/arrow-clone/python, inifile: collected <span class="m">198</span> items @@ -314,15 +319,16 @@ pyarrow/tests/test_schema.py ......... pyarrow/tests/test_table.py ............. pyarrow/tests/test_tensor.py ................ -<span class="o">======================</span> <span class="m">181</span> passed, <span class="m">17</span> skipped in 0.98 <span class="nv">seconds</span> <span class="o">===========</span> +<span class="o">======================</span> <span class="m">181</span> passed, <span class="m">17</span> skipped in <span class="m">0</span>.98 <span class="nv">seconds</span> <span class="o">===========</span> </pre></div> </div> <p>You can build a wheel by running:</p> <div class="highlight-shell"><div class="highlight"><pre><span></span>python setup.py build_ext --build-type<span class="o">=</span><span class="nv">$ARROW_BUILD_TYPE</span> <span class="se">\</span> - --with-parquet --bundle-arrow-cpp bdist_wheel + --with-parquet --with-plasma --bundle-arrow-cpp bdist_wheel </pre></div> </div> -<p>Again, if you did not build parquet-cpp, you should omit <code class="docutils literal"><span class="pre">--with-parquet</span></code>.</p> +<p>Again, if you did not build parquet-cpp, you should omit <code class="docutils literal"><span class="pre">--with-parquet</span></code> and +if you did not build with plasma, you should omit <code class="docutils literal"><span class="pre">--with-plasma</span></code>.</p> </div> </div> <div class="section" id="developing-on-windows"> @@ -353,7 +359,6 @@ cmake -G <span class="s2">"Visual Studio 14 2015 Win64"</span> ^ -DCMAKE_INSTALL_PREFIX<span class="o">=</span>%ARROW_HOME% ^ -DCMAKE_BUILD_TYPE<span class="o">=</span>Release ^ -DARROW_BUILD_TESTS<span class="o">=</span>off ^ - -DARROW_ZLIB_VENDORED<span class="o">=</span>off ^ -DARROW_PYTHON<span class="o">=</span>on .. cmake --build . --target INSTALL --config Release <span class="nb">cd</span> ..<span class="se">\.</span>. http://git-wip-us.apache.org/repos/asf/arrow-site/blob/4d4a3202/docs/python/filesystems.html ---------------------------------------------------------------------- diff --git a/docs/python/filesystems.html b/docs/python/filesystems.html index 38d7212..6336c2c 100644 --- a/docs/python/filesystems.html +++ b/docs/python/filesystems.html @@ -5,7 +5,7 @@ <head> <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> - <title>Filesystem Interfaces — pyarrow documentation</title> + <title>File System Interfaces — pyarrow documentation</title> <link rel="stylesheet" href="_static/bootstrap-sphinx.css" type="text/css" /> <link rel="stylesheet" href="_static/pygments.css" type="text/css" /> @@ -30,7 +30,7 @@ <script type="text/javascript" src="_static/bootstrap-sphinx.js"></script> <link rel="index" title="Index" href="genindex.html" /> <link rel="search" title="Search" href="search.html" /> - <link rel="next" title="Using PyArrow with pandas" href="pandas.html" /> + <link rel="next" title="pyarrow.hdfs.connect" href="generated/pyarrow.hdfs.connect.html" /> <link rel="prev" title="IPC: Fast Streaming and Serialization" href="ipc.html" /> <meta charset='utf-8'> <meta http-equiv='X-UA-Compatible' content='IE=edge,chrome=1'> @@ -73,7 +73,8 @@ <li class="toctree-l1"><a class="reference internal" href="memory.html">Memory and IO Interfaces</a></li> <li class="toctree-l1"><a class="reference internal" href="data.html">In-Memory Data Model</a></li> <li class="toctree-l1"><a class="reference internal" href="ipc.html">IPC: Fast Streaming and Serialization</a></li> -<li class="toctree-l1 current"><a class="current reference internal" href="#">Filesystem Interfaces</a></li> +<li class="toctree-l1 current"><a class="current reference internal" href="#">File System Interfaces</a></li> +<li class="toctree-l1"><a class="reference internal" href="plasma.html">The Plasma In-Memory Object Store</a></li> <li class="toctree-l1"><a class="reference internal" href="pandas.html">Using PyArrow with pandas</a></li> <li class="toctree-l1"><a class="reference internal" href="parquet.html">Reading and Writing the Apache Parquet Format</a></li> <li class="toctree-l1"><a class="reference internal" href="api.html">API Reference</a></li> @@ -91,8 +92,11 @@ <ul class="dropdown-menu localtoc" role="menu" aria-labelledby="dLabelLocalToc"><ul> -<li><a class="reference internal" href="#">Filesystem Interfaces</a><ul> -<li><a class="reference internal" href="#hadoop-file-system-hdfs">Hadoop File System (HDFS)</a></li> +<li><a class="reference internal" href="#">File System Interfaces</a><ul> +<li><a class="reference internal" href="#hadoop-file-system-hdfs">Hadoop File System (HDFS)</a><ul> +<li><a class="reference internal" href="#hdfs-api">HDFS API</a></li> +</ul> +</li> </ul> </li> </ul> @@ -108,7 +112,7 @@ </a> </li> <li> - <a href="pandas.html" title="Next Chapter: Using PyArrow with pandas"><span class="glyphicon glyphicon-chevron-right visible-sm"></span><span class="hidden-sm hidden-tablet">Using PyArrow... »</span> + <a href="generated/pyarrow.hdfs.connect.html" title="Next Chapter: pyarrow.hdfs.connect"><span class="glyphicon glyphicon-chevron-right visible-sm"></span><span class="hidden-sm hidden-tablet">pyarrow.hdfs.... »</span> </a> </li> @@ -142,21 +146,23 @@ <div class="row"> <div class="col-md-12 content"> - <div class="section" id="filesystem-interfaces"> -<h1>Filesystem Interfaces<a class="headerlink" href="#filesystem-interfaces" title="Permalink to this headline">¶</a></h1> + <div class="section" id="file-system-interfaces"> +<h1>File System Interfaces<a class="headerlink" href="#file-system-interfaces" title="Permalink to this headline">¶</a></h1> <p>In this section, we discuss filesystem-like interfaces in PyArrow.</p> <div class="section" id="hadoop-file-system-hdfs"> <span id="hdfs"></span><h2>Hadoop File System (HDFS)<a class="headerlink" href="#hadoop-file-system-hdfs" title="Permalink to this headline">¶</a></h2> <p>PyArrow comes with bindings to a C++-based interface to the Hadoop File System. You connect like so:</p> <div class="highlight-python"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">pyarrow</span> <span class="kn">as</span> <span class="nn">pa</span> -<span class="n">hdfs</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">HdfsClient</span><span class="p">(</span><span class="n">host</span><span class="p">,</span> <span class="n">port</span><span class="p">,</span> <span class="n">user</span><span class="o">=</span><span class="n">user</span><span class="p">,</span> <span class="n">kerb_ticket</span><span class="o">=</span><span class="n">ticket_cache_path</span><span class="p">)</span> +<span class="n">fs</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">hdfs</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">host</span><span class="p">,</span> <span class="n">port</span><span class="p">,</span> <span class="n">user</span><span class="o">=</span><span class="n">user</span><span class="p">,</span> <span class="n">kerb_ticket</span><span class="o">=</span><span class="n">ticket_cache_path</span><span class="p">)</span> +<span class="k">with</span> <span class="n">fs</span><span class="o">.</span><span class="n">open</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="s1">'rb'</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span> + <span class="c1"># Do something with f</span> </pre></div> </div> -<p>By default, <code class="docutils literal"><span class="pre">pyarrow.HdfsClient</span></code> uses libhdfs, a JNI-based interface to the -Java Hadoop client. This library is loaded <strong>at runtime</strong> (rather than at link -/ library load time, since the library may not be in your LD_LIBRARY_PATH), and -relies on some environment variables.</p> +<p>By default, <code class="docutils literal"><span class="pre">pyarrow.hdfs.HadoopFileSystem</span></code> uses libhdfs, a JNI-based +interface to the Java Hadoop client. This library is loaded <strong>at runtime</strong> +(rather than at link / library load time, since the library may not be in your +LD_LIBRARY_PATH), and relies on some environment variables.</p> <ul class="simple"> <li><code class="docutils literal"><span class="pre">HADOOP_HOME</span></code>: the root of your installed Hadoop distribution. Often has <cite>lib/native/libhdfs.so</cite>.</li> @@ -169,10 +175,78 @@ installed somewhere other than <code class="docutils literal"><span class="pre"> </pre></div> </div> <p>You can also use libhdfs3, a thirdparty C++ library for HDFS from Pivotal Labs:</p> -<div class="highlight-python"><div class="highlight"><pre><span></span><span class="n">hdfs3</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">HdfsClient</span><span class="p">(</span><span class="n">host</span><span class="p">,</span> <span class="n">port</span><span class="p">,</span> <span class="n">user</span><span class="o">=</span><span class="n">user</span><span class="p">,</span> <span class="n">kerb_ticket</span><span class="o">=</span><span class="n">ticket_cache_path</span><span class="p">,</span> - <span class="n">driver</span><span class="o">=</span><span class="s1">'libhdfs3'</span><span class="p">)</span> +<div class="highlight-python"><div class="highlight"><pre><span></span><span class="n">fs</span> <span class="o">=</span> <span class="n">pa</span><span class="o">.</span><span class="n">hdfs</span><span class="o">.</span><span class="n">connect</span><span class="p">(</span><span class="n">host</span><span class="p">,</span> <span class="n">port</span><span class="p">,</span> <span class="n">user</span><span class="o">=</span><span class="n">user</span><span class="p">,</span> <span class="n">kerb_ticket</span><span class="o">=</span><span class="n">ticket_cache_path</span><span class="p">,</span> + <span class="n">driver</span><span class="o">=</span><span class="s1">'libhdfs3'</span><span class="p">)</span> </pre></div> </div> +<div class="section" id="hdfs-api"> +<h3>HDFS API<a class="headerlink" href="#hdfs-api" title="Permalink to this headline">¶</a></h3> +<table border="1" class="longtable docutils"> +<colgroup> +<col width="10%" /> +<col width="90%" /> +</colgroup> +<tbody valign="top"> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.hdfs.connect.html#pyarrow.hdfs.connect" title="pyarrow.hdfs.connect"><code class="xref py py-obj docutils literal"><span class="pre">hdfs.connect</span></code></a>([host, port, user, ...])</td> +<td>Connect to an HDFS cluster.</td> +</tr> +<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.cat.html#pyarrow.HadoopFileSystem.cat" title="pyarrow.HadoopFileSystem.cat"><code class="xref py py-obj docutils literal"><span class="pre">HadoopFileSystem.cat</span></code></a>(path)</td> +<td>Return contents of file as a bytes object</td> +</tr> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.chmod.html#pyarrow.HadoopFileSystem.chmod" title="pyarrow.HadoopFileSystem.chmod"><code class="xref py py-obj docutils literal"><span class="pre">HadoopFileSystem.chmod</span></code></a>(self, path, mode)</td> +<td>Change file permissions</td> +</tr> +<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.chown.html#pyarrow.HadoopFileSystem.chown" title="pyarrow.HadoopFileSystem.chown"><code class="xref py py-obj docutils literal"><span class="pre">HadoopFileSystem.chown</span></code></a>(self, path[, owner, ...])</td> +<td>Change file permissions</td> +</tr> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.delete.html#pyarrow.HadoopFileSystem.delete" title="pyarrow.HadoopFileSystem.delete"><code class="xref py py-obj docutils literal"><span class="pre">HadoopFileSystem.delete</span></code></a>(path[, recursive])</td> +<td>Delete the indicated file or directory</td> +</tr> +<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.df.html#pyarrow.HadoopFileSystem.df" title="pyarrow.HadoopFileSystem.df"><code class="xref py py-obj docutils literal"><span class="pre">HadoopFileSystem.df</span></code></a>(self)</td> +<td>Return free space on disk, like the UNIX df command</td> +</tr> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.disk_usage.html#pyarrow.HadoopFileSystem.disk_usage" title="pyarrow.HadoopFileSystem.disk_usage"><code class="xref py py-obj docutils literal"><span class="pre">HadoopFileSystem.disk_usage</span></code></a>(path)</td> +<td>Compute bytes used by all contents under indicated path in file tree</td> +</tr> +<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.download.html#pyarrow.HadoopFileSystem.download" title="pyarrow.HadoopFileSystem.download"><code class="xref py py-obj docutils literal"><span class="pre">HadoopFileSystem.download</span></code></a>(self, path, stream)</td> +<td></td> +</tr> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.exists.html#pyarrow.HadoopFileSystem.exists" title="pyarrow.HadoopFileSystem.exists"><code class="xref py py-obj docutils literal"><span class="pre">HadoopFileSystem.exists</span></code></a>(self, path)</td> +<td>Returns True if the path is known to the cluster, False if it does not</td> +</tr> +<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.get_capacity.html#pyarrow.HadoopFileSystem.get_capacity" title="pyarrow.HadoopFileSystem.get_capacity"><code class="xref py py-obj docutils literal"><span class="pre">HadoopFileSystem.get_capacity</span></code></a>(self)</td> +<td>Get reported total capacity of file system</td> +</tr> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.get_space_used.html#pyarrow.HadoopFileSystem.get_space_used" title="pyarrow.HadoopFileSystem.get_space_used"><code class="xref py py-obj docutils literal"><span class="pre">HadoopFileSystem.get_space_used</span></code></a>(self)</td> +<td>Get space used on file system</td> +</tr> +<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.info.html#pyarrow.HadoopFileSystem.info" title="pyarrow.HadoopFileSystem.info"><code class="xref py py-obj docutils literal"><span class="pre">HadoopFileSystem.info</span></code></a>(self, path)</td> +<td>Return detailed HDFS information for path</td> +</tr> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.ls.html#pyarrow.HadoopFileSystem.ls" title="pyarrow.HadoopFileSystem.ls"><code class="xref py py-obj docutils literal"><span class="pre">HadoopFileSystem.ls</span></code></a>(path[, detail])</td> +<td>Retrieve directory contents and metadata, if requested.</td> +</tr> +<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.mkdir.html#pyarrow.HadoopFileSystem.mkdir" title="pyarrow.HadoopFileSystem.mkdir"><code class="xref py py-obj docutils literal"><span class="pre">HadoopFileSystem.mkdir</span></code></a>(path[, create_parents])</td> +<td></td> +</tr> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.open.html#pyarrow.HadoopFileSystem.open" title="pyarrow.HadoopFileSystem.open"><code class="xref py py-obj docutils literal"><span class="pre">HadoopFileSystem.open</span></code></a>(self, path[, mode, ...])</td> +<td>Open HDFS file for reading or writing</td> +</tr> +<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.rename.html#pyarrow.HadoopFileSystem.rename" title="pyarrow.HadoopFileSystem.rename"><code class="xref py py-obj docutils literal"><span class="pre">HadoopFileSystem.rename</span></code></a>(path, new_path)</td> +<td>Rename file, like UNIX mv command</td> +</tr> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.rm.html#pyarrow.HadoopFileSystem.rm" title="pyarrow.HadoopFileSystem.rm"><code class="xref py py-obj docutils literal"><span class="pre">HadoopFileSystem.rm</span></code></a>(path[, recursive])</td> +<td>Alias for FileSystem.delete</td> +</tr> +<tr class="row-even"><td><a class="reference internal" href="generated/pyarrow.HadoopFileSystem.upload.html#pyarrow.HadoopFileSystem.upload" title="pyarrow.HadoopFileSystem.upload"><code class="xref py py-obj docutils literal"><span class="pre">HadoopFileSystem.upload</span></code></a>(self, path, stream)</td> +<td>Upload file-like object to HDFS path</td> +</tr> +<tr class="row-odd"><td><a class="reference internal" href="generated/pyarrow.HdfsFile.html#pyarrow.HdfsFile" title="pyarrow.HdfsFile"><code class="xref py py-obj docutils literal"><span class="pre">HdfsFile</span></code></a></td> +<td></td> +</tr> +</tbody> +</table> +</div> </div> </div> http://git-wip-us.apache.org/repos/asf/arrow-site/blob/4d4a3202/docs/python/generated/pyarrow.ArrayValue.html ---------------------------------------------------------------------- diff --git a/docs/python/generated/pyarrow.ArrayValue.html b/docs/python/generated/pyarrow.ArrayValue.html index 051abd1..8ed771d 100644 --- a/docs/python/generated/pyarrow.ArrayValue.html +++ b/docs/python/generated/pyarrow.ArrayValue.html @@ -73,7 +73,8 @@ <li class="toctree-l1"><a class="reference internal" href="../memory.html">Memory and IO Interfaces</a></li> <li class="toctree-l1"><a class="reference internal" href="../data.html">In-Memory Data Model</a></li> <li class="toctree-l1"><a class="reference internal" href="../ipc.html">IPC: Fast Streaming and Serialization</a></li> -<li class="toctree-l1"><a class="reference internal" href="../filesystems.html">Filesystem Interfaces</a></li> +<li class="toctree-l1"><a class="reference internal" href="../filesystems.html">File System Interfaces</a></li> +<li class="toctree-l1"><a class="reference internal" href="../plasma.html">The Plasma In-Memory Object Store</a></li> <li class="toctree-l1"><a class="reference internal" href="../pandas.html">Using PyArrow with pandas</a></li> <li class="toctree-l1"><a class="reference internal" href="../parquet.html">Reading and Writing the Apache Parquet Format</a></li> <li class="toctree-l1 current"><a class="reference internal" href="../api.html">API Reference</a></li> @@ -105,7 +106,7 @@ </a> </li> <li> - <a href="pyarrow.BooleanValue.html" title="Next Chapter: pyarrow.BooleanValue"><span class="glyphicon glyphicon-chevron-right visible-sm"></span><span class="hidden-sm hidden-tablet">pyarrow.BooleanValue »</span> + <a href="pyarrow.BooleanValue.html" title="Next Chapter: pyarrow.BooleanValue"><span class="glyphicon glyphicon-chevron-right visible-sm"></span><span class="hidden-sm hidden-tablet">pyarrow.Boole... »</span> </a> </li> http://git-wip-us.apache.org/repos/asf/arrow-site/blob/4d4a3202/docs/python/generated/pyarrow.BinaryValue.html ---------------------------------------------------------------------- diff --git a/docs/python/generated/pyarrow.BinaryValue.html b/docs/python/generated/pyarrow.BinaryValue.html index 5adaff8..b06f182 100644 --- a/docs/python/generated/pyarrow.BinaryValue.html +++ b/docs/python/generated/pyarrow.BinaryValue.html @@ -73,7 +73,8 @@ <li class="toctree-l1"><a class="reference internal" href="../memory.html">Memory and IO Interfaces</a></li> <li class="toctree-l1"><a class="reference internal" href="../data.html">In-Memory Data Model</a></li> <li class="toctree-l1"><a class="reference internal" href="../ipc.html">IPC: Fast Streaming and Serialization</a></li> -<li class="toctree-l1"><a class="reference internal" href="../filesystems.html">Filesystem Interfaces</a></li> +<li class="toctree-l1"><a class="reference internal" href="../filesystems.html">File System Interfaces</a></li> +<li class="toctree-l1"><a class="reference internal" href="../plasma.html">The Plasma In-Memory Object Store</a></li> <li class="toctree-l1"><a class="reference internal" href="../pandas.html">Using PyArrow with pandas</a></li> <li class="toctree-l1"><a class="reference internal" href="../parquet.html">Reading and Writing the Apache Parquet Format</a></li> <li class="toctree-l1 current"><a class="reference internal" href="../api.html">API Reference</a></li> @@ -101,11 +102,11 @@ <li> - <a href="pyarrow.ListValue.html" title="Previous Chapter: pyarrow.ListValue"><span class="glyphicon glyphicon-chevron-left visible-sm"></span><span class="hidden-sm hidden-tablet">« pyarrow.ListValue</span> + <a href="pyarrow.ListValue.html" title="Previous Chapter: pyarrow.ListValue"><span class="glyphicon glyphicon-chevron-left visible-sm"></span><span class="hidden-sm hidden-tablet">« pyarrow.ListV...</span> </a> </li> <li> - <a href="pyarrow.StringValue.html" title="Next Chapter: pyarrow.StringValue"><span class="glyphicon glyphicon-chevron-right visible-sm"></span><span class="hidden-sm hidden-tablet">pyarrow.StringValue »</span> + <a href="pyarrow.StringValue.html" title="Next Chapter: pyarrow.StringValue"><span class="glyphicon glyphicon-chevron-right visible-sm"></span><span class="hidden-sm hidden-tablet">pyarrow.Strin... »</span> </a> </li> http://git-wip-us.apache.org/repos/asf/arrow-site/blob/4d4a3202/docs/python/generated/pyarrow.BooleanValue.html ---------------------------------------------------------------------- diff --git a/docs/python/generated/pyarrow.BooleanValue.html b/docs/python/generated/pyarrow.BooleanValue.html index 7a33c79..171111d 100644 --- a/docs/python/generated/pyarrow.BooleanValue.html +++ b/docs/python/generated/pyarrow.BooleanValue.html @@ -73,7 +73,8 @@ <li class="toctree-l1"><a class="reference internal" href="../memory.html">Memory and IO Interfaces</a></li> <li class="toctree-l1"><a class="reference internal" href="../data.html">In-Memory Data Model</a></li> <li class="toctree-l1"><a class="reference internal" href="../ipc.html">IPC: Fast Streaming and Serialization</a></li> -<li class="toctree-l1"><a class="reference internal" href="../filesystems.html">Filesystem Interfaces</a></li> +<li class="toctree-l1"><a class="reference internal" href="../filesystems.html">File System Interfaces</a></li> +<li class="toctree-l1"><a class="reference internal" href="../plasma.html">The Plasma In-Memory Object Store</a></li> <li class="toctree-l1"><a class="reference internal" href="../pandas.html">Using PyArrow with pandas</a></li> <li class="toctree-l1"><a class="reference internal" href="../parquet.html">Reading and Writing the Apache Parquet Format</a></li> <li class="toctree-l1 current"><a class="reference internal" href="../api.html">API Reference</a></li> @@ -101,11 +102,11 @@ <li> - <a href="pyarrow.ArrayValue.html" title="Previous Chapter: pyarrow.ArrayValue"><span class="glyphicon glyphicon-chevron-left visible-sm"></span><span class="hidden-sm hidden-tablet">« pyarrow.ArrayValue</span> + <a href="pyarrow.ArrayValue.html" title="Previous Chapter: pyarrow.ArrayValue"><span class="glyphicon glyphicon-chevron-left visible-sm"></span><span class="hidden-sm hidden-tablet">« pyarrow.Array...</span> </a> </li> <li> - <a href="pyarrow.Int8Value.html" title="Next Chapter: pyarrow.Int8Value"><span class="glyphicon glyphicon-chevron-right visible-sm"></span><span class="hidden-sm hidden-tablet">pyarrow.Int8Value »</span> + <a href="pyarrow.Int8Value.html" title="Next Chapter: pyarrow.Int8Value"><span class="glyphicon glyphicon-chevron-right visible-sm"></span><span class="hidden-sm hidden-tablet">pyarrow.Int8V... »</span> </a> </li> http://git-wip-us.apache.org/repos/asf/arrow-site/blob/4d4a3202/docs/python/generated/pyarrow.Buffer.html ---------------------------------------------------------------------- diff --git a/docs/python/generated/pyarrow.Buffer.html b/docs/python/generated/pyarrow.Buffer.html index 9d9a521..f4e571d 100644 --- a/docs/python/generated/pyarrow.Buffer.html +++ b/docs/python/generated/pyarrow.Buffer.html @@ -73,7 +73,8 @@ <li class="toctree-l1"><a class="reference internal" href="../memory.html">Memory and IO Interfaces</a></li> <li class="toctree-l1"><a class="reference internal" href="../data.html">In-Memory Data Model</a></li> <li class="toctree-l1"><a class="reference internal" href="../ipc.html">IPC: Fast Streaming and Serialization</a></li> -<li class="toctree-l1"><a class="reference internal" href="../filesystems.html">Filesystem Interfaces</a></li> +<li class="toctree-l1"><a class="reference internal" href="../filesystems.html">File System Interfaces</a></li> +<li class="toctree-l1"><a class="reference internal" href="../plasma.html">The Plasma In-Memory Object Store</a></li> <li class="toctree-l1"><a class="reference internal" href="../pandas.html">Using PyArrow with pandas</a></li> <li class="toctree-l1"><a class="reference internal" href="../parquet.html">Reading and Writing the Apache Parquet Format</a></li> <li class="toctree-l1 current"><a class="reference internal" href="../api.html">API Reference</a></li> @@ -105,7 +106,7 @@ </a> </li> <li> - <a href="pyarrow.BufferReader.html" title="Next Chapter: pyarrow.BufferReader"><span class="glyphicon glyphicon-chevron-right visible-sm"></span><span class="hidden-sm hidden-tablet">pyarrow.BufferReader »</span> + <a href="pyarrow.BufferReader.html" title="Next Chapter: pyarrow.BufferReader"><span class="glyphicon glyphicon-chevron-right visible-sm"></span><span class="hidden-sm hidden-tablet">pyarrow.Buffe... »</span> </a> </li> http://git-wip-us.apache.org/repos/asf/arrow-site/blob/4d4a3202/docs/python/generated/pyarrow.BufferOutputStream.html ---------------------------------------------------------------------- diff --git a/docs/python/generated/pyarrow.BufferOutputStream.html b/docs/python/generated/pyarrow.BufferOutputStream.html index 1b628c2..6cba8d5 100644 --- a/docs/python/generated/pyarrow.BufferOutputStream.html +++ b/docs/python/generated/pyarrow.BufferOutputStream.html @@ -73,7 +73,8 @@ <li class="toctree-l1"><a class="reference internal" href="../memory.html">Memory and IO Interfaces</a></li> <li class="toctree-l1"><a class="reference internal" href="../data.html">In-Memory Data Model</a></li> <li class="toctree-l1"><a class="reference internal" href="../ipc.html">IPC: Fast Streaming and Serialization</a></li> -<li class="toctree-l1"><a class="reference internal" href="../filesystems.html">Filesystem Interfaces</a></li> +<li class="toctree-l1"><a class="reference internal" href="../filesystems.html">File System Interfaces</a></li> +<li class="toctree-l1"><a class="reference internal" href="../plasma.html">The Plasma In-Memory Object Store</a></li> <li class="toctree-l1"><a class="reference internal" href="../pandas.html">Using PyArrow with pandas</a></li> <li class="toctree-l1"><a class="reference internal" href="../parquet.html">Reading and Writing the Apache Parquet Format</a></li> <li class="toctree-l1 current"><a class="reference internal" href="../api.html">API Reference</a></li> @@ -101,11 +102,11 @@ <li> - <a href="pyarrow.BufferReader.html" title="Previous Chapter: pyarrow.BufferReader"><span class="glyphicon glyphicon-chevron-left visible-sm"></span><span class="hidden-sm hidden-tablet">« pyarrow.BufferReader</span> + <a href="pyarrow.BufferReader.html" title="Previous Chapter: pyarrow.BufferReader"><span class="glyphicon glyphicon-chevron-left visible-sm"></span><span class="hidden-sm hidden-tablet">« pyarrow.Buffe...</span> </a> </li> <li> - <a href="pyarrow.NativeFile.html" title="Next Chapter: pyarrow.NativeFile"><span class="glyphicon glyphicon-chevron-right visible-sm"></span><span class="hidden-sm hidden-tablet">pyarrow.NativeFile »</span> + <a href="pyarrow.NativeFile.html" title="Next Chapter: pyarrow.NativeFile"><span class="glyphicon glyphicon-chevron-right visible-sm"></span><span class="hidden-sm hidden-tablet">pyarrow.Nativ... »</span> </a> </li> @@ -168,19 +169,19 @@ <td></td> </tr> <tr class="row-even"><td><a class="reference internal" href="#pyarrow.BufferOutputStream.read" title="pyarrow.BufferOutputStream.read"><code class="xref py py-obj docutils literal"><span class="pre">read</span></code></a>(self[, nbytes])</td> -<td></td> +<td>Read indicated number of bytes from file, or read all remaining bytes</td> </tr> <tr class="row-odd"><td><a class="reference internal" href="#pyarrow.BufferOutputStream.read_buffer" title="pyarrow.BufferOutputStream.read_buffer"><code class="xref py py-obj docutils literal"><span class="pre">read_buffer</span></code></a>(self[, nbytes])</td> <td></td> </tr> -<tr class="row-even"><td><a class="reference internal" href="#pyarrow.BufferOutputStream.seek" title="pyarrow.BufferOutputStream.seek"><code class="xref py py-obj docutils literal"><span class="pre">seek</span></code></a>(self, int64_t position)</td> -<td></td> +<tr class="row-even"><td><a class="reference internal" href="#pyarrow.BufferOutputStream.seek" title="pyarrow.BufferOutputStream.seek"><code class="xref py py-obj docutils literal"><span class="pre">seek</span></code></a>(self, int64_t position, int whence=0)</td> +<td>Change current file stream position</td> </tr> <tr class="row-odd"><td><a class="reference internal" href="#pyarrow.BufferOutputStream.size" title="pyarrow.BufferOutputStream.size"><code class="xref py py-obj docutils literal"><span class="pre">size</span></code></a>(self)</td> -<td></td> +<td>Return file size</td> </tr> <tr class="row-even"><td><a class="reference internal" href="#pyarrow.BufferOutputStream.tell" title="pyarrow.BufferOutputStream.tell"><code class="xref py py-obj docutils literal"><span class="pre">tell</span></code></a>(self)</td> -<td></td> +<td>Return current stream position</td> </tr> <tr class="row-odd"><td><a class="reference internal" href="#pyarrow.BufferOutputStream.upload" title="pyarrow.BufferOutputStream.upload"><code class="xref py py-obj docutils literal"><span class="pre">upload</span></code></a>(self, stream[, buffer_size])</td> <td>Pipe file-like object to file</td> @@ -233,7 +234,19 @@ memory). First seeks to the beginning of the file.</p> <dl class="method"> <dt id="pyarrow.BufferOutputStream.read"> <code class="descname">read</code><span class="sig-paren">(</span><em>self</em>, <em>nbytes=None</em><span class="sig-paren">)</span><a class="headerlink" href="#pyarrow.BufferOutputStream.read" title="Permalink to this definition">¶</a></dt> -<dd></dd></dl> +<dd><p>Read indicated number of bytes from file, or read all remaining bytes +if no argument passed</p> +<table class="docutils field-list" frame="void" rules="none"> +<col class="field-name" /> +<col class="field-body" /> +<tbody valign="top"> +<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><strong>nbytes</strong> (<em>int</em><em>, </em><em>default None</em>) – </td> +</tr> +<tr class="field-even field"><th class="field-name">Returns:</th><td class="field-body"><strong>data</strong> (<em>bytes</em>)</td> +</tr> +</tbody> +</table> +</dd></dl> <dl class="method"> <dt id="pyarrow.BufferOutputStream.read_buffer"> @@ -242,18 +255,46 @@ memory). First seeks to the beginning of the file.</p> <dl class="method"> <dt id="pyarrow.BufferOutputStream.seek"> -<code class="descname">seek</code><span class="sig-paren">(</span><em>self</em>, <em>int64_t position</em><span class="sig-paren">)</span><a class="headerlink" href="#pyarrow.BufferOutputStream.seek" title="Permalink to this definition">¶</a></dt> -<dd></dd></dl> +<code class="descname">seek</code><span class="sig-paren">(</span><em>self</em>, <em>int64_t position</em>, <em>int whence=0</em><span class="sig-paren">)</span><a class="headerlink" href="#pyarrow.BufferOutputStream.seek" title="Permalink to this definition">¶</a></dt> +<dd><p>Change current file stream position</p> +<table class="docutils field-list" frame="void" rules="none"> +<col class="field-name" /> +<col class="field-body" /> +<tbody valign="top"> +<tr class="field-odd field"><th class="field-name">Parameters:</th><td class="field-body"><ul class="first last simple"> +<li><strong>position</strong> (<em>int</em>) – Byte offset, interpreted relative to value of whence argument</li> +<li><strong>whence</strong> (<em>int</em><em>, </em><em>default 0</em>) – Point of reference for seek offset</li> +</ul> +</td> +</tr> +</tbody> +</table> +<p class="rubric">Notes</p> +<p>Values of whence: +* 0 – start of stream (the default); offset should be zero or positive +* 1 – current stream position; offset may be negative +* 2 – end of stream; offset is usually negative</p> +<table class="docutils field-list" frame="void" rules="none"> +<col class="field-name" /> +<col class="field-body" /> +<tbody valign="top"> +<tr class="field-odd field"><th class="field-name">Returns:</th><td class="field-body"><strong>new_position</strong> (<em>the new absolute stream position</em>)</td> +</tr> +</tbody> +</table> +</dd></dl> <dl class="method"> <dt id="pyarrow.BufferOutputStream.size"> <code class="descname">size</code><span class="sig-paren">(</span><em>self</em><span class="sig-paren">)</span><a class="headerlink" href="#pyarrow.BufferOutputStream.size" title="Permalink to this definition">¶</a></dt> -<dd></dd></dl> +<dd><p>Return file size</p> +</dd></dl> <dl class="method"> <dt id="pyarrow.BufferOutputStream.tell"> <code class="descname">tell</code><span class="sig-paren">(</span><em>self</em><span class="sig-paren">)</span><a class="headerlink" href="#pyarrow.BufferOutputStream.tell" title="Permalink to this definition">¶</a></dt> -<dd></dd></dl> +<dd><p>Return current stream position</p> +</dd></dl> <dl class="method"> <dt id="pyarrow.BufferOutputStream.upload">
