This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch asf-staging
in repository https://gitbox.apache.org/repos/asf/datafusion-python.git
The following commit(s) were added to refs/heads/asf-staging by this push:
new 0cbaf13 Publish built docs triggered by
5e32ada2565cebec3df54a1bbf9725f3a434b24d
0cbaf13 is described below
commit 0cbaf13d9de90f9a44038e901ed680e4ce6009fb
Author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
AuthorDate: Fri Nov 15 16:27:57 2024 +0000
Publish built docs triggered by 5e32ada2565cebec3df54a1bbf9725f3a434b24d
---
_sources/autoapi/datafusion/context/index.rst.txt | 9 ++++
_sources/user-guide/io/index.rst.txt | 1 +
_sources/user-guide/io/table_provider.rst.txt | 56 +++++++++++++++++++++
autoapi/datafusion/catalog/index.html | 5 ++
autoapi/datafusion/context/index.html | 22 ++++++++
autoapi/datafusion/dataframe/index.html | 5 ++
autoapi/datafusion/expr/index.html | 5 ++
autoapi/datafusion/functions/index.html | 5 ++
autoapi/datafusion/index.html | 5 ++
autoapi/datafusion/input/base/index.html | 5 ++
autoapi/datafusion/input/index.html | 5 ++
autoapi/datafusion/input/location/index.html | 5 ++
autoapi/datafusion/object_store/index.html | 5 ++
autoapi/datafusion/plan/index.html | 5 ++
autoapi/datafusion/record_batch/index.html | 5 ++
autoapi/datafusion/substrait/index.html | 5 ++
autoapi/datafusion/udf/index.html | 5 ++
autoapi/index.html | 5 ++
contributor-guide/introduction.html | 5 ++
genindex.html | 7 +++
index.html | 5 ++
objects.inv | Bin 5606 -> 5647 bytes
py-modindex.html | 5 ++
search.html | 5 ++
searchindex.js | 2 +-
user-guide/basics.html | 5 ++
user-guide/common-operations/aggregations.html | 43 +++++++++-------
user-guide/common-operations/basic-info.html | 39 +++++++-------
user-guide/common-operations/expressions.html | 5 ++
user-guide/common-operations/functions.html | 27 ++++++----
user-guide/common-operations/index.html | 5 ++
user-guide/common-operations/joins.html | 5 ++
.../common-operations/select-and-filter.html | 7 ++-
user-guide/common-operations/udf-and-udfa.html | 5 ++
user-guide/common-operations/windows.html | 7 ++-
user-guide/configuration.html | 5 ++
user-guide/introduction.html | 5 ++
user-guide/io/arrow.html | 5 ++
user-guide/io/avro.html | 5 ++
user-guide/io/csv.html | 5 ++
user-guide/io/index.html | 6 +++
user-guide/io/json.html | 5 ++
user-guide/io/parquet.html | 11 ++--
.../io/{parquet.html => table_provider.html} | 53 +++++++++++++------
user-guide/sql.html | 11 ++--
45 files changed, 376 insertions(+), 70 deletions(-)
diff --git a/_sources/autoapi/datafusion/context/index.rst.txt
b/_sources/autoapi/datafusion/context/index.rst.txt
index 368ddad..5c228cf 100644
--- a/_sources/autoapi/datafusion/context/index.rst.txt
+++ b/_sources/autoapi/datafusion/context/index.rst.txt
@@ -735,6 +735,15 @@ Module Contents
+ .. py:method:: register_table_provider(name: str, provider: Any) -> None
+
+ Register a table provider.
+
+ This table provider must have a method called
``__datafusion_table_provider__``
+ which returns a PyCapsule that exposes a ``FFI_TableProvider``.
+
+
+
.. py:method:: register_udaf(udaf: datafusion.udf.AggregateUDF) -> None
Register a user-defined aggregation function (UDAF) with the context.
diff --git a/_sources/user-guide/io/index.rst.txt
b/_sources/user-guide/io/index.rst.txt
index 0541132..b885cfe 100644
--- a/_sources/user-guide/io/index.rst.txt
+++ b/_sources/user-guide/io/index.rst.txt
@@ -26,3 +26,4 @@ IO
csv
json
parquet
+ table_provider
diff --git a/_sources/user-guide/io/table_provider.rst.txt
b/_sources/user-guide/io/table_provider.rst.txt
new file mode 100644
index 0000000..2ff9ae4
--- /dev/null
+++ b/_sources/user-guide/io/table_provider.rst.txt
@@ -0,0 +1,56 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements. See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership. The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License. You may obtain a copy of the License at
+
+.. http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied. See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+Custom Table Provider
+=====================
+
+If you have a custom data source that you want to integrate with DataFusion,
you can do so by
+implementing the `TableProvider
<https://datafusion.apache.org/library-user-guide/custom-table-providers.html>`_
+interface in Rust and then exposing it in Python. To do so,
+you must use DataFusion 43.0.0 or later and expose a `FFI_TableProvider
<https://crates.io/crates/datafusion-ffi>`_
+via `PyCapsule <https://pyo3.rs/main/doc/pyo3/types/struct.pycapsule>`_.
+
+A complete example can be found in the `examples folder
<https://github.com/apache/datafusion-python/tree/main/examples>`_.
+
+.. code-block:: rust
+
+ #[pymethods]
+ impl MyTableProvider {
+
+ fn __datafusion_table_provider__<'py>(
+ &self,
+ py: Python<'py>,
+ ) -> PyResult<Bound<'py, PyCapsule>> {
+ let name = CString::new("datafusion_table_provider").unwrap();
+
+ let provider = Arc::new(self.clone())
+ .map_err(|e| PyRuntimeError::new_err(e.to_string()))?;
+ let provider = FFI_TableProvider::new(Arc::new(provider), false);
+
+ PyCapsule::new_bound(py, provider, Some(name.clone()))
+ }
+ }
+
+Once you have this library available, in python you can register your table
provider
+to the ``SessionContext``.
+
+.. code-block:: python
+
+ provider = MyTableProvider()
+ ctx.register_table_provider("my_table", provider)
+
+ ctx.table("my_table").show()
diff --git a/autoapi/datafusion/catalog/index.html
b/autoapi/datafusion/catalog/index.html
index a523547..56aef0c 100644
--- a/autoapi/datafusion/catalog/index.html
+++ b/autoapi/datafusion/catalog/index.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal"
href="../../../user-guide/io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
diff --git a/autoapi/datafusion/context/index.html
b/autoapi/datafusion/context/index.html
index 2ef0e6b..5e83830 100644
--- a/autoapi/datafusion/context/index.html
+++ b/autoapi/datafusion/context/index.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal"
href="../../../user-guide/io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
@@ -910,6 +915,15 @@
</code>
</a>
</li>
+ <li class="toc-h4 nav-item toc-entry">
+ <a class="reference internal nav-link"
href="#datafusion.context.SessionContext.register_table_provider">
+ <code class="docutils literal notranslate">
+ <span class="pre">
+ SessionContext.register_table_provider()
+ </span>
+ </code>
+ </a>
+ </li>
<li class="toc-h4 nav-item toc-entry">
<a class="reference internal nav-link"
href="#datafusion.context.SessionContext.register_udaf">
<code class="docutils literal notranslate">
@@ -1945,6 +1959,14 @@ register it into the session using the given name.</p>
</dl>
</dd></dl>
+<dl class="py method">
+<dt class="sig sig-object py"
id="datafusion.context.SessionContext.register_table_provider">
+<span class="sig-name descname"><span
class="pre">register_table_provider</span></span><span
class="sig-paren">(</span><em class="sig-param"><span class="n"><span
class="pre">name</span></span><span class="p"><span
class="pre">:</span></span><span class="w"> </span><span class="n"><span
class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span
class="pre">provider</span></span><span class="p"><span
class="pre">:</span></span><span class="w"> </span><span class="n">< [...]
+<dd><p>Register a table provider.</p>
+<p>This table provider must have a method called <code class="docutils literal
notranslate"><span class="pre">__datafusion_table_provider__</span></code>
+which returns a PyCapsule that exposes a <code class="docutils literal
notranslate"><span class="pre">FFI_TableProvider</span></code>.</p>
+</dd></dl>
+
<dl class="py method">
<dt class="sig sig-object py"
id="datafusion.context.SessionContext.register_udaf">
<span class="sig-name descname"><span
class="pre">register_udaf</span></span><span class="sig-paren">(</span><em
class="sig-param"><span class="n"><span class="pre">udaf</span></span><span
class="p"><span class="pre">:</span></span><span class="w"> </span><span
class="n"><a class="reference internal"
href="../udf/index.html#datafusion.udf.AggregateUDF"
title="datafusion.udf.AggregateUDF"><span
class="pre">datafusion.udf.AggregateUDF</span></a></span></em><span
class="sig-paren">)</span> [...]
diff --git a/autoapi/datafusion/dataframe/index.html
b/autoapi/datafusion/dataframe/index.html
index ea8f791..3542259 100644
--- a/autoapi/datafusion/dataframe/index.html
+++ b/autoapi/datafusion/dataframe/index.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal"
href="../../../user-guide/io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
diff --git a/autoapi/datafusion/expr/index.html
b/autoapi/datafusion/expr/index.html
index 1ee6b74..3358f35 100644
--- a/autoapi/datafusion/expr/index.html
+++ b/autoapi/datafusion/expr/index.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal"
href="../../../user-guide/io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
diff --git a/autoapi/datafusion/functions/index.html
b/autoapi/datafusion/functions/index.html
index 5ccb68a..5b421ff 100644
--- a/autoapi/datafusion/functions/index.html
+++ b/autoapi/datafusion/functions/index.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal"
href="../../../user-guide/io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
diff --git a/autoapi/datafusion/index.html b/autoapi/datafusion/index.html
index 4b37677..240f060 100644
--- a/autoapi/datafusion/index.html
+++ b/autoapi/datafusion/index.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal"
href="../../user-guide/io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
diff --git a/autoapi/datafusion/input/base/index.html
b/autoapi/datafusion/input/base/index.html
index 7afd2a8..90b7ba1 100644
--- a/autoapi/datafusion/input/base/index.html
+++ b/autoapi/datafusion/input/base/index.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal"
href="../../../../user-guide/io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
diff --git a/autoapi/datafusion/input/index.html
b/autoapi/datafusion/input/index.html
index b59951e..afabb5c 100644
--- a/autoapi/datafusion/input/index.html
+++ b/autoapi/datafusion/input/index.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal"
href="../../../user-guide/io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
diff --git a/autoapi/datafusion/input/location/index.html
b/autoapi/datafusion/input/location/index.html
index 76db0da..aca64fd 100644
--- a/autoapi/datafusion/input/location/index.html
+++ b/autoapi/datafusion/input/location/index.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal"
href="../../../../user-guide/io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
diff --git a/autoapi/datafusion/object_store/index.html
b/autoapi/datafusion/object_store/index.html
index 4d9f31c..2c1b3ec 100644
--- a/autoapi/datafusion/object_store/index.html
+++ b/autoapi/datafusion/object_store/index.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal"
href="../../../user-guide/io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
diff --git a/autoapi/datafusion/plan/index.html
b/autoapi/datafusion/plan/index.html
index a5d25d6..3713b4c 100644
--- a/autoapi/datafusion/plan/index.html
+++ b/autoapi/datafusion/plan/index.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal"
href="../../../user-guide/io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
diff --git a/autoapi/datafusion/record_batch/index.html
b/autoapi/datafusion/record_batch/index.html
index d6a0131..88db99e 100644
--- a/autoapi/datafusion/record_batch/index.html
+++ b/autoapi/datafusion/record_batch/index.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal"
href="../../../user-guide/io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
diff --git a/autoapi/datafusion/substrait/index.html
b/autoapi/datafusion/substrait/index.html
index 7475708..01fe042 100644
--- a/autoapi/datafusion/substrait/index.html
+++ b/autoapi/datafusion/substrait/index.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal"
href="../../../user-guide/io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
diff --git a/autoapi/datafusion/udf/index.html
b/autoapi/datafusion/udf/index.html
index be1719e..a632dc2 100644
--- a/autoapi/datafusion/udf/index.html
+++ b/autoapi/datafusion/udf/index.html
@@ -203,6 +203,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal"
href="../../../user-guide/io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
diff --git a/autoapi/index.html b/autoapi/index.html
index b68660c..8534269 100644
--- a/autoapi/index.html
+++ b/autoapi/index.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="../user-guide/io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
diff --git a/contributor-guide/introduction.html
b/contributor-guide/introduction.html
index 033b601..08d928b 100644
--- a/contributor-guide/introduction.html
+++ b/contributor-guide/introduction.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="../user-guide/io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
diff --git a/genindex.html b/genindex.html
index 624ff1e..f5bffb6 100644
--- a/genindex.html
+++ b/genindex.html
@@ -201,6 +201,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="user-guide/io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
@@ -1867,6 +1872,8 @@
<li><a
href="autoapi/datafusion/context/index.html#datafusion.context.SessionContext.register_record_batches">register_record_batches()
(datafusion.context.SessionContext method)</a>
</li>
<li><a
href="autoapi/datafusion/context/index.html#datafusion.context.SessionContext.register_table">register_table()
(datafusion.context.SessionContext method)</a>
+</li>
+ <li><a
href="autoapi/datafusion/context/index.html#datafusion.context.SessionContext.register_table_provider">register_table_provider()
(datafusion.context.SessionContext method)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
diff --git a/index.html b/index.html
index 440adad..86e75d0 100644
--- a/index.html
+++ b/index.html
@@ -203,6 +203,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="user-guide/io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
diff --git a/objects.inv b/objects.inv
index 9b8fa4a..6a86b81 100644
Binary files a/objects.inv and b/objects.inv differ
diff --git a/py-modindex.html b/py-modindex.html
index e5fcf4c..79d243f 100644
--- a/py-modindex.html
+++ b/py-modindex.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="user-guide/io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
diff --git a/search.html b/search.html
index 80d58a8..0b04e1c 100644
--- a/search.html
+++ b/search.html
@@ -208,6 +208,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="user-guide/io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
diff --git a/searchindex.js b/searchindex.js
index 5e84516..4ae72ad 100644
--- a/searchindex.js
+++ b/searchindex.js
@@ -1 +1 @@
-Search.setIndex({"alltitles": {"API Reference": [[14, null]], "Aggregate
Functions": [[18, "aggregate-functions"], [25, "aggregate-functions"], [26,
"aggregate-functions"]], "Aggregation": [[18, null]], "Arrays": [[20,
"arrays"]], "Arrow": [[29, null]], "Attributes": [[3, "attributes"], [5,
"attributes"], [9, "attributes"], [13, "attributes"]], "Available Functions":
[[26, "available-functions"]], "Avro": [[30, null]], "Basic Operations": [[19,
null]], "Boolean": [[20, "boolean"]], "CSV" [...]
\ No newline at end of file
+Search.setIndex({"alltitles": {"API Reference": [[14, null]], "Aggregate
Functions": [[18, "aggregate-functions"], [25, "aggregate-functions"], [26,
"aggregate-functions"]], "Aggregation": [[18, null]], "Arrays": [[20,
"arrays"]], "Arrow": [[29, null]], "Attributes": [[3, "attributes"], [5,
"attributes"], [9, "attributes"], [13, "attributes"]], "Available Functions":
[[26, "available-functions"]], "Avro": [[30, null]], "Basic Operations": [[19,
null]], "Boolean": [[20, "boolean"]], "CSV" [...]
\ No newline at end of file
diff --git a/user-guide/basics.html b/user-guide/basics.html
index efb5992..c9f93ab 100644
--- a/user-guide/basics.html
+++ b/user-guide/basics.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
diff --git a/user-guide/common-operations/aggregations.html
b/user-guide/common-operations/aggregations.html
index 87872f9..833a884 100644
--- a/user-guide/common-operations/aggregations.html
+++ b/user-guide/common-operations/aggregations.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="../io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
@@ -421,7 +426,7 @@ to form a single summary value. For performing an
aggregation, DataFusion provid
<span class="o">...</span><span class="p">:</span> <span
class="s2">"pokemon.csv"</span><span class="p">,</span>
<span class="o">...</span><span class="p">:</span> <span class="p">)</span>
<span class="o">...</span><span class="p">:</span>
-<span class="n">Out</span><span class="p">[</span><span
class="mi">5</span><span class="p">]:</span> <span class="p">(</span><span
class="s1">'pokemon.csv'</span><span class="p">,</span> <span
class="o"><</span><span class="n">http</span><span class="o">.</span><span
class="n">client</span><span class="o">.</span><span
class="n">HTTPMessage</span> <span class="n">at</span> <span
class="mh">0x7f9b46516c10</span><span class="o">></span><span
class="p">)</span>
+<span class="n">Out</span><span class="p">[</span><span
class="mi">5</span><span class="p">]:</span> <span class="p">(</span><span
class="s1">'pokemon.csv'</span><span class="p">,</span> <span
class="o"><</span><span class="n">http</span><span class="o">.</span><span
class="n">client</span><span class="o">.</span><span
class="n">HTTPMessage</span> <span class="n">at</span> <span
class="mh">0x7f83cc069c10</span><span class="o">></span><span
class="p">)</span>
<span class="n">In</span> <span class="p">[</span><span
class="mi">6</span><span class="p">]:</span> <span class="n">ctx</span> <span
class="o">=</span> <span class="n">SessionContext</span><span
class="p">()</span>
@@ -470,6 +475,8 @@ For grouping the <code class="code docutils literal
notranslate"><span class="pr
<span class="o">+----------+-----------+-------------------+-----------+</span>
<span class="o">|</span> <span class="n">Type</span> <span class="mi">1</span>
<span class="o">|</span> <span class="n">Max</span> <span
class="n">Speed</span> <span class="o">|</span> <span class="n">Avg</span>
<span class="n">Speed</span> <span class="o">|</span> <span
class="n">Min</span> <span class="n">Speed</span> <span class="o">|</span>
<span class="o">+----------+-----------+-------------------+-----------+</span>
+<span class="o">|</span> <span class="n">Normal</span> <span
class="o">|</span> <span class="mi">121</span> <span class="o">|</span>
<span class="mf">72.75</span> <span class="o">|</span> <span
class="mi">20</span> <span class="o">|</span>
+<span class="o">|</span> <span class="n">Ice</span> <span
class="o">|</span> <span class="mi">95</span> <span class="o">|</span>
<span class="mf">90.0</span> <span class="o">|</span> <span
class="mi">85</span> <span class="o">|</span>
<span class="o">|</span> <span class="n">Grass</span> <span
class="o">|</span> <span class="mi">80</span> <span class="o">|</span>
<span class="mf">54.23076923076923</span> <span class="o">|</span> <span
class="mi">30</span> <span class="o">|</span>
<span class="o">|</span> <span class="n">Fire</span> <span
class="o">|</span> <span class="mi">105</span> <span class="o">|</span>
<span class="mf">86.28571428571429</span> <span class="o">|</span> <span
class="mi">60</span> <span class="o">|</span>
<span class="o">|</span> <span class="n">Water</span> <span
class="o">|</span> <span class="mi">115</span> <span class="o">|</span>
<span class="mf">67.25806451612904</span> <span class="o">|</span> <span
class="mi">15</span> <span class="o">|</span>
@@ -478,8 +485,6 @@ For grouping the <code class="code docutils literal
notranslate"><span class="pr
<span class="o">|</span> <span class="n">Psychic</span> <span
class="o">|</span> <span class="mi">150</span> <span class="o">|</span>
<span class="mf">99.25</span> <span class="o">|</span> <span
class="mi">42</span> <span class="o">|</span>
<span class="o">|</span> <span class="n">Rock</span> <span
class="o">|</span> <span class="mi">150</span> <span class="o">|</span>
<span class="mf">67.5</span> <span class="o">|</span> <span
class="mi">20</span> <span class="o">|</span>
<span class="o">|</span> <span class="n">Ghost</span> <span
class="o">|</span> <span class="mi">130</span> <span class="o">|</span>
<span class="mf">103.75</span> <span class="o">|</span> <span
class="mi">80</span> <span class="o">|</span>
-<span class="o">|</span> <span class="n">Dragon</span> <span
class="o">|</span> <span class="mi">80</span> <span class="o">|</span>
<span class="mf">66.66666666666667</span> <span class="o">|</span> <span
class="mi">50</span> <span class="o">|</span>
-<span class="o">|</span> <span class="n">Normal</span> <span
class="o">|</span> <span class="mi">121</span> <span class="o">|</span>
<span class="mf">72.75</span> <span class="o">|</span> <span
class="mi">20</span> <span class="o">|</span>
<span class="o">+----------+-----------+-------------------+-----------+</span>
</pre></div>
</div>
@@ -562,15 +567,15 @@ Pokemon set. Since there will be many entries of <code
class="docutils literal n
<span
class="o">+----------+--------------------------------------------------+</span>
<span class="o">|</span> <span class="n">Type</span> <span class="mi">1</span>
<span class="o">|</span> <span class="n">Type</span> <span
class="mi">2</span> <span class="n">List</span>
<span class="o">|</span>
<span
class="o">+----------+--------------------------------------------------+</span>
-<span class="o">|</span> <span class="n">Normal</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Flying</span><span
class="p">,</span> <span class="n">Fairy</span><span class="p">,</span> <span
class="p">]</span> <span class="o">|</span>
+<span class="o">|</span> <span class="n">Normal</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Flying</span><span
class="p">,</span> <span class="p">,</span> <span class="n">Fairy</span><span
class="p">]</span> <span class="o">|</span>
<span class="o">|</span> <span class="n">Ice</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Flying</span><span
class="p">,</span> <span class="n">Psychic</span><span class="p">]</span>
<span class="o">|</span>
-<span class="o">|</span> <span class="n">Bug</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Flying</span><span
class="p">,</span> <span class="n">Poison</span><span class="p">,</span> <span
class="p">,</span> <span class="n">Grass</span><span class="p">]</span>
<span class="o">|</span>
-<span class="o">|</span> <span class="n">Poison</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Ground</span><span
class="p">,</span> <span class="n">Flying</span><span class="p">,</span> <span
class="p">]</span> <span class="o">|</span>
-<span class="o">|</span> <span class="n">Electric</span> <span
class="o">|</span> <span class="p">[,</span> <span class="n">Flying</span><span
class="p">,</span> <span class="n">Steel</span><span class="p">]</span>
<span class="o">|</span>
+<span class="o">|</span> <span class="n">Bug</span> <span
class="o">|</span> <span class="p">[,</span> <span class="n">Grass</span><span
class="p">,</span> <span class="n">Flying</span><span class="p">,</span> <span
class="n">Poison</span><span class="p">]</span> <span
class="o">|</span>
+<span class="o">|</span> <span class="n">Poison</span> <span
class="o">|</span> <span class="p">[,</span> <span class="n">Flying</span><span
class="p">,</span> <span class="n">Ground</span><span class="p">]</span>
<span class="o">|</span>
+<span class="o">|</span> <span class="n">Electric</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Steel</span><span
class="p">,</span> <span class="p">,</span> <span class="n">Flying</span><span
class="p">]</span> <span class="o">|</span>
<span class="o">|</span> <span class="n">Fairy</span> <span
class="o">|</span> <span class="p">[]</span>
<span class="o">|</span>
<span class="o">|</span> <span class="n">Grass</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Poison</span><span
class="p">,</span> <span class="p">,</span> <span class="n">Psychic</span><span
class="p">]</span> <span class="o">|</span>
<span class="o">|</span> <span class="n">Fire</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Flying</span><span
class="p">,</span> <span class="n">Dragon</span><span class="p">,</span> <span
class="p">]</span> <span class="o">|</span>
-<span class="o">|</span> <span class="n">Water</span> <span
class="o">|</span> <span class="p">[,</span> <span
class="n">Psychic</span><span class="p">,</span> <span
class="n">Ice</span><span class="p">,</span> <span class="n">Dark</span><span
class="p">,</span> <span class="n">Fighting</span><span class="p">,</span>
<span class="n">Flying</span><span class="p">,</span> <span
class="n">Poison</span><span class="p">]</span> <span class="o">|</span>
+<span class="o">|</span> <span class="n">Water</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Poison</span><span
class="p">,</span> <span class="p">,</span> <span class="n">Dark</span><span
class="p">,</span> <span class="n">Fighting</span><span class="p">,</span>
<span class="n">Flying</span><span class="p">,</span> <span
class="n">Ice</span><span class="p">,</span> <span
class="n">Psychic</span><span class="p">]</span> <span class="o">|</span>
<span class="o">|</span> <span class="n">Ground</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Rock</span><span
class="p">,</span> <span class="p">]</span>
<span class="o">|</span>
<span
class="o">+----------+--------------------------------------------------+</span>
</pre></div>
@@ -585,15 +590,15 @@ entries entirely removed. The second is we can use the
<code class="docutils lit
<span
class="o">+----------+------------------------------------------------+</span>
<span class="o">|</span> <span class="n">Type</span> <span class="mi">1</span>
<span class="o">|</span> <span class="n">Type</span> <span
class="mi">2</span> <span class="n">List</span>
<span class="o">|</span>
<span
class="o">+----------+------------------------------------------------+</span>
-<span class="o">|</span> <span class="n">Normal</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Fairy</span><span
class="p">,</span> <span class="n">Flying</span><span class="p">]</span>
<span class="o">|</span>
-<span class="o">|</span> <span class="n">Ice</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Psychic</span><span
class="p">,</span> <span class="n">Flying</span><span class="p">]</span>
<span class="o">|</span>
<span class="o">|</span> <span class="n">Bug</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Poison</span><span
class="p">,</span> <span class="n">Grass</span><span class="p">,</span> <span
class="n">Flying</span><span class="p">]</span> <span
class="o">|</span>
-<span class="o">|</span> <span class="n">Poison</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Ground</span><span
class="p">,</span> <span class="n">Flying</span><span class="p">]</span>
<span class="o">|</span>
-<span class="o">|</span> <span class="n">Electric</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Steel</span><span
class="p">,</span> <span class="n">Flying</span><span class="p">]</span>
<span class="o">|</span>
+<span class="o">|</span> <span class="n">Poison</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Flying</span><span
class="p">,</span> <span class="n">Ground</span><span class="p">]</span>
<span class="o">|</span>
+<span class="o">|</span> <span class="n">Electric</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Flying</span><span
class="p">,</span> <span class="n">Steel</span><span class="p">]</span>
<span class="o">|</span>
+<span class="o">|</span> <span class="n">Normal</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Fairy</span><span
class="p">,</span> <span class="n">Flying</span><span class="p">]</span>
<span class="o">|</span>
+<span class="o">|</span> <span class="n">Ice</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Flying</span><span
class="p">,</span> <span class="n">Psychic</span><span class="p">]</span>
<span class="o">|</span>
<span class="o">|</span> <span class="n">Grass</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Poison</span><span
class="p">,</span> <span class="n">Psychic</span><span class="p">]</span>
<span class="o">|</span>
<span class="o">|</span> <span class="n">Fire</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Dragon</span><span
class="p">,</span> <span class="n">Flying</span><span class="p">]</span>
<span class="o">|</span>
-<span class="o">|</span> <span class="n">Water</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Dark</span><span
class="p">,</span> <span class="n">Ice</span><span class="p">,</span> <span
class="n">Flying</span><span class="p">,</span> <span
class="n">Poison</span><span class="p">,</span> <span
class="n">Fighting</span><span class="p">,</span> <span
class="n">Psychic</span><span class="p">]</span> <span class="o">|</span>
-<span class="o">|</span> <span class="n">Rock</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Water</span><span
class="p">,</span> <span class="n">Flying</span><span class="p">,</span> <span
class="n">Ground</span><span class="p">]</span> <span
class="o">|</span>
+<span class="o">|</span> <span class="n">Water</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Ice</span><span
class="p">,</span> <span class="n">Flying</span><span class="p">,</span> <span
class="n">Dark</span><span class="p">,</span> <span
class="n">Fighting</span><span class="p">,</span> <span
class="n">Poison</span><span class="p">,</span> <span
class="n">Psychic</span><span class="p">]</span> <span class="o">|</span>
+<span class="o">|</span> <span class="n">Rock</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Ground</span><span
class="p">,</span> <span class="n">Flying</span><span class="p">,</span> <span
class="n">Water</span><span class="p">]</span> <span
class="o">|</span>
<span class="o">|</span> <span class="n">Ghost</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Poison</span><span
class="p">]</span> <span
class="o">|</span>
<span
class="o">+----------+------------------------------------------------+</span>
@@ -603,15 +608,15 @@ entries entirely removed. The second is we can use the
<code class="docutils lit
<span
class="o">+----------+------------------------------------------------+</span>
<span class="o">|</span> <span class="n">Type</span> <span class="mi">1</span>
<span class="o">|</span> <span class="n">Type</span> <span
class="mi">2</span> <span class="n">List</span>
<span class="o">|</span>
<span
class="o">+----------+------------------------------------------------+</span>
-<span class="o">|</span> <span class="n">Normal</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Flying</span><span
class="p">,</span> <span class="n">Fairy</span><span class="p">]</span>
<span class="o">|</span>
+<span class="o">|</span> <span class="n">Normal</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Fairy</span><span
class="p">,</span> <span class="n">Flying</span><span class="p">]</span>
<span class="o">|</span>
<span class="o">|</span> <span class="n">Ice</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Flying</span><span
class="p">,</span> <span class="n">Psychic</span><span class="p">]</span>
<span class="o">|</span>
-<span class="o">|</span> <span class="n">Bug</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Poison</span><span
class="p">,</span> <span class="n">Grass</span><span class="p">,</span> <span
class="n">Flying</span><span class="p">]</span> <span
class="o">|</span>
+<span class="o">|</span> <span class="n">Bug</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Flying</span><span
class="p">,</span> <span class="n">Grass</span><span class="p">,</span> <span
class="n">Poison</span><span class="p">]</span> <span
class="o">|</span>
<span class="o">|</span> <span class="n">Poison</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Ground</span><span
class="p">,</span> <span class="n">Flying</span><span class="p">]</span>
<span class="o">|</span>
<span class="o">|</span> <span class="n">Electric</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Steel</span><span
class="p">,</span> <span class="n">Flying</span><span class="p">]</span>
<span class="o">|</span>
<span class="o">|</span> <span class="n">Fairy</span> <span
class="o">|</span> <span
class="o">|</span>
<span class="o">|</span> <span class="n">Grass</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Psychic</span><span
class="p">,</span> <span class="n">Poison</span><span class="p">]</span>
<span class="o">|</span>
<span class="o">|</span> <span class="n">Fire</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Flying</span><span
class="p">,</span> <span class="n">Dragon</span><span class="p">]</span>
<span class="o">|</span>
-<span class="o">|</span> <span class="n">Water</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Fighting</span><span
class="p">,</span> <span class="n">Dark</span><span class="p">,</span> <span
class="n">Poison</span><span class="p">,</span> <span class="n">Ice</span><span
class="p">,</span> <span class="n">Psychic</span><span class="p">,</span> <span
class="n">Flying</span><span class="p">]</span> <span class="o">|</span>
+<span class="o">|</span> <span class="n">Water</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Dark</span><span
class="p">,</span> <span class="n">Flying</span><span class="p">,</span> <span
class="n">Ice</span><span class="p">,</span> <span class="n">Poison</span><span
class="p">,</span> <span class="n">Fighting</span><span class="p">,</span>
<span class="n">Psychic</span><span class="p">]</span> <span class="o">|</span>
<span class="o">|</span> <span class="n">Ground</span> <span
class="o">|</span> <span class="p">[</span><span class="n">Rock</span><span
class="p">]</span> <span
class="o">|</span>
<span
class="o">+----------+------------------------------------------------+</span>
</pre></div>
@@ -696,12 +701,12 @@ aggregate function without filtering rows from the entire
DataFrame.</p>
<span class="o">|</span> <span class="n">Poison</span> <span
class="o">|</span> <span class="mf">58.785714285714285</span> <span
class="o">|</span> <span class="mf">48.0</span> <span
class="o">|</span>
<span class="o">|</span> <span class="n">Electric</span> <span
class="o">|</span> <span class="mf">98.88888888888889</span> <span
class="o">|</span> <span class="mf">72.5</span> <span
class="o">|</span>
<span class="o">|</span> <span class="n">Fairy</span> <span
class="o">|</span> <span class="mf">47.5</span> <span
class="o">|</span> <span class="mf">35.0</span> <span
class="o">|</span>
-<span class="o">|</span> <span class="n">Normal</span> <span
class="o">|</span> <span class="mf">72.75</span> <span
class="o">|</span> <span class="mf">52.8</span> <span
class="o">|</span>
-<span class="o">|</span> <span class="n">Ice</span> <span
class="o">|</span> <span class="mf">90.0</span> <span
class="o">|</span> <span class="o">|</span>
<span class="o">|</span> <span class="n">Grass</span> <span
class="o">|</span> <span class="mf">54.23076923076923</span> <span
class="o">|</span> <span class="mf">42.5</span> <span
class="o">|</span>
<span class="o">|</span> <span class="n">Fire</span> <span
class="o">|</span> <span class="mf">86.28571428571429</span> <span
class="o">|</span> <span class="mf">65.0</span> <span
class="o">|</span>
<span class="o">|</span> <span class="n">Water</span> <span
class="o">|</span> <span class="mf">67.25806451612904</span> <span
class="o">|</span> <span class="mf">63.833333333333336</span> <span
class="o">|</span>
<span class="o">|</span> <span class="n">Ground</span> <span
class="o">|</span> <span class="mf">58.125</span> <span
class="o">|</span> <span class="o">|</span>
+<span class="o">|</span> <span class="n">Fighting</span> <span
class="o">|</span> <span class="mf">66.14285714285714</span> <span
class="o">|</span> <span class="o">|</span>
+<span class="o">|</span> <span class="n">Psychic</span> <span
class="o">|</span> <span class="mf">99.25</span> <span
class="o">|</span> <span class="mf">81.75</span> <span
class="o">|</span>
<span class="o">+----------+--------------------+----------------------+</span>
</pre></div>
</div>
diff --git a/user-guide/common-operations/basic-info.html
b/user-guide/common-operations/basic-info.html
index 80c0acd..4d01001 100644
--- a/user-guide/common-operations/basic-info.html
+++ b/user-guide/common-operations/basic-info.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="../io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
@@ -389,11 +394,11 @@
<span class="o">+-----+---------+--------+--------+</span>
<span class="o">|</span> <span class="n">nrs</span> <span class="o">|</span>
<span class="n">names</span> <span class="o">|</span> <span
class="n">random</span> <span class="o">|</span> <span class="n">groups</span>
<span class="o">|</span>
<span class="o">+-----+---------+--------+--------+</span>
-<span class="o">|</span> <span class="mi">1</span> <span class="o">|</span>
<span class="n">python</span> <span class="o">|</span> <span
class="mi">222</span> <span class="o">|</span> <span class="n">A</span>
<span class="o">|</span>
-<span class="o">|</span> <span class="mi">2</span> <span class="o">|</span>
<span class="n">ruby</span> <span class="o">|</span> <span
class="mi">673</span> <span class="o">|</span> <span class="n">A</span>
<span class="o">|</span>
-<span class="o">|</span> <span class="mi">3</span> <span class="o">|</span>
<span class="n">java</span> <span class="o">|</span> <span
class="mi">469</span> <span class="o">|</span> <span class="n">B</span>
<span class="o">|</span>
-<span class="o">|</span> <span class="mi">4</span> <span class="o">|</span>
<span class="n">haskell</span> <span class="o">|</span> <span
class="mi">891</span> <span class="o">|</span> <span class="n">C</span>
<span class="o">|</span>
-<span class="o">|</span> <span class="mi">5</span> <span class="o">|</span>
<span class="n">go</span> <span class="o">|</span> <span
class="mi">640</span> <span class="o">|</span> <span class="n">B</span>
<span class="o">|</span>
+<span class="o">|</span> <span class="mi">1</span> <span class="o">|</span>
<span class="n">python</span> <span class="o">|</span> <span
class="mi">558</span> <span class="o">|</span> <span class="n">A</span>
<span class="o">|</span>
+<span class="o">|</span> <span class="mi">2</span> <span class="o">|</span>
<span class="n">ruby</span> <span class="o">|</span> <span
class="mi">690</span> <span class="o">|</span> <span class="n">A</span>
<span class="o">|</span>
+<span class="o">|</span> <span class="mi">3</span> <span class="o">|</span>
<span class="n">java</span> <span class="o">|</span> <span
class="mi">138</span> <span class="o">|</span> <span class="n">B</span>
<span class="o">|</span>
+<span class="o">|</span> <span class="mi">4</span> <span class="o">|</span>
<span class="n">haskell</span> <span class="o">|</span> <span
class="mi">134</span> <span class="o">|</span> <span class="n">C</span>
<span class="o">|</span>
+<span class="o">|</span> <span class="mi">5</span> <span class="o">|</span>
<span class="n">go</span> <span class="o">|</span> <span
class="mi">227</span> <span class="o">|</span> <span class="n">B</span>
<span class="o">|</span>
<span class="o">+-----+---------+--------+--------+</span>
</pre></div>
</div>
@@ -404,8 +409,8 @@
<span class="o">+-----+--------+--------+--------+</span>
<span class="o">|</span> <span class="n">nrs</span> <span class="o">|</span>
<span class="n">names</span> <span class="o">|</span> <span
class="n">random</span> <span class="o">|</span> <span class="n">groups</span>
<span class="o">|</span>
<span class="o">+-----+--------+--------+--------+</span>
-<span class="o">|</span> <span class="mi">1</span> <span class="o">|</span>
<span class="n">python</span> <span class="o">|</span> <span
class="mi">222</span> <span class="o">|</span> <span class="n">A</span>
<span class="o">|</span>
-<span class="o">|</span> <span class="mi">2</span> <span class="o">|</span>
<span class="n">ruby</span> <span class="o">|</span> <span
class="mi">673</span> <span class="o">|</span> <span class="n">A</span>
<span class="o">|</span>
+<span class="o">|</span> <span class="mi">1</span> <span class="o">|</span>
<span class="n">python</span> <span class="o">|</span> <span
class="mi">558</span> <span class="o">|</span> <span class="n">A</span>
<span class="o">|</span>
+<span class="o">|</span> <span class="mi">2</span> <span class="o">|</span>
<span class="n">ruby</span> <span class="o">|</span> <span
class="mi">690</span> <span class="o">|</span> <span class="n">A</span>
<span class="o">|</span>
<span class="o">+-----+--------+--------+--------+</span>
</pre></div>
</div>
@@ -423,11 +428,11 @@ passing them to an Arrow table, and then converting them
to a pandas DataFrame.<
<div class="highlight-ipython notranslate"><div
class="highlight"><pre><span></span><span class="n">In</span> <span
class="p">[</span><span class="mi">8</span><span class="p">]:</span> <span
class="n">df</span><span class="o">.</span><span
class="n">to_pandas</span><span class="p">()</span>
<span class="n">Out</span><span class="p">[</span><span
class="mi">8</span><span class="p">]:</span>
<span class="n">nrs</span> <span class="n">names</span> <span
class="n">random</span> <span class="n">groups</span>
-<span class="mi">0</span> <span class="mi">1</span> <span
class="n">python</span> <span class="mi">222</span> <span
class="n">A</span>
-<span class="mi">1</span> <span class="mi">2</span> <span
class="n">ruby</span> <span class="mi">673</span> <span
class="n">A</span>
-<span class="mi">2</span> <span class="mi">3</span> <span
class="n">java</span> <span class="mi">469</span> <span
class="n">B</span>
-<span class="mi">3</span> <span class="mi">4</span> <span
class="n">haskell</span> <span class="mi">891</span> <span
class="n">C</span>
-<span class="mi">4</span> <span class="mi">5</span> <span
class="n">go</span> <span class="mi">640</span> <span
class="n">B</span>
+<span class="mi">0</span> <span class="mi">1</span> <span
class="n">python</span> <span class="mi">558</span> <span
class="n">A</span>
+<span class="mi">1</span> <span class="mi">2</span> <span
class="n">ruby</span> <span class="mi">690</span> <span
class="n">A</span>
+<span class="mi">2</span> <span class="mi">3</span> <span
class="n">java</span> <span class="mi">138</span> <span
class="n">B</span>
+<span class="mi">3</span> <span class="mi">4</span> <span
class="n">haskell</span> <span class="mi">134</span> <span
class="n">C</span>
+<span class="mi">4</span> <span class="mi">5</span> <span
class="n">go</span> <span class="mi">227</span> <span
class="n">B</span>
</pre></div>
</div>
<p><a class="reference internal"
href="../../autoapi/datafusion/dataframe/index.html#datafusion.dataframe.DataFrame.describe"
title="datafusion.dataframe.DataFrame.describe"><code class="xref py py-func
docutils literal notranslate"><span class="pre">describe()</span></code></a>
shows a quick statistic summary of your data:</p>
@@ -439,11 +444,11 @@ passing them to an Arrow table, and then converting them
to a pandas DataFrame.<
<span
class="o">+------------+--------------------+-------+--------------------+--------+</span>
<span class="o">|</span> <span class="n">count</span> <span
class="o">|</span> <span class="mf">5.0</span> <span
class="o">|</span> <span class="mi">5</span> <span class="o">|</span> <span
class="mf">5.0</span> <span class="o">|</span> <span
class="mi">5</span> <span class="o">|</span>
<span class="o">|</span> <span class="n">null_count</span> <span
class="o">|</span> <span class="mf">0.0</span> <span
class="o">|</span> <span class="mi">0</span> <span class="o">|</span> <span
class="mf">0.0</span> <span class="o">|</span> <span
class="mi">0</span> <span class="o">|</span>
-<span class="o">|</span> <span class="n">mean</span> <span
class="o">|</span> <span class="mf">3.0</span> <span
class="o">|</span> <span class="n">null</span> <span class="o">|</span> <span
class="mf">579.0</span> <span class="o">|</span> <span
class="n">null</span> <span class="o">|</span>
-<span class="o">|</span> <span class="n">std</span> <span
class="o">|</span> <span class="mf">1.5811388300841898</span> <span
class="o">|</span> <span class="n">null</span> <span class="o">|</span> <span
class="mf">249.72484858339587</span> <span class="o">|</span> <span
class="n">null</span> <span class="o">|</span>
-<span class="o">|</span> <span class="nb">min</span> <span
class="o">|</span> <span class="mf">1.0</span> <span
class="o">|</span> <span class="n">go</span> <span class="o">|</span> <span
class="mf">222.0</span> <span class="o">|</span> <span
class="n">A</span> <span class="o">|</span>
-<span class="o">|</span> <span class="nb">max</span> <span
class="o">|</span> <span class="mf">5.0</span> <span
class="o">|</span> <span class="n">ruby</span> <span class="o">|</span> <span
class="mf">891.0</span> <span class="o">|</span> <span
class="n">C</span> <span class="o">|</span>
-<span class="o">|</span> <span class="n">median</span> <span
class="o">|</span> <span class="mf">3.0</span> <span
class="o">|</span> <span class="n">null</span> <span class="o">|</span> <span
class="mf">640.0</span> <span class="o">|</span> <span
class="n">null</span> <span class="o">|</span>
+<span class="o">|</span> <span class="n">mean</span> <span
class="o">|</span> <span class="mf">3.0</span> <span
class="o">|</span> <span class="n">null</span> <span class="o">|</span> <span
class="mf">349.4</span> <span class="o">|</span> <span
class="n">null</span> <span class="o">|</span>
+<span class="o">|</span> <span class="n">std</span> <span
class="o">|</span> <span class="mf">1.5811388300841898</span> <span
class="o">|</span> <span class="n">null</span> <span class="o">|</span> <span
class="mf">257.67770567125126</span> <span class="o">|</span> <span
class="n">null</span> <span class="o">|</span>
+<span class="o">|</span> <span class="nb">min</span> <span
class="o">|</span> <span class="mf">1.0</span> <span
class="o">|</span> <span class="n">go</span> <span class="o">|</span> <span
class="mf">134.0</span> <span class="o">|</span> <span
class="n">A</span> <span class="o">|</span>
+<span class="o">|</span> <span class="nb">max</span> <span
class="o">|</span> <span class="mf">5.0</span> <span
class="o">|</span> <span class="n">ruby</span> <span class="o">|</span> <span
class="mf">690.0</span> <span class="o">|</span> <span
class="n">C</span> <span class="o">|</span>
+<span class="o">|</span> <span class="n">median</span> <span
class="o">|</span> <span class="mf">3.0</span> <span
class="o">|</span> <span class="n">null</span> <span class="o">|</span> <span
class="mf">227.0</span> <span class="o">|</span> <span
class="n">null</span> <span class="o">|</span>
<span
class="o">+------------+--------------------+-------+--------------------+--------+</span>
</pre></div>
</div>
diff --git a/user-guide/common-operations/expressions.html
b/user-guide/common-operations/expressions.html
index 6bd6921..97123f6 100644
--- a/user-guide/common-operations/expressions.html
+++ b/user-guide/common-operations/expressions.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="../io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
diff --git a/user-guide/common-operations/functions.html
b/user-guide/common-operations/functions.html
index 2e0cec5..2498299 100644
--- a/user-guide/common-operations/functions.html
+++ b/user-guide/common-operations/functions.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="../io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
@@ -410,7 +415,7 @@ In here we will cover some of the more popular use cases.
If you want to view al
<span class="o">...</span><span class="p">:</span> <span
class="s2">"pokemon.csv"</span><span class="p">,</span>
<span class="o">...</span><span class="p">:</span> <span class="p">)</span>
<span class="o">...</span><span class="p">:</span>
-<span class="n">Out</span><span class="p">[</span><span
class="mi">3</span><span class="p">]:</span> <span class="p">(</span><span
class="s1">'pokemon.csv'</span><span class="p">,</span> <span
class="o"><</span><span class="n">http</span><span class="o">.</span><span
class="n">client</span><span class="o">.</span><span
class="n">HTTPMessage</span> <span class="n">at</span> <span
class="mh">0x7f9b462399d0</span><span class="o">></span><span
class="p">)</span>
+<span class="n">Out</span><span class="p">[</span><span
class="mi">3</span><span class="p">]:</span> <span class="p">(</span><span
class="s1">'pokemon.csv'</span><span class="p">,</span> <span
class="o"><</span><span class="n">http</span><span class="o">.</span><span
class="n">client</span><span class="o">.</span><span
class="n">HTTPMessage</span> <span class="n">at</span> <span
class="mh">0x7f83c757ee90</span><span class="o">></span><span
class="p">)</span>
<span class="n">In</span> <span class="p">[</span><span
class="mi">4</span><span class="p">]:</span> <span class="n">ctx</span> <span
class="o">=</span> <span class="n">SessionContext</span><span
class="p">()</span>
@@ -484,16 +489,16 @@ In here we will cover some of the more popular use cases.
If you want to view al
<span class="o">+--------------------------------+</span>
<span class="o">|</span> <span class="n">now</span><span class="p">()</span>
<span class="o">|</span>
<span class="o">+--------------------------------+</span>
-<span class="o">|</span> <span class="mi">2024</span><span
class="o">-</span><span class="mi">11</span><span class="o">-</span><span
class="mi">12</span><span class="n">T11</span><span class="p">:</span><span
class="mi">24</span><span class="p">:</span><span
class="mf">49.920770025</span><span class="n">Z</span> <span class="o">|</span>
-<span class="o">|</span> <span class="mi">2024</span><span
class="o">-</span><span class="mi">11</span><span class="o">-</span><span
class="mi">12</span><span class="n">T11</span><span class="p">:</span><span
class="mi">24</span><span class="p">:</span><span
class="mf">49.920770025</span><span class="n">Z</span> <span class="o">|</span>
-<span class="o">|</span> <span class="mi">2024</span><span
class="o">-</span><span class="mi">11</span><span class="o">-</span><span
class="mi">12</span><span class="n">T11</span><span class="p">:</span><span
class="mi">24</span><span class="p">:</span><span
class="mf">49.920770025</span><span class="n">Z</span> <span class="o">|</span>
-<span class="o">|</span> <span class="mi">2024</span><span
class="o">-</span><span class="mi">11</span><span class="o">-</span><span
class="mi">12</span><span class="n">T11</span><span class="p">:</span><span
class="mi">24</span><span class="p">:</span><span
class="mf">49.920770025</span><span class="n">Z</span> <span class="o">|</span>
-<span class="o">|</span> <span class="mi">2024</span><span
class="o">-</span><span class="mi">11</span><span class="o">-</span><span
class="mi">12</span><span class="n">T11</span><span class="p">:</span><span
class="mi">24</span><span class="p">:</span><span
class="mf">49.920770025</span><span class="n">Z</span> <span class="o">|</span>
-<span class="o">|</span> <span class="mi">2024</span><span
class="o">-</span><span class="mi">11</span><span class="o">-</span><span
class="mi">12</span><span class="n">T11</span><span class="p">:</span><span
class="mi">24</span><span class="p">:</span><span
class="mf">49.920770025</span><span class="n">Z</span> <span class="o">|</span>
-<span class="o">|</span> <span class="mi">2024</span><span
class="o">-</span><span class="mi">11</span><span class="o">-</span><span
class="mi">12</span><span class="n">T11</span><span class="p">:</span><span
class="mi">24</span><span class="p">:</span><span
class="mf">49.920770025</span><span class="n">Z</span> <span class="o">|</span>
-<span class="o">|</span> <span class="mi">2024</span><span
class="o">-</span><span class="mi">11</span><span class="o">-</span><span
class="mi">12</span><span class="n">T11</span><span class="p">:</span><span
class="mi">24</span><span class="p">:</span><span
class="mf">49.920770025</span><span class="n">Z</span> <span class="o">|</span>
-<span class="o">|</span> <span class="mi">2024</span><span
class="o">-</span><span class="mi">11</span><span class="o">-</span><span
class="mi">12</span><span class="n">T11</span><span class="p">:</span><span
class="mi">24</span><span class="p">:</span><span
class="mf">49.920770025</span><span class="n">Z</span> <span class="o">|</span>
-<span class="o">|</span> <span class="mi">2024</span><span
class="o">-</span><span class="mi">11</span><span class="o">-</span><span
class="mi">12</span><span class="n">T11</span><span class="p">:</span><span
class="mi">24</span><span class="p">:</span><span
class="mf">49.920770025</span><span class="n">Z</span> <span class="o">|</span>
+<span class="o">|</span> <span class="mi">2024</span><span
class="o">-</span><span class="mi">11</span><span class="o">-</span><span
class="mi">15</span><span class="n">T16</span><span class="p">:</span><span
class="mi">27</span><span class="p">:</span><span
class="mf">49.682334692</span><span class="n">Z</span> <span class="o">|</span>
+<span class="o">|</span> <span class="mi">2024</span><span
class="o">-</span><span class="mi">11</span><span class="o">-</span><span
class="mi">15</span><span class="n">T16</span><span class="p">:</span><span
class="mi">27</span><span class="p">:</span><span
class="mf">49.682334692</span><span class="n">Z</span> <span class="o">|</span>
+<span class="o">|</span> <span class="mi">2024</span><span
class="o">-</span><span class="mi">11</span><span class="o">-</span><span
class="mi">15</span><span class="n">T16</span><span class="p">:</span><span
class="mi">27</span><span class="p">:</span><span
class="mf">49.682334692</span><span class="n">Z</span> <span class="o">|</span>
+<span class="o">|</span> <span class="mi">2024</span><span
class="o">-</span><span class="mi">11</span><span class="o">-</span><span
class="mi">15</span><span class="n">T16</span><span class="p">:</span><span
class="mi">27</span><span class="p">:</span><span
class="mf">49.682334692</span><span class="n">Z</span> <span class="o">|</span>
+<span class="o">|</span> <span class="mi">2024</span><span
class="o">-</span><span class="mi">11</span><span class="o">-</span><span
class="mi">15</span><span class="n">T16</span><span class="p">:</span><span
class="mi">27</span><span class="p">:</span><span
class="mf">49.682334692</span><span class="n">Z</span> <span class="o">|</span>
+<span class="o">|</span> <span class="mi">2024</span><span
class="o">-</span><span class="mi">11</span><span class="o">-</span><span
class="mi">15</span><span class="n">T16</span><span class="p">:</span><span
class="mi">27</span><span class="p">:</span><span
class="mf">49.682334692</span><span class="n">Z</span> <span class="o">|</span>
+<span class="o">|</span> <span class="mi">2024</span><span
class="o">-</span><span class="mi">11</span><span class="o">-</span><span
class="mi">15</span><span class="n">T16</span><span class="p">:</span><span
class="mi">27</span><span class="p">:</span><span
class="mf">49.682334692</span><span class="n">Z</span> <span class="o">|</span>
+<span class="o">|</span> <span class="mi">2024</span><span
class="o">-</span><span class="mi">11</span><span class="o">-</span><span
class="mi">15</span><span class="n">T16</span><span class="p">:</span><span
class="mi">27</span><span class="p">:</span><span
class="mf">49.682334692</span><span class="n">Z</span> <span class="o">|</span>
+<span class="o">|</span> <span class="mi">2024</span><span
class="o">-</span><span class="mi">11</span><span class="o">-</span><span
class="mi">15</span><span class="n">T16</span><span class="p">:</span><span
class="mi">27</span><span class="p">:</span><span
class="mf">49.682334692</span><span class="n">Z</span> <span class="o">|</span>
+<span class="o">|</span> <span class="mi">2024</span><span
class="o">-</span><span class="mi">11</span><span class="o">-</span><span
class="mi">15</span><span class="n">T16</span><span class="p">:</span><span
class="mi">27</span><span class="p">:</span><span
class="mf">49.682334692</span><span class="n">Z</span> <span class="o">|</span>
<span class="o">+--------------------------------+</span>
</pre></div>
</div>
diff --git a/user-guide/common-operations/index.html
b/user-guide/common-operations/index.html
index eae847c..dde8d3b 100644
--- a/user-guide/common-operations/index.html
+++ b/user-guide/common-operations/index.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="../io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
diff --git a/user-guide/common-operations/joins.html
b/user-guide/common-operations/joins.html
index 435c096..45b07f9 100644
--- a/user-guide/common-operations/joins.html
+++ b/user-guide/common-operations/joins.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="../io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
diff --git a/user-guide/common-operations/select-and-filter.html
b/user-guide/common-operations/select-and-filter.html
index 9d07a0b..f6c40e7 100644
--- a/user-guide/common-operations/select-and-filter.html
+++ b/user-guide/common-operations/select-and-filter.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="../io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
@@ -378,7 +383,7 @@
<span class="n">In</span> <span class="p">[</span><span
class="mi">3</span><span class="p">]:</span> <span class="n">urllib</span><span
class="o">.</span><span class="n">request</span><span class="o">.</span><span
class="n">urlretrieve</span><span class="p">(</span><span
class="s2">"https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-01.parquet"</span><span
class="p">,</span>
<span class="o">...</span><span class="p">:</span>
<span class="s2">"yellow_trip_data.parquet"</span><span
class="p">)</span>
<span class="o">...</span><span class="p">:</span>
-<span class="n">Out</span><span class="p">[</span><span
class="mi">3</span><span class="p">]:</span> <span class="p">(</span><span
class="s1">'yellow_trip_data.parquet'</span><span class="p">,</span>
<span class="o"><</span><span class="n">http</span><span
class="o">.</span><span class="n">client</span><span class="o">.</span><span
class="n">HTTPMessage</span> <span class="n">at</span> <span
class="mh">0x7f9b46292b10</span><span class="o">></span><span
class="p">)</span>
+<span class="n">Out</span><span class="p">[</span><span
class="mi">3</span><span class="p">]:</span> <span class="p">(</span><span
class="s1">'yellow_trip_data.parquet'</span><span class="p">,</span>
<span class="o"><</span><span class="n">http</span><span
class="o">.</span><span class="n">client</span><span class="o">.</span><span
class="n">HTTPMessage</span> <span class="n">at</span> <span
class="mh">0x7f83e7f02ad0</span><span class="o">></span><span
class="p">)</span>
<span class="n">In</span> <span class="p">[</span><span
class="mi">4</span><span class="p">]:</span> <span class="n">ctx</span> <span
class="o">=</span> <span class="n">SessionContext</span><span
class="p">()</span>
diff --git a/user-guide/common-operations/udf-and-udfa.html
b/user-guide/common-operations/udf-and-udfa.html
index faa7a5f..75908be 100644
--- a/user-guide/common-operations/udf-and-udfa.html
+++ b/user-guide/common-operations/udf-and-udfa.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="../io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
diff --git a/user-guide/common-operations/windows.html
b/user-guide/common-operations/windows.html
index f5aa9cd..87c65dc 100644
--- a/user-guide/common-operations/windows.html
+++ b/user-guide/common-operations/windows.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="../io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
@@ -428,7 +433,7 @@ provides a single value for multiple rows.</p>
<span class="o">...</span><span class="p">:</span> <span
class="s2">"pokemon.csv"</span><span class="p">,</span>
<span class="o">...</span><span class="p">:</span> <span class="p">)</span>
<span class="o">...</span><span class="p">:</span>
-<span class="n">Out</span><span class="p">[</span><span
class="mi">5</span><span class="p">]:</span> <span class="p">(</span><span
class="s1">'pokemon.csv'</span><span class="p">,</span> <span
class="o"><</span><span class="n">http</span><span class="o">.</span><span
class="n">client</span><span class="o">.</span><span
class="n">HTTPMessage</span> <span class="n">at</span> <span
class="mh">0x7f9b4416b410</span><span class="o">></span><span
class="p">)</span>
+<span class="n">Out</span><span class="p">[</span><span
class="mi">5</span><span class="p">]:</span> <span class="p">(</span><span
class="s1">'pokemon.csv'</span><span class="p">,</span> <span
class="o"><</span><span class="n">http</span><span class="o">.</span><span
class="n">client</span><span class="o">.</span><span
class="n">HTTPMessage</span> <span class="n">at</span> <span
class="mh">0x7f83a5fb53d0</span><span class="o">></span><span
class="p">)</span>
<span class="n">In</span> <span class="p">[</span><span
class="mi">6</span><span class="p">]:</span> <span class="n">ctx</span> <span
class="o">=</span> <span class="n">SessionContext</span><span
class="p">()</span>
diff --git a/user-guide/configuration.html b/user-guide/configuration.html
index 105bc06..5fafa94 100644
--- a/user-guide/configuration.html
+++ b/user-guide/configuration.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
diff --git a/user-guide/introduction.html b/user-guide/introduction.html
index 9eef9b0..c1320db 100644
--- a/user-guide/introduction.html
+++ b/user-guide/introduction.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
diff --git a/user-guide/io/arrow.html b/user-guide/io/arrow.html
index c016bb6..fbeb5a1 100644
--- a/user-guide/io/arrow.html
+++ b/user-guide/io/arrow.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
diff --git a/user-guide/io/avro.html b/user-guide/io/avro.html
index 3921245..5076d0b 100644
--- a/user-guide/io/avro.html
+++ b/user-guide/io/avro.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
diff --git a/user-guide/io/csv.html b/user-guide/io/csv.html
index c127ccc..fe74a42 100644
--- a/user-guide/io/csv.html
+++ b/user-guide/io/csv.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
diff --git a/user-guide/io/index.html b/user-guide/io/index.html
index 04b30a8..1eb9815 100644
--- a/user-guide/io/index.html
+++ b/user-guide/io/index.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
@@ -379,6 +384,7 @@
<li class="toctree-l1"><a class="reference internal"
href="csv.html">CSV</a></li>
<li class="toctree-l1"><a class="reference internal"
href="json.html">JSON</a></li>
<li class="toctree-l1"><a class="reference internal"
href="parquet.html">Parquet</a></li>
+<li class="toctree-l1"><a class="reference internal"
href="table_provider.html">Custom Table Provider</a></li>
</ul>
</div>
</section>
diff --git a/user-guide/io/json.html b/user-guide/io/json.html
index 3196841..2ffcc62 100644
--- a/user-guide/io/json.html
+++ b/user-guide/io/json.html
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
diff --git a/user-guide/io/parquet.html b/user-guide/io/parquet.html
index b9f84ee..d2aa45a 100644
--- a/user-guide/io/parquet.html
+++ b/user-guide/io/parquet.html
@@ -34,7 +34,7 @@
<script src="../../_static/sphinx_highlight.js?v=dc90522c"></script>
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
- <link rel="next" title="SQL" href="../sql.html" />
+ <link rel="next" title="Custom Table Provider" href="table_provider.html"
/>
<link rel="prev" title="JSON" href="json.html" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="docsearch:language" content="en">
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1">
@@ -395,10 +400,10 @@
<p class="prev-next-title">JSON</p>
</div>
</a>
- <a class='right-next' id="next-link" href="../sql.html" title="next page">
+ <a class='right-next' id="next-link" href="table_provider.html"
title="next page">
<div class="prev-next-info">
<p class="prev-next-subtitle">next</p>
- <p class="prev-next-title">SQL</p>
+ <p class="prev-next-title">Custom Table Provider</p>
</div>
<i class="fas fa-angle-right"></i>
</a>
diff --git a/user-guide/io/parquet.html b/user-guide/io/table_provider.html
similarity index 68%
copy from user-guide/io/parquet.html
copy to user-guide/io/table_provider.html
index b9f84ee..f35d71c 100644
--- a/user-guide/io/parquet.html
+++ b/user-guide/io/table_provider.html
@@ -5,7 +5,7 @@
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0"
/><meta name="viewport" content="width=device-width, initial-scale=1" />
- <title>Parquet — Apache Arrow DataFusion documentation</title>
+ <title>Custom Table Provider — Apache Arrow DataFusion
documentation</title>
<link href="../../_static/styles/theme.css?digest=1999514e3f237ded88cf"
rel="stylesheet">
<link
href="../../_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf"
rel="stylesheet">
@@ -35,7 +35,7 @@
<link rel="index" title="Index" href="../../genindex.html" />
<link rel="search" title="Search" href="../../search.html" />
<link rel="next" title="SQL" href="../sql.html" />
- <link rel="prev" title="JSON" href="json.html" />
+ <link rel="prev" title="Parquet" href="parquet.html" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="docsearch:language" content="en">
@@ -199,9 +199,14 @@
JSON
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="parquet.html">
+ Parquet
+ </a>
+ </li>
<li class="toctree-l2 current active">
<a class="current reference internal" href="#">
- Parquet
+ Custom Table Provider
</a>
</li>
</ul>
@@ -366,18 +371,38 @@
<div>
- <section id="parquet">
-<h1>Parquet<a class="headerlink" href="#parquet" title="Link to this
heading">¶</a></h1>
-<p>It is quite simple to read a parquet file using the <a class="reference
internal"
href="../../autoapi/datafusion/context/index.html#datafusion.context.SessionContext.read_parquet"
title="datafusion.context.SessionContext.read_parquet"><code class="xref py
py-func docutils literal notranslate"><span
class="pre">read_parquet()</span></code></a> function.</p>
-<div class="highlight-python notranslate"><div
class="highlight"><pre><span></span><span class="kn">from</span> <span
class="nn">datafusion</span> <span class="kn">import</span> <span
class="n">SessionContext</span>
+ <section id="custom-table-provider">
+<h1>Custom Table Provider<a class="headerlink" href="#custom-table-provider"
title="Link to this heading">¶</a></h1>
+<p>If you have a custom data source that you want to integrate with
DataFusion, you can do so by
+implementing the <a class="reference external"
href="https://datafusion.apache.org/library-user-guide/custom-table-providers.html";>TableProvider</a>
+interface in Rust and then exposing it in Python. To do so,
+you must use DataFusion 43.0.0 or later and expose a <a class="reference
external" href="https://crates.io/crates/datafusion-ffi";>FFI_TableProvider</a>
+via <a class="reference external"
href="https://pyo3.rs/main/doc/pyo3/types/struct.pycapsule";>PyCapsule</a>.</p>
+<p>A complete example can be found in the <a class="reference external"
href="https://github.com/apache/datafusion-python/tree/main/examples";>examples
folder</a>.</p>
+<div class="highlight-rust notranslate"><div
class="highlight"><pre><span></span><span class="cp">#[pymethods]</span>
+<span class="k">impl</span><span class="w"> </span><span
class="n">MyTableProvider</span><span class="w"> </span><span class="p">{</span>
+
+<span class="w"> </span><span class="k">fn</span><span class="w">
</span><span class="nf">__datafusion_table_provider__</span><span
class="o"><'</span><span class="na">py</span><span
class="o">></span><span class="p">(</span>
+<span class="w"> </span><span class="o">&</span><span
class="bp">self</span><span class="p">,</span>
+<span class="w"> </span><span class="n">py</span><span
class="p">:</span><span class="w"> </span><span class="nc">Python</span><span
class="o"><'</span><span class="na">py</span><span
class="o">></span><span class="p">,</span>
+<span class="w"> </span><span class="p">)</span><span class="w">
</span><span class="p">-></span><span class="w"> </span><span
class="nc">PyResult</span><span class="o"><</span><span
class="n">Bound</span><span class="o"><'</span><span
class="na">py</span><span class="p">,</span><span class="w"> </span><span
class="n">PyCapsule</span><span class="o">>></span><span class="w">
</span><span class="p">{</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">name</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span class="n">CString</span><span
class="p">::</span><span class="n">new</span><span class="p">(</span><span
class="s">"datafusion_table_provider"</span><span
class="p">).</span><span class="n">unwrap</span><span class="p">();</span>
-<span class="n">ctx</span> <span class="o">=</span> <span
class="n">SessionContext</span><span class="p">()</span>
-<span class="n">df</span> <span class="o">=</span> <span
class="n">ctx</span><span class="o">.</span><span
class="n">read_parquet</span><span class="p">(</span><span
class="s2">"file.parquet"</span><span class="p">)</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">provider</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span class="n">Arc</span><span
class="p">::</span><span class="n">new</span><span class="p">(</span><span
class="bp">self</span><span class="p">.</span><span class="n">clone</span><span
class="p">())</span>
+<span class="w"> </span><span class="p">.</span><span
class="n">map_err</span><span class="p">(</span><span class="o">|</span><span
class="n">e</span><span class="o">|</span><span class="w"> </span><span
class="n">PyRuntimeError</span><span class="p">::</span><span
class="n">new_err</span><span class="p">(</span><span class="n">e</span><span
class="p">.</span><span class="n">to_string</span><span
class="p">()))</span><span class="o">?</span><span class="p">;</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">provider</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span
class="n">FFI_TableProvider</span><span class="p">::</span><span
class="n">new</span><span class="p">(</span><span class="n">Arc</span><span
class="p">::</span><span class="n">new</span><span class="p">(</span><span
class="n">provider</span><span class="p">),</span><span class="w"> </span><span
class [...]
+
+<span class="w"> </span><span class="n">PyCapsule</span><span
class="p">::</span><span class="n">new_bound</span><span
class="p">(</span><span class="n">py</span><span class="p">,</span><span
class="w"> </span><span class="n">provider</span><span class="p">,</span><span
class="w"> </span><span class="nb">Some</span><span class="p">(</span><span
class="n">name</span><span class="p">.</span><span class="n">clone</span><span
class="p">()))</span>
+<span class="w"> </span><span class="p">}</span>
+<span class="p">}</span>
</pre></div>
</div>
-<p>An alternative is to use <a class="reference internal"
href="../../autoapi/datafusion/context/index.html#datafusion.context.SessionContext.register_parquet"
title="datafusion.context.SessionContext.register_parquet"><code class="xref
py py-func docutils literal notranslate"><span
class="pre">register_parquet()</span></code></a></p>
-<div class="highlight-python notranslate"><div
class="highlight"><pre><span></span><span class="n">ctx</span><span
class="o">.</span><span class="n">register_parquet</span><span
class="p">(</span><span class="s2">"file"</span><span
class="p">,</span> <span class="s2">"file.parquet"</span><span
class="p">)</span>
-<span class="n">df</span> <span class="o">=</span> <span
class="n">ctx</span><span class="o">.</span><span class="n">table</span><span
class="p">(</span><span class="s2">"file"</span><span
class="p">)</span>
+<p>Once you have this library available, in python you can register your table
provider
+to the <code class="docutils literal notranslate"><span
class="pre">SessionContext</span></code>.</p>
+<div class="highlight-python notranslate"><div
class="highlight"><pre><span></span><span class="n">provider</span> <span
class="o">=</span> <span class="n">MyTableProvider</span><span
class="p">()</span>
+<span class="n">ctx</span><span class="o">.</span><span
class="n">register_table_provider</span><span class="p">(</span><span
class="s2">"my_table"</span><span class="p">,</span> <span
class="n">provider</span><span class="p">)</span>
+
+<span class="n">ctx</span><span class="o">.</span><span
class="n">table</span><span class="p">(</span><span
class="s2">"my_table"</span><span class="p">)</span><span
class="o">.</span><span class="n">show</span><span class="p">()</span>
</pre></div>
</div>
</section>
@@ -388,11 +413,11 @@
<!-- Previous / next buttons -->
<div class='prev-next-area'>
- <a class='left-prev' id="prev-link" href="json.html" title="previous page">
+ <a class='left-prev' id="prev-link" href="parquet.html" title="previous
page">
<i class="fas fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
- <p class="prev-next-title">JSON</p>
+ <p class="prev-next-title">Parquet</p>
</div>
</a>
<a class='right-next' id="next-link" href="../sql.html" title="next page">
diff --git a/user-guide/sql.html b/user-guide/sql.html
index 9420e7d..b1cdc19 100644
--- a/user-guide/sql.html
+++ b/user-guide/sql.html
@@ -35,7 +35,7 @@
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
<link rel="next" title="Introduction"
href="../contributor-guide/introduction.html" />
- <link rel="prev" title="Parquet" href="io/parquet.html" />
+ <link rel="prev" title="Custom Table Provider"
href="io/table_provider.html" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="docsearch:language" content="en">
@@ -204,6 +204,11 @@
Parquet
</a>
</li>
+ <li class="toctree-l2">
+ <a class="reference internal" href="io/table_provider.html">
+ Custom Table Provider
+ </a>
+ </li>
</ul>
</li>
<li class="toctree-l1 current active">
@@ -411,11 +416,11 @@
<!-- Previous / next buttons -->
<div class='prev-next-area'>
- <a class='left-prev' id="prev-link" href="io/parquet.html" title="previous
page">
+ <a class='left-prev' id="prev-link" href="io/table_provider.html"
title="previous page">
<i class="fas fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
- <p class="prev-next-title">Parquet</p>
+ <p class="prev-next-title">Custom Table Provider</p>
</div>
</a>
<a class='right-next' id="next-link"
href="../contributor-guide/introduction.html" title="next page">
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
