This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/asf-site by this push:
new 25af209cbc Publish built docs triggered by
a2db3f0d7ce0c3025ce1e5687e7133e3506ab81e
25af209cbc is described below
commit 25af209cbc10b0ee27a11d75cb3b0c8a0855b8e6
Author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
AuthorDate: Mon Mar 24 19:01:41 2025 +0000
Publish built docs triggered by a2db3f0d7ce0c3025ce1e5687e7133e3506ab81e
---
_sources/library-user-guide/adding-udfs.md.txt | 70 +++++++++++++++++++++++
library-user-guide/adding-udfs.html | 77 ++++++++++++++++++++++++++
searchindex.js | 2 +-
3 files changed, 148 insertions(+), 1 deletion(-)
diff --git a/_sources/library-user-guide/adding-udfs.md.txt
b/_sources/library-user-guide/adding-udfs.md.txt
index a62a9bb8c2..93a737b69b 100644
--- a/_sources/library-user-guide/adding-udfs.md.txt
+++ b/_sources/library-user-guide/adding-udfs.md.txt
@@ -719,6 +719,76 @@ async fn main() -> Result<()> {
// +---+
```
+## Custom Expression Planning
+
+DataFusion provides native support for common SQL operators by default such as
`+`, `-`, `||`. However it does not provide support for other operators such as
`@>`. To override DataFusion's default handling or support unsupported
operators, developers can extend DataFusion by implementing custom expression
planning, a core feature of DataFusion
+
+### Implementing Custom Expression Planning
+
+To extend DataFusion with support for custom operators not natively available,
you need to:
+
+1. Implement the `ExprPlanner` trait: This allows you to define custom logic
for planning expressions that DataFusion doesn't natively recognize. The trait
provides the necessary interface to translate SQL AST nodes into logical `Expr`.
+
+ For detailed documentation please see: [Trait
ExprPlanner](https://docs.rs/datafusion/latest/datafusion/logical_expr/planner/trait.ExprPlanner.html)
+
+2. Register your custom planner: Integrate your implementation with
DataFusion's `SessionContext` to ensure your custom planning logic is invoked
during the query optimization and execution planning phase.
+
+ For a detailed documentation see: [fn
register_expr_planner](https://docs.rs/datafusion/latest/datafusion/execution/trait.FunctionRegistry.html#method.register_expr_planner)
+
+See example below:
+
+```rust
+
+
+
+// Implement ExprPlanner to add support for the `->` custom operator
+impl ExprPlanner for MyCustomPlanner {
+ fn plan_binary_op(
+ &self,
+ expr: RawBinaryExpr,
+ _schema: &DFSchema,
+ ) -> Result<PlannerResult<RawBinaryExpr>> {
+ match &expr.op {
+ // Map `->` to string concatenation
+ BinaryOperator::Arrow => {
+ // Rewrite `->` as a string concatenation operation
+ // - `left` and `right` are the operands (e.g., 'hello' and
'world')
+ // - `Operator::StringConcat` tells DataFusion to concatenate
them
+ Ok(PlannerResult::Planned(Expr::BinaryExpr(BinaryExpr {
+ left: Box::new(expr.left.clone()),
+ right: Box::new(expr.right.clone()),
+ op: Operator::StringConcat,
+ })))
+ }
+ _ => Ok(PlannerResult::Original(expr)),
+ }
+ }
+}
+
+use datafusion::execution::context::SessionContext;
+use datafusion::arrow::util::pretty;
+
+#[tokio::main]
+async fn main() -> Result<()> {
+ let config = SessionConfig::new().set_str("datafusion.sql_parser.dialect",
"postgres");
+ let mut ctx = SessionContext::new_with_config(config);
+ ctx.register_expr_planner(Arc::new(MyCustomPlanner))?;
+ let results = ctx.sql("select 'foo'->'bar';").await?.collect().await?;
+
+ let expected = [
+ "+----------------------------+",
+ "| Utf8(\"foo\") || Utf8(\"bar\") |",
+ "+----------------------------+",
+ "| foobar |",
+ "+----------------------------+",
+ ];
+ assert_batches_eq!(&expected, &results);
+
+ pretty::print_batches(&results)?;
+ Ok(())
+}
+```
+
[1]:
https://github.com/apache/datafusion/blob/main/datafusion-examples/examples/simple_udf.rs
[2]:
https://github.com/apache/datafusion/blob/main/datafusion-examples/examples/simple_udwf.rs
[3]:
https://github.com/apache/datafusion/blob/main/datafusion-examples/examples/simple_udaf.rs
diff --git a/library-user-guide/adding-udfs.html
b/library-user-guide/adding-udfs.html
index 00a7b4ddcd..232e150dfa 100644
--- a/library-user-guide/adding-udfs.html
+++ b/library-user-guide/adding-udfs.html
@@ -605,6 +605,18 @@
</li>
</ul>
</li>
+ <li class="toc-h2 nav-item toc-entry">
+ <a class="reference internal nav-link" href="#custom-expression-planning">
+ Custom Expression Planning
+ </a>
+ <ul class="nav section-nav flex-column">
+ <li class="toc-h3 nav-item toc-entry">
+ <a class="reference internal nav-link"
href="#implementing-custom-expression-planning">
+ Implementing Custom Expression Planning
+ </a>
+ </li>
+ </ul>
+ </li>
</ul>
</nav>
@@ -1319,6 +1331,71 @@ validation of the input <code class="docutils literal
notranslate"><span class="
</div>
</section>
</section>
+<section id="custom-expression-planning">
+<h2>Custom Expression Planning<a class="headerlink"
href="#custom-expression-planning" title="Link to this heading">¶</a></h2>
+<p>DataFusion provides native support for common SQL operators by default such
as <code class="docutils literal notranslate"><span
class="pre">+</span></code>, <code class="docutils literal notranslate"><span
class="pre">-</span></code>, <code class="docutils literal notranslate"><span
class="pre">||</span></code>. However it does not provide support for other
operators such as <code class="docutils literal notranslate"><span
class="pre">@></span></code>. To override DataFusion’s [...]
+<section id="implementing-custom-expression-planning">
+<h3>Implementing Custom Expression Planning<a class="headerlink"
href="#implementing-custom-expression-planning" title="Link to this
heading">¶</a></h3>
+<p>To extend DataFusion with support for custom operators not natively
available, you need to:</p>
+<ol class="arabic">
+<li><p>Implement the <code class="docutils literal notranslate"><span
class="pre">ExprPlanner</span></code> trait: This allows you to define custom
logic for planning expressions that DataFusion doesn’t natively recognize. The
trait provides the necessary interface to translate SQL AST nodes into logical
<code class="docutils literal notranslate"><span
class="pre">Expr</span></code>.</p>
+<p>For detailed documentation please see: <a class="reference external"
href="https://docs.rs/datafusion/latest/datafusion/logical_expr/planner/trait.ExprPlanner.html";>Trait
ExprPlanner</a></p>
+</li>
+<li><p>Register your custom planner: Integrate your implementation with
DataFusion’s <code class="docutils literal notranslate"><span
class="pre">SessionContext</span></code> to ensure your custom planning logic
is invoked during the query optimization and execution planning phase.</p>
+<p>For a detailed documentation see: <a class="reference external"
href="https://docs.rs/datafusion/latest/datafusion/execution/trait.FunctionRegistry.html#method.register_expr_planner";>fn
register_expr_planner</a></p>
+</li>
+</ol>
+<p>See example below:</p>
+<div class="highlight-rust notranslate"><div
class="highlight"><pre><span></span><span class="c1">// Implement ExprPlanner
to add support for the `->` custom operator</span>
+<span class="k">impl</span><span class="w"> </span><span
class="n">ExprPlanner</span><span class="w"> </span><span
class="k">for</span><span class="w"> </span><span
class="n">MyCustomPlanner</span><span class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="k">fn</span><span class="w">
</span><span class="nf">plan_binary_op</span><span class="p">(</span>
+<span class="w"> </span><span class="o">&</span><span
class="bp">self</span><span class="p">,</span>
+<span class="w"> </span><span class="n">expr</span><span
class="p">:</span><span class="w"> </span><span
class="nc">RawBinaryExpr</span><span class="p">,</span>
+<span class="w"> </span><span class="n">_schema</span><span
class="p">:</span><span class="w"> </span><span class="kp">&</span><span
class="nc">DFSchema</span><span class="p">,</span>
+<span class="w"> </span><span class="p">)</span><span class="w">
</span><span class="p">-></span><span class="w"> </span><span
class="nb">Result</span><span class="o"><</span><span
class="n">PlannerResult</span><span class="o"><</span><span
class="n">RawBinaryExpr</span><span class="o">>></span><span class="w">
</span><span class="p">{</span>
+<span class="w"> </span><span class="k">match</span><span class="w">
</span><span class="o">&</span><span class="n">expr</span><span
class="p">.</span><span class="n">op</span><span class="w"> </span><span
class="p">{</span>
+<span class="w"> </span><span class="c1">// Map `->` to string
concatenation</span>
+<span class="w"> </span><span class="n">BinaryOperator</span><span
class="p">::</span><span class="n">Arrow</span><span class="w"> </span><span
class="o">=></span><span class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="c1">// Rewrite `->` as
a string concatenation operation</span>
+<span class="w"> </span><span class="c1">// - `left` and
`right` are the operands (e.g., 'hello' and 'world')</span>
+<span class="w"> </span><span class="c1">// -
`Operator::StringConcat` tells DataFusion to concatenate them</span>
+<span class="w"> </span><span class="nb">Ok</span><span
class="p">(</span><span class="n">PlannerResult</span><span
class="p">::</span><span class="n">Planned</span><span class="p">(</span><span
class="n">Expr</span><span class="p">::</span><span
class="n">BinaryExpr</span><span class="p">(</span><span
class="n">BinaryExpr</span><span class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="n">left</span><span
class="p">:</span><span class="w"> </span><span class="nb">Box</span><span
class="p">::</span><span class="n">new</span><span class="p">(</span><span
class="n">expr</span><span class="p">.</span><span class="n">left</span><span
class="p">.</span><span class="n">clone</span><span class="p">()),</span>
+<span class="w"> </span><span class="n">right</span><span
class="p">:</span><span class="w"> </span><span class="nb">Box</span><span
class="p">::</span><span class="n">new</span><span class="p">(</span><span
class="n">expr</span><span class="p">.</span><span class="n">right</span><span
class="p">.</span><span class="n">clone</span><span class="p">()),</span>
+<span class="w"> </span><span class="n">op</span><span
class="p">:</span><span class="w"> </span><span class="nc">Operator</span><span
class="p">::</span><span class="n">StringConcat</span><span class="p">,</span>
+<span class="w"> </span><span class="p">})))</span>
+<span class="w"> </span><span class="p">}</span>
+<span class="w"> </span><span class="n">_</span><span class="w">
</span><span class="o">=></span><span class="w"> </span><span
class="nb">Ok</span><span class="p">(</span><span
class="n">PlannerResult</span><span class="p">::</span><span
class="n">Original</span><span class="p">(</span><span
class="n">expr</span><span class="p">)),</span>
+<span class="w"> </span><span class="p">}</span>
+<span class="w"> </span><span class="p">}</span>
+<span class="p">}</span>
+
+<span class="k">use</span><span class="w"> </span><span
class="n">datafusion</span><span class="p">::</span><span
class="n">execution</span><span class="p">::</span><span
class="n">context</span><span class="p">::</span><span
class="n">SessionContext</span><span class="p">;</span>
+<span class="k">use</span><span class="w"> </span><span
class="n">datafusion</span><span class="p">::</span><span
class="n">arrow</span><span class="p">::</span><span class="n">util</span><span
class="p">::</span><span class="n">pretty</span><span class="p">;</span>
+
+<span class="cp">#[tokio::main]</span>
+<span class="k">async</span><span class="w"> </span><span
class="k">fn</span><span class="w"> </span><span class="nf">main</span><span
class="p">()</span><span class="w"> </span><span class="p">-></span><span
class="w"> </span><span class="nb">Result</span><span
class="o"><</span><span class="p">()</span><span class="o">></span><span
class="w"> </span><span class="p">{</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">config</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span
class="n">SessionConfig</span><span class="p">::</span><span
class="n">new</span><span class="p">().</span><span
class="n">set_str</span><span class="p">(</span><span
class="s">"datafusion.sql_parser.dialect"</span><span
class="p">,</span><span class="w"> </span><span class="s">"postgres&quo
[...]
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="k">mut</span><span class="w"> </span><span
class="n">ctx</span><span class="w"> </span><span class="o">=</span><span
class="w"> </span><span class="n">SessionContext</span><span
class="p">::</span><span class="n">new_with_config</span><span
class="p">(</span><span class="n">config</span><span class="p">);</span>
+<span class="w"> </span><span class="n">ctx</span><span
class="p">.</span><span class="n">register_expr_planner</span><span
class="p">(</span><span class="n">Arc</span><span class="p">::</span><span
class="n">new</span><span class="p">(</span><span
class="n">MyCustomPlanner</span><span class="p">))</span><span
class="o">?</span><span class="p">;</span>
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">results</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span class="n">ctx</span><span
class="p">.</span><span class="n">sql</span><span class="p">(</span><span
class="s">"select 'foo'->'bar';"</span><span
class="p">).</span><span class="k">await</span><span class="o">?</span><span
class="p">.</span><span class="n">collect</span><span c [...]
+
+<span class="w"> </span><span class="kd">let</span><span class="w">
</span><span class="n">expected</span><span class="w"> </span><span
class="o">=</span><span class="w"> </span><span class="p">[</span>
+<span class="w"> </span><span
class="s">"+----------------------------+"</span><span
class="p">,</span>
+<span class="w"> </span><span class="s">"| Utf8(</span><span
class="se">\"</span><span class="s">foo</span><span
class="se">\"</span><span class="s">) || Utf8(</span><span
class="se">\"</span><span class="s">bar</span><span
class="se">\"</span><span class="s">) |"</span><span
class="p">,</span>
+<span class="w"> </span><span
class="s">"+----------------------------+"</span><span
class="p">,</span>
+<span class="w"> </span><span class="s">"| foobar
|"</span><span class="p">,</span>
+<span class="w"> </span><span
class="s">"+----------------------------+"</span><span
class="p">,</span>
+<span class="w"> </span><span class="p">];</span>
+<span class="w"> </span><span class="n">assert_batches_eq</span><span
class="o">!</span><span class="p">(</span><span class="o">&</span><span
class="n">expected</span><span class="p">,</span><span class="w"> </span><span
class="o">&</span><span class="n">results</span><span class="p">);</span>
+
+<span class="w"> </span><span class="n">pretty</span><span
class="p">::</span><span class="n">print_batches</span><span
class="p">(</span><span class="o">&</span><span
class="n">results</span><span class="p">)</span><span class="o">?</span><span
class="p">;</span>
+<span class="w"> </span><span class="nb">Ok</span><span
class="p">(())</span>
+<span class="p">}</span>
+</pre></div>
+</div>
+</section>
+</section>
</section>
diff --git a/searchindex.js b/searchindex.js
index ef2eb49904..7a1be2e8e1 100644
--- a/searchindex.js
+++ b/searchindex.js
@@ -1 +1 @@
-Search.setIndex({"alltitles":{"!=":[[52,"op-neq"]],"!~":[[52,"op-re-not-match"]],"!~*":[[52,"op-re-not-match-i"]],"!~~":[[52,"id19"]],"!~~*":[[52,"id20"]],"#":[[52,"op-bit-xor"]],"%":[[52,"op-modulo"]],"&":[[52,"op-bit-and"]],"(relation,
name) tuples in logical fields and logical columns are
unique":[[12,"relation-name-tuples-in-logical-fields-and-logical-columns-are-unique"]],"*":[[52,"op-multiply"]],"+":[[52,"op-plus"]],"-":[[52,"op-minus"]],"/":[[52,"op-divide"]],"2022
Q2":[[10,"q2"]] [...]
\ No newline at end of file
+Search.setIndex({"alltitles":{"!=":[[52,"op-neq"]],"!~":[[52,"op-re-not-match"]],"!~*":[[52,"op-re-not-match-i"]],"!~~":[[52,"id19"]],"!~~*":[[52,"id20"]],"#":[[52,"op-bit-xor"]],"%":[[52,"op-modulo"]],"&":[[52,"op-bit-and"]],"(relation,
name) tuples in logical fields and logical columns are
unique":[[12,"relation-name-tuples-in-logical-fields-and-logical-columns-are-unique"]],"*":[[52,"op-multiply"]],"+":[[52,"op-plus"]],"-":[[52,"op-minus"]],"/":[[52,"op-divide"]],"2022
Q2":[[10,"q2"]] [...]
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
