This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new d1a305821f perf: Apply logical regexp optimizations to Utf8View and
LargeUtf8 inputs (#20581)
d1a305821f is described below
commit d1a305821f4d883a2a1e13d0bbc54af43d6d0441
Author: Peter Nguyen <[email protected]>
AuthorDate: Tue Mar 3 03:22:08 2026 -0800
perf: Apply logical regexp optimizations to Utf8View and LargeUtf8 inputs
(#20581)
## Which issue does this PR close?
<!--
We generally require a GitHub issue to be filed for all bug fixes and
enhancements and this helps us generate change logs for our releases.
You can link an issue to this PR using the GitHub syntax. For example
`Closes #123` indicates that this PR will close issue #123.
-->
- Closes #20580
## Rationale for this change
<!--
Why are you proposing this change? If this is already explained clearly
in the issue then this section is not needed.
Explaining clearly why changes are proposed helps reviewers understand
your changes and offer better suggestions for fixes.
-->
I ran into a bug that prevented some regexp optimizations from working
that were introduced in https://github.com/apache/datafusion/pull/15299.
After https://github.com/apache/datafusion/pull/16290, some SQL types
were updated to `utf8view`. As part of that PR, some expected query
plans in sqllogictest were updated to expect the unoptimized version.
I need this fixed to avoid additional test failures while implementing a
new regexp optimization for
https://github.com/apache/datafusion/issues/20579.
## What changes are included in this PR?
<!--
There is no need to duplicate the description in the issue here but it
is sometimes worth providing a summary of the individual changes in this
PR.
-->
- Add support for Utf8View and LargeUtf8 in `regex.rs`.
- Properly return `Transformed::no()` on cases when the plan isn't
modified (previously, it was always returning `Transformed::yes()`
- Updates the tests back to expect the optimized query plans
## Are these changes tested?
<!--
We typically require tests for all PRs in order to:
1. Prevent the code from being accidentally broken by subsequent changes
2. Serve as another way to document the expected behavior of the code
If tests are not included in your PR, please explain why (for example,
are they covered by existing tests)?
-->
Fixed existing tests that previously weren't working. Now they reflect
the optimization being reflected properly.
## Are there any user-facing changes?
<!--
If there are user-facing changes then we may require documentation to be
updated before approving the PR.
-->
<!--
If there are any breaking changes to public APIs, please add the `api
change` label.
-->
No. Just applying the optimizations to more cases.
---
.../src/simplify_expressions/expr_simplifier.rs | 76 ++++++++----
.../optimizer/src/simplify_expressions/regex.rs | 133 +++++++++++++--------
.../sqllogictest/test_files/simplify_expr.slt | 8 +-
.../sqllogictest/test_files/string/string_view.slt | 2 +-
4 files changed, 146 insertions(+), 73 deletions(-)
diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
index c6644e0086..fe2e1a3b04 100644
--- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
+++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs
@@ -1660,17 +1660,19 @@ impl TreeNodeRewriter for Simplifier<'_> {
left,
op: op @ (RegexMatch | RegexNotMatch | RegexIMatch |
RegexNotIMatch),
right,
- }) => Transformed::yes(simplify_regex_expr(left, op, right)?),
+ }) => simplify_regex_expr(left, op, right)?,
// Rules for Like
Expr::Like(like) => {
// `\` is implicit escape, see
https://github.com/apache/datafusion/issues/13291
let escape_char = like.escape_char.unwrap_or('\\');
- match as_string_scalar(&like.pattern) {
- Some((data_type, pattern_str)) => {
+
+ match StringScalar::try_from_expr(&like.pattern) {
+ Some(string_scalar) => {
+ let pattern_str = string_scalar.as_str();
match pattern_str {
None => return
Ok(Transformed::yes(lit_bool_null())),
- Some(pattern_str) if pattern_str == "%" => {
+ Some("%") => {
// exp LIKE '%' is
// - when exp is not NULL, it's true
// - when exp is NULL, it's NULL
@@ -1702,10 +1704,9 @@ impl TreeNodeRewriter for Simplifier<'_> {
.replace_all(pattern_str, "%")
.to_string();
Transformed::yes(Expr::Like(Like {
- pattern: Box::new(to_string_scalar(
- &data_type,
- Some(simplified_pattern),
- )),
+ pattern: Box::new(
+
string_scalar.to_expr(&simplified_pattern),
+ ),
..like
}))
}
@@ -2126,21 +2127,54 @@ fn is_literal_or_literal_cast(expr: &Expr) -> bool {
}
}
-fn as_string_scalar(expr: &Expr) -> Option<(DataType, &Option<String>)> {
- match expr {
- Expr::Literal(ScalarValue::Utf8(s), _) => Some((DataType::Utf8, s)),
- Expr::Literal(ScalarValue::LargeUtf8(s), _) =>
Some((DataType::LargeUtf8, s)),
- Expr::Literal(ScalarValue::Utf8View(s), _) =>
Some((DataType::Utf8View, s)),
- _ => None,
- }
+/// Helper for working with string scalar values (Utf8, LargeUtf8, Utf8View)
+pub(crate) enum StringScalar<'a> {
+ Utf8(&'a ScalarValue),
+ LargeUtf8(&'a ScalarValue),
+ Utf8View(&'a ScalarValue),
}
-fn to_string_scalar(data_type: &DataType, value: Option<String>) -> Expr {
- match data_type {
- DataType::Utf8 => Expr::Literal(ScalarValue::Utf8(value), None),
- DataType::LargeUtf8 => Expr::Literal(ScalarValue::LargeUtf8(value),
None),
- DataType::Utf8View => Expr::Literal(ScalarValue::Utf8View(value),
None),
- _ => unreachable!(),
+impl<'a> StringScalar<'a> {
+ /// Create a `StringScalar` view from an `Expr` if it is a supported
string literal.
+ /// Returns `None` if the expression is not a string literal.
+ pub(crate) fn try_from_expr(expr: &'a Expr) -> Option<Self> {
+ match expr {
+ Expr::Literal(scalar, _) => Self::try_from_scalar(scalar),
+ _ => None,
+ }
+ }
+
+ /// Create a `StringScalar` view from a `ScalarValue` if it is a supported
string type.
+ /// Returns `None` if the scalar value is not a supported string type.
+ fn try_from_scalar(scalar: &'a ScalarValue) -> Option<Self> {
+ match scalar {
+ ScalarValue::Utf8(_) => Some(Self::Utf8(scalar)),
+ ScalarValue::LargeUtf8(_) => Some(Self::LargeUtf8(scalar)),
+ ScalarValue::Utf8View(_) => Some(Self::Utf8View(scalar)),
+ _ => None,
+ }
+ }
+
+ /// Returns the underlying string slice.
+ pub(crate) fn as_str(&self) -> Option<&'a str> {
+ match self {
+ Self::Utf8(scalar) | Self::LargeUtf8(scalar) |
Self::Utf8View(scalar) => {
+ scalar.try_as_str().flatten()
+ }
+ }
+ }
+
+ /// Build a new `Expr` of the same string type with the given value.
+ pub(crate) fn to_expr(&self, val: &str) -> Expr {
+ match self {
+ Self::Utf8(_) =>
Expr::Literal(ScalarValue::Utf8(Some(val.to_owned())), None),
+ Self::LargeUtf8(_) => {
+ Expr::Literal(ScalarValue::LargeUtf8(Some(val.to_owned())),
None)
+ }
+ Self::Utf8View(_) => {
+ Expr::Literal(ScalarValue::Utf8View(Some(val.to_owned())),
None)
+ }
+ }
}
}
diff --git a/datafusion/optimizer/src/simplify_expressions/regex.rs
b/datafusion/optimizer/src/simplify_expressions/regex.rs
index d388aaf74c..6c2492d054 100644
--- a/datafusion/optimizer/src/simplify_expressions/regex.rs
+++ b/datafusion/optimizer/src/simplify_expressions/regex.rs
@@ -15,10 +15,13 @@
// specific language governing permissions and limitations
// under the License.
-use datafusion_common::{DataFusionError, Result, ScalarValue};
+use datafusion_common::tree_node::Transformed;
+use datafusion_common::{DataFusionError, Result};
use datafusion_expr::{BinaryExpr, Expr, Like, Operator, lit};
use regex_syntax::hir::{Capture, Hir, HirKind, Literal, Look};
+use crate::simplify_expressions::expr_simplifier::StringScalar;
+
/// Maximum number of regex alternations (`foo|bar|...`) that will be expanded
into multiple `LIKE` expressions.
const MAX_REGEX_ALTERNATIONS_EXPANSION: usize = 4;
@@ -43,52 +46,70 @@ pub fn simplify_regex_expr(
left: Box<Expr>,
op: Operator,
right: Box<Expr>,
-) -> Result<Expr> {
- let mode = OperatorMode::new(&op);
+) -> Result<Transformed<Expr>> {
+ // Check if the right operand is a supported string literal
+ let Some(string_scalar) = StringScalar::try_from_expr(right.as_ref()) else
{
+ return Ok(Transformed::no(Expr::BinaryExpr(BinaryExpr {
+ left,
+ op,
+ right,
+ })));
+ };
+ let pattern = string_scalar.as_str();
+ let Some(pattern) = pattern else {
+ return Ok(Transformed::no(Expr::BinaryExpr(BinaryExpr {
+ left,
+ op,
+ right,
+ })));
+ };
- if let Expr::Literal(ScalarValue::Utf8(Some(pattern)), _) = right.as_ref()
{
- // Handle the special case for ".*" pattern
- if pattern == ANY_CHAR_REGEX_PATTERN {
- let new_expr = if mode.not {
- // not empty
- let empty_lit = Box::new(lit(""));
- Expr::BinaryExpr(BinaryExpr {
- left,
- op: Operator::Eq,
- right: empty_lit,
- })
- } else {
- // not null
- left.is_not_null()
- };
- return Ok(new_expr);
- }
+ let mode = OperatorMode::new(&op);
+ // Handle the special case for ".*" pattern
+ if pattern == ANY_CHAR_REGEX_PATTERN {
+ let new_expr = if mode.not {
+ // not empty
+ let empty_lit = Box::new(string_scalar.to_expr(""));
+ Expr::BinaryExpr(BinaryExpr {
+ left,
+ op: Operator::Eq,
+ right: empty_lit,
+ })
+ } else {
+ // not null
+ left.is_not_null()
+ };
+ return Ok(Transformed::yes(new_expr));
+ }
- match regex_syntax::Parser::new().parse(pattern) {
- Ok(hir) => {
- let kind = hir.kind();
- if let HirKind::Alternation(alts) = kind {
- if alts.len() <= MAX_REGEX_ALTERNATIONS_EXPANSION
- && let Some(expr) = lower_alt(&mode, &left, alts)
- {
- return Ok(expr);
- }
- } else if let Some(expr) = lower_simple(&mode, &left, &hir) {
- return Ok(expr);
+ match regex_syntax::Parser::new().parse(pattern) {
+ Ok(hir) => {
+ let kind = hir.kind();
+ if let HirKind::Alternation(alts) = kind {
+ if alts.len() <= MAX_REGEX_ALTERNATIONS_EXPANSION
+ && let Some(expr) = lower_alt(&mode, &left, alts,
&string_scalar)
+ {
+ return Ok(Transformed::yes(expr));
}
- }
- Err(e) => {
- // error out early since the execution may fail anyways
- return Err(DataFusionError::Context(
- "Invalid regex".to_owned(),
- Box::new(DataFusionError::External(Box::new(e))),
- ));
+ } else if let Some(expr) = lower_simple(&mode, &left, &hir,
&string_scalar) {
+ return Ok(Transformed::yes(expr));
}
}
+ Err(e) => {
+ // error out early since the execution may fail anyways
+ return Err(DataFusionError::Context(
+ "Invalid regex".to_owned(),
+ Box::new(DataFusionError::External(Box::new(e))),
+ ));
+ }
}
// Leave untouched if optimization didn't work
- Ok(Expr::BinaryExpr(BinaryExpr { left, op, right }))
+ Ok(Transformed::no(Expr::BinaryExpr(BinaryExpr {
+ left,
+ op,
+ right,
+ })))
}
#[derive(Debug)]
@@ -117,11 +138,11 @@ impl OperatorMode {
}
/// Creates an [`LIKE`](Expr::Like) from the given `LIKE` pattern.
- fn expr(&self, expr: Box<Expr>, pattern: String) -> Expr {
+ fn expr(&self, expr: Box<Expr>, pattern: Box<Expr>) -> Expr {
let like = Like {
negated: self.not,
expr,
- pattern: Box::new(Expr::Literal(ScalarValue::from(pattern), None)),
+ pattern,
escape_char: None,
case_insensitive: self.i,
};
@@ -311,14 +332,24 @@ fn anchored_alternation_to_exprs(v: &[Hir]) ->
Option<Vec<Expr>> {
}
/// Tries to lower (transform) a simple regex pattern to a LIKE expression.
-fn lower_simple(mode: &OperatorMode, left: &Expr, hir: &Hir) -> Option<Expr> {
+fn lower_simple(
+ mode: &OperatorMode,
+ left: &Expr,
+ hir: &Hir,
+ string_scalar: &StringScalar,
+) -> Option<Expr> {
match hir.kind() {
HirKind::Empty => {
- return Some(mode.expr(Box::new(left.clone()), "%".to_owned()));
+ return Some(
+ mode.expr(Box::new(left.clone()),
Box::new(string_scalar.to_expr("%"))),
+ );
}
HirKind::Literal(l) => {
let s = like_str_from_literal(l)?;
- return Some(mode.expr(Box::new(left.clone()), format!("%{s}%")));
+ return Some(mode.expr(
+ Box::new(left.clone()),
+ Box::new(string_scalar.to_expr(&format!("%{s}%"))),
+ ));
}
HirKind::Concat(inner) if is_anchored_literal(inner) => {
return anchored_literal_to_expr(inner).map(|right| {
@@ -333,7 +364,10 @@ fn lower_simple(mode: &OperatorMode, left: &Expr, hir:
&Hir) -> Option<Expr> {
if let Some(pattern) = partial_anchored_literal_to_like(inner)
.or_else(|| collect_concat_to_like_string(inner))
{
- return Some(mode.expr(Box::new(left.clone()), pattern));
+ return Some(mode.expr(
+ Box::new(left.clone()),
+ Box::new(string_scalar.to_expr(&pattern)),
+ ));
}
}
_ => {}
@@ -344,11 +378,16 @@ fn lower_simple(mode: &OperatorMode, left: &Expr, hir:
&Hir) -> Option<Expr> {
/// Calls [`lower_simple`] for each alternative and combine the results with
`or` or `and`
/// based on [`OperatorMode`]. Any fail attempt to lower an alternative will
makes this
/// function to return `None`.
-fn lower_alt(mode: &OperatorMode, left: &Expr, alts: &[Hir]) -> Option<Expr> {
+fn lower_alt(
+ mode: &OperatorMode,
+ left: &Expr,
+ alts: &[Hir],
+ string_scalar: &StringScalar,
+) -> Option<Expr> {
let mut accu: Option<Expr> = None;
for part in alts {
- if let Some(expr) = lower_simple(mode, left, part) {
+ if let Some(expr) = lower_simple(mode, left, part, string_scalar) {
accu = match accu {
Some(accu) => {
if mode.not {
diff --git a/datafusion/sqllogictest/test_files/simplify_expr.slt
b/datafusion/sqllogictest/test_files/simplify_expr.slt
index 99fc9900ef..f8c219e052 100644
--- a/datafusion/sqllogictest/test_files/simplify_expr.slt
+++ b/datafusion/sqllogictest/test_files/simplify_expr.slt
@@ -34,20 +34,20 @@ query TT
explain select b from t where b ~ '.*'
----
logical_plan
-01)Filter: t.b ~ Utf8View(".*")
+01)Filter: t.b IS NOT NULL
02)--TableScan: t projection=[b]
physical_plan
-01)FilterExec: b@0 ~ .*
+01)FilterExec: b@0 IS NOT NULL
02)--DataSourceExec: partitions=1, partition_sizes=[1]
query TT
explain select b from t where b !~ '.*'
----
logical_plan
-01)Filter: t.b !~ Utf8View(".*")
+01)Filter: t.b = Utf8View("")
02)--TableScan: t projection=[b]
physical_plan
-01)FilterExec: b@0 !~ .*
+01)FilterExec: b@0 =
02)--DataSourceExec: partitions=1, partition_sizes=[1]
query T
diff --git a/datafusion/sqllogictest/test_files/string/string_view.slt
b/datafusion/sqllogictest/test_files/string/string_view.slt
index 13b0aba653..4dcc2f663a 100644
--- a/datafusion/sqllogictest/test_files/string/string_view.slt
+++ b/datafusion/sqllogictest/test_files/string/string_view.slt
@@ -1100,7 +1100,7 @@ EXPLAIN SELECT
FROM test;
----
logical_plan
-01)Projection: test.column1_utf8view ~ Utf8View("an") AS c1
+01)Projection: test.column1_utf8view LIKE Utf8View("%an%") AS c1
02)--TableScan: test projection=[column1_utf8view]
# `~*` operator (regex match case-insensitive)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
