This is an automated email from the ASF dual-hosted git repository.
philo-he pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new b8ffd91a00 [GLUTEN-13476][VL] Fall back regex patterns with character
class intersection (#13477)
b8ffd91a00 is described below
commit b8ffd91a0021864ce2006fd8bf07dfee49f0488e
Author: YangJie <[email protected]>
AuthorDate: Wed Nov 11 23:44:05 2026 -0500
[GLUTEN-13476][VL] Fall back regex patterns with character class
intersection (#13477)
---
.../execution/VeloxStringFunctionsSuite.scala | 22 ++++++++++++++++++++++
cpp/bolt/utils/Common.cc | 15 ++++++++++++---
cpp/velox/utils/Common.cc | 15 ++++++++++++---
3 files changed, 46 insertions(+), 6 deletions(-)
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxStringFunctionsSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxStringFunctionsSuite.scala
index 22d3540bc0..38f0838a07 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxStringFunctionsSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/VeloxStringFunctionsSuite.scala
@@ -20,6 +20,7 @@ import org.apache.spark.SparkConf
import org.apache.spark.sql.catalyst.expressions.{Alias, Literal}
import org.apache.spark.sql.catalyst.optimizer.{ConstantFolding,
NullPropagation}
import org.apache.spark.sql.classic.ClassicColumn
+import org.apache.spark.sql.execution.GlutenImplicits._
import org.apache.spark.sql.functions.col
import org.apache.spark.sql.types.StringType
@@ -518,6 +519,27 @@ class VeloxStringFunctionsSuite extends
VeloxWholeStageTransformerSuite {
false)(_ => {})
}
+ test("regex with character class intersection falls back") {
+ // Java reads "&&" inside a character class as intersection, RE2 as two
literal '&'.
+ // The last pattern also covers a ']' right after "[^", which is a class
member.
+ Seq(
+ s"select l_orderkey, rlike(l_comment, '[a&&d]') from $LINEITEM_TABLE
limit $LENGTH",
+ s"select l_orderkey, regexp_extract(l_comment, '[a-z&&l-p]', 0) " +
+ s"from $LINEITEM_TABLE limit $LENGTH",
+ s"select l_orderkey, regexp_replace(l_comment, '[^]&&x]', '') " +
+ s"from $LINEITEM_TABLE limit $LENGTH"
+ ).foreach {
+ query =>
+ val df = runQueryAndCompare(query, noFallBack = false)(_ => {})
+ val reasons =
df.fallbackSummary().fallbackNodeToReason.flatMap(_.values)
+ assert(reasons.exists(_.contains("Not support character class")),
reasons.mkString("\n"))
+ }
+ // A single '&' is an ordinary class member in both engines, so this stays
native.
+ runQueryAndCompare(
+ s"select l_orderkey, rlike(l_comment, '[a&d]') from $LINEITEM_TABLE
limit $LENGTH")(
+ checkGlutenPlan[ProjectExecTransformer])
+ }
+
test("replace") {
runQueryAndCompare(
s"select l_orderkey, replace(l_comment, ' ', 'hello') " +
diff --git a/cpp/bolt/utils/Common.cc b/cpp/bolt/utils/Common.cc
index 1ec7a699ac..82a4b56b20 100644
--- a/cpp/bolt/utils/Common.cc
+++ b/cpp/bolt/utils/Common.cc
@@ -35,14 +35,23 @@ bool ensureRegexIsCompatible(const std::string& pattern,
std::string& error) {
if (charClassStart != pattern.cend()) {
error =
"Not support character class union, intersection, "
- "or difference ([a[b]], [a&&[b]], [a&&[^b]])";
+ "or difference ([a[b]], [a&&b], [a&&[b]], [a&&[^b]])";
return false;
}
charClassStart = c;
- // A ] immediately after a [ does not end the character class, and is
+ // A ] immediately after a [ or [^ does not end the character class, and
// instead adds the character ].
- } else if (*c == ']' && charClassStart != pattern.cend() && charClassStart
+ 1 != c) {
+ } else if (
+ *c == ']' && charClassStart != pattern.cend() && charClassStart + 1 !=
c &&
+ !(charClassStart + 2 == c && *(charClassStart + 1) == '^')) {
charClassStart = pattern.cend();
+ } else if (*c == '&' && c + 1 < pattern.cend() && *(c + 1) == '&' &&
charClassStart != pattern.cend()) {
+ // Java reads "&&" inside a class as set intersection while RE2 reads it
as
+ // two literal ampersands, so results diverge; fall back instead.
+ error =
+ "Not support character class union, intersection, "
+ "or difference ([a[b]], [a&&b], [a&&[b]], [a&&[^b]])";
+ return false;
}
}
return true;
diff --git a/cpp/velox/utils/Common.cc b/cpp/velox/utils/Common.cc
index 7dfffd5079..001ed9054c 100644
--- a/cpp/velox/utils/Common.cc
+++ b/cpp/velox/utils/Common.cc
@@ -35,14 +35,23 @@ bool ensureRegexIsCompatible(const std::string& pattern,
std::string& error) {
if (charClassStart != pattern.cend()) {
error =
"Not support character class union, intersection, "
- "or difference ([a[b]], [a&&[b]], [a&&[^b]])";
+ "or difference ([a[b]], [a&&b], [a&&[b]], [a&&[^b]])";
return false;
}
charClassStart = c;
- // A ] immediately after a [ does not end the character class, and is
+ // A ] immediately after a [ or [^ does not end the character class, and
// instead adds the character ].
- } else if (*c == ']' && charClassStart != pattern.cend() && charClassStart
+ 1 != c) {
+ } else if (
+ *c == ']' && charClassStart != pattern.cend() && charClassStart + 1 !=
c &&
+ !(charClassStart + 2 == c && *(charClassStart + 1) == '^')) {
charClassStart = pattern.cend();
+ } else if (*c == '&' && c + 1 < pattern.cend() && *(c + 1) == '&' &&
charClassStart != pattern.cend()) {
+ // Java reads "&&" inside a class as set intersection while RE2 reads it
as
+ // two literal ampersands, so results diverge; fall back instead.
+ error =
+ "Not support character class union, intersection, "
+ "or difference ([a[b]], [a&&b], [a&&[b]], [a&&[^b]])";
+ return false;
}
}
return true;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]