Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/22052#discussion_r208970323
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
---
@@ -356,6 +356,52 @@ case class ArrayFilter(
override def prettyName: String = "filter"
}
+/**
+ * Tests whether a predicate holds for one or more elements in the array.
+ */
+@ExpressionDescription(usage =
+ "_FUNC_(expr, pred) - Tests whether a predicate holds for one or more
elements in the array.",
+ examples = """
+ Examples:
+ > SELECT _FUNC_(array(1, 2, 3), x -> x % 2 == 0);
+ true
+ """,
+ since = "2.4.0")
+case class ArrayExists(
+ input: Expression,
+ function: Expression)
+ extends ArrayBasedSimpleHigherOrderFunction with CodegenFallback {
+
+ override def nullable: Boolean = input.nullable
+
+ override def dataType: DataType = BooleanType
+
+ override def expectingFunctionType: AbstractDataType = BooleanType
+
+ override def bind(f: (Expression, Seq[(DataType, Boolean)]) =>
LambdaFunction): ArrayExists = {
+ val elem = HigherOrderFunction.arrayArgumentType(input.dataType)
+ copy(function = f(function, elem :: Nil))
+ }
+
+ @transient lazy val LambdaFunction(_, Seq(elementVar:
NamedLambdaVariable), _) = function
+
+ override def nullSafeEval(inputRow: InternalRow, value: Any): Any = {
+ val arr = value.asInstanceOf[ArrayData]
+ val f = functionForEval
+ var i = 0
+ while (i < arr.numElements) {
+ elementVar.value.set(arr.get(i, elementVar.dataType))
+ if (f.eval(inputRow).asInstanceOf[Boolean]) {
+ return true
--- End diff --
shall we use a `var exists = false` to keep the result, and stop the loop
when result is true `while (i < arr.numElements & !exists)`?
IIUC return in Scala is implemented by throwing an exception, which may
have performance issue.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
