Kimahriman commented on code in PR #34558:
URL: https://github.com/apache/spark/pull/34558#discussion_r3293466426
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala:
##########
@@ -354,6 +442,49 @@ case class ArrayTransform(
result
}
+ override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
+ ctx.withLambdaVars(Seq(elementVar) ++ indexVar, varCodes => {
+ val elementCode = varCodes.head
+ val indexCode = varCodes.tail.headOption
+
+ nullSafeCodeGen(ctx, ev, arg => {
+ val numElements = ctx.freshName("numElements")
+ val arrayData = ctx.freshName("arrayData")
+ val i = ctx.freshName("i")
+
+ val initialization = CodeGenerator.createArrayData(
+ arrayData, dataType.elementType, numElements, s" $prettyName
failed.")
+
+ val functionCode = function.genCode(ctx)
+
+ val elementAssignment = assignArrayElement(ctx, arg, elementCode,
elementVar, i)
+ val indexAssignment = indexCode.map(c => assignIndex(ctx, c,
indexVar.get, i))
+ val varAssignments = (Seq(elementAssignment) ++
indexAssignment).mkString("\n")
+
+ // Some expressions return internal buffers that we have to copy
+ val copy = if (CodeGenerator.isPrimitiveType(function.dataType)) {
+ s"${functionCode.value}"
+ } else {
+ s"InternalRow.copyValue(${functionCode.value})"
+ }
+ val resultNull = if (function.nullable)
Some(functionCode.isNull.toString) else None
+ val resultAssignment = CodeGenerator.setArrayElement(arrayData,
dataType.elementType,
Review Comment:
Fixed in `eecc672cb44`.
`CodeGenerator.setArrayElement` now honors the provided `isNull` bit for
non-primitive element types as well, so nullable complex array writes call
`setNullAt` instead of writing a stale complex value.
I moved the regression to Catalyst with custom `GenericArrayData` that
simulates the vectorized-reader shape: `isNullAt(0)` reports null while the
typed complex getter can still expose a stale nested value. The test covers
both generated `transform` output and `filter` output.
Verified with:
```bash
build/sbt 'catalyst/testOnly
org.apache.spark.sql.catalyst.expressions.HigherOrderFunctionsSuite -- -z
"generated code"'
```
Result: passed (`2` tests, `0` failures).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
