srielau commented on code in PR #58584:
URL: https://github.com/apache/spark/pull/58584#discussion_r4027846797
##########
sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala:
##########
@@ -897,6 +953,19 @@ private[hive] trait HiveInspectors {
(value: Any, row: InternalRow, ordinal: Int) => row(ordinal) =
unwrapper(value)
}
+ /**
+ * Builds an in-place unwrapper that also honors target-type-specific
conversions.
Review Comment:
Done: the in-place unwrapper Scaladoc now states that only
CHAR/VARCHAR-bearing targets use the DataType-aware path; other types,
including nanosecond timestamps, still use `unwrapperFor(field)`.
##########
sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDAFSuite.scala:
##########
@@ -200,6 +201,25 @@ class HiveUDAFSuite extends QueryTest
}
}
+ test("SPARK-59277: Hive UDAF supports first-class CHAR/VARCHAR") {
Review Comment:
Done: added `MockPartialStringFinalCharUDAF`, whose PARTIAL inspector is
STRING and whose FINAL inspector is CHAR(5), plus a shuffle-covering test so
the `(partial, final)` serde pair cannot silently use one type for both.
##########
sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala:
##########
@@ -292,6 +300,140 @@ class HiveInspectorSuite extends SparkFunSuite with
HiveInspectors {
assert(typeInfo2.scale() === 10)
}
+ test("SPARK-59277: Hive object inspectors preserve CHAR/VARCHAR type
information") {
+ withFirstClassCharVarchar(enabled = true) {
+ Seq[DataType](CharType(5), VarcharType(7)).foreach { dataType =>
+ val inspector =
toInspector(dataType).asInstanceOf[PrimitiveObjectInspector]
+ assert(inspectorToDataType(inspector) === dataType)
+ dataType match {
+ case c: CharType =>
+ assert(inspector.getTypeInfo.asInstanceOf[CharTypeInfo].getLength
=== c.length)
+ case v: VarcharType =>
+
assert(inspector.getTypeInfo.asInstanceOf[VarcharTypeInfo].getLength ===
v.length)
+ }
+ }
+ }
+ }
+
+ test("SPARK-59277: Hive object inspectors accept collated CHAR/VARCHAR
values") {
+ withFirstClassCharVarchar(enabled = true) {
+ Seq[DataType](
+ CharType(5, "UTF8_LCASE"),
+ VarcharType(7, "UNICODE_CI")).foreach { dataType =>
+ val inspector = toInspector(dataType)
+ val value = UTF8String.fromString(dataType match {
+ case _: CharType => "ab"
+ case _: VarcharType => "abc"
+ })
+ val expectedValue = dataType match {
+ case _: CharType => UTF8String.fromString("ab ")
+ case _: VarcharType => value
+ }
+ val expectedType = dataType match {
+ case c: CharType => CharType(c.length)
+ case v: VarcharType => VarcharType(v.length)
+ }
+ assert(inspectorToDataType(inspector) === expectedType)
+ assert(unwrap(wrap(value, inspector, dataType), inspector) ===
expectedValue)
+ }
+ }
+ }
+
+ test("SPARK-59277: Hive object inspectors support nested CHAR/VARCHAR") {
+ withFirstClassCharVarchar(enabled = true) {
+ val dataType = StructType(Seq(
+ StructField("chars", ArrayType(CharType(4))),
+ StructField("varchars", MapType(IntegerType, VarcharType(8)))))
+ val inspector = toInspector(dataType)
+ assert(inspectorToDataType(inspector) === dataType)
+
+ val input = InternalRow(
+ new GenericArrayData(Array[Any](UTF8String.fromString("a"))),
+ ArrayBasedMapData(
+ Array[Any](1),
+ Array[Any](UTF8String.fromString("value"))))
+ val result = unwrapperFor(inspector, dataType)(
+ wrap(input, inspector, dataType)).asInstanceOf[InternalRow]
+ assert(result.getArray(0).getUTF8String(0) === UTF8String.fromString("a
"))
+ assert(result.getMap(1).valueArray().getUTF8String(0) ===
UTF8String.fromString("value"))
+
+ val outerType = StructType(Seq(StructField("nested", dataType)))
+ val outerInspector =
toInspector(outerType).asInstanceOf[StructObjectInspector]
+ val field = outerInspector.getAllStructFieldRefs.get(0)
+ val targetRow = new SpecificInternalRow(Seq(dataType))
+ unwrapperFor(field, dataType)(wrap(input, inspector, dataType),
targetRow, 0)
+ val nestedResult = targetRow.getStruct(0, dataType.length)
+ assert(nestedResult.getArray(0).getUTF8String(0) ===
UTF8String.fromString("a "))
+ assert(
+ nestedResult.getMap(1).valueArray().getUTF8String(0) ===
UTF8String.fromString("value"))
+ }
+ }
+
+ test("SPARK-59277: Hive constant inspectors preserve CHAR/VARCHAR type
information") {
Review Comment:
Done: the constant-inspector CHAR/VARCHAR cases now also wrap and unwrap a
null literal through the new helpers.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
