cloud-fan commented on code in PR #58584:
URL: https://github.com/apache/spark/pull/58584#discussion_r4056972670
##########
sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala:
##########
@@ -829,6 +869,26 @@ private[hive] trait HiveInspectors {
null
}
}
+ case (_, c: CharType) =>
+ val unwrapper = unwrapperFor(objectInspector)
+ data: Any => {
+ val value = unwrapper(data).asInstanceOf[UTF8String]
+ if (value == null) {
+ null
+ } else {
+ CharVarcharCodegenUtils.charTypeReadSideCheck(value, c.length)
Review Comment:
**Non-blocking (P2):** This recursive key conversion can collapse distinct
runtime keys before the map is built. For example, STRING keys `a` and `a `
both become `a ` for an analyzed `MAP<CHAR(2), ...>`, but this path constructs
`ArrayBasedMapData` directly, so the duplicate bypasses `MAP_KEY_DEDUP_POLICY`
and leaves undefined lookup behavior. Please rebuild the converted entries
through a fresh `ArrayBasedMapBuilder` and add bounded-key coverage for both
Java and writable inspectors.
##########
sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala:
##########
@@ -554,4 +592,77 @@ private[hive] case class HiveUDAFFunction(
copy(children = newChildren)
}
+object HiveUDAFFunction extends HiveInspectors {
+ private[hive] case class InitializedEvaluators(
+ partialEvaluator: GenericUDAFEvaluator,
+ partialInspector: ObjectInspector,
+ finalEvaluator: GenericUDAFEvaluator,
+ finalInspector: ObjectInspector)
+
+ def apply(
+ name: String,
+ funcWrapper: HiveFunctionWrapper,
+ children: Seq[Expression]): HiveUDAFFunction = {
+ apply(name, funcWrapper, children, isUDAFBridgeRequired = false)
+ }
+
+ def apply(
+ name: String,
+ funcWrapper: HiveFunctionWrapper,
+ children: Seq[Expression],
+ isUDAFBridgeRequired: Boolean): HiveUDAFFunction = {
+ val (partialType, resultType) =
+ inferResolvedTypes(funcWrapper, children, isUDAFBridgeRequired)
+ HiveUDAFFunction(
+ name,
+ funcWrapper,
+ children,
+ isUDAFBridgeRequired,
+ mutableAggBufferOffset = 0,
+ inputAggBufferOffset = 0,
+ partialType,
+ resultType)
+ }
+
+ private[hive] def initializeEvaluators(
+ funcWrapper: HiveFunctionWrapper,
+ children: Seq[Expression],
+ isUDAFBridgeRequired: Boolean,
+ expectedPartialType: Option[DataType] = None,
+ expectedResultType: Option[DataType] = None): InitializedEvaluators = {
+ val inputInspectors = children.map(toInspector).toArray
+ def newEvaluator(): GenericUDAFEvaluator = {
+ val resolver = if (isUDAFBridgeRequired) {
+ new SparkGenericUDAFBridge(funcWrapper.createFunction[UDAF]())
+ } else {
+ funcWrapper.createFunction[AbstractGenericUDAFResolver]()
+ }
+ val parameterInfo = new SimpleGenericUDAFParameterInfo(
+ inputInspectors, false, false, false)
+ resolver.getEvaluator(parameterInfo)
+ }
+ val partial1 = newEvaluator()
+ val partialInspector = partial1.init(GenericUDAFEvaluator.Mode.PARTIAL1,
inputInspectors)
+ val finalEvaluator = newEvaluator()
+ val finalInspector =
+ finalEvaluator.init(GenericUDAFEvaluator.Mode.FINAL,
Array(partialInspector))
+
expectedPartialType.foreach(checkCompatibleHiveReturnType(partialInspector, _))
Review Comment:
**Non-blocking (P2):** The current UDAF cases cover compatible copied
children and a valid STRING-partial/CHAR-final pair, but neither runtime
inspector is incompatible with its corresponding snapshot. Removing either of
these checks would therefore leave the suite green while shuffle serialization
or final conversion uses the wrong captured type. Please add an end-to-end
fixture that independently triggers partial and final mismatches.
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala:
##########
@@ -201,7 +218,15 @@ trait BaseScriptTransformationExec extends UnaryExecNode {
private lazy val outputFieldWriters: Seq[String => Any] = output.map { attr
=>
val converter =
CatalystTypeConverters.createToCatalystConverter(attr.dataType)
attr.dataType match {
- case StringType => wrapperConvertException(data => data, converter)
+ case _: CharType | _: VarcharType =>
+ // First-class CHAR/VARCHAR must not use Hive LazySimpleSerDe's
null-on-error path.
+ (data: String) =>
+ if (data == ioschema.outputRowFormatMap("TOK_TABLEROWFORMATNULL")) {
Review Comment:
**Non-blocking (P2):** Please add scalar CHAR and VARCHAR cases for the
configured output null token. Without a direct assertion, removing this check
leaves the focused tests green while the token is padded as a literal value or
rejected for length instead of producing SQL null; a custom token would also
prove the configuration is honored.
##########
sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala:
##########
@@ -1122,6 +1244,51 @@ private[hive] trait HiveInspectors {
case _: JavaVoidObjectInspector => NullType
}
+ /**
+ * Analysis snapshots the Catalyst return type, but runtime inspectors are
rebuilt from the
+ * current children (including foldability and session CHAR/VARCHAR
settings). STRING may drift
+ * to or from a bounded string type across that boundary. Two bounded types
must match exactly:
+ * accepting a different kind or length would apply the snapshotted
conversion to an incompatible
+ * runtime value.
+ */
+ def checkCompatibleHiveReturnType(
+ inspector: ObjectInspector,
+ expectedType: DataType): Unit = {
+ checkCompatibleHiveReturnType(
+ inspectorToDataType(inspector, preserveCharVarchar = true),
+ expectedType)
+ }
+
+ def checkCompatibleHiveReturnType(
+ runtimeType: DataType,
+ expectedType: DataType): Unit = {
+ if (!compatibleHiveReturnType(runtimeType, expectedType)) {
+ throw SparkException.internalError(
+ s"Hive function runtime type ${runtimeType.catalogString} is
incompatible " +
+ s"with analysis type ${expectedType.catalogString}.")
+ }
+ }
+
+ private def compatibleHiveReturnType(
+ runtimeType: DataType,
+ expectedType: DataType): Boolean = {
+ (runtimeType, expectedType) match {
+ case (rt: CharType, et: CharType) => rt == et
+ case (rt: VarcharType, et: VarcharType) => rt == et
+ case (_: CharType | _: VarcharType, _: CharType | _: VarcharType) =>
false
+ case (_: StringType, _: CharType | _: VarcharType) => true
+ case (_: CharType | _: VarcharType, _: StringType) => true
+ case (ArrayType(rt, _), ArrayType(et, _)) =>
compatibleHiveReturnType(rt, et)
+ case (MapType(rk, rv, _), MapType(ek, ev, _)) =>
Review Comment:
Confirmed. The compatibility suite now exercises map key/value recursion and
struct field recursion, including arity and bounded-length mismatches. Thanks.
<!-- SPARK_DEV_REVIEW_REPLY
{"feedback_id":"inline:4036387242","thread_id":"inline:4036387242","verdict_sha256":"686cf8b8db16e9edfdf7fa644425fede25c5c3f146e894c6f1f37b7f07580e18"}
-->
##########
sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala:
##########
@@ -155,16 +157,42 @@ class HiveGenericUDFEvaluator(
oi
}
}
+}
+
+private[hive] class HiveGenericUDFEvaluator(
+ funcWrapper: HiveFunctionWrapper,
+ children: Seq[Expression],
+ catalystReturnType: DataType)
+ extends HiveUDFEvaluatorBase[GenericUDF](funcWrapper, children) {
+
+ // SPARK-58792: copied expression nodes (e.g. via withNewChildrenInternal)
share one
+ // HiveFunctionWrapper, whose cached GenericUDF instance is mutable:
initialize()
+ // rewrites its converters and output holders based on the arguments of
whichever
+ // copy initialized it last. Give every evaluator its own clone so copied
nodes
+ // cannot corrupt each other.
+ @transient
+ override lazy val function: GenericUDF =
+
HiveFunctionRegistryUtils.cloneGenericUDF(funcWrapper.createFunction[GenericUDF]())
+
+ @transient
+ private lazy val argumentInspectors = children.map(toInspector).toArray
+
+ @transient
+ lazy val returnInspector = {
+ val inspector = HiveGenericUDFEvaluator.initialize(function,
argumentInspectors)
+ checkCompatibleHiveReturnType(inspector, catalystReturnType)
Review Comment:
**Non-blocking (P2):** The helper-level mismatch matrix does not prove this
consuming guard remains wired in, and the current child-constantness cases
preserve the same CHAR/VARCHAR kind and length. Please add custom GenericUDF
and GenericUDTF cases whose runtime inspector is incompatible with the analysis
snapshot, so deleting either call-site check fails its owning execution test.
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala:
##########
@@ -241,6 +266,29 @@ trait BaseScriptTransformationExec extends UnaryExecNode {
data => IntervalUtils.microsToDuration(
IntervalUtils.castStringToDTInterval(UTF8String.fromString(data),
start, end)),
converter)
+ case dt @ (_: ArrayType | _: MapType | _: StructType)
+ if CharVarcharUtils.hasCharVarchar(dt) =>
+ val physicalType =
ScriptTransformationIOSchema.toUnboundedStringType(dt)
+ // JSON object keys are strings. Cast them to the declared map key
type after parsing.
+ val jsonType =
ScriptTransformationIOSchema.toJsonMapKeyType(physicalType)
+ val complexTypeFactory = JsonToStructs(
+ jsonType,
+ ioschema.outputSerdeProps.toMap,
+ Literal(null),
+ Some(conf.sessionLocalTimeZone))
+ val parsedToPhysical = if (jsonType.sameType(physicalType)) {
+ identity[Any] _
+ } else {
+ val restoreMapKeys =
ScriptTransformationIOSchema.makeJsonMapKeyRestorer(
+ physicalType, Some(conf.sessionLocalTimeZone))
+ value: Any => restoreMapKeys(value)
+ }
+ val toScala =
CatalystTypeConverters.createToScalaConverter(physicalType)
+ val parser = wrapperConvertException(
+ data => parsedToPhysical(
+ complexTypeFactory.nullSafeEval(UTF8String.fromString(data))),
+ identity)
+ data => converter(toScala(parser(data)))
Review Comment:
**Non-blocking (P2):** The fresh builder above validates only restoration to
the unbounded physical key type. This final conversion can still collapse
distinct keys: `a` and `a ` both pad to `CHAR(2)`, and permitted VARCHAR
trailing-space trimming has the same issue. `CatalystTypeConverters` then
creates `ArrayBasedMapData` without duplicate validation. Please make the final
bounded-key conversion use validated map construction recursively so collisions
follow Spark's configured duplicate-key behavior.
See **Shared repair plan 1** in the review body.
##########
sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala:
##########
@@ -111,34 +111,36 @@ class HiveSimpleUDFEvaluator(
}
}
-class HiveGenericUDFEvaluator(
- funcWrapper: HiveFunctionWrapper, children: Seq[Expression])
- extends HiveUDFEvaluatorBase[GenericUDF](funcWrapper, children) {
-
- // SPARK-58792: copied expression nodes (e.g. via withNewChildrenInternal)
share one
- // HiveFunctionWrapper, whose cached GenericUDF instance is mutable:
initialize()
- // rewrites its converters and output holders based on the arguments of
whichever
- // copy initialized it last. Give every evaluator its own clone so copied
nodes
- // cannot corrupt each other.
- @transient
- override lazy val function: GenericUDF =
-
HiveFunctionRegistryUtils.cloneGenericUDF(funcWrapper.createFunction[GenericUDF]())
-
- @transient
- private lazy val argumentInspectors = children.map(toInspector).toArray
+private[hive] object HiveGenericUDFEvaluator extends HiveInspectors {
+
+ /**
+ * Driver-side Hive initialize for `SELECT hive_udf(...)`. Returns the
Catalyst type
Review Comment:
Confirmed. The Scaladoc now clearly says `Driver-side Hive initialization`.
Thanks.
<!-- SPARK_DEV_REVIEW_REPLY
{"feedback_id":"inline:4036387252","thread_id":"inline:4036387252","verdict_sha256":"686cf8b8db16e9edfdf7fa644425fede25c5c3f146e894c6f1f37b7f07580e18"}
-->
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala:
##########
@@ -201,7 +211,15 @@ trait BaseScriptTransformationExec extends UnaryExecNode {
private lazy val outputFieldWriters: Seq[String => Any] = output.map { attr
=>
val converter =
CatalystTypeConverters.createToCatalystConverter(attr.dataType)
attr.dataType match {
- case StringType => wrapperConvertException(data => data, converter)
+ case _: CharType | _: VarcharType =>
+ // First-class CHAR/VARCHAR must not use Hive LazySimpleSerde's
null-on-error path.
Review Comment:
Confirmed. The cited comment now spells `LazySimpleSerDe` correctly. Thanks.
<!-- SPARK_DEV_REVIEW_REPLY
{"feedback_id":"inline:4036387257","thread_id":"inline:4036387257","verdict_sha256":"686cf8b8db16e9edfdf7fa644425fede25c5c3f146e894c6f1f37b7f07580e18"}
-->
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/BaseScriptTransformationExec.scala:
##########
@@ -241,6 +259,31 @@ trait BaseScriptTransformationExec extends UnaryExecNode {
data => IntervalUtils.microsToDuration(
IntervalUtils.castStringToDTInterval(UTF8String.fromString(data),
start, end)),
converter)
+ case dt @ (_: ArrayType | _: MapType | _: StructType)
+ if CharVarcharUtils.hasCharVarchar(dt) =>
+ val physicalType =
ScriptTransformationIOSchema.toUnboundedStringType(dt)
+ // JSON object keys are strings. Cast them to the declared map key
type after parsing.
+ val jsonType =
ScriptTransformationIOSchema.toJsonMapKeyType(physicalType)
+ val complexTypeFactory = JsonToStructs(
+ jsonType,
+ ioschema.outputSerdeProps.toMap,
+ Literal(null),
+ Some(conf.sessionLocalTimeZone))
+ val parsedToPhysical = if (jsonType.sameType(physicalType)) {
+ identity[Any] _
+ } else {
+ val cast = Cast(
Review Comment:
Confirmed. JSON map keys are now restored recursively through a fresh
validated builder, and the same-partition regression proves a failed row cannot
contaminate the next row. Thanks.
<!-- SPARK_DEV_REVIEW_REPLY
{"feedback_id":"inline:4036387263","thread_id":"inline:4036387263","verdict_sha256":"686cf8b8db16e9edfdf7fa644425fede25c5c3f146e894c6f1f37b7f07580e18"}
-->
##########
sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveInspectorSuite.scala:
##########
@@ -292,6 +301,240 @@ class HiveInspectorSuite extends SparkFunSuite with
HiveInspectors {
assert(typeInfo2.scale() === 10)
}
+ test("SPARK-59277: Hive object inspectors preserve CHAR/VARCHAR type
information") {
+ withFirstClassCharVarchar(enabled = true) {
+ Seq[DataType](CharType(5), VarcharType(7)).foreach { dataType =>
+ val inspector =
toInspector(dataType).asInstanceOf[PrimitiveObjectInspector]
+ assert(inspectorToDataType(inspector) === dataType)
+ dataType match {
+ case c: CharType =>
+ assert(inspector.getTypeInfo.asInstanceOf[CharTypeInfo].getLength
=== c.length)
+ case v: VarcharType =>
+
assert(inspector.getTypeInfo.asInstanceOf[VarcharTypeInfo].getLength ===
v.length)
+ }
+ }
+ }
+ }
+
+ test("SPARK-59277: Hive object inspectors accept collated CHAR/VARCHAR
values") {
+ withFirstClassCharVarchar(enabled = true) {
+ Seq[DataType](
+ CharType(5, "UTF8_LCASE"),
+ VarcharType(7, "UNICODE_CI")).foreach { dataType =>
+ val inspector = toInspector(dataType)
+ val value = UTF8String.fromString(dataType match {
+ case _: CharType => "ab"
+ case _: VarcharType => "abc"
+ })
+ val expectedValue = dataType match {
+ case _: CharType => UTF8String.fromString("ab ")
+ case _: VarcharType => value
+ }
+ val expectedType = dataType match {
+ case c: CharType => CharType(c.length)
+ case v: VarcharType => VarcharType(v.length)
+ }
+ assert(inspectorToDataType(inspector) === expectedType)
+ assert(unwrap(wrap(value, inspector, dataType), inspector) ===
expectedValue)
+ }
+ }
+ }
+
+ test("SPARK-59277: writable CHAR/VARCHAR inspectors round-trip values and
nulls") {
+ withFirstClassCharVarchar(enabled = true) {
+ val charType = CharType(5)
+ val charInspector =
PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
+ new CharTypeInfo(charType.length))
+ val charValue = UTF8String.fromString("abc")
+ val wrappedChar = wrap(charValue, charInspector, charType)
+ assert(wrappedChar.isInstanceOf[HiveCharWritable])
+
assert(wrappedChar.asInstanceOf[HiveCharWritable].getHiveChar.getPaddedValue
=== "abc ")
+ assert(unwrapperFor(charInspector, charType)(wrappedChar) ===
+ UTF8String.fromString("abc "))
+ assert(wrap(null, charInspector, charType) === null)
+ assert(unwrapperFor(charInspector, charType)(null) === null)
+ checkError(
+ exception = intercept[SparkRuntimeException] {
+ wrap(UTF8String.fromString("abcdef"), charInspector, charType)
+ },
+ condition = "EXCEED_LIMIT_LENGTH",
+ parameters = Map("limit" -> "5"))
+
+ val varcharType = VarcharType(7)
+ val varcharInspector =
PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
+ new VarcharTypeInfo(varcharType.length))
+ val varcharValue = UTF8String.fromString("abc")
+ val wrappedVarchar = wrap(varcharValue, varcharInspector, varcharType)
+ assert(wrappedVarchar.isInstanceOf[HiveVarcharWritable])
+
assert(wrappedVarchar.asInstanceOf[HiveVarcharWritable].getHiveVarchar.getValue
=== "abc")
+ assert(unwrapperFor(varcharInspector, varcharType)(wrappedVarchar) ===
varcharValue)
+ assert(wrap(null, varcharInspector, varcharType) === null)
+ assert(unwrapperFor(varcharInspector, varcharType)(null) === null)
+ checkError(
+ exception = intercept[SparkRuntimeException] {
+ wrap(UTF8String.fromString("abcdefgh"), varcharInspector,
varcharType)
+ },
+ condition = "EXCEED_LIMIT_LENGTH",
+ parameters = Map("limit" -> "7"))
+ }
+ }
+
+ test("SPARK-59277: Hive object inspectors support nested CHAR/VARCHAR") {
+ withFirstClassCharVarchar(enabled = true) {
+ val dataType = StructType(Seq(
+ StructField("chars", ArrayType(CharType(4))),
+ StructField("varchars", MapType(IntegerType, VarcharType(8)))))
+ val inspector = toInspector(dataType)
+ assert(inspectorToDataType(inspector) === dataType)
+
+ val input = InternalRow(
+ new GenericArrayData(Array[Any](UTF8String.fromString("a"))),
+ ArrayBasedMapData(
+ Array[Any](1),
+ Array[Any](UTF8String.fromString("value"))))
+ val result = unwrapperFor(inspector, dataType)(
+ wrap(input, inspector, dataType)).asInstanceOf[InternalRow]
+ assert(result.getArray(0).getUTF8String(0) === UTF8String.fromString("a
"))
+ assert(result.getMap(1).valueArray().getUTF8String(0) ===
UTF8String.fromString("value"))
+
+ val outerType = StructType(Seq(StructField("nested", dataType)))
+ val outerInspector =
toInspector(outerType).asInstanceOf[StructObjectInspector]
+ val field = outerInspector.getAllStructFieldRefs.get(0)
+ val targetRow = new SpecificInternalRow(Seq(dataType))
+ unwrapperFor(field, dataType)(wrap(input, inspector, dataType),
targetRow, 0)
+ val nestedResult = targetRow.getStruct(0, dataType.length)
+ assert(nestedResult.getArray(0).getUTF8String(0) ===
UTF8String.fromString("a "))
+ assert(
+ nestedResult.getMap(1).valueArray().getUTF8String(0) ===
UTF8String.fromString("value"))
+ }
+ }
+
+ test("SPARK-59277: typed field unwrappers preserve nanosecond timestamps") {
+ val value = Timestamp.valueOf("2026-09-16 12:34:56.123456789")
+ Seq(
+ TimestampNTZNanosType(9) ->
+ DateTimeUtils.localDateTimeToTimestampNanos(value.toLocalDateTime, 9),
+ TimestampLTZNanosType(9) ->
+ DateTimeUtils.instantToTimestampNanos(value.toInstant, 9)).foreach {
+ case (dataType, expected) =>
+ val inspector =
ObjectInspectorFactory.getStandardStructObjectInspector(
+ util.Arrays.asList("value"),
+
util.Arrays.asList(PrimitiveObjectInspectorFactory.javaTimestampObjectInspector))
+ val field = inspector.getAllStructFieldRefs.get(0)
+ val targetRow = new SpecificInternalRow(Seq(dataType))
+ unwrapperFor(field, dataType)(value, targetRow, 0)
+ assert(targetRow.get(0, dataType) === expected)
+ }
+ }
+
+ test("SPARK-59277: Hive constant inspectors preserve CHAR/VARCHAR type
information") {
+ withFirstClassCharVarchar(enabled = true) {
+ Seq[DataType](CharType(5), VarcharType(7)).foreach { dataType =>
+ val value = UTF8String.fromString("abc")
+ val inspector = toInspector(Literal.create(value, dataType))
+ assert(inspector.isInstanceOf[ConstantObjectInspector])
+ assert(inspectorToDataType(inspector) === dataType)
+ val expected = dataType match {
+ case _: CharType => UTF8String.fromString("abc ")
+ case _: VarcharType => value
+ }
+ assert(unwrapperFor(inspector, dataType)(
+
inspector.asInstanceOf[ConstantObjectInspector].getWritableConstantValue) ===
expected)
+
+ val nullInspector = toInspector(Literal.create(null, dataType))
+ assert(nullInspector.isInstanceOf[ConstantObjectInspector])
+ assert(inspectorToDataType(nullInspector) === dataType)
+ assert(unwrapperFor(nullInspector, dataType)(
+
nullInspector.asInstanceOf[ConstantObjectInspector].getWritableConstantValue)
=== null)
+ }
+ }
+ }
+
+ test("SPARK-59277: Hive CHAR/VARCHAR inspectors remain STRING under legacy
semantics") {
+ withFirstClassCharVarchar(enabled = false) {
+ Seq[DataType](CharType(5), VarcharType(7)).foreach { dataType =>
+ val inspector = toInspector(dataType)
+ assert(inspectorToDataType(inspector) === StringType)
+ assert(inspectorToDataType(inspector, preserveCharVarchar = true) ===
dataType)
+ }
+ }
+ }
+
+ test("SPARK-59277: Hive return type compatibility allows only STRING
boundary drift") {
+ checkCompatibleHiveReturnType(StringType, CharType(5))
+ checkCompatibleHiveReturnType(CharType(5), StringType)
+ checkCompatibleHiveReturnType(ArrayType(StringType),
ArrayType(VarcharType(7)))
+ checkCompatibleHiveReturnType(CharType(5), CharType(5))
+ checkCompatibleHiveReturnType(VarcharType(3), VarcharType(3))
+
+ Seq[(DataType, DataType)](
+ VarcharType(3) -> CharType(5),
+ CharType(4) -> CharType(5),
+ VarcharType(4) -> VarcharType(5),
+ IntegerType -> CharType(5)).foreach { case (runtimeType, expectedType) =>
+ intercept[SparkException] {
+ checkCompatibleHiveReturnType(runtimeType, expectedType)
+ }
+ }
+ }
+
+ test("SPARK-59277: Hive CHAR/VARCHAR boundaries enforce Spark length
semantics") {
+ withFirstClassCharVarchar(enabled = true) {
+ val varchar = VarcharType(3)
+ val varcharInspector = toInspector(varchar)
+ checkError(
+ exception = intercept[SparkRuntimeException] {
+ wrap(UTF8String.fromString("abcd"), varcharInspector, varchar)
+ },
+ condition = "EXCEED_LIMIT_LENGTH",
+ parameters = Map("limit" -> "3"))
+
+ val char = CharType(5)
+ val charInspector = toInspector(char)
+ checkError(
+ exception = intercept[SparkRuntimeException] {
+ wrap(UTF8String.fromString("abcdef"), charInspector, char)
+ },
+ condition = "EXCEED_LIMIT_LENGTH",
+ parameters = Map("limit" -> "5"))
+
+ val varcharUnwrapper =
+
unwrapperFor(PrimitiveObjectInspectorFactory.javaStringObjectInspector, varchar)
+ checkError(
+ exception = intercept[SparkRuntimeException] {
+ varcharUnwrapper("abcd")
+ },
+ condition = "EXCEED_LIMIT_LENGTH",
+ parameters = Map("limit" -> "3"))
+
+ val charUnwrapper =
+
unwrapperFor(PrimitiveObjectInspectorFactory.javaStringObjectInspector, char)
+ assert(charUnwrapper("ab") === UTF8String.fromString("ab "))
+ checkError(
+ exception = intercept[SparkRuntimeException] {
+ charUnwrapper("abcdef")
+ },
+ condition = "EXCEED_LIMIT_LENGTH",
+ parameters = Map("limit" -> "5"))
+ }
+ }
+
+ test("SPARK-59277: Hive object inspectors reject unsupported CHAR/VARCHAR
lengths") {
+ withFirstClassCharVarchar(enabled = true) {
+ Seq[DataType](CharType(0), CharType(256), VarcharType(65536)).foreach {
dataType =>
Review Comment:
Confirmed. `VarcharType(0)` now exercises both the object-inspector and
type-info lower-bound checks. Thanks.
<!-- SPARK_DEV_REVIEW_REPLY
{"feedback_id":"inline:4036387274","thread_id":"inline:4036387274","verdict_sha256":"686cf8b8db16e9edfdf7fa644425fede25c5c3f146e894c6f1f37b7f07580e18"}
-->
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
