srielau commented on code in PR #58317:
URL: https://github.com/apache/spark/pull/58317#discussion_r3905478699
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala:
##########
@@ -427,16 +427,29 @@ object OrcUtils extends Logging {
* Given a `StructType` object, this methods converts it to corresponding
string representation
* in ORC.
*/
- def getOrcSchemaString(dt: DataType): String = dt match {
+ def getOrcSchemaString(dt: DataType): String = {
+ getOrcSchemaString(dt, SQLConf.get.charVarcharStandardSemantics)
+ }
+
+ private def getOrcSchemaString(
+ dt: DataType,
+ charVarcharStandardSemantics: Boolean): String = dt match {
case s: StructType =>
val fieldTypes = s.fields.map { f =>
- s"${quoteIdentifier(f.name)}:${getOrcSchemaString(f.dataType)}"
+ s"${quoteIdentifier(f.name)}:" +
+ s"${getOrcSchemaString(f.dataType, charVarcharStandardSemantics)}"
}
s"struct<${fieldTypes.mkString(",")}>"
case a: ArrayType =>
- s"array<${getOrcSchemaString(a.elementType)}>"
+ s"array<${getOrcSchemaString(a.elementType,
charVarcharStandardSemantics)}>"
case m: MapType =>
-
s"map<${getOrcSchemaString(m.keyType)},${getOrcSchemaString(m.valueType)}>"
+ s"map<${getOrcSchemaString(m.keyType, charVarcharStandardSemantics)}," +
+ s"${getOrcSchemaString(m.valueType, charVarcharStandardSemantics)}>"
+ // Under standard semantics, keep Spark responsible for CHAR/VARCHAR
assignment and scan
+ // checks. Native ORC would truncate or pad before Spark can validate the
original value.
+ // Preserve-only mode retains the native constrained schema and its legacy
enforcement.
+ case _: CharType | _: VarcharType if charVarcharStandardSemantics =>
Review Comment:
Added in 5117aafe2b4. The preserve-only regression writes first-class
`CHAR(4)` and `VARCHAR(4)` values through ORC, reloads without a user schema,
asserts the inferred constrained types, and verifies native padding/truncation
(`\"ab \"`, `\"abcd\"`) in both V1 and V2 modes.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
