dongjoon-hyun commented on code in PR #59965:
URL: https://github.com/apache/spark/pull/59965#discussion_r4461357692
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala:
##########
@@ -4594,6 +4594,16 @@ object SQLConf {
.booleanConf
.createWithDefault(false)
+ val USE_CURRENT_SQL_CONFIGS_FOR_GENERATED_COLUMN =
+ buildConf("spark.sql.legacy.useCurrentConfigsForGeneratedColumn")
+ .internal()
+ .doc("When true, a write parses and analyzes the SQL text of a
generation expression of a " +
+ "Data Source V2 table under the SQL configs of the current active
SparkSession, and " +
+ "CREATE TABLE and REPLACE TABLE do not reject a text that the session
reads differently.")
+ .version("4.4.0")
Review Comment:
Thank you. Applied your suggestion in 7baee5d1ad6. `build/sbt -Phive
'hive/testOnly *SparkConfigBindingPolicySuite'` passes now.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableConstraints.scala:
##########
@@ -119,27 +135,46 @@ class ResolveTableConstraints(val catalogManager:
CatalogManager) extends Rule[L
Option(col.columnGenerationExpression())
.filter(_ => !autoFilledGenCols.exists(n => resolver(n, col.name)))
.map { genExpr =>
- val catalystExpr = buildGenerationCatalystExpression(genExpr)
+ val catalystExpr = buildGenerationCatalystExpression(r, col.name,
genExpr)
val colRef = UnresolvedAttribute.quoted(col.name)
val columnExtractors = Seq(col.name -> colRef)
val genExprSql = Option(genExpr.getSql()).getOrElse(catalystExpr.sql)
- CheckInvariant(
- EqualNullSafe(colRef, catalystExpr),
+ // Compare with the value an auto-fill stores, which store
assignment casts to the column
+ // type. Otherwise, the comparison can cast the provided value to
the expression type.
+ val colType =
CharVarcharUtils.replaceCharVarcharWithString(col.dataType())
+ def check(expr: Expression): Expression = CheckInvariant(
+ EqualNullSafe(
+ colRef, Cast(expr, colType, Option(conf.sessionLocalTimeZone),
ansiEnabled = true)),
Review Comment:
I reproduced this on both the base and this PR: the auto-fill stores `{x: 2,
y: 1}`, and both the re-insert and the provided value fail with
`CHECK_CONSTRAINT_VIOLATION`. Since it predates this PR, I kept it out of this
one. I am working on a fix separately, and will file a JIRA ticket for it.
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveTableConstraints.scala:
##########
@@ -119,27 +135,46 @@ class ResolveTableConstraints(val catalogManager:
CatalogManager) extends Rule[L
Option(col.columnGenerationExpression())
.filter(_ => !autoFilledGenCols.exists(n => resolver(n, col.name)))
.map { genExpr =>
- val catalystExpr = buildGenerationCatalystExpression(genExpr)
+ val catalystExpr = buildGenerationCatalystExpression(r, col.name,
genExpr)
val colRef = UnresolvedAttribute.quoted(col.name)
val columnExtractors = Seq(col.name -> colRef)
val genExprSql = Option(genExpr.getSql()).getOrElse(catalystExpr.sql)
- CheckInvariant(
- EqualNullSafe(colRef, catalystExpr),
+ // Compare with the value an auto-fill stores, which store
assignment casts to the column
+ // type. Otherwise, the comparison can cast the provided value to
the expression type.
+ val colType =
CharVarcharUtils.replaceCharVarcharWithString(col.dataType())
+ def check(expr: Expression): Expression = CheckInvariant(
+ EqualNullSafe(
+ colRef, Cast(expr, colType, Option(conf.sessionLocalTimeZone),
ansiEnabled = true)),
columnExtractors,
"Generated Column",
s"${col.name} <=> $genExprSql")
+ // Analyze the check under the settings an auto-filled value is
analyzed under, so that a
+ // row Spark auto-filled passes it when a later write carries it.
+ StoredExpressionText.analyze(
+ check(catalystExpr),
+
check(Option(genExpr.getSql).map(StoredExpressionText.parseWithSessionAnsi)
+ .getOrElse(catalystExpr)),
+ v2Write.query.output,
+ analyze)
}
}.toSeq
}
/**
- * Convert a V2 GenerationExpression to a Catalyst expression.
- * Try the V2 expression first, fall back to parsing the SQL string.
+ * Convert a V2 GenerationExpression to a Catalyst expression. Read the SQL
string first, as an
+ * auto-filled value does, and fall back to the V2 expression.
*/
- private def buildGenerationCatalystExpression(genExpr:
GenerationExpression): Expression = {
- Option(genExpr.getExpression)
- .flatMap(V2ExpressionUtils.toCatalyst)
-
.getOrElse(catalogManager.v1SessionCatalog.parser.parseExpression(genExpr.getSql))
+ private def buildGenerationCatalystExpression(
+ r: DataSourceV2Relation,
+ colName: String,
+ genExpr: GenerationExpression): Expression = {
+ Option(genExpr.getSql)
+ .map(StoredExpressionText.parse)
+
.orElse(Option(genExpr.getExpression).flatMap(V2ExpressionUtils.toCatalyst))
+ .getOrElse(throw QueryCompilationErrors.unsupportedTableOperationError(
+ r.catalog.get, r.identifier.get,
Review Comment:
Thank you. I reproduced it with a `TableProvider` and `save()`, which failed
with `NoSuchElementException: None.get`. Applied your fix in 7baee5d1ad6, with
the new `unsupportedTableOperationError(table, operation)` overload. The same
write now fails with ``Table `probe_provider_table` does not support a value
for the generated column `b` ...``.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
