[
https://issues.apache.org/jira/browse/SPARK-55588?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Ammar Chalifah updated SPARK-55588:
-----------------------------------
Description:
h3. Apache Iceberg version
Tried both:
* iceberg-spark-runtime-3.5_2.12-1.10.1.jar
* /usr/share/aws/iceberg/lib/iceberg-spark3-runtime.jar (from AWS EMR)
h3. Query engine
Spark 3.5.6
h3. Please describe the bug 🐞
I have a target table & input table, both are written in Iceberg. Based on the
manifest, the partition spec of target table is:
{code:java}
"partition-specs":[{"spec-id":0,"fields":[
{"name":"collected_date","transform":"identity","source-id":29,"field-id":1000}
,{"name":"user_id_bucket","transform":"bucket[32]","source-id":2,"field-id":1001}]}]
{code}
And partition spec of input table is
{code:java}
"partition-specs":[{"spec-id":0,"fields":[
{"name":"collected_date","transform":"identity","source-id":29,"field-id":1000}
,{"name":"user_id_bucket","transform":"bucket[32]","source-id":2,"field-id":1001}]}]
{code}
The field {{user_id}} is originally a string column, but the bucketed value
turns to Integer. I'm running SPJ by doing a {{MERGE INTO}} with join on
{{user_id}}
{code:java}
MERGE INTO target AS t
USING source AS s
ON t.post_id = s.post_id AND t.user_id = s.user_id
WHEN MATCHED AND (t.collected_at IS NULL OR t.collected_at <= s.collected_at)
THEN UPDATE SET *
WHEN NOT MATCHED
THEN INSERT *{code}
The {{MERGE INTO}} fails with this error
pyspark.errors.exceptions.captured.IllegalArgumentException: Wrong class,
expected java.lang.CharSequence, but was java.lang.Integer, for object: 1
Here's the trace
JVM stacktrace:
{code:java}
java.lang.IllegalArgumentException: Wrong class, expected
java.lang.CharSequence, but was java.lang.Integer, for object: 5
at org.apache.iceberg.PartitionData.get(PartitionData.java:138)
at
org.apache.iceberg.spark.source.StructInternalRow.getUTF8StringInternal(StructInternalRow.java:177)
at
org.apache.iceberg.spark.source.StructInternalRow.getUTF8String(StructInternalRow.java:173)
at
org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificOrdering.compare(Unknown
Source)
at
org.apache.spark.sql.catalyst.expressions.BaseOrdering.compare(ordering.scala:30)
at scala.math.Ordering$$anon$1.compare(Ordering.scala:122)
at java.base/java.util.TimSort.countRunAndMakeAscending(TimSort.java:355)
at java.base/java.util.TimSort.sort(TimSort.java:234)
at java.base/java.util.Arrays.sort(Arrays.java:1233)
at scala.collection.SeqLike.sorted(SeqLike.scala:659)
at scala.collection.SeqLike.sorted$(SeqLike.scala:647)
at scala.collection.AbstractSeq.sorted(Seq.scala:45)
at
org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExecBase.$anonfun$groupPartitions$1(DataSourceV2ScanExecBase.scala:164)
at scala.Option.flatMap(Option.scala:271)
at
org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExecBase.groupPartitions(DataSourceV2ScanExecBase.scala:133)
at
org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExecBase.groupPartitions$(DataSourceV2ScanExecBase.scala:126)
at
org.apache.spark.sql.execution.datasources.v2.BatchScanExec.groupPartitions(BatchScanExec.scala:41)
at
org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExecBase.groupedPartitions(DataSourceV2ScanExecBase.scala:109)
at
org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExecBase.groupedPartitions$(DataSourceV2ScanExecBase.scala:106)
at
org.apache.spark.sql.execution.datasources.v2.BatchScanExec.groupedPartitions$lzycompute(BatchScanExec.scala:41)
at
org.apache.spark.sql.execution.datasources.v2.BatchScanExec.groupedPartitions(BatchScanExec.scala:41)
at
org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExecBase.outputPartitioning(DataSourceV2ScanExecBase.scala:96)
at
org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExecBase.outputPartitioning$(DataSourceV2ScanExecBase.scala:93)
at
org.apache.spark.sql.execution.datasources.v2.BatchScanExec.outputPartitioning(BatchScanExec.scala:129)
at
org.apache.spark.sql.execution.exchange.BaseEnsureRequirements.$anonfun$ensureDistributionAndOrdering$1(EnsureRequirements.scala:103)
at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:286)
at scala.collection.Iterator.foreach(Iterator.scala:943)
at scala.collection.Iterator.foreach$(Iterator.scala:943)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
at scala.collection.IterableLike.foreach(IterableLike.scala:74)
at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
at scala.collection.TraversableLike.map(TraversableLike.scala:286)
at scala.collection.TraversableLike.map$(TraversableLike.scala:279)
at scala.collection.AbstractTraversable.map(Traversable.scala:108)
at
org.apache.spark.sql.execution.exchange.BaseEnsureRequirements.org$apache$spark$sql$execution$exchange$BaseEnsureRequirements$$ensureDistributionAndOrdering(EnsureRequirements.scala:102)
at
org.apache.spark.sql.execution.exchange.BaseEnsureRequirements$$anonfun$1.applyOrElse(EnsureRequirements.scala:712)
at
org.apache.spark.sql.execution.exchange.BaseEnsureRequirements$$anonfun$1.applyOrElse(EnsureRequirements.scala:687)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformUpWithPruning$2(TreeNode.scala:575)
at
org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:77)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformUpWithPruning(TreeNode.scala:575)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformUpWithPruning$1(TreeNode.scala:572)
at
org.apache.spark.sql.catalyst.trees.BinaryLike.mapChildren(TreeNode.scala:1450)
at
org.apache.spark.sql.catalyst.trees.BinaryLike.mapChildren$(TreeNode.scala:1447)
at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.mapChildren(BroadcastHashJoinExec.scala:40)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformUpWithPruning(TreeNode.scala:572)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformUpWithPruning$1(TreeNode.scala:572)
at
org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren(TreeNode.scala:1422)
at
org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren$(TreeNode.scala:1421)
at
org.apache.spark.sql.execution.ProjectExec.mapChildren(basicPhysicalOperators.scala:46)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformUpWithPruning(TreeNode.scala:572)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformUpWithPruning$1(TreeNode.scala:572)
at
org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren(TreeNode.scala:1422)
at
org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren$(TreeNode.scala:1421)
at
org.apache.spark.sql.execution.aggregate.HashAggregateExec.mapChildren(HashAggregateExec.scala:57)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformUpWithPruning(TreeNode.scala:572)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformUpWithPruning$1(TreeNode.scala:572)
at
org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren(TreeNode.scala:1422)
at
org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren$(TreeNode.scala:1421)
at
org.apache.spark.sql.execution.aggregate.HashAggregateExec.mapChildren(HashAggregateExec.scala:57)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformUpWithPruning(TreeNode.scala:572)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:548)
at
org.apache.spark.sql.execution.exchange.BaseEnsureRequirements.apply(EnsureRequirements.scala:687)
at
org.apache.spark.sql.execution.exchange.BaseEnsureRequirements.apply(EnsureRequirements.scala:78)
at
org.apache.spark.sql.execution.adaptive.RuleHelper$RuleSeq.$anonfun$applyAll$1(RuleHelper.scala:28)
at scala.collection.LinearSeqOptimized.foldLeft(LinearSeqOptimized.scala:126)
at scala.collection.LinearSeqOptimized.foldLeft$(LinearSeqOptimized.scala:122)
at scala.collection.immutable.List.foldLeft(List.scala:91)
at
org.apache.spark.sql.execution.adaptive.RuleHelper$RuleSeq.applyAll(RuleHelper.scala:27)
at
org.apache.spark.sql.execution.adaptive.PreprocessingRule$.$anonfun$apply$2(PreprocessingRule.scala:60)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan$PreprocessingRuleExecutor.$anonfun$doExecute$1(InsertAdaptiveSparkPlan.scala:280)
at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan$PreprocessingRuleExecutor.doExecute(InsertAdaptiveSparkPlan.scala:279)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan$PreprocessingRuleExecutor.execute(InsertAdaptiveSparkPlan.scala:250)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.preprocessAndCreateAdaptivePlan(InsertAdaptiveSparkPlan.scala:185)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.applyInternal(InsertAdaptiveSparkPlan.scala:140)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.prepareSubquery(InsertAdaptiveSparkPlan.scala:416)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.compileSubquery(InsertAdaptiveSparkPlan.scala:404)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.$anonfun$buildSubqueryMap$4(InsertAdaptiveSparkPlan.scala:363)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.$anonfun$buildSubqueryMap$4$adapted(InsertAdaptiveSparkPlan.scala:359)
at org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:281)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1(TreeNode.scala:282)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1$adapted(TreeNode.scala:282)
at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
at org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:282)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1(TreeNode.scala:282)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1$adapted(TreeNode.scala:282)
at scala.collection.Iterator.foreach(Iterator.scala:943)
at scala.collection.Iterator.foreach$(Iterator.scala:943)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
at scala.collection.IterableLike.foreach(IterableLike.scala:74)
at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
at org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:282)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.$anonfun$buildSubqueryMap$3(InsertAdaptiveSparkPlan.scala:359)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.$anonfun$buildSubqueryMap$3$adapted(InsertAdaptiveSparkPlan.scala:359)
at scala.collection.immutable.Stream.foreach(Stream.scala:533)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.$anonfun$buildSubqueryMap$1(InsertAdaptiveSparkPlan.scala:359)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.$anonfun$buildSubqueryMap$1$adapted(InsertAdaptiveSparkPlan.scala:359)
at org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:281)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1(TreeNode.scala:282)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1$adapted(TreeNode.scala:282)
at scala.collection.Iterator.foreach(Iterator.scala:943)
at scala.collection.Iterator.foreach$(Iterator.scala:943)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
at scala.collection.IterableLike.foreach(IterableLike.scala:74)
at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
at org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:282)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1(TreeNode.scala:282)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1$adapted(TreeNode.scala:282)
at scala.collection.Iterator.foreach(Iterator.scala:943)
at scala.collection.Iterator.foreach$(Iterator.scala:943)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
at scala.collection.IterableLike.foreach(IterableLike.scala:74)
at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
at org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:282)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1(TreeNode.scala:282)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1$adapted(TreeNode.scala:282)
at scala.collection.Iterator.foreach(Iterator.scala:943)
at scala.collection.Iterator.foreach$(Iterator.scala:943)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
at scala.collection.IterableLike.foreach(IterableLike.scala:74)
at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
at org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:282)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1(TreeNode.scala:282)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1$adapted(TreeNode.scala:282)
at scala.collection.Iterator.foreach(Iterator.scala:943)
at scala.collection.Iterator.foreach$(Iterator.scala:943)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
at scala.collection.IterableLike.foreach(IterableLike.scala:74)
at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
at org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:282)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1(TreeNode.scala:282)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1$adapted(TreeNode.scala:282)
at scala.collection.Iterator.foreach(Iterator.scala:943)
at scala.collection.Iterator.foreach$(Iterator.scala:943)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
at scala.collection.IterableLike.foreach(IterableLike.scala:74)
at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
at org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:282)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.org$apache$spark$sql$execution$adaptive$InsertAdaptiveSparkPlan$$buildSubqueryMap(InsertAdaptiveSparkPlan.scala:359)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan$PreprocessSubqueryPlanning$.apply(InsertAdaptiveSparkPlan.scala:337)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan$PreprocessingRuleExecutor.$anonfun$doExecute$1(InsertAdaptiveSparkPlan.scala:280)
at scala.collection.immutable.List.foreach(List.scala:431)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan$PreprocessingRuleExecutor.doExecute(InsertAdaptiveSparkPlan.scala:279)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan$PreprocessingRuleExecutor.execute(InsertAdaptiveSparkPlan.scala:232)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.preprocessAndCreateAdaptivePlan(InsertAdaptiveSparkPlan.scala:185)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.applyInternal(InsertAdaptiveSparkPlan.scala:140)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.apply(InsertAdaptiveSparkPlan.scala:123)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.$anonfun$applyInternal$1(InsertAdaptiveSparkPlan.scala:133)
at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:286)
at scala.collection.Iterator.foreach(Iterator.scala:943)
at scala.collection.Iterator.foreach$(Iterator.scala:943)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
at scala.collection.IterableLike.foreach(IterableLike.scala:74)
at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
at scala.collection.TraversableLike.map(TraversableLike.scala:286)
at scala.collection.TraversableLike.map$(TraversableLike.scala:279)
at scala.collection.AbstractTraversable.map(Traversable.scala:108)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.applyInternal(InsertAdaptiveSparkPlan.scala:133)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.apply(InsertAdaptiveSparkPlan.scala:123)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.apply(InsertAdaptiveSparkPlan.scala:51)
at
org.apache.spark.sql.execution.QueryExecution$.$anonfun$prepareForExecution$1(QueryExecution.scala:673)
at scala.collection.LinearSeqOptimized.foldLeft(LinearSeqOptimized.scala:126)
at scala.collection.LinearSeqOptimized.foldLeft$(LinearSeqOptimized.scala:122)
at scala.collection.immutable.List.foldLeft(List.scala:91)
at
org.apache.spark.sql.execution.QueryExecution$.prepareForExecution(QueryExecution.scala:672)
at
org.apache.spark.sql.execution.QueryExecution.$anonfun$getExecutedPlan$1(QueryExecution.scala:227)
at
org.apache.spark.sql.catalyst.QueryPlanningTracker.measurePhase(QueryPlanningTracker.scala:227)
at
org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$2(QueryExecution.scala:293)
at
org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:748)
at
org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$1(QueryExecution.scala:293)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:996)
at
org.apache.spark.sql.execution.QueryExecution.executePhase(QueryExecution.scala:292)
at
org.apache.spark.sql.execution.QueryExecution.getExecutedPlan(QueryExecution.scala:224)
at
org.apache.spark.sql.execution.QueryExecution.executedPlan$lzycompute(QueryExecution.scala:219)
at
org.apache.spark.sql.execution.QueryExecution.executedPlan(QueryExecution.scala:219)
at
org.apache.spark.sql.execution.QueryExecution.$anonfun$writeProcessedPlans$10(QueryExecution.scala:448)
at org.apache.spark.sql.catalyst.plans.QueryPlan$.append(QueryPlan.scala:750)
at
org.apache.spark.sql.execution.QueryExecution.writeProcessedPlans(QueryExecution.scala:449)
at
org.apache.spark.sql.execution.QueryExecution.writePlans(QueryExecution.scala:420)
at
org.apache.spark.sql.execution.QueryExecution.toString(QueryExecution.scala:466)
at
org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$explainString(QueryExecution.scala:361)
at
org.apache.spark.sql.execution.QueryExecution.explainStringWithoutSanitization(QueryExecution.scala:337)
at
org.apache.spark.sql.execution.SQLExecution$.executeQuery$1(SQLExecution.scala:148)
at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$11(SQLExecution.scala:227)
at
org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:108)
at
org.apache.spark.sql.execution.SQLExecution$.withTracker(SQLExecution.scala:391)
at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$10(SQLExecution.scala:227)
at
org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:412)
at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:226)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:996)
at
org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:84)
at
org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:75)
at
org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:124)
at
org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:115)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:521)
at
org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:77)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:521)
at
org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:39)
at
org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:303)
at
org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:299)
at
org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:39)
at
org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:39)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:497)
at
org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:115)
at
org.apache.spark.sql.execution.QueryExecution.commandExecuted$lzycompute(QueryExecution.scala:102)
at
org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:100)
at org.apache.spark.sql.Dataset.<init>(Dataset.scala:223)
at org.apache.spark.sql.Dataset$.$anonfun$ofRows$2(Dataset.scala:103)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:996)
at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:99)
at org.apache.spark.sql.SparkSession.doAnalyze(SparkSession.scala:783)
at org.apache.spark.sql.SparkSession.$anonfun$sql$1(SparkSession.scala:631)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:996)
at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:631)
at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:652)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
at py4j.Gateway.invoke(Gateway.java:282)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
at py4j.ClientServerConnection.run(ClientServerConnection.java:106)
at java.base/java.lang.Thread.run(Thread.java:840){code}
Both target & source are stored with
{code:java}
`format-version=2`{code}
was:
h3. Apache Iceberg version
Tried both:
* iceberg-spark-runtime-3.5_2.12-1.10.1.jar
* /usr/share/aws/iceberg/lib/iceberg-spark3-runtime.jar (from AWS EMR)
h3. Query engine
Spark 3.5.6
h3. Please describe the bug 🐞
I have a target table & input table, both are written in Iceberg. Based on the
manifest, the partition spec of target table is:
"partition-specs":[\{"spec-id":0,"fields":[{"name":"collected_date","transform":"identity","source-id":29,"field-id":1000},\{"name":"user_id_bucket","transform":"bucket[32]","source-id":2,"field-id":1001}]}]
And partition spec of input table is
"partition-specs":[\{"spec-id":0,"fields":[{"name":"collected_date","transform":"identity","source-id":29,"field-id":1000},\{"name":"user_id_bucket","transform":"bucket[32]","source-id":2,"field-id":1001}]}]
The field {{user_id}} is originally a string column, but the bucketed value
turns to Integer. I'm running SPJ by doing a {{MERGE INTO}} with join on
{{user_id}}
{{}}
MERGE INTO target AS t
USING source AS s
ON t.post_id = s.post_id AND t.user_id = s.user_id
WHEN MATCHED AND (t.collected_at IS NULL OR t.collected_at <=
s.collected_at)
THEN UPDATE SET *
WHEN NOT MATCHED
THEN INSERT *
The {{MERGE INTO}} fails with this error
pyspark.errors.exceptions.captured.IllegalArgumentException: Wrong class,
expected java.lang.CharSequence, but was java.lang.Integer, for object: 1
Here's the trace
JVM stacktrace:
java.lang.IllegalArgumentException: Wrong class, expected
java.lang.CharSequence, but was java.lang.Integer, for object: 5
at org.apache.iceberg.PartitionData.get(PartitionData.java:138)
at
org.apache.iceberg.spark.source.StructInternalRow.getUTF8StringInternal(StructInternalRow.java:177)
at
org.apache.iceberg.spark.source.StructInternalRow.getUTF8String(StructInternalRow.java:173)
at
org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificOrdering.compare(Unknown
Source)
at
org.apache.spark.sql.catalyst.expressions.BaseOrdering.compare(ordering.scala:30)
at scala.math.Ordering$$anon$1.compare(Ordering.scala:122)
at
java.base/java.util.TimSort.countRunAndMakeAscending(TimSort.java:355)
at java.base/java.util.TimSort.sort(TimSort.java:234)
at java.base/java.util.Arrays.sort(Arrays.java:1233)
at scala.collection.SeqLike.sorted(SeqLike.scala:659)
at scala.collection.SeqLike.sorted$(SeqLike.scala:647)
at scala.collection.AbstractSeq.sorted(Seq.scala:45)
at
org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExecBase.$anonfun$groupPartitions$1(DataSourceV2ScanExecBase.scala:164)
at scala.Option.flatMap(Option.scala:271)
at
org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExecBase.groupPartitions(DataSourceV2ScanExecBase.scala:133)
at
org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExecBase.groupPartitions$(DataSourceV2ScanExecBase.scala:126)
at
org.apache.spark.sql.execution.datasources.v2.BatchScanExec.groupPartitions(BatchScanExec.scala:41)
at
org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExecBase.groupedPartitions(DataSourceV2ScanExecBase.scala:109)
at
org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExecBase.groupedPartitions$(DataSourceV2ScanExecBase.scala:106)
at
org.apache.spark.sql.execution.datasources.v2.BatchScanExec.groupedPartitions$lzycompute(BatchScanExec.scala:41)
at
org.apache.spark.sql.execution.datasources.v2.BatchScanExec.groupedPartitions(BatchScanExec.scala:41)
at
org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExecBase.outputPartitioning(DataSourceV2ScanExecBase.scala:96)
at
org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExecBase.outputPartitioning$(DataSourceV2ScanExecBase.scala:93)
at
org.apache.spark.sql.execution.datasources.v2.BatchScanExec.outputPartitioning(BatchScanExec.scala:129)
at
org.apache.spark.sql.execution.exchange.BaseEnsureRequirements.$anonfun$ensureDistributionAndOrdering$1(EnsureRequirements.scala:103)
at
scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:286)
at scala.collection.Iterator.foreach(Iterator.scala:943)
at scala.collection.Iterator.foreach$(Iterator.scala:943)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
at scala.collection.IterableLike.foreach(IterableLike.scala:74)
at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
at scala.collection.TraversableLike.map(TraversableLike.scala:286)
at scala.collection.TraversableLike.map$(TraversableLike.scala:279)
at scala.collection.AbstractTraversable.map(Traversable.scala:108)
at
org.apache.spark.sql.execution.exchange.BaseEnsureRequirements.org$apache$spark$sql$execution$exchange$BaseEnsureRequirements$$ensureDistributionAndOrdering(EnsureRequirements.scala:102)
at
org.apache.spark.sql.execution.exchange.BaseEnsureRequirements$$anonfun$1.applyOrElse(EnsureRequirements.scala:712)
at
org.apache.spark.sql.execution.exchange.BaseEnsureRequirements$$anonfun$1.applyOrElse(EnsureRequirements.scala:687)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformUpWithPruning$2(TreeNode.scala:575)
at
org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:77)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformUpWithPruning(TreeNode.scala:575)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformUpWithPruning$1(TreeNode.scala:572)
at
org.apache.spark.sql.catalyst.trees.BinaryLike.mapChildren(TreeNode.scala:1450)
at
org.apache.spark.sql.catalyst.trees.BinaryLike.mapChildren$(TreeNode.scala:1447)
at
org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.mapChildren(BroadcastHashJoinExec.scala:40)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformUpWithPruning(TreeNode.scala:572)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformUpWithPruning$1(TreeNode.scala:572)
at
org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren(TreeNode.scala:1422)
at
org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren$(TreeNode.scala:1421)
at
org.apache.spark.sql.execution.ProjectExec.mapChildren(basicPhysicalOperators.scala:46)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformUpWithPruning(TreeNode.scala:572)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformUpWithPruning$1(TreeNode.scala:572)
at
org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren(TreeNode.scala:1422)
at
org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren$(TreeNode.scala:1421)
at
org.apache.spark.sql.execution.aggregate.HashAggregateExec.mapChildren(HashAggregateExec.scala:57)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformUpWithPruning(TreeNode.scala:572)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformUpWithPruning$1(TreeNode.scala:572)
at
org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren(TreeNode.scala:1422)
at
org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren$(TreeNode.scala:1421)
at
org.apache.spark.sql.execution.aggregate.HashAggregateExec.mapChildren(HashAggregateExec.scala:57)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformUpWithPruning(TreeNode.scala:572)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:548)
at
org.apache.spark.sql.execution.exchange.BaseEnsureRequirements.apply(EnsureRequirements.scala:687)
at
org.apache.spark.sql.execution.exchange.BaseEnsureRequirements.apply(EnsureRequirements.scala:78)
at
org.apache.spark.sql.execution.adaptive.RuleHelper$RuleSeq.$anonfun$applyAll$1(RuleHelper.scala:28)
at
scala.collection.LinearSeqOptimized.foldLeft(LinearSeqOptimized.scala:126)
at
scala.collection.LinearSeqOptimized.foldLeft$(LinearSeqOptimized.scala:122)
at scala.collection.immutable.List.foldLeft(List.scala:91)
at
org.apache.spark.sql.execution.adaptive.RuleHelper$RuleSeq.applyAll(RuleHelper.scala:27)
at
org.apache.spark.sql.execution.adaptive.PreprocessingRule$.$anonfun$apply$2(PreprocessingRule.scala:60)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan$PreprocessingRuleExecutor.$anonfun$doExecute$1(InsertAdaptiveSparkPlan.scala:280)
at
scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
at
scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan$PreprocessingRuleExecutor.doExecute(InsertAdaptiveSparkPlan.scala:279)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan$PreprocessingRuleExecutor.execute(InsertAdaptiveSparkPlan.scala:250)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.preprocessAndCreateAdaptivePlan(InsertAdaptiveSparkPlan.scala:185)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.applyInternal(InsertAdaptiveSparkPlan.scala:140)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.prepareSubquery(InsertAdaptiveSparkPlan.scala:416)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.compileSubquery(InsertAdaptiveSparkPlan.scala:404)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.$anonfun$buildSubqueryMap$4(InsertAdaptiveSparkPlan.scala:363)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.$anonfun$buildSubqueryMap$4$adapted(InsertAdaptiveSparkPlan.scala:359)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:281)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1(TreeNode.scala:282)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1$adapted(TreeNode.scala:282)
at
scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
at
scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:282)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1(TreeNode.scala:282)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1$adapted(TreeNode.scala:282)
at scala.collection.Iterator.foreach(Iterator.scala:943)
at scala.collection.Iterator.foreach$(Iterator.scala:943)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
at scala.collection.IterableLike.foreach(IterableLike.scala:74)
at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:282)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.$anonfun$buildSubqueryMap$3(InsertAdaptiveSparkPlan.scala:359)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.$anonfun$buildSubqueryMap$3$adapted(InsertAdaptiveSparkPlan.scala:359)
at scala.collection.immutable.Stream.foreach(Stream.scala:533)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.$anonfun$buildSubqueryMap$1(InsertAdaptiveSparkPlan.scala:359)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.$anonfun$buildSubqueryMap$1$adapted(InsertAdaptiveSparkPlan.scala:359)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:281)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1(TreeNode.scala:282)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1$adapted(TreeNode.scala:282)
at scala.collection.Iterator.foreach(Iterator.scala:943)
at scala.collection.Iterator.foreach$(Iterator.scala:943)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
at scala.collection.IterableLike.foreach(IterableLike.scala:74)
at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:282)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1(TreeNode.scala:282)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1$adapted(TreeNode.scala:282)
at scala.collection.Iterator.foreach(Iterator.scala:943)
at scala.collection.Iterator.foreach$(Iterator.scala:943)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
at scala.collection.IterableLike.foreach(IterableLike.scala:74)
at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:282)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1(TreeNode.scala:282)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1$adapted(TreeNode.scala:282)
at scala.collection.Iterator.foreach(Iterator.scala:943)
at scala.collection.Iterator.foreach$(Iterator.scala:943)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
at scala.collection.IterableLike.foreach(IterableLike.scala:74)
at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:282)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1(TreeNode.scala:282)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1$adapted(TreeNode.scala:282)
at scala.collection.Iterator.foreach(Iterator.scala:943)
at scala.collection.Iterator.foreach$(Iterator.scala:943)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
at scala.collection.IterableLike.foreach(IterableLike.scala:74)
at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:282)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1(TreeNode.scala:282)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1$adapted(TreeNode.scala:282)
at scala.collection.Iterator.foreach(Iterator.scala:943)
at scala.collection.Iterator.foreach$(Iterator.scala:943)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
at scala.collection.IterableLike.foreach(IterableLike.scala:74)
at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
at
org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:282)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.org$apache$spark$sql$execution$adaptive$InsertAdaptiveSparkPlan$$buildSubqueryMap(InsertAdaptiveSparkPlan.scala:359)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan$PreprocessSubqueryPlanning$.apply(InsertAdaptiveSparkPlan.scala:337)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan$PreprocessingRuleExecutor.$anonfun$doExecute$1(InsertAdaptiveSparkPlan.scala:280)
at scala.collection.immutable.List.foreach(List.scala:431)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan$PreprocessingRuleExecutor.doExecute(InsertAdaptiveSparkPlan.scala:279)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan$PreprocessingRuleExecutor.execute(InsertAdaptiveSparkPlan.scala:232)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.preprocessAndCreateAdaptivePlan(InsertAdaptiveSparkPlan.scala:185)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.applyInternal(InsertAdaptiveSparkPlan.scala:140)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.apply(InsertAdaptiveSparkPlan.scala:123)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.$anonfun$applyInternal$1(InsertAdaptiveSparkPlan.scala:133)
at
scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:286)
at scala.collection.Iterator.foreach(Iterator.scala:943)
at scala.collection.Iterator.foreach$(Iterator.scala:943)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
at scala.collection.IterableLike.foreach(IterableLike.scala:74)
at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
at scala.collection.TraversableLike.map(TraversableLike.scala:286)
at scala.collection.TraversableLike.map$(TraversableLike.scala:279)
at scala.collection.AbstractTraversable.map(Traversable.scala:108)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.applyInternal(InsertAdaptiveSparkPlan.scala:133)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.apply(InsertAdaptiveSparkPlan.scala:123)
at
org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.apply(InsertAdaptiveSparkPlan.scala:51)
at
org.apache.spark.sql.execution.QueryExecution$.$anonfun$prepareForExecution$1(QueryExecution.scala:673)
at
scala.collection.LinearSeqOptimized.foldLeft(LinearSeqOptimized.scala:126)
at
scala.collection.LinearSeqOptimized.foldLeft$(LinearSeqOptimized.scala:122)
at scala.collection.immutable.List.foldLeft(List.scala:91)
at
org.apache.spark.sql.execution.QueryExecution$.prepareForExecution(QueryExecution.scala:672)
at
org.apache.spark.sql.execution.QueryExecution.$anonfun$getExecutedPlan$1(QueryExecution.scala:227)
at
org.apache.spark.sql.catalyst.QueryPlanningTracker.measurePhase(QueryPlanningTracker.scala:227)
at
org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$2(QueryExecution.scala:293)
at
org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:748)
at
org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$1(QueryExecution.scala:293)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:996)
at
org.apache.spark.sql.execution.QueryExecution.executePhase(QueryExecution.scala:292)
at
org.apache.spark.sql.execution.QueryExecution.getExecutedPlan(QueryExecution.scala:224)
at
org.apache.spark.sql.execution.QueryExecution.executedPlan$lzycompute(QueryExecution.scala:219)
at
org.apache.spark.sql.execution.QueryExecution.executedPlan(QueryExecution.scala:219)
at
org.apache.spark.sql.execution.QueryExecution.$anonfun$writeProcessedPlans$10(QueryExecution.scala:448)
at
org.apache.spark.sql.catalyst.plans.QueryPlan$.append(QueryPlan.scala:750)
at
org.apache.spark.sql.execution.QueryExecution.writeProcessedPlans(QueryExecution.scala:449)
at
org.apache.spark.sql.execution.QueryExecution.writePlans(QueryExecution.scala:420)
at
org.apache.spark.sql.execution.QueryExecution.toString(QueryExecution.scala:466)
at
org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$explainString(QueryExecution.scala:361)
at
org.apache.spark.sql.execution.QueryExecution.explainStringWithoutSanitization(QueryExecution.scala:337)
at
org.apache.spark.sql.execution.SQLExecution$.executeQuery$1(SQLExecution.scala:148)
at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$11(SQLExecution.scala:227)
at
org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:108)
at
org.apache.spark.sql.execution.SQLExecution$.withTracker(SQLExecution.scala:391)
at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$10(SQLExecution.scala:227)
at
org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:412)
at
org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:226)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:996)
at
org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:84)
at
org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:75)
at
org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:124)
at
org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:115)
at
org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:521)
at
org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:77)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:521)
at
org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:39)
at
org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:303)
at
org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:299)
at
org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:39)
at
org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:39)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:497)
at
org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:115)
at
org.apache.spark.sql.execution.QueryExecution.commandExecuted$lzycompute(QueryExecution.scala:102)
at
org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:100)
at org.apache.spark.sql.Dataset.<init>(Dataset.scala:223)
at org.apache.spark.sql.Dataset$.$anonfun$ofRows$2(Dataset.scala:103)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:996)
at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:99)
at org.apache.spark.sql.SparkSession.doAnalyze(SparkSession.scala:783)
at
org.apache.spark.sql.SparkSession.$anonfun$sql$1(SparkSession.scala:631)
at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:996)
at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:631)
at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:652)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
at py4j.Gateway.invoke(Gateway.java:282)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
at py4j.ClientServerConnection.run(ClientServerConnection.java:106)
at java.base/java.lang.Thread.run(Thread.java:840)
Both target & source are stored with `{{{}format-version{}}}={{{}2`{}}}
> SPJ with Bucket Partition Key: Error: Wrong class, expected
> java.lang.CharSequence, but was java.lang.Integer
> -------------------------------------------------------------------------------------------------------------
>
> Key: SPARK-55588
> URL: https://issues.apache.org/jira/browse/SPARK-55588
> Project: Spark
> Issue Type: Wish
> Components: Spark Core
> Affects Versions: 3.5.6
> Reporter: Ammar Chalifah
> Priority: Minor
> Labels: bug
>
> h3. Apache Iceberg version
> Tried both:
> * iceberg-spark-runtime-3.5_2.12-1.10.1.jar
> * /usr/share/aws/iceberg/lib/iceberg-spark3-runtime.jar (from AWS EMR)
> h3. Query engine
> Spark 3.5.6
> h3. Please describe the bug 🐞
> I have a target table & input table, both are written in Iceberg. Based on
> the manifest, the partition spec of target table is:
> {code:java}
> "partition-specs":[{"spec-id":0,"fields":[
> {"name":"collected_date","transform":"identity","source-id":29,"field-id":1000}
> ,{"name":"user_id_bucket","transform":"bucket[32]","source-id":2,"field-id":1001}]}]
> {code}
>
> And partition spec of input table is
> {code:java}
> "partition-specs":[{"spec-id":0,"fields":[
> {"name":"collected_date","transform":"identity","source-id":29,"field-id":1000}
> ,{"name":"user_id_bucket","transform":"bucket[32]","source-id":2,"field-id":1001}]}]
> {code}
>
> The field {{user_id}} is originally a string column, but the bucketed value
> turns to Integer. I'm running SPJ by doing a {{MERGE INTO}} with join on
> {{user_id}}
> {code:java}
> MERGE INTO target AS t
> USING source AS s
> ON t.post_id = s.post_id AND t.user_id = s.user_id
> WHEN MATCHED AND (t.collected_at IS NULL OR t.collected_at <= s.collected_at)
> THEN UPDATE SET *
> WHEN NOT MATCHED
> THEN INSERT *{code}
>
> The {{MERGE INTO}} fails with this error
> pyspark.errors.exceptions.captured.IllegalArgumentException: Wrong class,
> expected java.lang.CharSequence, but was java.lang.Integer, for object: 1
>
> Here's the trace
> JVM stacktrace:
> {code:java}
> java.lang.IllegalArgumentException: Wrong class, expected
> java.lang.CharSequence, but was java.lang.Integer, for object: 5
> at org.apache.iceberg.PartitionData.get(PartitionData.java:138)
> at
> org.apache.iceberg.spark.source.StructInternalRow.getUTF8StringInternal(StructInternalRow.java:177)
> at
> org.apache.iceberg.spark.source.StructInternalRow.getUTF8String(StructInternalRow.java:173)
> at
> org.apache.spark.sql.catalyst.expressions.GeneratedClass$SpecificOrdering.compare(Unknown
> Source)
> at
> org.apache.spark.sql.catalyst.expressions.BaseOrdering.compare(ordering.scala:30)
> at scala.math.Ordering$$anon$1.compare(Ordering.scala:122)
> at java.base/java.util.TimSort.countRunAndMakeAscending(TimSort.java:355)
> at java.base/java.util.TimSort.sort(TimSort.java:234)
> at java.base/java.util.Arrays.sort(Arrays.java:1233)
> at scala.collection.SeqLike.sorted(SeqLike.scala:659)
> at scala.collection.SeqLike.sorted$(SeqLike.scala:647)
> at scala.collection.AbstractSeq.sorted(Seq.scala:45)
> at
> org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExecBase.$anonfun$groupPartitions$1(DataSourceV2ScanExecBase.scala:164)
> at scala.Option.flatMap(Option.scala:271)
> at
> org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExecBase.groupPartitions(DataSourceV2ScanExecBase.scala:133)
> at
> org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExecBase.groupPartitions$(DataSourceV2ScanExecBase.scala:126)
> at
> org.apache.spark.sql.execution.datasources.v2.BatchScanExec.groupPartitions(BatchScanExec.scala:41)
> at
> org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExecBase.groupedPartitions(DataSourceV2ScanExecBase.scala:109)
> at
> org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExecBase.groupedPartitions$(DataSourceV2ScanExecBase.scala:106)
> at
> org.apache.spark.sql.execution.datasources.v2.BatchScanExec.groupedPartitions$lzycompute(BatchScanExec.scala:41)
> at
> org.apache.spark.sql.execution.datasources.v2.BatchScanExec.groupedPartitions(BatchScanExec.scala:41)
> at
> org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExecBase.outputPartitioning(DataSourceV2ScanExecBase.scala:96)
> at
> org.apache.spark.sql.execution.datasources.v2.DataSourceV2ScanExecBase.outputPartitioning$(DataSourceV2ScanExecBase.scala:93)
> at
> org.apache.spark.sql.execution.datasources.v2.BatchScanExec.outputPartitioning(BatchScanExec.scala:129)
> at
> org.apache.spark.sql.execution.exchange.BaseEnsureRequirements.$anonfun$ensureDistributionAndOrdering$1(EnsureRequirements.scala:103)
> at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:286)
> at scala.collection.Iterator.foreach(Iterator.scala:943)
> at scala.collection.Iterator.foreach$(Iterator.scala:943)
> at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
> at scala.collection.IterableLike.foreach(IterableLike.scala:74)
> at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
> at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
> at scala.collection.TraversableLike.map(TraversableLike.scala:286)
> at scala.collection.TraversableLike.map$(TraversableLike.scala:279)
> at scala.collection.AbstractTraversable.map(Traversable.scala:108)
> at
> org.apache.spark.sql.execution.exchange.BaseEnsureRequirements.org$apache$spark$sql$execution$exchange$BaseEnsureRequirements$$ensureDistributionAndOrdering(EnsureRequirements.scala:102)
> at
> org.apache.spark.sql.execution.exchange.BaseEnsureRequirements$$anonfun$1.applyOrElse(EnsureRequirements.scala:712)
> at
> org.apache.spark.sql.execution.exchange.BaseEnsureRequirements$$anonfun$1.applyOrElse(EnsureRequirements.scala:687)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformUpWithPruning$2(TreeNode.scala:575)
> at
> org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:77)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.transformUpWithPruning(TreeNode.scala:575)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformUpWithPruning$1(TreeNode.scala:572)
> at
> org.apache.spark.sql.catalyst.trees.BinaryLike.mapChildren(TreeNode.scala:1450)
> at
> org.apache.spark.sql.catalyst.trees.BinaryLike.mapChildren$(TreeNode.scala:1447)
> at
> org.apache.spark.sql.execution.joins.BroadcastHashJoinExec.mapChildren(BroadcastHashJoinExec.scala:40)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.transformUpWithPruning(TreeNode.scala:572)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformUpWithPruning$1(TreeNode.scala:572)
> at
> org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren(TreeNode.scala:1422)
> at
> org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren$(TreeNode.scala:1421)
> at
> org.apache.spark.sql.execution.ProjectExec.mapChildren(basicPhysicalOperators.scala:46)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.transformUpWithPruning(TreeNode.scala:572)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformUpWithPruning$1(TreeNode.scala:572)
> at
> org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren(TreeNode.scala:1422)
> at
> org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren$(TreeNode.scala:1421)
> at
> org.apache.spark.sql.execution.aggregate.HashAggregateExec.mapChildren(HashAggregateExec.scala:57)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.transformUpWithPruning(TreeNode.scala:572)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformUpWithPruning$1(TreeNode.scala:572)
> at
> org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren(TreeNode.scala:1422)
> at
> org.apache.spark.sql.catalyst.trees.UnaryLike.mapChildren$(TreeNode.scala:1421)
> at
> org.apache.spark.sql.execution.aggregate.HashAggregateExec.mapChildren(HashAggregateExec.scala:57)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.transformUpWithPruning(TreeNode.scala:572)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:548)
> at
> org.apache.spark.sql.execution.exchange.BaseEnsureRequirements.apply(EnsureRequirements.scala:687)
> at
> org.apache.spark.sql.execution.exchange.BaseEnsureRequirements.apply(EnsureRequirements.scala:78)
> at
> org.apache.spark.sql.execution.adaptive.RuleHelper$RuleSeq.$anonfun$applyAll$1(RuleHelper.scala:28)
> at scala.collection.LinearSeqOptimized.foldLeft(LinearSeqOptimized.scala:126)
> at scala.collection.LinearSeqOptimized.foldLeft$(LinearSeqOptimized.scala:122)
> at scala.collection.immutable.List.foldLeft(List.scala:91)
> at
> org.apache.spark.sql.execution.adaptive.RuleHelper$RuleSeq.applyAll(RuleHelper.scala:27)
> at
> org.apache.spark.sql.execution.adaptive.PreprocessingRule$.$anonfun$apply$2(PreprocessingRule.scala:60)
> at
> org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan$PreprocessingRuleExecutor.$anonfun$doExecute$1(InsertAdaptiveSparkPlan.scala:280)
> at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
> at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
> at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
> at
> org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan$PreprocessingRuleExecutor.doExecute(InsertAdaptiveSparkPlan.scala:279)
> at
> org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan$PreprocessingRuleExecutor.execute(InsertAdaptiveSparkPlan.scala:250)
> at
> org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.preprocessAndCreateAdaptivePlan(InsertAdaptiveSparkPlan.scala:185)
> at
> org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.applyInternal(InsertAdaptiveSparkPlan.scala:140)
> at
> org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.prepareSubquery(InsertAdaptiveSparkPlan.scala:416)
> at
> org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.compileSubquery(InsertAdaptiveSparkPlan.scala:404)
> at
> org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.$anonfun$buildSubqueryMap$4(InsertAdaptiveSparkPlan.scala:363)
> at
> org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.$anonfun$buildSubqueryMap$4$adapted(InsertAdaptiveSparkPlan.scala:359)
> at org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:281)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1(TreeNode.scala:282)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1$adapted(TreeNode.scala:282)
> at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)
> at scala.collection.mutable.ResizableArray.foreach$(ResizableArray.scala:55)
> at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49)
> at org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:282)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1(TreeNode.scala:282)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1$adapted(TreeNode.scala:282)
> at scala.collection.Iterator.foreach(Iterator.scala:943)
> at scala.collection.Iterator.foreach$(Iterator.scala:943)
> at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
> at scala.collection.IterableLike.foreach(IterableLike.scala:74)
> at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
> at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
> at org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:282)
> at
> org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.$anonfun$buildSubqueryMap$3(InsertAdaptiveSparkPlan.scala:359)
> at
> org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.$anonfun$buildSubqueryMap$3$adapted(InsertAdaptiveSparkPlan.scala:359)
> at scala.collection.immutable.Stream.foreach(Stream.scala:533)
> at
> org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.$anonfun$buildSubqueryMap$1(InsertAdaptiveSparkPlan.scala:359)
> at
> org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.$anonfun$buildSubqueryMap$1$adapted(InsertAdaptiveSparkPlan.scala:359)
> at org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:281)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1(TreeNode.scala:282)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1$adapted(TreeNode.scala:282)
> at scala.collection.Iterator.foreach(Iterator.scala:943)
> at scala.collection.Iterator.foreach$(Iterator.scala:943)
> at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
> at scala.collection.IterableLike.foreach(IterableLike.scala:74)
> at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
> at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
> at org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:282)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1(TreeNode.scala:282)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1$adapted(TreeNode.scala:282)
> at scala.collection.Iterator.foreach(Iterator.scala:943)
> at scala.collection.Iterator.foreach$(Iterator.scala:943)
> at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
> at scala.collection.IterableLike.foreach(IterableLike.scala:74)
> at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
> at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
> at org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:282)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1(TreeNode.scala:282)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1$adapted(TreeNode.scala:282)
> at scala.collection.Iterator.foreach(Iterator.scala:943)
> at scala.collection.Iterator.foreach$(Iterator.scala:943)
> at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
> at scala.collection.IterableLike.foreach(IterableLike.scala:74)
> at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
> at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
> at org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:282)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1(TreeNode.scala:282)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1$adapted(TreeNode.scala:282)
> at scala.collection.Iterator.foreach(Iterator.scala:943)
> at scala.collection.Iterator.foreach$(Iterator.scala:943)
> at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
> at scala.collection.IterableLike.foreach(IterableLike.scala:74)
> at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
> at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
> at org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:282)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1(TreeNode.scala:282)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$foreach$1$adapted(TreeNode.scala:282)
> at scala.collection.Iterator.foreach(Iterator.scala:943)
> at scala.collection.Iterator.foreach$(Iterator.scala:943)
> at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
> at scala.collection.IterableLike.foreach(IterableLike.scala:74)
> at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
> at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
> at org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:282)
> at
> org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.org$apache$spark$sql$execution$adaptive$InsertAdaptiveSparkPlan$$buildSubqueryMap(InsertAdaptiveSparkPlan.scala:359)
> at
> org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan$PreprocessSubqueryPlanning$.apply(InsertAdaptiveSparkPlan.scala:337)
> at
> org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan$PreprocessingRuleExecutor.$anonfun$doExecute$1(InsertAdaptiveSparkPlan.scala:280)
> at scala.collection.immutable.List.foreach(List.scala:431)
> at
> org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan$PreprocessingRuleExecutor.doExecute(InsertAdaptiveSparkPlan.scala:279)
> at
> org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan$PreprocessingRuleExecutor.execute(InsertAdaptiveSparkPlan.scala:232)
> at
> org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.preprocessAndCreateAdaptivePlan(InsertAdaptiveSparkPlan.scala:185)
> at
> org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.applyInternal(InsertAdaptiveSparkPlan.scala:140)
> at
> org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.apply(InsertAdaptiveSparkPlan.scala:123)
> at
> org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.$anonfun$applyInternal$1(InsertAdaptiveSparkPlan.scala:133)
> at scala.collection.TraversableLike.$anonfun$map$1(TraversableLike.scala:286)
> at scala.collection.Iterator.foreach(Iterator.scala:943)
> at scala.collection.Iterator.foreach$(Iterator.scala:943)
> at scala.collection.AbstractIterator.foreach(Iterator.scala:1431)
> at scala.collection.IterableLike.foreach(IterableLike.scala:74)
> at scala.collection.IterableLike.foreach$(IterableLike.scala:73)
> at scala.collection.AbstractIterable.foreach(Iterable.scala:56)
> at scala.collection.TraversableLike.map(TraversableLike.scala:286)
> at scala.collection.TraversableLike.map$(TraversableLike.scala:279)
> at scala.collection.AbstractTraversable.map(Traversable.scala:108)
> at
> org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.applyInternal(InsertAdaptiveSparkPlan.scala:133)
> at
> org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.apply(InsertAdaptiveSparkPlan.scala:123)
> at
> org.apache.spark.sql.execution.adaptive.InsertAdaptiveSparkPlan.apply(InsertAdaptiveSparkPlan.scala:51)
> at
> org.apache.spark.sql.execution.QueryExecution$.$anonfun$prepareForExecution$1(QueryExecution.scala:673)
> at scala.collection.LinearSeqOptimized.foldLeft(LinearSeqOptimized.scala:126)
> at scala.collection.LinearSeqOptimized.foldLeft$(LinearSeqOptimized.scala:122)
> at scala.collection.immutable.List.foldLeft(List.scala:91)
> at
> org.apache.spark.sql.execution.QueryExecution$.prepareForExecution(QueryExecution.scala:672)
> at
> org.apache.spark.sql.execution.QueryExecution.$anonfun$getExecutedPlan$1(QueryExecution.scala:227)
> at
> org.apache.spark.sql.catalyst.QueryPlanningTracker.measurePhase(QueryPlanningTracker.scala:227)
> at
> org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$2(QueryExecution.scala:293)
> at
> org.apache.spark.sql.execution.QueryExecution$.withInternalError(QueryExecution.scala:748)
> at
> org.apache.spark.sql.execution.QueryExecution.$anonfun$executePhase$1(QueryExecution.scala:293)
> at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:996)
> at
> org.apache.spark.sql.execution.QueryExecution.executePhase(QueryExecution.scala:292)
> at
> org.apache.spark.sql.execution.QueryExecution.getExecutedPlan(QueryExecution.scala:224)
> at
> org.apache.spark.sql.execution.QueryExecution.executedPlan$lzycompute(QueryExecution.scala:219)
> at
> org.apache.spark.sql.execution.QueryExecution.executedPlan(QueryExecution.scala:219)
> at
> org.apache.spark.sql.execution.QueryExecution.$anonfun$writeProcessedPlans$10(QueryExecution.scala:448)
> at org.apache.spark.sql.catalyst.plans.QueryPlan$.append(QueryPlan.scala:750)
> at
> org.apache.spark.sql.execution.QueryExecution.writeProcessedPlans(QueryExecution.scala:449)
> at
> org.apache.spark.sql.execution.QueryExecution.writePlans(QueryExecution.scala:420)
> at
> org.apache.spark.sql.execution.QueryExecution.toString(QueryExecution.scala:466)
> at
> org.apache.spark.sql.execution.QueryExecution.org$apache$spark$sql$execution$QueryExecution$$explainString(QueryExecution.scala:361)
> at
> org.apache.spark.sql.execution.QueryExecution.explainStringWithoutSanitization(QueryExecution.scala:337)
> at
> org.apache.spark.sql.execution.SQLExecution$.executeQuery$1(SQLExecution.scala:148)
> at
> org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$11(SQLExecution.scala:227)
> at
> org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:108)
> at
> org.apache.spark.sql.execution.SQLExecution$.withTracker(SQLExecution.scala:391)
> at
> org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$10(SQLExecution.scala:227)
> at
> org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:412)
> at
> org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$1(SQLExecution.scala:226)
> at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:996)
> at
> org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:84)
> at
> org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:75)
> at
> org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:124)
> at
> org.apache.spark.sql.execution.QueryExecution$$anonfun$eagerlyExecuteCommands$1.applyOrElse(QueryExecution.scala:115)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:521)
> at
> org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(origin.scala:77)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.transformDownWithPruning(TreeNode.scala:521)
> at
> org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDownWithPruning(LogicalPlan.scala:39)
> at
> org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning(AnalysisHelper.scala:303)
> at
> org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper.transformDownWithPruning$(AnalysisHelper.scala:299)
> at
> org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:39)
> at
> org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDownWithPruning(LogicalPlan.scala:39)
> at
> org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:497)
> at
> org.apache.spark.sql.execution.QueryExecution.eagerlyExecuteCommands(QueryExecution.scala:115)
> at
> org.apache.spark.sql.execution.QueryExecution.commandExecuted$lzycompute(QueryExecution.scala:102)
> at
> org.apache.spark.sql.execution.QueryExecution.commandExecuted(QueryExecution.scala:100)
> at org.apache.spark.sql.Dataset.<init>(Dataset.scala:223)
> at org.apache.spark.sql.Dataset$.$anonfun$ofRows$2(Dataset.scala:103)
> at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:996)
> at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:99)
> at org.apache.spark.sql.SparkSession.doAnalyze(SparkSession.scala:783)
> at org.apache.spark.sql.SparkSession.$anonfun$sql$1(SparkSession.scala:631)
> at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:996)
> at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:631)
> at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:652)
> at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> at
> java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
> at
> java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.base/java.lang.reflect.Method.invoke(Method.java:569)
> at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
> at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
> at py4j.Gateway.invoke(Gateway.java:282)
> at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
> at py4j.commands.CallCommand.execute(CallCommand.java:79)
> at
> py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
> at py4j.ClientServerConnection.run(ClientServerConnection.java:106)
> at java.base/java.lang.Thread.run(Thread.java:840){code}
>
> Both target & source are stored with
> {code:java}
> `format-version=2`{code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
