[ https://issues.apache.org/jira/browse/SPARK-12777?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Sean Owen updated SPARK-12777: ------------------------------ Component/s: SQL > Dataset fields can't be Scala tuples > ------------------------------------ > > Key: SPARK-12777 > URL: https://issues.apache.org/jira/browse/SPARK-12777 > Project: Spark > Issue Type: Bug > Components: SQL > Affects Versions: 1.6.0, 1.6.1, 2.0.0 > Reporter: Chris Jansen > > Datasets can't seem to handle scala tuples as fields of case classes in > datasets. > {code} > Seq((1,2), (3,4)).toDS().show() //works > {code} > When including a tuple as a field, the code fails: > {code} > case class Test(v: (Int, Int)) > Seq(Test((1,2)), Test((3,4)).toDS().show //fails > {code} > {code} > UnresolvedException: : Invalid call to dataType on unresolved object, tree: > 'name (unresolved.scala:59) > > org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute.dataType(unresolved.scala:59) > > org.apache.spark.sql.catalyst.expressions.GetStructField.org$apache$spark$sql$catalyst$expressions$GetStructField$$field$lzycompute(complexTypeExtractors.scala:107) > > org.apache.spark.sql.catalyst.expressions.GetStructField.org$apache$spark$sql$catalyst$expressions$GetStructField$$field(complexTypeExtractors.scala:107) > > org.apache.spark.sql.catalyst.expressions.GetStructField$$anonfun$toString$1.apply(complexTypeExtractors.scala:111) > > org.apache.spark.sql.catalyst.expressions.GetStructField$$anonfun$toString$1.apply(complexTypeExtractors.scala:111) > > org.apache.spark.sql.catalyst.expressions.GetStructField.toString(complexTypeExtractors.scala:111) > > org.apache.spark.sql.catalyst.expressions.Expression.toString(Expression.scala:217) > > org.apache.spark.sql.catalyst.expressions.Expression.toString(Expression.scala:217) > > org.apache.spark.sql.catalyst.expressions.If.toString(conditionalExpressions.scala:76) > > org.apache.spark.sql.catalyst.expressions.Expression.toString(Expression.scala:217) > > org.apache.spark.sql.catalyst.expressions.Alias.toString(namedExpressions.scala:155) > > org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$argString$1.apply(TreeNode.scala:385) > > org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$argString$1.apply(TreeNode.scala:381) > org.apache.spark.sql.catalyst.trees.TreeNode.argString(TreeNode.scala:388) > org.apache.spark.sql.catalyst.trees.TreeNode.simpleString(TreeNode.scala:391) > > org.apache.spark.sql.catalyst.plans.QueryPlan.simpleString(QueryPlan.scala:172) > > org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:441) > org.apache.spark.sql.catalyst.trees.TreeNode.treeString(TreeNode.scala:396) > > org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1$$anonfun$apply$5.apply(RuleExecutor.scala:118) > > org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1$$anonfun$apply$5.apply(RuleExecutor.scala:119) > org.apache.spark.Logging$class.logDebug(Logging.scala:62) > > org.apache.spark.sql.catalyst.rules.RuleExecutor.logDebug(RuleExecutor.scala:44) > > org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1.apply(RuleExecutor.scala:115) > > org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1.apply(RuleExecutor.scala:72) > > org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(RuleExecutor.scala:72) > > org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.resolve(ExpressionEncoder.scala:253) > org.apache.spark.sql.Dataset.<init>(Dataset.scala:78) > org.apache.spark.sql.Dataset.<init>(Dataset.scala:89) > org.apache.spark.sql.SQLContext.createDataset(SQLContext.scala:507) > > org.apache.spark.sql.SQLImplicits.localSeqToDatasetHolder(SQLImplicits.scala:80) > {code} > When providing a type alias, the code fails in a different way: > {code} > type TwoInt = (Int, Int) > case class Test(v: TwoInt) > Seq(Test((1,2)), Test((3,4)).toDS().show //fails > {code} > {code} > NoSuchElementException: : head of empty list (ScalaReflection.scala:504) > > org.apache.spark.sql.catalyst.ScalaReflection$$anonfun$org$apache$spark$sql$catalyst$ScalaReflection$$extractorFor$1.apply(ScalaReflection.scala:504) > > org.apache.spark.sql.catalyst.ScalaReflection$$anonfun$org$apache$spark$sql$catalyst$ScalaReflection$$extractorFor$1.apply(ScalaReflection.scala:502) > > org.apache.spark.sql.catalyst.ScalaReflection$.org$apache$spark$sql$catalyst$ScalaReflection$$extractorFor(ScalaReflection.scala:502) > > org.apache.spark.sql.catalyst.ScalaReflection$$anonfun$org$apache$spark$sql$catalyst$ScalaReflection$$extractorFor$1.apply(ScalaReflection.scala:509) > > org.apache.spark.sql.catalyst.ScalaReflection$$anonfun$org$apache$spark$sql$catalyst$ScalaReflection$$extractorFor$1.apply(ScalaReflection.scala:502) > > org.apache.spark.sql.catalyst.ScalaReflection$.org$apache$spark$sql$catalyst$ScalaReflection$$extractorFor(ScalaReflection.scala:502) > > org.apache.spark.sql.catalyst.ScalaReflection$.extractorsFor(ScalaReflection.scala:394) > > org.apache.spark.sql.catalyst.encoders.ExpressionEncoder$.apply(ExpressionEncoder.scala:54) > org.apache.spark.sql.SQLImplicits.newProductEncoder(SQLImplicits.scala:41) > > com.intenthq.pipeline.actions.ActionsJobIntegrationSpec.enrich(ActionsJobIntegrationSpec.scala:63) > > com.intenthq.pipeline.actions.ActionsJobIntegrationSpec$$anonfun$is$2.apply(ActionsJobIntegrationSpec.scala:45) > > com.intenthq.pipeline.actions.ActionsJobIntegrationSpec$$anonfun$is$2.apply(ActionsJobIntegrationSpec.scala:45) > {code} -- This message was sent by Atlassian JIRA (v6.3.4#6332) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org