Chris Jansen created SPARK-12777: ------------------------------------ Summary: Dataset fields can't be Scala tuples Key: SPARK-12777 URL: https://issues.apache.org/jira/browse/SPARK-12777 Project: Spark Issue Type: Bug Affects Versions: 1.6.0, 1.6.1, 2.0.0 Reporter: Chris Jansen
Datasets can't seem to handle scala tuples as fields of case classes in datasets. {code} Seq((1,2), (3,4)).toDS().show() //works {code} When including a tuple as a field, the code fails: {code} case class Test(v: (Int, Int)) Seq(Test((1,2)), Test((3,4)).toDS().show //fails {code} {code} UnresolvedException: : Invalid call to dataType on unresolved object, tree: 'name (unresolved.scala:59) org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute.dataType(unresolved.scala:59) org.apache.spark.sql.catalyst.expressions.GetStructField.org$apache$spark$sql$catalyst$expressions$GetStructField$$field$lzycompute(complexTypeExtractors.scala:107) org.apache.spark.sql.catalyst.expressions.GetStructField.org$apache$spark$sql$catalyst$expressions$GetStructField$$field(complexTypeExtractors.scala:107) org.apache.spark.sql.catalyst.expressions.GetStructField$$anonfun$toString$1.apply(complexTypeExtractors.scala:111) org.apache.spark.sql.catalyst.expressions.GetStructField$$anonfun$toString$1.apply(complexTypeExtractors.scala:111) org.apache.spark.sql.catalyst.expressions.GetStructField.toString(complexTypeExtractors.scala:111) org.apache.spark.sql.catalyst.expressions.Expression.toString(Expression.scala:217) org.apache.spark.sql.catalyst.expressions.Expression.toString(Expression.scala:217) org.apache.spark.sql.catalyst.expressions.If.toString(conditionalExpressions.scala:76) org.apache.spark.sql.catalyst.expressions.Expression.toString(Expression.scala:217) org.apache.spark.sql.catalyst.expressions.Alias.toString(namedExpressions.scala:155) org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$argString$1.apply(TreeNode.scala:385) org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$argString$1.apply(TreeNode.scala:381) org.apache.spark.sql.catalyst.trees.TreeNode.argString(TreeNode.scala:388) org.apache.spark.sql.catalyst.trees.TreeNode.simpleString(TreeNode.scala:391) org.apache.spark.sql.catalyst.plans.QueryPlan.simpleString(QueryPlan.scala:172) org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:441) org.apache.spark.sql.catalyst.trees.TreeNode.treeString(TreeNode.scala:396) org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1$$anonfun$apply$5.apply(RuleExecutor.scala:118) org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1$$anonfun$apply$5.apply(RuleExecutor.scala:119) org.apache.spark.Logging$class.logDebug(Logging.scala:62) org.apache.spark.sql.catalyst.rules.RuleExecutor.logDebug(RuleExecutor.scala:44) org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1.apply(RuleExecutor.scala:115) org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1.apply(RuleExecutor.scala:72) org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(RuleExecutor.scala:72) org.apache.spark.sql.catalyst.encoders.ExpressionEncoder.resolve(ExpressionEncoder.scala:253) org.apache.spark.sql.Dataset.<init>(Dataset.scala:78) org.apache.spark.sql.Dataset.<init>(Dataset.scala:89) org.apache.spark.sql.SQLContext.createDataset(SQLContext.scala:507) org.apache.spark.sql.SQLImplicits.localSeqToDatasetHolder(SQLImplicits.scala:80) {code} When providing a type alias, the code fails in a different way: {code} type TwoInt = (Int, Int) case class Test(v: TwoInt) Seq(Test((1,2)), Test((3,4)).toDS().show //fails {code} {code} NoSuchElementException: : head of empty list (ScalaReflection.scala:504) org.apache.spark.sql.catalyst.ScalaReflection$$anonfun$org$apache$spark$sql$catalyst$ScalaReflection$$extractorFor$1.apply(ScalaReflection.scala:504) org.apache.spark.sql.catalyst.ScalaReflection$$anonfun$org$apache$spark$sql$catalyst$ScalaReflection$$extractorFor$1.apply(ScalaReflection.scala:502) org.apache.spark.sql.catalyst.ScalaReflection$.org$apache$spark$sql$catalyst$ScalaReflection$$extractorFor(ScalaReflection.scala:502) org.apache.spark.sql.catalyst.ScalaReflection$$anonfun$org$apache$spark$sql$catalyst$ScalaReflection$$extractorFor$1.apply(ScalaReflection.scala:509) org.apache.spark.sql.catalyst.ScalaReflection$$anonfun$org$apache$spark$sql$catalyst$ScalaReflection$$extractorFor$1.apply(ScalaReflection.scala:502) org.apache.spark.sql.catalyst.ScalaReflection$.org$apache$spark$sql$catalyst$ScalaReflection$$extractorFor(ScalaReflection.scala:502) org.apache.spark.sql.catalyst.ScalaReflection$.extractorsFor(ScalaReflection.scala:394) org.apache.spark.sql.catalyst.encoders.ExpressionEncoder$.apply(ExpressionEncoder.scala:54) org.apache.spark.sql.SQLImplicits.newProductEncoder(SQLImplicits.scala:41) com.intenthq.pipeline.actions.ActionsJobIntegrationSpec.enrich(ActionsJobIntegrationSpec.scala:63) com.intenthq.pipeline.actions.ActionsJobIntegrationSpec$$anonfun$is$2.apply(ActionsJobIntegrationSpec.scala:45) com.intenthq.pipeline.actions.ActionsJobIntegrationSpec$$anonfun$is$2.apply(ActionsJobIntegrationSpec.scala:45) {code} -- This message was sent by Atlassian JIRA (v6.3.4#6332) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org