Can you take a look at SPARK-5278 where ambiguity is shown between field names which differ only by case ?
Cheers On Sat, Sep 12, 2015 at 3:40 AM, Fengdong Yu <fengdo...@everstring.com> wrote: > Hi Ted, > I checked the JSON, there aren't duplicated key in JSON. > > > Azuryy Yu > Sr. Infrastructure Engineer > > cel: 158-0164-9103 > wetchat: azuryy > > > On Sat, Sep 12, 2015 at 5:52 PM, Ted Yu <yuzhih...@gmail.com> wrote: > >> Is it possible that Canonical_URL occurs more than once in your json ? >> >> Can you check your json input ? >> >> Thanks >> >> On Sat, Sep 12, 2015 at 2:05 AM, Fengdong Yu <fengdo...@everstring.com> >> wrote: >> >>> Hi, >>> >>> I am using spark1.4.1 data frame, read JSON data, then save it to orc. >>> the code is very simple: >>> >>> DataFrame json = sqlContext.read().json(input); >>> >>> json.write().format("orc").save(output); >>> >>> the job failed. what's wrong with this exception? Thanks. >>> >>> Exception in thread "main" org.apache.spark.sql.AnalysisException: >>> Reference 'Canonical_URL' is ambiguous, could be: Canonical_URL#960, >>> Canonical_URL#1010.; at >>> org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.resolve(LogicalPlan.scala:279) >>> at >>> org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.resolveChildren(LogicalPlan.scala:116) >>> at >>> org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveReferences$$anonfun$apply$8$$anonfun$applyOrElse$4$$anonfun$16.apply(Analyzer.scala:350) >>> at >>> org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveReferences$$anonfun$apply$8$$anonfun$applyOrElse$4$$anonfun$16.apply(Analyzer.scala:350) >>> at >>> org.apache.spark.sql.catalyst.analysis.package$.withPosition(package.scala:48) >>> at >>> org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveReferences$$anonfun$apply$8$$anonfun$applyOrElse$4.applyOrElse(Analyzer.scala:350) >>> at >>> org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveReferences$$anonfun$apply$8$$anonfun$applyOrElse$4.applyOrElse(Analyzer.scala:341) >>> at >>> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:286) >>> at >>> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:286) >>> at >>> org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:51) >>> at >>> org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:285) >>> at >>> org.apache.spark.sql.catalyst.plans.QueryPlan.org$apache$spark$sql$catalyst$plans$QueryPlan$$transformExpressionUp$1(QueryPlan.scala:108) >>> at >>> org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$2$$anonfun$apply$2.apply(QueryPlan.scala:123) >>> at >>> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244) >>> at >>> scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244) >>> at scala.collection.immutable.List.foreach(List.scala:318) at >>> scala.collection.TraversableLike$class.map(TraversableLike.scala:244) at >>> scala.collection.AbstractTraversable.map(Traversable.scala:105) at >>> org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$2.apply(QueryPlan.scala:122) >>> at scala.collection.Iterator$$anon$11.next(Iterator.scala:328) at >>> scala.collection.Iterator$class.foreach(Iterator.scala:727) at >>> scala.collection.AbstractIterator.foreach(Iterator.scala:1157) at >>> scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:48) at >>> scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:103) >>> at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:47) >>> at scala.collection.TraversableOnce$class.to(TraversableOnce.scala:273) >>> at scala.collection.AbstractIterator.to(Iterator.scala:1157) at >>> scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:265) >>> at scala.collection.AbstractIterator.toBuffer(Iterator.scala:1157) at >>> scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:252) >>> at scala.collection.AbstractIterator.toArray(Iterator.scala:1157) at >>> org.apache.spark.sql.catalyst.plans.QueryPlan.transformExpressionsUp(QueryPlan.scala:127) >>> at >>> org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveReferences$$anonfun$apply$8.applyOrElse(Analyzer.scala:341) >>> at >>> org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveReferences$$anonfun$apply$8.applyOrElse(Analyzer.scala:243) >>> at >>> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:286) >>> at >>> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformUp$1.apply(TreeNode.scala:286) >>> at >>> org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:51) >>> at >>> org.apache.spark.sql.catalyst.trees.TreeNode.transformUp(TreeNode.scala:285) >>> at >>> org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveReferences$.apply(Analyzer.scala:243) >>> at >>> org.apache.spark.sql.catalyst.analysis.Analyzer$ResolveReferences$.apply(Analyzer.scala:242) >>> at >>> org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1$$anonfun$apply$1.apply(RuleExecutor.scala:61) >>> at >>> org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1$$anonfun$apply$1.apply(RuleExecutor.scala:59) >>> at >>> scala.collection.LinearSeqOptimized$class.foldLeft(LinearSeqOptimized.scala:111) >>> at scala.collection.immutable.List.foldLeft(List.scala:84) at >>> org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1.apply(RuleExecutor.scala:59) >>> at >>> org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1.apply(RuleExecutor.scala:51) >>> at scala.collection.immutable.List.foreach(List.scala:318) at >>> org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(RuleExecutor.scala:51) >>> at >>> org.apache.spark.sql.SQLContext$QueryExecution.analyzed$lzycompute(SQLContext.scala:933) >>> at >>> org.apache.spark.sql.SQLContext$QueryExecution.analyzed(SQLContext.scala:933) >>> at >>> org.apache.spark.sql.SQLContext$QueryExecution.assertAnalyzed(SQLContext.scala:931) >>> at org.apache.spark.sql.DataFrame.(DataFrame.scala:131) at >>> org.apache.spark.sql.DataFrame$.apply(DataFrame.scala:51) at >>> org.apache.spark.sql.sources.InsertIntoHadoopFsRelation.run(commands.scala:132) >>> at >>> org.apache.spark.sql.execution.ExecutedCommand.sideEffectResult$lzycompute(commands.scala:57) >>> at >>> org.apache.spark.sql.execution.ExecutedCommand.sideEffectResult(commands.scala:57) >>> at >>> org.apache.spark.sql.execution.ExecutedCommand.doExecute(commands.scala:68) >>> at >>> org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:88) >>> at >>> org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:88) >>> at >>> org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:147) >>> at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:87) at >>> org.apache.spark.sql.SQLContext$QueryExecution.toRdd$lzycompute(SQLContext.scala:950) >>> at >>> org.apache.spark.sql.SQLContext$QueryExecution.toRdd(SQLContext.scala:950) >>> at org.apache.spark.sql.sources.ResolvedDataSource$.apply(ddl.scala:336) at >>> org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:144) at >>> org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:135) at >>> com.es.infrastructure.spark.orc.transformer.JsonTransformer.run(JsonTransformer.java:22) >>> at Main.main(Main.java:70) at >>> sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at >>> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) >>> at >>> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) >>> at java.lang.reflect.Method.invoke(Method.java:606) at >>> org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:665) >>> at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:170) >>> at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:193) at >>> org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:112) at >>> org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) >>> >>> >> >