Any update? I encountered same issue in my environment.
Here are my steps as usual: git clone https://github.com/apache/spark mvn -Pyarn -Phadoop-2.4 -Dhadoop.version=2.4.0 -Phive -DskipTests clean package build successfully by maven. import into IDEA as a maven project, click Build->Make Project,. 2 compile errors found, one of them are DataTypeConversions.scala Error:scala: while compiling: /opt/Development/spark/sql/core/src/main/scala/org/apache/spark/sql/types/util/DataTypeConversions.scala during phase: jvm library version: version 2.10.4 compiler version: version 2.10.4 reconstructed args: -nobootcp -javabootclasspath : -deprecation -feature -Xplugin:/home/shawguo/host/maven_repo/org/scalamacros/paradise_2.10.4/2.0.1/paradise_2.10.4-2.0.1.jar -unchecked -classpath /opt/Development/jdk1.6.0_45/jre/lib/jce.jar:/opt/Development/jdk1.6.0_45/jre/lib/javaws.jar:/opt/Development/jdk1.6.0_45/jre/lib/management-agent.jar:/opt/Development/jdk1.6.0_45/jre/lib/jsse.jar:/opt/Development/jdk1.6.0_45/jre/lib/deploy.jar:/opt/Development/jdk1.6.0_45/jre/lib/plugin.jar:/opt/Development/jdk1.6.0_45/jre/lib/resources.jar:/opt/Development/jdk1.6.0_45/jre/lib/rt.jar:/opt/Development/jdk1.6.0_45/jre/lib/charsets.jar:/opt/Development/jdk1.6.0_45/jre/lib/ext/sunjce_provider.jar:/opt/Development/jdk1.6.0_45/jre/lib/ext/localedata.jar:/opt/Development/jdk1.6.0_45/jre/lib/ext/dnsns.jar:/opt/Development/jdk1.6.0_45/jre/lib/ext/sunpkcs11.jar:/opt/Development/spark/sql/core/target/scala-2.10/classes:/opt/Development/spark/core/target/scala-2.10/classes:/home/shawguo/host/maven_repo/org/apache/hadoop/hadoop-client/2.4.0/hadoop-client-2.4.0.jar:/home/shawguo/host/maven_repo/org/apache/hadoop/hadoop-common/2.4.0/hadoop-common-2.4.0.jar:/home/shawguo/host/maven_repo/org/apache/hadoop/hadoop-annotations/2.4.0/hadoop-annotations-2.4.0.jar:/home/shawguo/host/maven_repo/com/google/guava/guava/14.0.1/guava-14.0.1.jar:/home/shawguo/host/maven_repo/commons-cli/commons-cli/1.2/commons-cli-1.2.jar:/home/shawguo/host/maven_repo/xmlenc/xmlenc/0.52/xmlenc-0.52.jar:/home/shawguo/host/maven_repo/commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar:/home/shawguo/host/maven_repo/commons-codec/commons-codec/1.5/commons-codec-1.5.jar:/home/shawguo/host/maven_repo/commons-io/commons-io/2.4/commons-io-2.4.jar:/home/shawguo/host/maven_repo/commons-net/commons-net/2.2/commons-net-2.2.jar:/home/shawguo/host/maven_repo/commons-collections/commons-collections/3.2.1/commons-collections-3.2.1.jar:/home/shawguo/host/maven_repo/log4j/log4j/1.2.17/log4j-1.2.17.jar:/home/shawguo/host/maven_repo/commons-lang/commons-lang/2.6/commons-lang-2.6.jar:/home/shawguo/host/maven_repo/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar:/home/shawguo/host/maven_repo/commons-digester/commons-digester/1.8/commons-digester-1.8.jar:/home/shawguo/host/maven_repo/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar:/home/shawguo/host/maven_repo/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar:/home/shawguo/host/maven_repo/org/slf4j/slf4j-api/1.7.5/slf4j-api-1.7.5.jar:/home/shawguo/host/maven_repo/org/slf4j/slf4j-log4j12/1.7.5/slf4j-log4j12-1.7.5.jar:/home/shawguo/host/maven_repo/org/codehaus/jackson/jackson-core-asl/1.9.11/jackson-core-asl-1.9.11.jar:/home/shawguo/host/maven_repo/org/codehaus/jackson/jackson-mapper-asl/1.8.8/jackson-mapper-asl-1.8.8.jar:/home/shawguo/host/maven_repo/org/apache/avro/avro/1.7.6/avro-1.7.6.jar:/home/shawguo/host/maven_repo/com/thoughtworks/paranamer/paranamer/2.6/paranamer-2.6.jar:/home/shawguo/host/maven_repo/org/xerial/snappy/snappy-java/1.1.1.3/snappy-java-1.1.1.3.jar:/home/shawguo/host/maven_repo/org/apache/commons/commons-compress/1.4.1/commons-compress-1.4.1.jar:/home/shawguo/host/maven_repo/org/tukaani/xz/1.0/xz-1.0.jar:/home/shawguo/host/maven_repo/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar:/home/shawguo/host/maven_repo/org/apache/hadoop/hadoop-auth/2.4.0/hadoop-auth-2.4.0.jar:/home/shawguo/host/maven_repo/org/apache/httpcomponents/httpclient/4.1.2/httpclient-4.1.2.jar:/home/shawguo/host/maven_repo/org/apache/httpcomponents/httpcore/4.1.2/httpcore-4.1.2.jar:/home/shawguo/host/maven_repo/com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar:/home/shawguo/host/maven_repo/org/apache/zookeeper/zookeeper/3.4.5/zookeeper-3.4.5.jar:/home/shawguo/host/maven_repo/org/apache/hadoop/hadoop-hdfs/2.4.0/hadoop-hdfs-2.4.0.jar:/home/shawguo/host/maven_repo/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar:/home/shawguo/host/maven_repo/org/apache/hadoop/hadoop-mapreduce-client-app/2.4.0/hadoop-mapreduce-client-app-2.4.0.jar:/home/shawguo/host/maven_repo/org/apache/hadoop/hadoop-mapreduce-client-common/2.4.0/hadoop-mapreduce-client-common-2.4.0.jar:/home/shawguo/host/maven_repo/org/apache/hadoop/hadoop-yarn-common/2.4.0/hadoop-yarn-common-2.4.0.jar:/home/shawguo/host/maven_repo/org/apache/hadoop/hadoop-yarn-api/2.4.0/hadoop-yarn-api-2.4.0.jar:/home/shawguo/host/maven_repo/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar:/home/shawguo/host/maven_repo/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar:/home/shawguo/host/maven_repo/javax/activation/activation/1.1/activation-1.1.jar:/home/shawguo/host/maven_repo/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar:/home/shawguo/host/maven_repo/org/apache/hadoop/hadoop-yarn-client/2.4.0/hadoop-yarn-client-2.4.0.jar:/home/shawguo/host/maven_repo/com/sun/jersey/jersey-client/1.9/jersey-client-1.9.jar:/home/shawguo/host/maven_repo/org/apache/hadoop/hadoop-mapreduce-client-core/2.4.0/hadoop-mapreduce-client-core-2.4.0.jar:/home/shawguo/host/maven_repo/org/apache/hadoop/hadoop-yarn-server-common/2.4.0/hadoop-yarn-server-common-2.4.0.jar:/home/shawguo/host/maven_repo/org/apache/hadoop/hadoop-mapreduce-client-shuffle/2.4.0/hadoop-mapreduce-client-shuffle-2.4.0.jar:/home/shawguo/host/maven_repo/org/apache/hadoop/hadoop-mapreduce-client-jobclient/2.4.0/hadoop-mapreduce-client-jobclient-2.4.0.jar:/home/shawguo/host/maven_repo/net/java/dev/jets3t/jets3t/0.9.0/jets3t-0.9.0.jar:/home/shawguo/host/maven_repo/com/jamesmurty/utils/java-xmlbuilder/0.4/java-xmlbuilder-0.4.jar:/home/shawguo/host/maven_repo/org/apache/curator/curator-recipes/2.4.0/curator-recipes-2.4.0.jar:/home/shawguo/host/maven_repo/org/apache/curator/curator-framework/2.4.0/curator-framework-2.4.0.jar:/home/shawguo/host/maven_repo/org/apache/curator/curator-client/2.4.0/curator-client-2.4.0.jar:/home/shawguo/host/maven_repo/jline/jline/0.9.94/jline-0.9.94.jar:/home/shawguo/host/maven_repo/org/eclipse/jetty/jetty-plus/8.1.14.v20131031/jetty-plus-8.1.14.v20131031.jar:/home/shawguo/host/maven_repo/org/eclipse/jetty/orbit/javax.transaction/1.1.1.v201105210645/javax.transaction-1.1.1.v201105210645.jar:/home/shawguo/host/maven_repo/org/eclipse/jetty/jetty-webapp/8.1.14.v20131031/jetty-webapp-8.1.14.v20131031.jar:/home/shawguo/host/maven_repo/org/eclipse/jetty/jetty-xml/8.1.14.v20131031/jetty-xml-8.1.14.v20131031.jar:/home/shawguo/host/maven_repo/org/eclipse/jetty/jetty-util/8.1.14.v20131031/jetty-util-8.1.14.v20131031.jar:/home/shawguo/host/maven_repo/org/eclipse/jetty/jetty-servlet/8.1.14.v20131031/jetty-servlet-8.1.14.v20131031.jar:/home/shawguo/host/maven_repo/org/eclipse/jetty/jetty-security/8.1.14.v20131031/jetty-security-8.1.14.v20131031.jar:/home/shawguo/host/maven_repo/org/eclipse/jetty/jetty-server/8.1.14.v20131031/jetty-server-8.1.14.v20131031.jar:/home/shawguo/host/maven_repo/org/eclipse/jetty/orbit/javax.servlet/3.0.0.v201112011016/javax.servlet-3.0.0.v201112011016.jar:/home/shawguo/host/maven_repo/org/eclipse/jetty/jetty-continuation/8.1.14.v20131031/jetty-continuation-8.1.14.v20131031.jar:/home/shawguo/host/maven_repo/org/eclipse/jetty/jetty-http/8.1.14.v20131031/jetty-http-8.1.14.v20131031.jar:/home/shawguo/host/maven_repo/org/eclipse/jetty/jetty-io/8.1.14.v20131031/jetty-io-8.1.14.v20131031.jar:/home/shawguo/host/maven_repo/org/eclipse/jetty/jetty-jndi/8.1.14.v20131031/jetty-jndi-8.1.14.v20131031.jar:/home/shawguo/host/maven_repo/org/eclipse/jetty/orbit/javax.mail.glassfish/1.4.1.v201005082020/javax.mail.glassfish-1.4.1.v201005082020.jar:/home/shawguo/host/maven_repo/org/eclipse/jetty/orbit/javax.activation/1.1.0.v201105071233/javax.activation-1.1.0.v201105071233.jar:/home/shawguo/host/maven_repo/org/apache/commons/commons-lang3/3.3.2/commons-lang3-3.3.2.jar:/home/shawguo/host/maven_repo/org/slf4j/jul-to-slf4j/1.7.5/jul-to-slf4j-1.7.5.jar:/home/shawguo/host/maven_repo/org/slf4j/jcl-over-slf4j/1.7.5/jcl-over-slf4j-1.7.5.jar:/home/shawguo/host/maven_repo/com/ning/compress-lzf/1.0.0/compress-lzf-1.0.0.jar:/home/shawguo/host/maven_repo/net/jpountz/lz4/lz4/1.2.0/lz4-1.2.0.jar:/home/shawguo/host/maven_repo/com/twitter/chill_2.10/0.3.6/chill_2.10-0.3.6.jar:/home/shawguo/host/maven_repo/org/scala-lang/scala-library/2.10.4/scala-library-2.10.4.jar:/home/shawguo/host/maven_repo/com/twitter/chill-java/0.3.6/chill-java-0.3.6.jar:/home/shawguo/host/maven_repo/com/esotericsoftware/kryo/kryo/2.21/kryo-2.21.jar:/home/shawguo/host/maven_repo/com/esotericsoftware/reflectasm/reflectasm/1.07/reflectasm-1.07-shaded.jar:/home/shawguo/host/maven_repo/com/esotericsoftware/minlog/minlog/1.2/minlog-1.2.jar:/home/shawguo/host/maven_repo/org/objenesis/objenesis/1.2/objenesis-1.2.jar:/home/shawguo/host/maven_repo/org/roaringbitmap/RoaringBitmap/0.4.1/RoaringBitmap-0.4.1.jar:/home/shawguo/host/maven_repo/org/spark-project/akka/akka-remote_2.10/2.3.4-spark/akka-remote_2.10-2.3.4-spark.jar:/home/shawguo/host/maven_repo/org/spark-project/akka/akka-actor_2.10/2.3.4-spark/akka-actor_2.10-2.3.4-spark.jar:/home/shawguo/host/maven_repo/com/typesafe/config/1.2.1/config-1.2.1.jar:/home/shawguo/host/maven_repo/io/netty/netty/3.8.0.Final/netty-3.8.0.Final.jar:/home/shawguo/host/maven_repo/org/spark-project/protobuf/protobuf-java/2.5.0-spark/protobuf-java-2.5.0-spark.jar:/home/shawguo/host/maven_repo/org/uncommons/maths/uncommons-maths/1.2.2a/uncommons-maths-1.2.2a.jar:/home/shawguo/host/maven_repo/org/spark-project/akka/akka-slf4j_2.10/2.3.4-spark/akka-slf4j_2.10-2.3.4-spark.jar:/home/shawguo/host/maven_repo/org/json4s/json4s-jackson_2.10/3.2.10/json4s-jackson_2.10-3.2.10.jar:/home/shawguo/host/maven_repo/org/json4s/json4s-core_2.10/3.2.10/json4s-core_2.10-3.2.10.jar:/home/shawguo/host/maven_repo/org/json4s/json4s-ast_2.10/3.2.10/json4s-ast_2.10-3.2.10.jar:/home/shawguo/host/maven_repo/org/scala-lang/scalap/2.10.4/scalap-2.10.4.jar:/home/shawguo/host/maven_repo/org/scala-lang/scala-compiler/2.10.4/scala-compiler-2.10.4.jar:/home/shawguo/host/maven_repo/org/scala-lang/scala-reflect/2.10.4/scala-reflect-2.10.4.jar:/home/shawguo/host/maven_repo/com/fasterxml/jackson/core/jackson-databind/2.3.0/jackson-databind-2.3.0.jar:/home/shawguo/host/maven_repo/com/fasterxml/jackson/core/jackson-annotations/2.3.0/jackson-annotations-2.3.0.jar:/home/shawguo/host/maven_repo/com/fasterxml/jackson/core/jackson-core/2.3.0/jackson-core-2.3.0.jar:/home/shawguo/host/maven_repo/colt/colt/1.2.0/colt-1.2.0.jar:/home/shawguo/host/maven_repo/concurrent/concurrent/1.3.4/concurrent-1.3.4.jar:/home/shawguo/host/maven_repo/org/apache/mesos/mesos/0.18.1/mesos-0.18.1-shaded-protobuf.jar:/home/shawguo/host/maven_repo/io/netty/netty-all/4.0.23.Final/netty-all-4.0.23.Final.jar:/home/shawguo/host/maven_repo/com/clearspring/analytics/stream/2.7.0/stream-2.7.0.jar:/home/shawguo/host/maven_repo/com/codahale/metrics/metrics-core/3.0.0/metrics-core-3.0.0.jar:/home/shawguo/host/maven_repo/com/codahale/metrics/metrics-jvm/3.0.0/metrics-jvm-3.0.0.jar:/home/shawguo/host/maven_repo/com/codahale/metrics/metrics-json/3.0.0/metrics-json-3.0.0.jar:/home/shawguo/host/maven_repo/com/codahale/metrics/metrics-graphite/3.0.0/metrics-graphite-3.0.0.jar:/home/shawguo/host/maven_repo/org/tachyonproject/tachyon-client/0.5.0/tachyon-client-0.5.0.jar:/home/shawguo/host/maven_repo/org/tachyonproject/tachyon/0.5.0/tachyon-0.5.0.jar:/home/shawguo/host/maven_repo/org/spark-project/pyrolite/2.0.1/pyrolite-2.0.1.jar:/home/shawguo/host/maven_repo/net/sf/py4j/py4j/0.8.2.1/py4j-0.8.2.1.jar:/home/shawguo/host/maven_repo/org/spark-project/spark/unused/1.0.0/unused-1.0.0.jar:/opt/Development/spark/sql/catalyst/target/scala-2.10/classes:/home/shawguo/host/maven_repo/org/scalamacros/quasiquotes_2.10/2.0.1/quasiquotes_2.10-2.0.1.jar:/home/shawguo/host/maven_repo/com/twitter/parquet-column/1.4.3/parquet-column-1.4.3.jar:/home/shawguo/host/maven_repo/com/twitter/parquet-common/1.4.3/parquet-common-1.4.3.jar:/home/shawguo/host/maven_repo/com/twitter/parquet-encoding/1.4.3/parquet-encoding-1.4.3.jar:/home/shawguo/host/maven_repo/com/twitter/parquet-generator/1.4.3/parquet-generator-1.4.3.jar:/home/shawguo/host/maven_repo/com/twitter/parquet-hadoop/1.4.3/parquet-hadoop-1.4.3.jar:/home/shawguo/host/maven_repo/com/twitter/parquet-format/2.0.0/parquet-format-2.0.0.jar:/home/shawguo/host/maven_repo/com/twitter/parquet-jackson/1.4.3/parquet-jackson-1.4.3.jar last tree to typer: Literal(Constant(org.apache.spark.sql.catalyst.types.PrimitiveType)) symbol: null symbol definition: null tpe: Class(classOf[org.apache.spark.sql.catalyst.types.PrimitiveType]) symbol owners: context owners: anonymous class anonfun$asScalaDataType$1 -> package util == Enclosing template or block == Template( // val <local $anonfun>: <notype>, tree.tpe=org.apache.spark.sql.types.util.anonfun$asScalaDataType$1 "scala.runtime.AbstractFunction1", "scala.Serializable" // parents ValDef( private "_" <tpt> <empty> ) // 3 statements DefDef( // final def apply(javaStructField: org.apache.spark.sql.api.java.StructField): org.apache.spark.sql.catalyst.types.StructField <method> final <triedcooking> "apply" [] // 1 parameter list ValDef( // javaStructField: org.apache.spark.sql.api.java.StructField <synthetic> <triedcooking> "javaStructField" <tpt> // tree.tpe=org.apache.spark.sql.api.java.StructField <empty> ) <tpt> // tree.tpe=org.apache.spark.sql.catalyst.types.StructField Apply( // def asScalaStructField(javaStructField: org.apache.spark.sql.api.java.StructField): org.apache.spark.sql.catalyst.types.StructField in object DataTypeConversions, tree.tpe=org.apache.spark.sql.catalyst.types.StructField DataTypeConversions.this."asScalaStructField" // def asScalaStructField(javaStructField: org.apache.spark.sql.api.java.StructField): org.apache.spark.sql.catalyst.types.StructField in object DataTypeConversions, tree.tpe=(javaStructField: org.apache.spark.sql.api.java.StructField)org.apache.spark.sql.catalyst.types.StructField "javaStructField" // javaStructField: org.apache.spark.sql.api.java.StructField, tree.tpe=org.apache.spark.sql.api.java.StructField ) ) DefDef( // final def apply(v1: Object): Object <method> final <bridge> "apply" [] // 1 parameter list ValDef( // v1: Object <triedcooking> "v1" <tpt> // tree.tpe=Object <empty> ) <tpt> // tree.tpe=Object Apply( // final def apply(javaStructField: org.apache.spark.sql.api.java.StructField): org.apache.spark.sql.catalyst.types.StructField, tree.tpe=org.apache.spark.sql.catalyst.types.StructField DataTypeConversions$$anonfun$asScalaDataType$1.this."apply" // final def apply(javaStructField: org.apache.spark.sql.api.java.StructField): org.apache.spark.sql.catalyst.types.StructField, tree.tpe=(javaStructField: org.apache.spark.sql.api.java.StructField)org.apache.spark.sql.catalyst.types.StructField Apply( // final def $asInstanceOf[T0 >: ? <: ?](): T0 in class Object, tree.tpe=org.apache.spark.sql.api.java.StructField TypeApply( // final def $asInstanceOf[T0 >: ? <: ?](): T0 in class Object, tree.tpe=()org.apache.spark.sql.api.java.StructField "v1"."$asInstanceOf" // final def $asInstanceOf[T0 >: ? <: ?](): T0 in class Object, tree.tpe=[T0 >: ? <: ?]()T0 <tpt> // tree.tpe=org.apache.spark.sql.api.java.StructField ) Nil ) ) ) DefDef( // def <init>(): org.apache.spark.sql.types.util.anonfun$asScalaDataType$1 <method> <triedcooking> "<init>" [] List(Nil) <tpt> // tree.tpe=org.apache.spark.sql.types.util.anonfun$asScalaDataType$1 Block( // tree.tpe=Unit Apply( // def <init>(): scala.runtime.AbstractFunction1 in class AbstractFunction1, tree.tpe=scala.runtime.AbstractFunction1 DataTypeConversions$$anonfun$asScalaDataType$1.super."<init>" // def <init>(): scala.runtime.AbstractFunction1 in class AbstractFunction1, tree.tpe=()scala.runtime.AbstractFunction1 Nil ) () ) ) ) == Expanded type of tree == ConstantType( value = Constant(org.apache.spark.sql.catalyst.types.PrimitiveType) ) uncaught exception during compilation: java.lang.AssertionError -- View this message in context: http://apache-spark-user-list.1001560.n3.nabble.com/scalac-crash-when-compiling-DataTypeConversions-scala-tp17083p17303.html Sent from the Apache Spark User List mailing list archive at Nabble.com. --------------------------------------------------------------------- To unsubscribe, e-mail: user-unsubscr...@spark.apache.org For additional commands, e-mail: user-h...@spark.apache.org
