Hi Arkadiusz, Try 'rooting' your import. It looks like the import is being interpreted as being relative to the previous. 'rooting; is done by adding the '_root_' prefix to your import:
import org.apache.spark.streaming.kafka.KafkaUtils import _root_.io.confluent.kafka.serializers.KafkaAvroDecoder kr, Gerard. On Wed, Dec 13, 2017 at 6:05 PM, Arkadiusz Bicz <arkadiusz.b...@gmail.com> wrote: > Hi, > > I try to test spark streaming 2.2.0 version with confluent 3.3.0 > > I have got lot of error during compilation this is my sbt: > > lazy val sparkstreaming = (project in file(".")) > .settings( > name := "sparkstreaming", > organization := "org.arek", > version := "0.1-SNAPSHOT", > scalaVersion := "2.11.8", > libraryDependencies ++= Seq( > "org.apache.spark" %% "spark-streaming" % "2.2.0", > "org.apache.spark" %% "spark-streaming-kafka-0-8" % "2.2.0", > "io.confluent" % "kafka-avro-serializer" % "3.3.0" > ) > ) > > > import org.apache.spark._ > import org.apache.spark.streaming._ > import org.apache.spark.streaming.kafka.KafkaUtils > import io.confluent.kafka.serializers.KafkaAvroDecoder > > object Transformation extends Serializable { > > def main(args: Array[String]) = { > val conf = new SparkConf().setAppName("StreamingTranformation"). > setMaster("local[*]") > val streamingContext = new StreamingContext(conf, Seconds(1)) > > val kafkaParams = Map[String, String]("metadata.broker.list" -> > "local:2181", > "schema.registry.url" -> "http://local:8081", > "auto.offset.reset" -> "smallest") > > val topicSet = Set("GEXPPROD_ROUTE") > val messages = KafkaUtils.createDirectStream[Object, Object, > KafkaAvroDecoder, KafkaAvroDecoder](streamingContext, kafkaParams, > topicSet).map(_._2) > > val lines = messages.foreachRDD(rdd => { > rdd.foreach({ avroRecord => > println(avroRecord) > }) > }) > } > > > [warn] Found version conflict(s) in library dependencies; some are > suspected to be binary incompatible: > [warn] * io.netty:netty:3.9.9.Final is selected over {3.6.2.Final, > 3.7.0.Final} > [warn] +- org.apache.spark:spark-core_2.11:2.2.0 > (depends on 3.7.0.Final) > [warn] +- org.apache.zookeeper:zookeeper:3.4.8 > (depends on 3.7.0.Final) > [warn] +- org.apache.zookeeper:zookeeper:3.4.6 > (depends on 3.6.2.Final) > [warn] +- org.apache.hadoop:hadoop-hdfs:2.6.5 > (depends on 3.6.2.Final) > [warn] * commons-net:commons-net:2.2 is selected over 3.1 > [warn] +- org.apache.spark:spark-core_2.11:2.2.0 > (depends on 3.1) > [warn] +- org.apache.hadoop:hadoop-common:2.6.5 > (depends on 3.1) > [warn] * com.google.guava:guava:11.0.2 is selected over {12.0.1, 16.0.1} > [warn] +- org.apache.hadoop:hadoop-yarn-client:2.6.5 > (depends on 11.0.2) > [warn] +- org.apache.hadoop:hadoop-yarn-api:2.6.5 > (depends on 11.0.2) > [warn] +- org.apache.hadoop:hadoop-yarn-common:2.6.5 > (depends on 11.0.2) > [warn] +- org.apache.hadoop:hadoop-yarn-server-nodemanager:2.6.5 > (depends on 11.0.2) > [warn] +- org.apache.hadoop:hadoop-common:2.6.5 > (depends on 11.0.2) > [warn] +- org.apache.hadoop:hadoop-yarn-server-common:2.6.5 > (depends on 11.0.2) > [warn] +- org.apache.hadoop:hadoop-hdfs:2.6.5 > (depends on 11.0.2) > [warn] +- org.apache.curator:curator-framework:2.6.0 > (depends on 16.0.1) > [warn] +- org.apache.curator:curator-client:2.6.0 > (depends on 16.0.1) > [warn] +- org.apache.curator:curator-recipes:2.6.0 > (depends on 16.0.1) > [warn] +- org.htrace:htrace-core:3.0.4 (depends > on 12.0.1) > [warn] Run 'evicted' to see detailed eviction warnings > [info] Compiling 1 Scala source to /home/adminuser/data- > streaming-platform/sparkstreaming/target/scala-2.11/classes ... > [error] /home/adminuser/data-streaming-platform/ > sparkstreaming/src/main/scala/com/arek/streaming/spark/Transformation.scala:6:11: > object confluent is not a member of package org.apache.spark.io > [error] import io.confluent.kafka.serializers.KafkaAvroDecoder > [error] ^ > [error] /home/adminuser/data-streaming-platform/ > sparkstreaming/src/main/scala/com/arek/streaming/spark/Transformation.scala:19:66: > not found: type KafkaAvroDecoder > [error] val messages = KafkaUtils.createDirectStream[Object, Object, > KafkaAvroDecoder, KafkaAvroDecoder](streamingContext, kafkaParams, > topicSet).map(_._2) > [error] ^ > [error] /home/adminuser/data-streaming-platform/ > sparkstreaming/src/main/scala/com/arek/streaming/spark/Transformation.scala:19:84: > not found: type KafkaAvroDecoder > [error] val messages = KafkaUtils.createDirectStream[Object, Object, > KafkaAvroDecoder, KafkaAvroDecoder](streamingContext, kafkaParams, > topicSet).map(_._2) > [error] > > > When changing to library "org.apache.spark" %% > "spark-streaming-kafka-0-10" % "2.2.0" : > > > [warn] Found version conflict(s) in library dependencies; some are > suspected to be binary incompatible: > [warn] * com.101tec:zkclient:0.10 is selected over 0.8 > [warn] +- io.confluent:common-utils:3.3.0 > (depends on 0.10) > [warn] +- org.apache.kafka:kafka_2.11:0.10.0.1 > (depends on 0.8) > [warn] * io.netty:netty:3.9.9.Final is selected over {3.6.2.Final, > 3.7.0.Final} > [warn] +- org.apache.spark:spark-core_2.11:2.2.0 > (depends on 3.7.0.Final) > [warn] +- org.apache.zookeeper:zookeeper:3.4.8 > (depends on 3.7.0.Final) > [warn] +- org.apache.zookeeper:zookeeper:3.4.6 > (depends on 3.6.2.Final) > [warn] +- org.apache.hadoop:hadoop-hdfs:2.6.5 > (depends on 3.6.2.Final) > [warn] * commons-net:commons-net:2.2 is selected over 3.1 > [warn] +- org.apache.spark:spark-core_2.11:2.2.0 > (depends on 3.1) > [warn] +- org.apache.hadoop:hadoop-common:2.6.5 > (depends on 3.1) > [warn] * com.google.guava:guava:11.0.2 is selected over {12.0.1, 16.0.1} > [warn] +- org.apache.hadoop:hadoop-yarn-client:2.6.5 > (depends on 11.0.2) > [warn] +- org.apache.hadoop:hadoop-yarn-api:2.6.5 > (depends on 11.0.2) > [warn] +- org.apache.hadoop:hadoop-yarn-common:2.6.5 > (depends on 11.0.2) > [warn] +- org.apache.hadoop:hadoop-yarn-server-nodemanager:2.6.5 > (depends on 11.0.2) > [warn] +- org.apache.hadoop:hadoop-common:2.6.5 > (depends on 11.0.2) > [warn] +- org.apache.hadoop:hadoop-yarn-server-common:2.6.5 > (depends on 11.0.2) > [warn] +- org.apache.hadoop:hadoop-hdfs:2.6.5 > (depends on 11.0.2) > [warn] +- org.apache.curator:curator-framework:2.6.0 > (depends on 16.0.1) > [warn] +- org.apache.curator:curator-client:2.6.0 > (depends on 16.0.1) > [warn] +- org.apache.curator:curator-recipes:2.6.0 > (depends on 16.0.1) > [warn] +- org.htrace:htrace-core:3.0.4 (depends > on 12.0.1) > [warn] Run 'evicted' to see detailed eviction warnings > [info] Compiling 1 Scala source to /home/adminuser/data- > streaming-platform/sparkstreaming/target/scala-2.11/classes ... > [error] /home/adminuser/data-streaming-platform/ > sparkstreaming/src/main/scala/com/arek/streaming/spark/Transformation.scala:18:11: > object confluent is not a member of package org.apache.spark.io > [error] import io.confluent.kafka.serializers.KafkaAvroDecoder > > Regards, > > Arek > ^ >