Hi,
I try to test spark streaming 2.2.0 version with confluent 3.3.0
I have got lot of error during compilation this is my sbt:
lazy val sparkstreaming = (project in file("."))
.settings(
name := "sparkstreaming",
organization := "org.arek",
version := "0.1-SNAPSHOT",
scalaVersion := "2.11.8",
libraryDependencies ++= Seq(
"org.apache.spark" %% "spark-streaming" % "2.2.0",
"org.apache.spark" %% "spark-streaming-kafka-0-8" % "2.2.0",
"io.confluent" % "kafka-avro-serializer" % "3.3.0"
)
)
import org.apache.spark._
import org.apache.spark.streaming._
import org.apache.spark.streaming.kafka.KafkaUtils
import io.confluent.kafka.serializers.KafkaAvroDecoder
object Transformation extends Serializable {
def main(args: Array[String]) = {
val conf = new
SparkConf().setAppName("StreamingTranformation").setMaster("local[*]")
val streamingContext = new StreamingContext(conf, Seconds(1))
val kafkaParams = Map[String, String]("metadata.broker.list" ->
"local:2181",
"schema.registry.url" -> "http://local:8081";,
"auto.offset.reset" -> "smallest")
val topicSet = Set("GEXPPROD_ROUTE")
val messages = KafkaUtils.createDirectStream[Object, Object,
KafkaAvroDecoder, KafkaAvroDecoder](streamingContext, kafkaParams,
topicSet).map(_._2)
val lines = messages.foreachRDD(rdd => {
rdd.foreach({ avroRecord =>
println(avroRecord)
})
})
}
[warn] Found version conflict(s) in library dependencies; some are
suspected to be binary incompatible:
[warn] * io.netty:netty:3.9.9.Final is selected over {3.6.2.Final,
3.7.0.Final}
[warn] +- org.apache.spark:spark-core_2.11:2.2.0 (depends
on 3.7.0.Final)
[warn] +- org.apache.zookeeper:zookeeper:3.4.8 (depends
on 3.7.0.Final)
[warn] +- org.apache.zookeeper:zookeeper:3.4.6 (depends
on 3.6.2.Final)
[warn] +- org.apache.hadoop:hadoop-hdfs:2.6.5 (depends
on 3.6.2.Final)
[warn] * commons-net:commons-net:2.2 is selected over 3.1
[warn] +- org.apache.spark:spark-core_2.11:2.2.0 (depends
on 3.1)
[warn] +- org.apache.hadoop:hadoop-common:2.6.5 (depends
on 3.1)
[warn] * com.google.guava:guava:11.0.2 is selected over {12.0.1, 16.0.1}
[warn] +- org.apache.hadoop:hadoop-yarn-client:2.6.5 (depends
on 11.0.2)
[warn] +- org.apache.hadoop:hadoop-yarn-api:2.6.5 (depends
on 11.0.2)
[warn] +- org.apache.hadoop:hadoop-yarn-common:2.6.5 (depends
on 11.0.2)
[warn] +- org.apache.hadoop:hadoop-yarn-server-nodemanager:2.6.5
(depends on 11.0.2)
[warn] +- org.apache.hadoop:hadoop-common:2.6.5 (depends
on 11.0.2)
[warn] +- org.apache.hadoop:hadoop-yarn-server-common:2.6.5 (depends
on 11.0.2)
[warn] +- org.apache.hadoop:hadoop-hdfs:2.6.5 (depends
on 11.0.2)
[warn] +- org.apache.curator:curator-framework:2.6.0 (depends
on 16.0.1)
[warn] +- org.apache.curator:curator-client:2.6.0 (depends
on 16.0.1)
[warn] +- org.apache.curator:curator-recipes:2.6.0 (depends
on 16.0.1)
[warn] +- org.htrace:htrace-core:3.0.4 (depends
on 12.0.1)
[warn] Run 'evicted' to see detailed eviction warnings
[info] Compiling 1 Scala source to
/home/adminuser/data-streaming-platform/sparkstreaming/target/scala-2.11/classes
...
[error]
/home/adminuser/data-streaming-platform/sparkstreaming/src/main/scala/com/arek/streaming/spark/Transformation.scala:6:11:
object confluent is not a member of package org.apache.spark.io
[error] import io.confluent.kafka.serializers.KafkaAvroDecoder
[error] ^
[error]
/home/adminuser/data-streaming-platform/sparkstreaming/src/main/scala/com/arek/streaming/spark/Transformation.scala:19:66:
not found: type KafkaAvroDecoder
[error] val messages = KafkaUtils.createDirectStream[Object, Object,
KafkaAvroDecoder, KafkaAvroDecoder](streamingContext, kafkaParams,
topicSet).map(_._2)
[error] ^
[error]
/home/adminuser/data-streaming-platform/sparkstreaming/src/main/scala/com/arek/streaming/spark/Transformation.scala:19:84:
not found: type KafkaAvroDecoder
[error] val messages = KafkaUtils.createDirectStream[Object, Object,
KafkaAvroDecoder, KafkaAvroDecoder](streamingContext, kafkaParams,
topicSet).map(_._2)
[error]
When changing to library "org.apache.spark" %%
"spark-streaming-kafka-0-10" % "2.2.0" :
[warn] Found version conflict(s) in library dependencies; some are
suspected to be binary incompatible:
[warn] * com.101tec:zkclient:0.10 is selected over 0.8
[warn] +- io.confluent:common-utils:3.3.0 (depends
on 0.10)
[warn] +- org.apache.kafka:kafka_2.11:0.10.0.1 (depends
on 0.8)
[warn] * io.netty:netty:3.9.9.Final is selected over {3.6.2.Final,
3.7.0.Final}
[warn] +- org.apache.spark:spark-core_2.11:2.2.0 (depends
on 3.7.0.Final)
[warn] +- org.apache.zookeeper:zookeeper:3.4.8 (depends
on 3.7.0.Final)
[warn] +- org.apache.zookeeper:zookeeper:3.4.6 (depends
on 3.6.2.Final)
[warn] +- org.apache.hadoop:hadoop-hdfs:2.6.5 (depends
on 3.6.2.Final)
[warn] * commons-net:commons-net:2.2 is selected over 3.1
[warn] +- org.apache.spark:spark-core_2.11:2.2.0 (depends
on 3.1)
[warn] +- org.apache.hadoop:hadoop-common:2.6.5 (depends
on 3.1)
[warn] * com.google.guava:guava:11.0.2 is selected over {12.0.1, 16.0.1}
[warn] +- org.apache.hadoop:hadoop-yarn-client:2.6.5 (depends
on 11.0.2)
[warn] +- org.apache.hadoop:hadoop-yarn-api:2.6.5 (depends
on 11.0.2)
[warn] +- org.apache.hadoop:hadoop-yarn-common:2.6.5 (depends
on 11.0.2)
[warn] +- org.apache.hadoop:hadoop-yarn-server-nodemanager:2.6.5
(depends on 11.0.2)
[warn] +- org.apache.hadoop:hadoop-common:2.6.5 (depends
on 11.0.2)
[warn] +- org.apache.hadoop:hadoop-yarn-server-common:2.6.5 (depends
on 11.0.2)
[warn] +- org.apache.hadoop:hadoop-hdfs:2.6.5 (depends
on 11.0.2)
[warn] +- org.apache.curator:curator-framework:2.6.0 (depends
on 16.0.1)
[warn] +- org.apache.curator:curator-client:2.6.0 (depends
on 16.0.1)
[warn] +- org.apache.curator:curator-recipes:2.6.0 (depends
on 16.0.1)
[warn] +- org.htrace:htrace-core:3.0.4 (depends
on 12.0.1)
[warn] Run 'evicted' to see detailed eviction warnings
[info] Compiling 1 Scala source to
/home/adminuser/data-streaming-platform/sparkstreaming/target/scala-2.11/classes
...
[error]
/home/adminuser/data-streaming-platform/sparkstreaming/src/main/scala/com/arek/streaming/spark/Transformation.scala:18:11:
object confluent is not a member of package org.apache.spark.io
[error] import io.confluent.kafka.serializers.KafkaAvroDecoder
Regards,
Arek
^
