What I noticied was that, if I remove the dependency on
flink-connector-kafka so it is clearly to do something with that
dependency.
On Mon, Mar 14, 2016 at 3:46 PM, Balaji Rajagopalan <
balaji.rajagopa...@olacabs.com> wrote:
> Robert,
> I have moved on to latest version of flink of 1.0.0 hoping that will
> solve my problem with kafka connector . Here is my pom.xml but now I cannot
> get the code compiled.
>
> [ERROR] Failed to execute goal
> net.alchim31.maven:scala-maven-plugin:3.2.1:compile (scala-compile-first)
> on project flink-streaming-demo: Execution scala-compile-first of goal
> net.alchim31.maven:scala-maven-plugin:3.2.1:compile failed: For artifact
> {null:null:null:jar}: The groupId cannot be empty. -> [Help 1]
>
> I read about the above errors in most cases people where able to overcome
> is by deleting the .m2 directory, and that did not fix the issue for me.
>
> What I noticied was that, if I remove the dependency on
>
> Here is my pom.xml
>
> <?xml version="1.0" encoding="UTF-8"?>
> <!--
> Copyright 2015 data Artisans GmbH
>
> Licensed under the Apache License, Version 2.0 (the "License");
> you may not use this file except in compliance with the License.
> You may obtain a copy of the License at
>
> http://www.apache.org/licenses/LICENSE-2.0
>
> Unless required by applicable law or agreed to in writing, software
> distributed under the License is distributed on an "AS IS" BASIS,
> WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> See the License for the specific language governing permissions and
> limitations under the License.
> -->
> <project xmlns="http://maven.apache.org/POM/4.0.0";
> xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
> xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
> http://maven.apache.org/xsd/maven-4.0.0.xsd";>
> <modelVersion>4.0.0</modelVersion>
>
> <groupId>com.dataArtisans</groupId>
> <artifactId>flink-streaming-demo</artifactId>
> <version>0.1</version>
> <packaging>jar</packaging>
>
> <name>Flink Streaming Demo</name>
> <url>http://www.data-artisans.com</url>
>
> <properties>
> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
> <slf4j.version>1.7.12</slf4j.version>
> <flink.version>1.0.0</flink.version>
> <scala.version>2.10</scala.version>
> </properties>
>
> <dependencies>
>
>
>
> <dependency>
> <groupId>org.apache.flink</groupId>
> <artifactId>flink-streaming-scala_${scala.version}</artifactId>
> <version>${flink.version}</version>
> </dependency>
>
> <dependency>
> <groupId>org.apache.flink</groupId>
> <artifactId>flink-runtime-web_${scala.version}</artifactId>
> <version>${flink.version}</version>
> </dependency>
>
> <dependency>
> <groupId>org.elasticsearch</groupId>
> <artifactId>elasticsearch</artifactId>
> <version>1.7.3</version>
> <scope>compile</scope>
> </dependency>
>
> <dependency>
> <groupId>joda-time</groupId>
> <artifactId>joda-time</artifactId>
> <version>2.7</version>
> </dependency>
>
> <dependency>
> <groupId>org.apache.kafka</groupId>
> <artifactId>kafka_${scala.version}</artifactId>
> <version>0.8.2.0</version>
> </dependency>
>
> <dependency>
> <groupId>org.apache.flink</groupId>
> <artifactId>flink-connector-kafka_${scala.version}</artifactId>
> <version>${flink.version}</version>
> </dependency>
>
> <dependency>
> <groupId>org.json4s</groupId>
> <artifactId>json4s-native_${scala.version}</artifactId>
> <version>3.3.0</version>
> </dependency>
>
>
> </dependencies>
>
> <build>
> <plugins>
>
> <!-- Scala Compiler -->
> <plugin>
> <groupId>net.alchim31.maven</groupId>
> <artifactId>scala-maven-plugin</artifactId>
> <version>3.2.1</version>
> <executions>
> <!-- Run scala compiler in the process-resources phase, so
> that dependencies on
> scala classes can be resolved later in the (Java) compile
> phase -->
> <execution>
> <id>scala-compile-first</id>
> <phase>process-resources</phase>
> <goals>
> <goal>compile</goal>
> </goals>
> </execution>
>
> <!-- Run scala compiler in the process-test-resources phase,
> so that dependencies on
> scala classes can be resolved later in the (Java)
> test-compile phase -->
> <execution>
> <id>scala-test-compile</id>
> <phase>process-test-resources</phase>
> <goals>
> <goal>testCompile</goal>
> </goals>
> </execution>
> </executions>
> <configuration>
> <jvmArgs>
> <jvmArg>-Xms128m</jvmArg>
> <jvmArg>-Xmx512m</jvmArg>
> </jvmArgs>
> </configuration>
> </plugin>
>
> <plugin>
> <groupId>org.apache.maven.plugins</groupId>
> <artifactId>maven-dependency-plugin</artifactId>
> <version>2.9</version>
> <executions>
> <execution>
> <id>unpack</id>
> <!-- executed just before the package phase -->
> <phase>prepare-package</phase>
> <goals>
> <goal>unpack</goal>
> </goals>
> <configuration>
> <artifactItems>
> <!-- For Flink connector classes -->
> <artifactItem>
> <groupId>org.apache.flink</groupId>
>
> <artifactId>flink-connector-kafka_${scala.version}</artifactId>
> <version>1.0.0</version>
> <type>jar</type>
> <overWrite>false</overWrite>
>
> <outputDirectory>${project.build.directory}/classes</outputDirectory>
> <includes>org/apache/flink/**</includes>
> </artifactItem>
> <!-- For Kafka API classes -->
> <artifactItem>
> <groupId>org.apache.kafka</groupId>
> <artifactId>kafka_${scala.version}</artifactId>
> <version>0.8.2.0</version>
> <type>jar</type>
> <overWrite>false</overWrite>
>
> <outputDirectory>${project.build.directory}/classes</outputDirectory>
> <includes>kafka/**</includes>
> </artifactItem>
> </artifactItems>
> </configuration>
> </execution>
> </executions>
> </plugin>
>
> <!--plugin>
> <groupId>org.apache.maven.plugins</groupId>
> <artifactId>maven-jar-plugin</artifactId>
> <executions>
>
> <execution>
> <id>MBoxParser</id>
> <phase>package</phase>
> <goals>
> <goal>jar</goal>
> </goals>
>
> <configuration>
> <classifier>MBoxParser</classifier>
>
> <archive>
> <manifestEntries>
>
> <main-class>com.dataartisans.flinkTraining.dataSetPreparation.MBoxParser</main-class>
> </manifestEntries>
> </archive>
>
> <includes>
> <include>**/MBoxParser.class</include>
> <include>**/MBoxParser$*.class</include>
> </includes>
> </configuration>
> </execution>
>
> </executions>
> </plugin-->
>
> <plugin>
> <groupId>org.apache.maven.plugins</groupId>
> <artifactId>maven-compiler-plugin</artifactId>
> <version>3.1</version>
> <configuration>
> <source>1.8</source> <!-- If you want to use Java 8, change
> this to "1.8" -->
> <target>1.8</target> <!-- If you want to use Java 8, change
> this to "1.8" -->
> </configuration>
> </plugin>
>
> <plugin>
> <groupId>org.apache.rat</groupId>
> <artifactId>apache-rat-plugin</artifactId>
> <version>0.10</version><!--$NO-MVN-MAN-VER$-->
> <inherited>false</inherited>
> <executions>
> <execution>
> <phase>verify</phase>
> <goals>
> <goal>check</goal>
> </goals>
> </execution>
> </executions>
> <configuration>
> <excludeSubProjects>false</excludeSubProjects>
> <numUnapprovedLicenses>0</numUnapprovedLicenses>
> <licenses>
> <!-- Enforce this license:
> Copyright 2015 data Artisans GmbH
>
> Licensed under the Apache License, Version 2.0 (the
> "License");
> you may not use this file except in compliance with the
> License.
> You may obtain a copy of the License at
>
> http://www.apache.org/licenses/LICENSE-2.0
>
> Unless required by applicable law or agreed to in
> writing, software
> distributed under the License is distributed on an "AS
> IS" BASIS,
> WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
> express or implied.
> See the License for the specific language governing
> permissions and
> limitations under the License.
> -->
> <license
> implementation="org.apache.rat.analysis.license.SimplePatternBasedLicense">
> <licenseFamilyCategory>AL2 </licenseFamilyCategory>
> <licenseFamilyName>Apache License 2.0</licenseFamilyName>
> <notes />
> <patterns>
> <pattern>Copyright 2015 data Artisans GmbH</pattern>
> <pattern>Licensed under the Apache License, Version
> 2.0 (the "License");</pattern>
> </patterns>
> </license>
> </licenses>
> <licenseFamilies>
> <licenseFamily
> implementation="org.apache.rat.license.SimpleLicenseFamily">
> <familyName>Apache License 2.0</familyName>
> </licenseFamily>
> </licenseFamilies>
> <excludes>
> <!-- Additional files like .gitignore etc.-->
> <exclude>**/.*</exclude>
> <exclude>**/*.prefs</exclude>
> <exclude>**/*.properties</exclude>
> <exclude>**/*.log</exclude>
> <exclude>*.txt/**</exclude>
> <!-- Administrative files in the main trunk. -->
> <exclude>**/README.md</exclude>
> <exclude>CHANGELOG</exclude>
> <!-- Build files -->
> <exclude>**/*.iml</exclude>
> <!-- Generated content -->
> <exclude>**/target/**</exclude>
> <exclude>**/build/**</exclude>
> </excludes>
> </configuration>
> </plugin>
>
> <plugin>
> <groupId>org.apache.maven.plugins</groupId>
> <artifactId>maven-checkstyle-plugin</artifactId>
> <version>2.12.1</version>
> <executions>
> <execution>
> <id>validate</id>
> <phase>validate</phase>
> <goals>
> <goal>check</goal>
> </goals>
> </execution>
> </executions>
> <configuration>
> <configLocation>/tools/maven/checkstyle.xml</configLocation>
> <logViolationsToConsole>true</logViolationsToConsole>
> </configuration>
> </plugin>
>
> </plugins>
>
> </build>
> </project>
>
>
> On Mon, Mar 14, 2016 at 3:15 PM, Robert Metzger <rmetz...@apache.org>
> wrote:
>
>> Can you send me the full build file to further investigate the issue?
>>
>> On Fri, Mar 11, 2016 at 4:56 PM, Balaji Rajagopalan <
>> balaji.rajagopa...@olacabs.com> wrote:
>>
>>> Robert,
>>> That did not fix it ( using flink and connector same version) . Tried
>>> with scala version 2.11, so will try to see scala 2.10 makes any
>>> difference.
>>>
>>> balaji
>>>
>>> On Fri, Mar 11, 2016 at 8:06 PM, Robert Metzger <rmetz...@apache.org>
>>> wrote:
>>>
>>>> Hi,
>>>>
>>>> you have to use the same version for all dependencies from the
>>>> "org.apache.flink" group.
>>>>
>>>> You said these are the versions you are using:
>>>>
>>>> flink.version = 0.10.2
>>>> kafka.verison = 0.8.2
>>>> flink.kafka.connection.verion=0.9.1
>>>>
>>>> For the connector, you also need to use 0.10.2.
>>>>
>>>>
>>>>
>>>> On Fri, Mar 11, 2016 at 9:56 AM, Balaji Rajagopalan <
>>>> balaji.rajagopa...@olacabs.com> wrote:
>>>>
>>>>> I am tyring to use the flink kafka connector, for this I have
>>>>> specified the kafka connector dependency and created a fat jar since
>>>>> default flink installation does not contain kafka connector jars. I have
>>>>> made sure that flink-streaming-demo-0.1.jar has the
>>>>> kafka.javaapi.consumer.SimpleConsumer.class but still I see the class not
>>>>> found exception.
>>>>>
>>>>> The code for kafka connector in flink.
>>>>>
>>>>> val env = StreamExecutionEnvironment.getExecutionEnvironment
>>>>> val prop:Properties = new Properties()
>>>>> prop.setProperty("zookeeper.connect","somezookeer:2181")
>>>>> prop.setProperty("group.id","some-group")
>>>>> prop.setProperty("bootstrap.servers","somebroker:9092")
>>>>>
>>>>> val stream = env
>>>>> .addSource(new FlinkKafkaConsumer082[String]("location", new
>>>>> SimpleStringSchema, prop))
>>>>>
>>>>> jar tvf flink-streaming-demo-0.1.jar | grep
>>>>> kafka.javaapi.consumer.SimpleConsumer
>>>>>
>>>>> 5111 Fri Mar 11 14:18:36 UTC 2016
>>>>> *kafka/javaapi/consumer/SimpleConsumer*.class
>>>>>
>>>>> flink.version = 0.10.2
>>>>> kafka.verison = 0.8.2
>>>>> flink.kafka.connection.verion=0.9.1
>>>>>
>>>>> The command that I use to run the flink program in yarn cluster is
>>>>> below,
>>>>>
>>>>> HADOOP_CONF_DIR=/etc/hadoop/conf /usr/share/flink/bin/flink run -c
>>>>> com.dataartisans.flink_demo.examples.DriverEventConsumer -m yarn-cluster
>>>>> -yn 2 /home/balajirajagopalan/flink-streaming-demo-0.1.jar
>>>>>
>>>>> java.lang.NoClassDefFoundError: kafka/javaapi/consumer/SimpleConsumer
>>>>>
>>>>> at
>>>>> org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer.getPartitionsForTopic(FlinkKafkaConsumer.java:691)
>>>>>
>>>>> at
>>>>> org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer.<init>(FlinkKafkaConsumer.java:281)
>>>>>
>>>>> at
>>>>> org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer082.<init>(FlinkKafkaConsumer082.java:49)
>>>>>
>>>>> at
>>>>> com.dataartisans.flink_demo.examples.DriverEventConsumer$.main(DriverEventConsumer.scala:53)
>>>>>
>>>>> at
>>>>> com.dataartisans.flink_demo.examples.DriverEventConsumer.main(DriverEventConsumer.scala)
>>>>>
>>>>> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>>>>>
>>>>> at
>>>>> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>>>>>
>>>>> at
>>>>> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>>>>
>>>>> at java.lang.reflect.Method.invoke(Method.java:497)
>>>>>
>>>>> at
>>>>> org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:497)
>>>>>
>>>>> at
>>>>> org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:395)
>>>>>
>>>>> at org.apache.flink.client.program.Client.runBlocking(Client.java:252)
>>>>>
>>>>> at
>>>>> org.apache.flink.client.CliFrontend.executeProgramBlocking(CliFrontend.java:676)
>>>>>
>>>>> at org.apache.flink.client.CliFrontend.run(CliFrontend.java:326)
>>>>>
>>>>> at
>>>>> org.apache.flink.client.CliFrontend.parseParameters(CliFrontend.java:978)
>>>>>
>>>>> at org.apache.flink.client.CliFrontend.main(CliFrontend.java:1028)
>>>>>
>>>>> Caused by: java.lang.ClassNotFoundException:
>>>>> kafka.javaapi.consumer.SimpleConsumer
>>>>>
>>>>> at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
>>>>>
>>>>> at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
>>>>>
>>>>> at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
>>>>>
>>>>> at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
>>>>>
>>>>> ... 16 more
>>>>>
>>>>>
>>>>> Any help appreciated.
>>>>>
>>>>>
>>>>> balaji
>>>>>
>>>>
>>>>
>>>
>>
>
