flink-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Robert Metzger <rmetz...@apache.org>
Subject Re: kafka.javaapi.consumer.SimpleConsumer class not found
Date Mon, 14 Mar 2016 10:44:01 GMT
Hi,

flink-connector-kafka_ doesn't exist for 1.0.0. You have to use either
flink-connector-kafka-0.8_ or flink-connector-kafka-0.9_


On Mon, Mar 14, 2016 at 11:17 AM, Balaji Rajagopalan <
balaji.rajagopalan@olacabs.com> wrote:

> What I noticied was that, if I remove the dependency on
> flink-connector-kafka so it is clearly to do something with that
> dependency.
>
>
> On Mon, Mar 14, 2016 at 3:46 PM, Balaji Rajagopalan <
> balaji.rajagopalan@olacabs.com> wrote:
>
>> Robert,
>>    I have  moved on to latest version of flink of 1.0.0 hoping that will
>> solve my problem with kafka connector . Here is my pom.xml but now I cannot
>> get the code compiled.
>>
>> [ERROR] Failed to execute goal
>> net.alchim31.maven:scala-maven-plugin:3.2.1:compile (scala-compile-first)
>> on project flink-streaming-demo: Execution scala-compile-first of goal
>> net.alchim31.maven:scala-maven-plugin:3.2.1:compile failed: For artifact
>> {null:null:null:jar}: The groupId cannot be empty. -> [Help 1]
>>
>> I read about the above errors in most cases people where able to overcome
>> is by deleting the .m2 directory, and that did not fix the issue for me.
>>
>> What I noticied was that, if I remove the dependency on
>>
>> Here is my pom.xml
>>
>> <?xml version="1.0" encoding="UTF-8"?>
>> <!--
>>  Copyright 2015 data Artisans GmbH
>>
>>  Licensed under the Apache License, Version 2.0 (the "License");
>>  you may not use this file except in compliance with the License.
>>  You may obtain a copy of the License at
>>
>>   http://www.apache.org/licenses/LICENSE-2.0
>>
>>  Unless required by applicable law or agreed to in writing, software
>>  distributed under the License is distributed on an "AS IS" BASIS,
>>  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
>>  See the License for the specific language governing permissions and
>>  limitations under the License.
>> -->
>> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
>>    <modelVersion>4.0.0</modelVersion>
>>
>>    <groupId>com.dataArtisans</groupId>
>>    <artifactId>flink-streaming-demo</artifactId>
>>    <version>0.1</version>
>>    <packaging>jar</packaging>
>>
>>    <name>Flink Streaming Demo</name>
>>    <url>http://www.data-artisans.com</url>
>>
>>    <properties>
>>       <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
>>       <slf4j.version>1.7.12</slf4j.version>
>>       <flink.version>1.0.0</flink.version>
>>       <scala.version>2.10</scala.version>
>>    </properties>
>>
>>    <dependencies>
>>
>>
>>
>>       <dependency>
>>          <groupId>org.apache.flink</groupId>
>>          <artifactId>flink-streaming-scala_${scala.version}</artifactId>
>>          <version>${flink.version}</version>
>>       </dependency>
>>
>>       <dependency>
>>          <groupId>org.apache.flink</groupId>
>>          <artifactId>flink-runtime-web_${scala.version}</artifactId>
>>          <version>${flink.version}</version>
>>       </dependency>
>>
>>       <dependency>
>>          <groupId>org.elasticsearch</groupId>
>>          <artifactId>elasticsearch</artifactId>
>>          <version>1.7.3</version>
>>          <scope>compile</scope>
>>       </dependency>
>>
>>       <dependency>
>>          <groupId>joda-time</groupId>
>>          <artifactId>joda-time</artifactId>
>>          <version>2.7</version>
>>       </dependency>
>>
>>       <dependency>
>>          <groupId>org.apache.kafka</groupId>
>>          <artifactId>kafka_${scala.version}</artifactId>
>>          <version>0.8.2.0</version>
>>       </dependency>
>>
>>         <dependency>
>>           <groupId>org.apache.flink</groupId>
>>           <artifactId>flink-connector-kafka_${scala.version}</artifactId>
>>           <version>${flink.version}</version>
>>       </dependency>
>>
>>       <dependency>
>>         <groupId>org.json4s</groupId>
>>         <artifactId>json4s-native_${scala.version}</artifactId>
>>         <version>3.3.0</version>
>>       </dependency>
>>
>>
>>    </dependencies>
>>
>>    <build>
>>       <plugins>
>>
>>          <!-- Scala Compiler -->
>>          <plugin>
>>             <groupId>net.alchim31.maven</groupId>
>>             <artifactId>scala-maven-plugin</artifactId>
>>             <version>3.2.1</version>
>>             <executions>
>>                <!-- Run scala compiler in the process-resources phase, so that
dependencies on
>>                   scala classes can be resolved later in the (Java) compile phase
-->
>>                <execution>
>>                   <id>scala-compile-first</id>
>>                   <phase>process-resources</phase>
>>                   <goals>
>>                      <goal>compile</goal>
>>                   </goals>
>>                </execution>
>>
>>                <!-- Run scala compiler in the process-test-resources phase, so
that dependencies on
>>                    scala classes can be resolved later in the (Java) test-compile
phase -->
>>                <execution>
>>                   <id>scala-test-compile</id>
>>                   <phase>process-test-resources</phase>
>>                   <goals>
>>                      <goal>testCompile</goal>
>>                   </goals>
>>                </execution>
>>             </executions>
>>             <configuration>
>>                <jvmArgs>
>>                   <jvmArg>-Xms128m</jvmArg>
>>                   <jvmArg>-Xmx512m</jvmArg>
>>                </jvmArgs>
>>             </configuration>
>>          </plugin>
>>
>>          <plugin>
>>             <groupId>org.apache.maven.plugins</groupId>
>>             <artifactId>maven-dependency-plugin</artifactId>
>>             <version>2.9</version>
>>             <executions>
>>                <execution>
>>                   <id>unpack</id>
>>                   <!-- executed just before the package phase -->
>>                   <phase>prepare-package</phase>
>>                   <goals>
>>                      <goal>unpack</goal>
>>                   </goals>
>>                   <configuration>
>>                      <artifactItems>
>>                         <!-- For Flink connector classes -->
>>                         <artifactItem>
>>                            <groupId>org.apache.flink</groupId>
>>                            <artifactId>flink-connector-kafka_${scala.version}</artifactId>
>>                            <version>1.0.0</version>
>>                            <type>jar</type>
>>                            <overWrite>false</overWrite>
>>                            <outputDirectory>${project.build.directory}/classes</outputDirectory>
>>                            <includes>org/apache/flink/**</includes>
>>                         </artifactItem>
>>                         <!-- For Kafka API classes -->
>>                         <artifactItem>
>>                            <groupId>org.apache.kafka</groupId>
>>                            <artifactId>kafka_${scala.version}</artifactId>
>>                            <version>0.8.2.0</version>
>>                            <type>jar</type>
>>                            <overWrite>false</overWrite>
>>                            <outputDirectory>${project.build.directory}/classes</outputDirectory>
>>                            <includes>kafka/**</includes>
>>                         </artifactItem>
>>                      </artifactItems>
>>                   </configuration>
>>                </execution>
>>             </executions>
>>          </plugin>
>>
>>          <!--plugin>
>>             <groupId>org.apache.maven.plugins</groupId>
>>             <artifactId>maven-jar-plugin</artifactId>
>>             <executions>
>>
>>                <execution>
>>                   <id>MBoxParser</id>
>>                   <phase>package</phase>
>>                   <goals>
>>                      <goal>jar</goal>
>>                   </goals>
>>
>>                   <configuration>
>>                      <classifier>MBoxParser</classifier>
>>
>>                      <archive>
>>                         <manifestEntries>
>>                            <main-class>com.dataartisans.flinkTraining.dataSetPreparation.MBoxParser</main-class>
>>                         </manifestEntries>
>>                      </archive>
>>
>>                      <includes>
>>                         <include>**/MBoxParser.class</include>
>>                         <include>**/MBoxParser$*.class</include>
>>                      </includes>
>>                   </configuration>
>>                </execution>
>>
>>             </executions>
>>          </plugin-->
>>
>>          <plugin>
>>             <groupId>org.apache.maven.plugins</groupId>
>>             <artifactId>maven-compiler-plugin</artifactId>
>>             <version>3.1</version>
>>             <configuration>
>>                <source>1.8</source> <!-- If you want to use Java 8,
change this to "1.8" -->
>>                <target>1.8</target> <!-- If you want to use Java 8,
change this to "1.8" -->
>>             </configuration>
>>          </plugin>
>>
>>          <plugin>
>>             <groupId>org.apache.rat</groupId>
>>             <artifactId>apache-rat-plugin</artifactId>
>>             <version>0.10</version><!--$NO-MVN-MAN-VER$-->
>>             <inherited>false</inherited>
>>             <executions>
>>                <execution>
>>                   <phase>verify</phase>
>>                   <goals>
>>                      <goal>check</goal>
>>                   </goals>
>>                </execution>
>>             </executions>
>>             <configuration>
>>                <excludeSubProjects>false</excludeSubProjects>
>>                <numUnapprovedLicenses>0</numUnapprovedLicenses>
>>                <licenses>
>>                   <!-- Enforce this license:
>>                      Copyright 2015 data Artisans GmbH
>>
>>                      Licensed under the Apache License, Version 2.0 (the "License");
>>                      you may not use this file except in compliance with the License.
>>                      You may obtain a copy of the License at
>>
>>                        http://www.apache.org/licenses/LICENSE-2.0
>>
>>                      Unless required by applicable law or agreed to in writing, software
>>                      distributed under the License is distributed on an "AS IS" BASIS,
>>                      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
or implied.
>>                      See the License for the specific language governing permissions
and
>>                      limitations under the License.
>>                   -->
>>                   <license implementation="org.apache.rat.analysis.license.SimplePatternBasedLicense">
>>                      <licenseFamilyCategory>AL2 </licenseFamilyCategory>
>>                      <licenseFamilyName>Apache License 2.0</licenseFamilyName>
>>                      <notes />
>>                      <patterns>
>>                         <pattern>Copyright 2015 data Artisans GmbH</pattern>
>>                         <pattern>Licensed under the Apache License, Version
2.0 (the "License");</pattern>
>>                      </patterns>
>>                   </license>
>>                </licenses>
>>                <licenseFamilies>
>>                   <licenseFamily implementation="org.apache.rat.license.SimpleLicenseFamily">
>>                      <familyName>Apache License 2.0</familyName>
>>                   </licenseFamily>
>>                </licenseFamilies>
>>                <excludes>
>>                   <!-- Additional files like .gitignore etc.-->
>>                   <exclude>**/.*</exclude>
>>                   <exclude>**/*.prefs</exclude>
>>                   <exclude>**/*.properties</exclude>
>>                   <exclude>**/*.log</exclude>
>>                   <exclude>*.txt/**</exclude>
>>                   <!-- Administrative files in the main trunk. -->
>>                   <exclude>**/README.md</exclude>
>>                   <exclude>CHANGELOG</exclude>
>>                   <!-- Build files -->
>>                   <exclude>**/*.iml</exclude>
>>                   <!-- Generated content -->
>>                   <exclude>**/target/**</exclude>
>>                   <exclude>**/build/**</exclude>
>>                </excludes>
>>             </configuration>
>>          </plugin>
>>
>>          <plugin>
>>             <groupId>org.apache.maven.plugins</groupId>
>>             <artifactId>maven-checkstyle-plugin</artifactId>
>>             <version>2.12.1</version>
>>             <executions>
>>                <execution>
>>                   <id>validate</id>
>>                   <phase>validate</phase>
>>                   <goals>
>>                      <goal>check</goal>
>>                   </goals>
>>                </execution>
>>             </executions>
>>             <configuration>
>>                <configLocation>/tools/maven/checkstyle.xml</configLocation>
>>                <logViolationsToConsole>true</logViolationsToConsole>
>>             </configuration>
>>          </plugin>
>>
>>       </plugins>
>>
>>    </build>
>> </project>
>>
>>
>> On Mon, Mar 14, 2016 at 3:15 PM, Robert Metzger <rmetzger@apache.org>
>> wrote:
>>
>>> Can you send me the full build file to further investigate the issue?
>>>
>>> On Fri, Mar 11, 2016 at 4:56 PM, Balaji Rajagopalan <
>>> balaji.rajagopalan@olacabs.com> wrote:
>>>
>>>> Robert,
>>>>   That did not fix it ( using flink and connector same version) . Tried
>>>> with scala version 2.11, so will try to see scala 2.10 makes any
>>>> difference.
>>>>
>>>> balaji
>>>>
>>>> On Fri, Mar 11, 2016 at 8:06 PM, Robert Metzger <rmetzger@apache.org>
>>>> wrote:
>>>>
>>>>> Hi,
>>>>>
>>>>> you have to use the same version for all dependencies from the
>>>>> "org.apache.flink" group.
>>>>>
>>>>> You said these are the versions you are using:
>>>>>
>>>>> flink.version = 0.10.2
>>>>> kafka.verison = 0.8.2
>>>>> flink.kafka.connection.verion=0.9.1
>>>>>
>>>>> For the connector, you also need to use 0.10.2.
>>>>>
>>>>>
>>>>>
>>>>> On Fri, Mar 11, 2016 at 9:56 AM, Balaji Rajagopalan <
>>>>> balaji.rajagopalan@olacabs.com> wrote:
>>>>>
>>>>>> I am tyring to use the flink kafka connector, for this I have
>>>>>> specified the kafka connector dependency and created a fat jar since
>>>>>> default flink installation does not contain kafka connector jars.
I have
>>>>>> made sure that flink-streaming-demo-0.1.jar has the
>>>>>> kafka.javaapi.consumer.SimpleConsumer.class but still I see the class
not
>>>>>> found exception.
>>>>>>
>>>>>> The code for kafka connector in flink.
>>>>>>
>>>>>> val env = StreamExecutionEnvironment.getExecutionEnvironment
>>>>>> val prop:Properties = new Properties()
>>>>>> prop.setProperty("zookeeper.connect","somezookeer:2181")
>>>>>> prop.setProperty("group.id","some-group")
>>>>>> prop.setProperty("bootstrap.servers","somebroker:9092")
>>>>>>
>>>>>> val stream = env
>>>>>>   .addSource(new FlinkKafkaConsumer082[String]("location", new SimpleStringSchema,
prop))
>>>>>>
>>>>>> jar tvf flink-streaming-demo-0.1.jar | grep
>>>>>> kafka.javaapi.consumer.SimpleConsumer
>>>>>>
>>>>>>   5111 Fri Mar 11 14:18:36 UTC 2016
>>>>>> *kafka/javaapi/consumer/SimpleConsumer*.class
>>>>>>
>>>>>> flink.version = 0.10.2
>>>>>> kafka.verison = 0.8.2
>>>>>> flink.kafka.connection.verion=0.9.1
>>>>>>
>>>>>> The command that I use to run the flink program in yarn cluster is
>>>>>> below,
>>>>>>
>>>>>> HADOOP_CONF_DIR=/etc/hadoop/conf /usr/share/flink/bin/flink run -c
>>>>>> com.dataartisans.flink_demo.examples.DriverEventConsumer  -m yarn-cluster
>>>>>> -yn 2 /home/balajirajagopalan/flink-streaming-demo-0.1.jar
>>>>>>
>>>>>> java.lang.NoClassDefFoundError: kafka/javaapi/consumer/SimpleConsumer
>>>>>>
>>>>>> at
>>>>>> org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer.getPartitionsForTopic(FlinkKafkaConsumer.java:691)
>>>>>>
>>>>>> at
>>>>>> org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer.<init>(FlinkKafkaConsumer.java:281)
>>>>>>
>>>>>> at
>>>>>> org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer082.<init>(FlinkKafkaConsumer082.java:49)
>>>>>>
>>>>>> at
>>>>>> com.dataartisans.flink_demo.examples.DriverEventConsumer$.main(DriverEventConsumer.scala:53)
>>>>>>
>>>>>> at
>>>>>> com.dataartisans.flink_demo.examples.DriverEventConsumer.main(DriverEventConsumer.scala)
>>>>>>
>>>>>> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>>>>>>
>>>>>> at
>>>>>> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>>>>>>
>>>>>> at
>>>>>> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>>>>>
>>>>>> at java.lang.reflect.Method.invoke(Method.java:497)
>>>>>>
>>>>>> at
>>>>>> org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:497)
>>>>>>
>>>>>> at
>>>>>> org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:395)
>>>>>>
>>>>>> at org.apache.flink.client.program.Client.runBlocking(Client.java:252)
>>>>>>
>>>>>> at
>>>>>> org.apache.flink.client.CliFrontend.executeProgramBlocking(CliFrontend.java:676)
>>>>>>
>>>>>> at org.apache.flink.client.CliFrontend.run(CliFrontend.java:326)
>>>>>>
>>>>>> at
>>>>>> org.apache.flink.client.CliFrontend.parseParameters(CliFrontend.java:978)
>>>>>>
>>>>>> at org.apache.flink.client.CliFrontend.main(CliFrontend.java:1028)
>>>>>>
>>>>>> Caused by: java.lang.ClassNotFoundException:
>>>>>> kafka.javaapi.consumer.SimpleConsumer
>>>>>>
>>>>>> at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
>>>>>>
>>>>>> at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
>>>>>>
>>>>>> at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
>>>>>>
>>>>>> at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
>>>>>>
>>>>>> ... 16 more
>>>>>>
>>>>>>
>>>>>> Any help appreciated.
>>>>>>
>>>>>>
>>>>>> balaji
>>>>>>
>>>>>
>>>>>
>>>>
>>>
>>
>

Mime
View raw message