flink-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Tarandeep Singh <tarand...@gmail.com>
Subject Re: Flink 1.2 and Cassandra Connector
Date Mon, 06 Mar 2017 10:32:14 GMT
Hi Robert & Nico,

I am facing the same problem (java.lang.NoClassDefFoundError:
com/codahale/metrics/Metric)
Can you help me identify shading issue in pom.xml file.

My pom.xml content-
---------------------------------------------


<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
   xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>

   <groupId>rfk-dataplatform</groupId>
   <artifactId>stream-processing</artifactId>
   <version>0.1.0</version>
   <packaging>jar</packaging>

   <name>Stream processing</name>

   <properties>
      <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
      <flink.version>1.2.0</flink.version>
      <slf4j.version>1.7.7</slf4j.version>
      <log4j.version>1.2.17</log4j.version>
   </properties>

   <dependencies>
      <dependency>
         <groupId>org.apache.flink</groupId>
         <artifactId>flink-streaming-java_2.10</artifactId>
         <version>${flink.version}</version>
      </dependency>
      <dependency>
         <groupId>org.apache.flink</groupId>
         <artifactId>flink-clients_2.10</artifactId>
         <version>${flink.version}</version>
      </dependency>
      <dependency>
         <groupId>org.apache.flink</groupId>
         <artifactId>flink-connector-cassandra_2.10</artifactId>
         <version>1.2.0</version>
      </dependency>
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-statebackend-rocksdb_2.10</artifactId>
            <version>1.2.0</version>
        </dependency>

      <dependency>
         <groupId>org.slf4j</groupId>
         <artifactId>slf4j-log4j12</artifactId>
         <version>${slf4j.version}</version>
      </dependency>
      <dependency>
         <groupId>log4j</groupId>
         <artifactId>log4j</artifactId>
         <version>${log4j.version}</version>
      </dependency>

        <dependency>
            <groupId>org.apache.avro</groupId>
            <artifactId>avro</artifactId>
            <version>1.8.1</version>
        </dependency>

        <dependency>
            <groupId>org.testng</groupId>
            <artifactId>testng</artifactId>
            <version>6.8</version>
            <scope>test</scope>
        </dependency>

        <!-- flink kafka connector -->
        <dependency>
            <groupId>org.apache.flink</groupId>
            <artifactId>flink-connector-kafka-0.8_2.10</artifactId>
            <version>1.2.0</version>
        </dependency>

        <!-- java influxdb client -->
        <dependency>
            <groupId>org.influxdb</groupId>
            <artifactId>influxdb-java</artifactId>
            <version>2.5</version>
        </dependency>

    </dependencies>

   <profiles>
      <profile>
         <!-- Profile for packaging correct JAR files -->
         <id>build-jar</id>

         <activation>
            <activeByDefault>false</activeByDefault>
         </activation>

         <dependencies>
            <dependency>
               <groupId>org.apache.flink</groupId>
               <artifactId>flink-java</artifactId>
               <version>${flink.version}</version>
               <scope>provided</scope>
            </dependency>
            <dependency>
               <groupId>org.apache.flink</groupId>
               <artifactId>flink-streaming-java_2.10</artifactId>
               <version>${flink.version}</version>
               <scope>provided</scope>
            </dependency>
            <dependency>
               <groupId>org.apache.flink</groupId>
               <artifactId>flink-clients_2.10</artifactId>
               <version>${flink.version}</version>
               <scope>provided</scope>
            </dependency>
            <dependency>
               <groupId>org.slf4j</groupId>
               <artifactId>slf4j-log4j12</artifactId>
               <version>${slf4j.version}</version>
               <scope>provided</scope>
            </dependency>
            <dependency>
               <groupId>log4j</groupId>
               <artifactId>log4j</artifactId>
               <version>${log4j.version}</version>
               <scope>provided</scope>
            </dependency>
         </dependencies>


            <build>
            <plugins>
               <!-- disable the exclusion rules -->
               <plugin>
                  <groupId>org.apache.maven.plugins</groupId>
                  <artifactId>maven-shade-plugin</artifactId>
                  <version>2.4.1</version>
                  <executions>
                     <execution>
                        <phase>package</phase>
                        <goals>
                           <goal>shade</goal>
                        </goals>
                        <configuration>
                           <artifactSet>
                              <excludes combine.self="override"></excludes>
                           </artifactSet>
                        </configuration>
                     </execution>
                  </executions>
               </plugin>
            </plugins>
         </build>
      </profile>
   </profiles>

   <build>
      <plugins>
         <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-shade-plugin</artifactId>
            <version>2.4.1</version>
            <executions>
               <!-- Run shade goal on package phase -->
               <execution>
                  <phase>package</phase>
                  <goals>
                     <goal>shade</goal>
                  </goals>
                  <configuration>
                     <artifactSet>
                        <excludes>
                           <!-- This list contains all dependencies of
flink-dist
                           Everything else will be packaged into the fat-jar
                           -->
                           <exclude>org.apache.flink:flink-annotations</exclude>

<exclude>org.apache.flink:flink-shaded-hadoop2</exclude>

<exclude>org.apache.flink:flink-shaded-curator-recipes</exclude>
                           <exclude>org.apache.flink:flink-core</exclude>
                           <exclude>org.apache.flink:flink-java</exclude>
                           <exclude>org.apache.flink:flink-scala_2.10</exclude>

<exclude>org.apache.flink:flink-runtime_2.10</exclude>

<exclude>org.apache.flink:flink-optimizer_2.10</exclude>

<exclude>org.apache.flink:flink-clients_2.10</exclude>
                           <exclude>org.apache.flink:flink-avro_2.10</exclude>

<exclude>org.apache.flink:flink-examples-batch_2.10</exclude>

<exclude>org.apache.flink:flink-examples-streaming_2.10</exclude>

<exclude>org.apache.flink:flink-streaming-java_2.10</exclude>

<exclude>org.apache.flink:flink-streaming-scala_2.10</exclude>

<exclude>org.apache.flink:flink-scala-shell_2.10</exclude>
                           <exclude>org.apache.flink:flink-python</exclude>

<exclude>org.apache.flink:flink-metrics-core</exclude>
                           <exclude>org.apache.flink:flink-metrics-jmx</exclude>

<exclude>org.apache.flink:flink-statebackend-rocksdb_2.10</exclude>

                           <!-- Also exclude very big transitive
dependencies of Flink

                           WARNING: You have to remove these excludes
if your code relies on other
                           versions of these dependencies.

                           -->

                           <exclude>log4j:log4j</exclude>
                           <exclude>org.scala-lang:scala-library</exclude>
                           <exclude>org.scala-lang:scala-compiler</exclude>
                           <exclude>org.scala-lang:scala-reflect</exclude>
                           <exclude>com.data-artisans:flakka-actor_*</exclude>
                           <exclude>com.data-artisans:flakka-remote_*</exclude>
                           <exclude>com.data-artisans:flakka-slf4j_*</exclude>
                           <exclude>io.netty:netty-all</exclude>
                           <exclude>io.netty:netty</exclude>

<exclude>commons-fileupload:commons-fileupload</exclude>
                           <exclude>org.apache.avro:avro</exclude>

<exclude>commons-collections:commons-collections</exclude>

<exclude>org.codehaus.jackson:jackson-core-asl</exclude>

<exclude>org.codehaus.jackson:jackson-mapper-asl</exclude>

<exclude>com.thoughtworks.paranamer:paranamer</exclude>
                           <exclude>org.xerial.snappy:snappy-java</exclude>

<exclude>org.apache.commons:commons-compress</exclude>
                           <exclude>org.tukaani:xz</exclude>
                           <exclude>com.esotericsoftware.kryo:kryo</exclude>
                           <exclude>com.esotericsoftware.minlog:minlog</exclude>
                           <exclude>org.objenesis:objenesis</exclude>
                           <exclude>com.twitter:chill_*</exclude>
                           <exclude>com.twitter:chill-java</exclude>
                           <exclude>commons-lang:commons-lang</exclude>
                           <exclude>junit:junit</exclude>
                           <exclude>org.apache.commons:commons-lang3</exclude>
                           <exclude>org.slf4j:slf4j-api</exclude>
                           <exclude>org.slf4j:slf4j-log4j12</exclude>
                           <exclude>log4j:log4j</exclude>
                           <exclude>org.apache.commons:commons-math</exclude>

<exclude>org.apache.sling:org.apache.sling.commons.json</exclude>
                           <exclude>commons-logging:commons-logging</exclude>
                           <exclude>commons-codec:commons-codec</exclude>

<exclude>com.fasterxml.jackson.core:jackson-core</exclude>

<exclude>com.fasterxml.jackson.core:jackson-databind</exclude>

<exclude>com.fasterxml.jackson.core:jackson-annotations</exclude>
                           <exclude>stax:stax-api</exclude>
                           <exclude>com.typesafe:config</exclude>

<exclude>org.uncommons.maths:uncommons-maths</exclude>
                           <exclude>com.github.scopt:scopt_*</exclude>
                           <exclude>commons-io:commons-io</exclude>
                           <exclude>commons-cli:commons-cli</exclude>
                        </excludes>
                     </artifactSet>
                     <filters>
                        <filter>
                           <artifact>org.apache.flink:*</artifact>
                           <excludes>
                              <!-- exclude shaded google but include
shaded curator -->
                              <exclude>org/apache/flink/shaded/com/**</exclude>
                              <exclude>web-docs/**</exclude>
                           </excludes>
                        </filter>
                        <filter>
                           <artifact>*:*</artifact>
                           <excludes>
                              <exclude>META-INF/*.SF</exclude>
                              <exclude>META-INF/*.DSA</exclude>
                              <exclude>META-INF/*.RSA</exclude>
                           </excludes>
                        </filter>
                     </filters>

<createDependencyReducedPom>false</createDependencyReducedPom>
                  </configuration>
               </execution>
            </executions>
         </plugin>

         <plugin>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-compiler-plugin</artifactId>
            <version>3.1</version>
            <configuration>
               <source>1.8</source>
               <target>1.8</target>
            </configuration>
         </plugin>

            <plugin>
                <groupId>org.apache.avro</groupId>
                <artifactId>avro-maven-plugin</artifactId>
                <version>1.8.1</version>
                <executions>
                    <execution>
                        <phase>generate-sources</phase>
                        <goals>
                            <goal>schema</goal>
                        </goals>
                        <configuration>
                            <stringType>String</stringType>
                            <fieldVisibility>PRIVATE</fieldVisibility>

<sourceDirectory>${project.basedir}/src/main/schema/</sourceDirectory>

<outputDirectory>${project.basedir}/src/main/java/</outputDirectory>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                </configuration>
            </plugin>
      </plugins>

   </build>
</project>


---------------------------------------------


On Sun, Feb 12, 2017 at 1:56 AM, Robert Metzger <rmetzger@apache.org> wrote:

> Hi Nico,
> The cassandra connector should be available on Maven central:
> http://search.maven.org/#artifactdetails%7Corg.apache.flink%7Cflink-
> connector-cassandra_2.10%7C1.2.0%7Cjar
>
> Potentially, the issue you've mentioned is due to some shading issue. Is
> the "com/codahale/metrics/Metric" class in your user code jar?
>
> On Thu, Feb 9, 2017 at 2:56 PM, Nico <nico.franzeck@gmail.com> wrote:
>
>> Hi,
>>
>> I would like to upgrade to the new stable version 1.2 - but i get an
>> ClassNotFound exception when i start the application.
>>
>> Caused by: java.lang.NoClassDefFoundError: com/codahale/metrics/Metric
>>     at com.datastax.driver.core.Cluster$Manager.init(Cluster.java:1367)
>>     at com.datastax.driver.core.Cluster.init(Cluster.java:162)
>>     at com.datastax.driver.core.Cluster.connectAsync(Cluster.java:333)
>>     at com.datastax.driver.core.Cluster.connectAsync(Cluster.java:308)
>>     at com.datastax.driver.core.Cluster.connect(Cluster.java:250)
>>     at org.apache.flink.streaming.connectors.cassandra.CassandraSin
>> kBase.open(CassandraSinkBase.java:67)
>>     at org.apache.flink.streaming.connectors.cassandra.CassandraTup
>> leSink.open(CassandraTupleSink.java:42)
>>     at org.apache.flink.api.common.functions.util.FunctionUtils.ope
>> nFunction(FunctionUtils.java:36)
>>     at org.apache.flink.streaming.api.operators.AbstractUdfStreamOp
>> erator.open(AbstractUdfStreamOperator.java:112)
>>     at org.apache.flink.streaming.runtime.tasks.StreamTask.openAllO
>> perators(StreamTask.java:386)
>>     at org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(
>> StreamTask.java:262)
>>     at org.apache.flink.runtime.taskmanager.Task.run(Task.java:655)
>>     at java.lang.Thread.run(Thread.java:745)
>>
>>
>> So I think the cassandra connector is the reason for it. Moreover, i
>> don't see a version 1.2 in the maven repository for the connector as
>> mentioned in the doc.
>>
>> <dependency>
>>   <groupId>org.apache.flink</groupId>
>>   <artifactId>flink-connector-cassandra_2.10</artifactId>
>>   <version>1.2.0</version>
>> </dependency>
>>
>> Is there a plan to release a new version?
>>
>> Best,
>> Nico
>>
>
>

Mime
View raw message