Return-Path: X-Original-To: apmail-flink-user-archive@minotaur.apache.org Delivered-To: apmail-flink-user-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id DF9D218E13 for ; Wed, 20 May 2015 08:54:21 +0000 (UTC) Received: (qmail 68505 invoked by uid 500); 20 May 2015 08:54:21 -0000 Delivered-To: apmail-flink-user-archive@flink.apache.org Received: (qmail 68440 invoked by uid 500); 20 May 2015 08:54:21 -0000 Mailing-List: contact user-help@flink.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: user@flink.apache.org Delivered-To: mailing list user@flink.apache.org Received: (qmail 68431 invoked by uid 99); 20 May 2015 08:54:21 -0000 Received: from mail-relay.apache.org (HELO mail-relay.apache.org) (140.211.11.15) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 20 May 2015 08:54:21 +0000 Received: from mail-la0-f51.google.com (mail-la0-f51.google.com [209.85.215.51]) by mail-relay.apache.org (ASF Mail Server at mail-relay.apache.org) with ESMTPSA id E5B8F1A06E7 for ; Wed, 20 May 2015 08:54:20 +0000 (UTC) Received: by laat2 with SMTP id t2so63135568laa.1 for ; Wed, 20 May 2015 01:54:19 -0700 (PDT) X-Gm-Message-State: ALoCoQl2bmQ/Z+d8mpOODXU1UgSVbCY4w6CYkEia0ZVQlUvk1Rdha3w6Sv5L2EbvtHKttpWRhfGs X-Received: by 10.112.129.132 with SMTP id nw4mr25544347lbb.122.1432112059280; Wed, 20 May 2015 01:54:19 -0700 (PDT) MIME-Version: 1.0 Received: by 10.25.31.15 with HTTP; Wed, 20 May 2015 01:53:58 -0700 (PDT) In-Reply-To: References: From: Maximilian Michels Date: Wed, 20 May 2015 10:53:58 +0200 Message-ID: Subject: Re: ClassReader could not be created To: "user@flink.apache.org" Content-Type: multipart/alternative; boundary=047d7b3a88bc92885505167f925e --047d7b3a88bc92885505167f925e Content-Type: text/plain; charset=UTF-8 Hi Flavio, It would be helpful, if we knew which class could not be found. In the ClosureCleaner, can you change line 42 to include the class name in the error message? Like in this example: private static ClassReader getClassReader(Class cls) { String className = cls.getName().replaceFirst("^.*\\.", "") + ".class"; try { return new ClassReader(cls.getResourceAsStream(className)); } catch (IOException e) { throw new RuntimeException("Could not create ClassReader for class " + cls.getName() + ":" + e); } } Could it be that you're running an old job on the latest snapshot version? This could cause class-related problems... Cheers, Max On Wed, May 20, 2015 at 9:41 AM, Flavio Pompermaier wrote: > Any insight about this..? > > On Tue, May 19, 2015 at 12:49 PM, Flavio Pompermaier > wrote: > >> Hi to all, >> >> I tried to run my job on a brand new Flink cluster (0.9-SNAPSHOT) from >> the web client UI using the shading strategy of the quickstart example but >> I get this exception: >> >> Caused by: java.lang.RuntimeException: Could not create ClassReader: >> java.io.IOException: Class not found >> at >> org.apache.flink.api.java.ClosureCleaner.getClassReader(ClosureCleaner.java:42) >> at >> org.apache.flink.api.java.ClosureCleaner.cleanThis0(ClosureCleaner.java:67) >> at org.apache.flink.api.java.ClosureCleaner.clean(ClosureCleaner.java:54) >> >> It seems that it cannot find some kryo class..how do I fix this? this is >> my shade plugin section of pom.xml: >> >> >> >> org.apache.maven.plugins >> maven-shade-plugin >> 1.4 >> >> >> package >> >> shade >> >> >> >> >> >> org.apache.flink:flink-shaded-* >> org.apache.flink:flink-core >> org.apache.flink:flink-java >> org.apache.flink:flink-scala >> org.apache.flink:flink-runtime >> org.apache.flink:flink-optimizer >> org.apache.flink:flink-clients >> org.apache.flink:flink-spargel >> org.apache.flink:flink-avro >> org.apache.flink:flink-java-examples >> org.apache.flink:flink-scala-examples >> org.apache.flink:flink-streaming-examples >> org.apache.flink:flink-streaming-core >> >> >> org.scala-lang:scala-library >> org.scala-lang:scala-compiler >> org.scala-lang:scala-reflect >> com.amazonaws:aws-java-sdk >> com.typesafe.akka:akka-actor_* >> com.typesafe.akka:akka-remote_* >> com.typesafe.akka:akka-slf4j_* >> io.netty:netty-all >> io.netty:netty >> org.eclipse.jetty:jetty-server >> org.eclipse.jetty:jetty-continuation >> org.eclipse.jetty:jetty-http >> org.eclipse.jetty:jetty-io >> org.eclipse.jetty:jetty-util >> org.eclipse.jetty:jetty-security >> org.eclipse.jetty:jetty-servlet >> commons-fileupload:commons-fileupload >> org.apache.avro:avro >> commons-collections:commons-collections >> org.codehaus.jackson:jackson-core-asl >> org.codehaus.jackson:jackson-mapper-asl >> com.thoughtworks.paranamer:paranamer >> org.xerial.snappy:snappy-java >> org.apache.commons:commons-compress >> org.tukaani:xz >> com.esotericsoftware.kryo:kryo >> com.esotericsoftware.minlog:minlog >> org.objenesis:objenesis >> com.twitter:chill_* >> com.twitter:chill-java >> com.twitter:chill-avro_* >> com.twitter:chill-bijection_* >> com.twitter:bijection-core_* >> com.twitter:bijection-avro_* >> com.twitter:chill-protobuf >> com.google.protobuf:protobuf-java >> com.twitter:chill-thrift >> org.apache.thrift:libthrift >> commons-lang:commons-lang >> junit:junit >> de.javakaffee:kryo-serializers >> joda-time:joda-time >> org.apache.commons:commons-lang3 >> org.slf4j:slf4j-api >> org.slf4j:slf4j-log4j12 >> log4j:log4j >> org.apache.commons:commons-math >> org.apache.sling:org.apache.sling.commons.json >> commons-logging:commons-logging >> org.apache.httpcomponents:httpclient >> org.apache.httpcomponents:httpcore >> commons-codec:commons-codec >> com.fasterxml.jackson.core:jackson-core >> com.fasterxml.jackson.core:jackson-databind >> com.fasterxml.jackson.core:jackson-annotations >> org.codehaus.jettison:jettison >> stax:stax-api >> com.typesafe:config >> org.uncommons.maths:uncommons-maths >> com.github.scopt:scopt_* >> org.mortbay.jetty:servlet-api >> commons-io:commons-io >> commons-cli:commons-cli >> >> >> >> >> org.apache.flink:* >> >> org/apache/flink/shaded/** >> web-docs/** >> >> >> >> false >> XXXX >> >> >> > >> implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> >> >> XXX >> >> >> >> >> >> >> >> > > --047d7b3a88bc92885505167f925e Content-Type: text/html; charset=UTF-8 Content-Transfer-Encoding: quoted-printable
Hi Flavio,

It would be helpful, if we knew whi= ch class could not be found. In the ClosureCleaner, can you change line 42 = to include the class name in the error message? Like in this example:
private static ClassReader getClassReade= r(Class<?> cls) {
String className =3D cls.getName().replaceFir= st("^.*\\.", "") + ".class";
try {
return new ClassReader(cls.getRe= sourceAsStream(className));
} catch (IOException e) {
throw new RuntimeException("Could not create ClassReader= for class " + cls.getName() + ":" + e);
}
}=
Could it be that you're running an old job on the late= st snapshot version? This could cause class-related problems...

Chee= rs,
Max

On Wed, May 20, 2015 at 9:41 AM, Flavio Pompermaier = <pompermaier@o= kkam.it> wrote:
Any insight about this..?

On Tue, May 19, 2015 at 12:49 PM, Flavio Pom= permaier <pompermaier@okkam.it> wrote:
Hi to all,

=

I tried to run my job on a brand new Flink cluster (0.9-SNAPSHOT) from= the web client UI using the shading strategy of the quickstart example but= I get this exception:

Caused by: java.lang.RuntimeException: Could not create ClassReader: java.= io.IOException: Class not found
at org.apache.flink.api.java.ClosureCleaner.getClassReader(Clos= ureCleaner.java:42)
= at org.apache.flink.api.java.ClosureCleaner.cleanThis0(ClosureCleaner.java:= 67)
at org.apache.fl= ink.api.java.ClosureCleaner.clean(ClosureCleaner.java:54)
<= br>
It seems that it cannot find some kryo class..how do I fix th= is? this is my shade plugin section of pom.xml:


<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>mav= en-shade-plugin</artifactId>
<version>1.4</version>
<executions>
<execution>
<phase>package</phase>
= <goals>
<goal>shade</goal><= /div>
</goals>
<configuration>=
<artifactSet&= gt;
<excludes= >
<!-- Th= is list contains all dependencies of flink-dist Everything=C2=A0
= else will be packaged in= to the fat-jar -->
= <exclude>org.apache.flink:flink-shaded-*</exclude>
=
<exclude>org.a= pache.flink:flink-core</exclude>
<exclude>org.apache.flink:flink-java</excl= ude>
<exc= lude>org.apache.flink:flink-scala</exclude>
<exclude>org.apache.flink:flin= k-runtime</exclude>
= <exclude>org.apache.flink:flink-optimizer</exclude><= /div>
<exclude>= org.apache.flink:flink-clients</exclude>
<exclude>org.apache.flink:flink-sparg= el</exclude>
<exclude>org.apache.flink:flink-avro</exclude>
<exclude>org.apache.f= link:flink-java-examples</exclude>
<exclude>org.apache.flink:flink-scala-examp= les</exclude>
<exclude>org.apache.flink:flink-streaming-examples</exclude&g= t;
<exclude&= gt;org.apache.flink:flink-streaming-core</exclude>

<!-- Also exclu= de very big transitive dependencies of Flink WARNING:=C2=A0
You have to remove these excl= udes if your code relies on other versions of=C2=A0
these dependencies. -->
<exclude>org.scal= a-lang:scala-library</exclude>
<exclude>org.scala-lang:scala-compiler</excl= ude>
<exc= lude>org.scala-lang:scala-reflect</exclude>
<exclude>com.amazonaws:aws-jav= a-sdk</exclude>
= <exclude>com.typesafe.akka:akka-actor_*</exclude>
<= div> <exclude>com.ty= pesafe.akka:akka-remote_*</exclude>
<exclude>com.typesafe.akka:akka-slf4j_*<= ;/exclude>
&= lt;exclude>io.netty:netty-all</exclude>
<exclude>io.netty:netty</exclude= >
<exclud= e>org.eclipse.jetty:jetty-server</exclude>
<exclude>org.eclipse.jetty:jet= ty-continuation</exclude>
<exclude>org.eclipse.jetty:jetty-http</exclude>= ;
<exclude&g= t;org.eclipse.jetty:jetty-io</exclude>
<exclude>org.eclipse.jetty:jetty-util&l= t;/exclude>
= <exclude>org.eclipse.jetty:jetty-security</exclude>
<= span style=3D"white-space:pre-wrap"> <exclude>org.eclipse= .jetty:jetty-servlet</exclude>
<exclude>commons-fileupload:commons-fileupload&= lt;/exclude>
<exclude>org.apache.avro:avro</exclude>
<exclude>commons-collections:c= ommons-collections</exclude>
<exclude>org.codehaus.jackson:jackson-core-asl<= ;/exclude>
&= lt;exclude>org.codehaus.jackson:jackson-mapper-asl</exclude>
=
<exclude>com.t= houghtworks.paranamer:paranamer</exclude>
<exclude>org.xerial.snappy:snappy-ja= va</exclude>
<exclude>org.apache.commons:commons-compress</exclude>
<exclude>org.= tukaani:xz</exclude>
= <exclude>com.esotericsoftware.kryo:kryo</exclude>
<exclude>c= om.esotericsoftware.minlog:minlog</exclude>
<exclude>org.objenesis:objenesis&l= t;/exclude>
= <exclude>com.twitter:chill_*</exclude>
<exclude>com.twitter:chill-java&l= t;/exclude>
= <exclude>com.twitter:chill-avro_*</exclude>
<exclude>com.twitter:chill-b= ijection_*</exclude>
= <exclude>com.twitter:bijection-core_*</exclude>
<exclude>com= .twitter:bijection-avro_*</exclude>
<exclude>com.twitter:chill-protobuf</ex= clude>
<e= xclude>com.google.protobuf:protobuf-java</exclude>
<exclude>com.twitter:chi= ll-thrift</exclude>
= <exclude>org.apache.thrift:libthrift</exclude>
=
<exclude>commo= ns-lang:commons-lang</exclude>
<exclude>junit:junit</exclude>
= <exclude>de.javakaf= fee:kryo-serializers</exclude>
<exclude>joda-time:joda-time</exclude>
<exclude>or= g.apache.commons:commons-lang3</exclude>
<exclude>org.slf4j:slf4j-api</excl= ude>
<exc= lude>org.slf4j:slf4j-log4j12</exclude>
<exclude>log4j:log4j</exclude>=
<exclude>= ;org.apache.commons:commons-math</exclude>
<exclude>org.apache.sling:org.apach= e.sling.commons.json</exclude>
<exclude>commons-logging:commons-logging</ex= clude>
<e= xclude>org.apache.httpcomponents:httpclient</exclude>
<exclude>org.apache.h= ttpcomponents:httpcore</exclude>
<exclude>commons-codec:commons-codec</excl= ude>
<exc= lude>com.fasterxml.jackson.core:jackson-core</exclude>
<= span style=3D"white-space:pre-wrap"> <exclude>com.fasterx= ml.jackson.core:jackson-databind</exclude>
<exclude>com.fasterxml.jackson.core= :jackson-annotations</exclude>
<exclude>org.codehaus.jettison:jettison</exc= lude>
<ex= clude>stax:stax-api</exclude>
<exclude>com.typesafe:config</exclude><= /div>
<exclude>= org.uncommons.maths:uncommons-maths</exclude>
<exclude>com.github.scopt:scop= t_*</exclude>
<exclude>org.mortbay.jetty:servlet-api</exclude>
<exclude>commons-i= o:commons-io</exclude>
<exclude>commons-cli:commons-cli</exclude>
<= div> </excludes>
</artifactSet><= /div>
<filters>
<filter>
<artifact>o= rg.apache.flink:*</artifact>
<excludes>
<exclude>org/apache/flink/shaded/**</exclud= e>
<excl= ude>web-docs/**</exclude>
</excludes>
</filter>
</filters>
<createDependencyReducedPom>false</createDepe= ndencyReducedPom>
<finalName>XXXX</finalName>
<transformers>
<!-- add Main-Class to manifest file --&= gt;
<transfor= mer
implementat= ion=3D"org.apache.maven.plugins.shade.resource.ManifestResourceTransfo= rmer">
= <manifestEntries>
= <Main-Class>XXX</Main-Class>
</manifestEntries>
</transformer>
</transformers>
</configuration>=
</execution>=
</executions>=
</plugin>


--047d7b3a88bc92885505167f925e--