spark-dev mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Josh Rosen <joshro...@databricks.com>
Subject Re: Apache Spark Exception in thread “main” java.lang.NoClassDefFoundError: scala/collection/GenTraversableOnce$class
Date Thu, 17 Mar 2016 02:08:39 GMT
Err, whoops, looks like this is a user app and not building Spark itself,
so you'll have to change your deps to use the 2.11 versions of Spark.
e.g. spark-streaming_2.10 -> spark-streaming_2.11.

On Wed, Mar 16, 2016 at 7:07 PM Josh Rosen <joshrosen@databricks.com> wrote:

> See the instructions in the Spark documentation:
> https://spark.apache.org/docs/latest/building-spark.html#building-for-scala-211
>
> On Wed, Mar 16, 2016 at 7:05 PM satyajit vegesna <
> satyajit.apasprk@gmail.com> wrote:
>
>>
>>
>> Hi,
>>
>> Scala version:2.11.7(had to upgrade the scala verison to enable case
>> clasess to accept more than 22 parameters.)
>>
>> Spark version:1.6.1.
>>
>> PFB pom.xml
>>
>> Getting below error when trying to setup spark on intellij IDE,
>>
>> 16/03/16 18:36:44 INFO spark.SparkContext: Running Spark version 1.6.1
>> Exception in thread "main" java.lang.NoClassDefFoundError:
>> scala/collection/GenTraversableOnce$class at
>> org.apache.spark.util.TimeStampedWeakValueHashMap.(TimeStampedWeakValueHashMap.scala:42)
>> at org.apache.spark.SparkContext.(SparkContext.scala:298) at
>> com.examples.testSparkPost$.main(testSparkPost.scala:27) at
>> com.examples.testSparkPost.main(testSparkPost.scala) at
>> sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at
>> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>> at
>> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>> at java.lang.reflect.Method.invoke(Method.java:606) at
>> com.intellij.rt.execution.application.AppMain.main(AppMain.java:140) Caused
>> by: java.lang.ClassNotFoundException:
>> scala.collection.GenTraversableOnce$class at
>> java.net.URLClassLoader$1.run(URLClassLoader.java:366) at
>> java.net.URLClassLoader$1.run(URLClassLoader.java:355) at
>> java.security.AccessController.doPrivileged(Native Method) at
>> java.net.URLClassLoader.findClass(URLClassLoader.java:354) at
>> java.lang.ClassLoader.loadClass(ClassLoader.java:425) at
>> sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:308) at
>> java.lang.ClassLoader.loadClass(ClassLoader.java:358) ... 9 more
>>
>> pom.xml:
>>
>> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
>>          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
>>     <modelVersion>4.0.0</modelVersion>
>>     <groupId>StreamProcess</groupId>
>>     <artifactId>StreamProcess</artifactId>
>>     <version>0.0.1-SNAPSHOT</version>
>>     <name>${project.artifactId}</name>
>>     <description>This is a boilerplate maven project to start using Spark in
Scala</description>
>>     <inceptionYear>2010</inceptionYear>
>>
>>     <properties>
>>         <maven.compiler.source>1.6</maven.compiler.source>
>>         <maven.compiler.target>1.6</maven.compiler.target>
>>         <encoding>UTF-8</encoding>
>>         <scala.tools.version>2.10</scala.tools.version>
>>         <!-- Put the Scala version of the cluster -->
>>         <scala.version>2.11.7</scala.version>
>>     </properties>
>>
>>     <!-- repository to add org.apache.spark -->
>>     <repositories>
>>         <repository>
>>             <id>cloudera-repo-releases</id>
>>             <url>https://repository.cloudera.com/artifactory/repo/</url>
>>         </repository>
>>     </repositories>
>>
>>     <build>
>>         <sourceDirectory>src/main/scala</sourceDirectory>
>>         <testSourceDirectory>src/test/scala</testSourceDirectory>
>>         <plugins>
>>                 <!-- any other plugins -->
>>                 <plugin>
>>                     <artifactId>maven-assembly-plugin</artifactId>
>>                     <executions>
>>                         <execution>
>>                             <phase>package</phase>
>>                             <goals>
>>                                 <goal>single</goal>
>>                             </goals>
>>                         </execution>
>>                     </executions>
>>                     <configuration>
>>                         <descriptorRefs>
>>                             <descriptorRef>jar-with-dependencies</descriptorRef>
>>                         </descriptorRefs>
>>                     </configuration>
>>                 </plugin>
>>             <plugin>
>>                 <!-- see http://davidb.github.com/scala-maven-plugin -->
>>                 <groupId>net.alchim31.maven</groupId>
>>                 <artifactId>scala-maven-plugin</artifactId>
>>                 <version>3.2.2</version>
>>                 <executions>
>>                     <execution>
>>                         <goals>
>>                             <goal>compile</goal>
>>                             <goal>testCompile</goal>
>>                         </goals>
>>                         <configuration>
>>                             <args>
>>                                 <!--<arg>-make:transitive</arg>-->
>>                                 <arg>-dependencyfile</arg>
>>                                 <arg>${project.build.directory}/.scala_dependencies</arg>
>>                             </args>
>>                         </configuration>
>>                     </execution>
>>                 </executions>
>>             </plugin>
>>
>>             <!-- "package" command plugin -->
>>             <plugin>
>>                 <artifactId>maven-assembly-plugin</artifactId>
>>                 <version>2.4.1</version>
>>                 <configuration>
>>                     <descriptorRefs>
>>                         <descriptorRef>jar-with-dependencies</descriptorRef>
>>                     </descriptorRefs>
>>                 </configuration>
>>                 <executions>
>>                     <execution>
>>                         <id>make-assembly</id>
>>                         <phase>package</phase>
>>                         <goals>
>>                             <goal>single</goal>
>>                         </goals>
>>                     </execution>
>>                 </executions>
>>             </plugin>
>>         </plugins>
>>     </build>
>>     <dependencies>
>>         <dependency>
>>             <groupId>org.scala-lang</groupId>
>>             <artifactId>scala-library</artifactId>
>>             <version>${scala.version}</version>
>>         </dependency>
>>         <dependency>
>>             <groupId>org.mongodb.mongo-hadoop</groupId>
>>             <artifactId>mongo-hadoop-core</artifactId>
>>             <version>1.4.2</version>
>>             <exclusions>
>>                 <exclusion>
>>                     <groupId>javax.servlet</groupId>
>>                     <artifactId>servlet-api</artifactId>
>>                 </exclusion>
>>             </exclusions>
>>         </dependency>
>>         <dependency>
>>                 <groupId>org.mongodb</groupId>
>>                 <artifactId>mongodb-driver</artifactId>
>>                 <version>3.2.2</version>
>>             <exclusions>
>>                 <exclusion>
>>                     <groupId>javax.servlet</groupId>
>>                     <artifactId>servlet-api</artifactId>
>>                 </exclusion>
>>             </exclusions>
>>         </dependency>
>>         <dependency>
>>                 <groupId>org.mongodb</groupId>
>>                 <artifactId>mongodb-driver</artifactId>
>>                 <version>3.2.2</version>
>>             <exclusions>
>>                 <exclusion>
>>                     <groupId>javax.servlet</groupId>
>>                     <artifactId>servlet-api</artifactId>
>>                 </exclusion>
>>             </exclusions>
>>             </dependency>
>>         <dependency>
>>             <groupId>org.apache.spark</groupId>
>>             <artifactId>spark-streaming_2.10</artifactId>
>>             <version>1.6.1</version>
>>         </dependency>
>>         <dependency>
>>             <groupId>org.apache.spark</groupId>
>>             <artifactId>spark-core_2.10</artifactId>
>>             <version>1.6.1</version>
>>         </dependency>
>>         <dependency>
>>             <groupId>org.apache.spark</groupId>
>>             <artifactId>spark-sql_2.10</artifactId>
>>             <version>1.6.1</version>
>>         </dependency>
>>         <dependency>
>>             <groupId>org.apache.hadoop</groupId>
>>             <artifactId>hadoop-hdfs</artifactId>
>>             <version>2.6.0</version>
>>         </dependency>
>>         <dependency>
>>             <groupId>org.apache.hadoop</groupId>
>>             <artifactId>hadoop-auth</artifactId>
>>             <version>2.6.0</version>
>>         </dependency>
>>         <dependency>
>>             <groupId>org.apache.hadoop</groupId>
>>             <artifactId>hadoop-common</artifactId>
>>             <version>2.6.0</version>
>>         </dependency>
>>         <dependency>
>>             <groupId>org.apache.hadoop</groupId>
>>             <artifactId>hadoop-core</artifactId>
>>             <version>1.2.1</version>
>>         </dependency>
>>     </dependencies>
>> </project>
>>
>> Would like to know , what to be changed in pom to get things going.
>>
>> Regards,
>> Satyajit.
>>
>>

Mime
View raw message