hudi-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From GitBox <...@apache.org>
Subject [GitHub] [incubator-hudi] yanbigot opened a new issue #1230: java.lang.NoSuchMethodError: org.apache.avro.Schema.getLogicalType()Lorg/apache/avro/LogicalType
Date Wed, 15 Jan 2020 14:04:14 GMT
yanbigot opened a new issue #1230: java.lang.NoSuchMethodError: org.apache.avro.Schema.getLogicalType()Lorg/apache/avro/LogicalType
URL: https://github.com/apache/incubator-hudi/issues/1230
 
 
   While running on yarn cluster with:
   - spark 2.2.0
   - hdp 2.6.4
   
   Works on local env.
   
   Here is my pom.xml
   `<properties>
           <jdk.version.source>1.8</jdk.version.source>
           <jdk.version.target>1.8</jdk.version.target>
           <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
           <build.id>NA</build.id><!-- need to have a default value -->
   
           <!-- Plugins versions -->
           <maven.jar-plugin.version>2.6</maven.jar-plugin.version>
           <maven.compiler-plugin.version>3.5.1</maven.compiler-plugin.version>
           <scoverage-plugin.version>1.3.0</scoverage-plugin.version>
           <scala-maven-plugin.version>3.2.0</scala-maven-plugin.version>
           <maven-surefire-plugin.version>2.7</maven-surefire-plugin.version>
           <scalatest-maven-plugin.version>1.0</scalatest-maven-plugin.version>
           <maven-assembly-plugin.version>2.6</maven-assembly-plugin.version>
           <maven-resources-plugin.version>3.0.1</maven-resources-plugin.version>
           <buildnumber-maven-plugin.version>1.4</buildnumber-maven-plugin.version>
   
           <!-- Dependency versions -->
           <commons-configuration.version>1.10</commons-configuration.version>
           <spark.version>2.2.0</spark.version>
           <logback.version>1.2.3</logback.version>
           <joda-time.version>2.9.4</joda-time.version>
           <joda-convert.version>1.8</joda-convert.version>
           <hadoop.version>2.9.0</hadoop.version>
           <scala.tool.version>2.11</scala.tool.version>
           <scala.version>2.11.12</scala.version>
           <jackson.version>2.9.6</jackson.version>
           <jackson-asl.version>1.9.13</jackson-asl.version>
           <scopt.version>3.4.0</scopt.version>
           <scalaj.version>2.3.0</scalaj.version>
           <oozie.version>4.2.0.2.4.2.0-258</oozie.version>
           <junit.version>4.12</junit.version>
           <specs2.version>2.3.13</specs2.version>
           <assertj.version>3.8.0</assertj.version>
           <lift.version>2.6</lift.version>
           <spark-streaming-kafka.version>2.0.2</spark-streaming-kafka.version>
           <scalatest.version>3.0.1</scalatest.version>
           <scalacheck.version>1.13.4</scalacheck.version>
           <derby.version>10.10.2.0</derby.version> <!-- 10.14.1.0 -->
           <curator.version>2.5.0</curator.version>
           <kafka.version>0.10.0.0</kafka.version>
           <log4j.version>1.2.17</log4j.version>
           <slf4j.version>1.7.10</slf4j.version>
           <objenesis.version>1.2</objenesis.version>
           <commons-lang.version>2.6</commons-lang.version>
           <commons-logging.version>1.1.3</commons-logging.version>
           <xml-apis.version>1.4.01</xml-apis.version>
           <xercesImpl.version>2.10.0</xercesImpl.version>
           <jetty.version>6.1.26</jetty.version>
           <servlet-api.version>2.5</servlet-api.version>
           <guava.version>11.0.2</guava.version>
           <simplyscala.version>0.5</simplyscala.version>
           <bonecp.version>0.8.0.RELEASE</bonecp.version>
           <scoverage.plugin.version>1.3.0</scoverage.plugin.version>
           <typesafe-config.version>1.3.0</typesafe-config.version>
           <postgresql.version>42.1.4</postgresql.version>
           <spark-testing-base.version>0.7.4</spark-testing-base.version>
           <mvel.version>2.4.0.Final</mvel.version>
           <jdbc-slim.version>1.2.2</jdbc-slim.version>
           <postgresql.version>42.1.4</postgresql.version>
           <dbfit.version>3.2.0</dbfit.version>
           <smartrics-RestFixture.version>4.4</smartrics-RestFixture.version>
           <commons-email.version>1.5</commons-email.version>
           <commons-dbcp2.version>2.3.0</commons-dbcp2.version>
       </properties>
       <dependencies>
           <!-- Avro -->
           <dependency>
               <groupId>org.apache.parquet</groupId>
               <artifactId>parquet-avro</artifactId>
               <version>1.8.2</version>
           </dependency>
   
           <!-- Hudi -->
           <dependency>
               <groupId>org.apache.hudi</groupId>
               <artifactId>hudi-spark</artifactId>
               <version>0.5.0-incubating</version>
           </dependency>
   
           <!-- Spark -->
           <dependency>
               <groupId>org.apache.spark</groupId>
               <artifactId>spark-core_${scala.tool.version}</artifactId>
               <version>${spark.version}</version>
               <exclusions>
                   <exclusion>
                       <groupId>org.slf4j</groupId>
                       <artifactId>slf4j-log4j12</artifactId>
                   </exclusion>
               </exclusions>
           </dependency>
           <dependency>
               <groupId>org.apache.spark</groupId>
               <artifactId>spark-sql_${scala.tool.version}</artifactId>
               <version>${spark.version}</version>
               <exclusions>
                   <exclusion>
                       <groupId>org.slf4j</groupId>
                       <artifactId>slf4j-log4j12</artifactId>
                   </exclusion>
               </exclusions>
           </dependency>
           <dependency>
               <groupId>org.apache.spark</groupId>
               <artifactId>spark-hive_${scala.tool.version}</artifactId>
               <version>${spark.version}</version>
               <!--<scope>provided</scope>-->
           </dependency>
   
   
   
           <!-- Dhr -->
           <dependency>
               <groupId>com.socgen.dhr</groupId>
               <artifactId>dhr-configuration</artifactId>
               <version>4.0.0</version>
           </dependency>
   
           <!-- Spark testing base-->
           <dependency>
               <groupId>com.holdenkarau</groupId>
               <artifactId>spark-testing-base_2.11</artifactId>
               <scope>test</scope>
               <exclusions>
                   <exclusion>
                       <groupId>org.mockito</groupId>
                       <artifactId>mockito-core</artifactId>
                   </exclusion>
               </exclusions>
           </dependency>
   
           <!-- Scala -->
           <dependency>
               <groupId>org.scala-lang</groupId>
               <artifactId>scala-library</artifactId>
               <version>${scala.version}</version>
           </dependency>
           <dependency>
               <groupId>org.scala-lang</groupId>
               <artifactId>scala-reflect</artifactId>
               <version>${scala.version}</version>
           </dependency>
           <dependency>
               <groupId>org.scala-lang</groupId>
               <artifactId>scala-compiler</artifactId>
               <version>${scala.version}</version>
           </dependency>
           <dependency>
               <groupId>org.scala-lang</groupId>
               <artifactId>scalap</artifactId>
               <version>${scala.version}</version>
           </dependency>
   
           <dependency>
               <groupId>org.slf4j</groupId>
               <artifactId>slf4j-api</artifactId>
               <version>${slf4j.version}</version>
           </dependency>
           <dependency>
               <groupId>org.slf4j</groupId>
               <artifactId>slf4j-log4j12</artifactId>
               <version>${slf4j.version}</version>
           </dependency>
           <dependency>
               <groupId>ch.qos.logback</groupId>
               <artifactId>logback-classic</artifactId>
               <version>${logback.version}</version>
           </dependency>
   
           <dependency>
               <groupId>org.scalatest</groupId>
               <artifactId>scalatest_2.11</artifactId>
               <version>${scalatest.version}</version>
               <scope>test</scope>
           </dependency>
       </dependencies>
   
       <dependencyManagement>
           <dependencies>
               <dependency>
                   <groupId>com.holdenkarau</groupId>
                   <artifactId>spark-testing-base_${scala.tool.version}</artifactId>
                   <version>${spark.version}_${spark-testing-base.version}</version>
                   <scope>test</scope>
               </dependency>
           </dependencies>
       </dependencyManagement>`
   
   The stack trace:
   ```
   Caused by: java.lang.NoSuchMethodError: org.apache.avro.Schema.getLogicalType()Lorg/apache/avro/LogicalType;
           at org.apache.parquet.avro.AvroSchemaConverter.convertField(AvroSchemaConverter.java:178)
           at org.apache.parquet.avro.AvroSchemaConverter.convertUnion(AvroSchemaConverter.java:214)
           at org.apache.parquet.avro.AvroSchemaConverter.convertField(AvroSchemaConverter.java:171)
           at org.apache.parquet.avro.AvroSchemaConverter.convertField(AvroSchemaConverter.java:130)
           at org.apache.parquet.avro.AvroSchemaConverter.convertField(AvroSchemaConverter.java:227)
           at org.apache.parquet.avro.AvroSchemaConverter.convertFields(AvroSchemaConverter.java:124)
           at org.apache.parquet.avro.AvroSchemaConverter.convert(AvroSchemaConverter.java:115)
           at org.apache.hudi.io.storage.HoodieStorageWriterFactory.newParquetStorageWriter(HoodieStorageWriterFactory.java:60)
           at org.apache.hudi.io.storage.HoodieStorageWriterFactory.getStorageWriter(HoodieStorageWriterFactory.java:47)
           at org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:72)
           at org.apache.hudi.func.CopyOnWriteLazyInsertIterable$CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteLazyInser
                                 tIterable.java:139)
           at org.apache.hudi.func.CopyOnWriteLazyInsertIterable$CopyOnWriteInsertHandler.consumeOneRecord(CopyOnWriteLazyInser
                                 tIterable.java:127)
           at org.apache.hudi.common.util.queue.BoundedInMemoryQueueConsumer.consume(BoundedInMemoryQueueConsumer.java:37)
           at org.apache.hudi.common.util.queue.BoundedInMemoryExecutor.lambda$null$192(BoundedInMemoryExecutor.java:121)
           at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   ```

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

Mime
View raw message