parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tomwh...@apache.org
Subject git commit: PARQUET-121: Allow Parquet to build with Java 8
Date Mon, 03 Nov 2014 14:12:17 GMT
Repository: incubator-parquet-mr
Updated Branches:
  refs/heads/master a29815abf -> f1da5e927


PARQUET-121: Allow Parquet to build with Java 8

There are test failures running with Java 8 due to http://openjdk.java.net/jeps/180 which
changed retrieval order for HashMap.

Here's how I tested this:

```bash
use-java8
mvn clean install -DskipTests -Dmaven.javadoc.skip=true
mvn test
mvn test -P hadoop-2
```

I also compiled the main code with Java 7 (target=1.6 bytecode), and compiled the tests with
Java 8, and ran them with Java 8. The idea here is to simulate users who want to run Parquet
with JRE 8.
```bash
use-java7
mvn clean install -DskipTests -Dmaven.javadoc.skip=true
use-java8
find . -name test-classes | grep target/test-classes | grep -v 'parquet-scrooge' | xargs rm
-rf
mvn test -DtargetJavaVersion=1.8 -Dmaven.main.skip=true -Dscala.maven.test.skip=true
```
A couple of notes about this:
* The targetJavaVersion property is used since other Hadoop projects use the same name.
* I couldn’t get parquet-scrooge to compile with target=1.8, which is why I introduced scala.maven.test.skip
(and updated scala-maven-plugin to the latest version which supports the property). Compiling
with target=1.8 should be fixed in another JIRA as it looks pretty involved.

Author: Tom White <tom@cloudera.com>

Closes #77 from tomwhite/PARQUET-121-java8 and squashes the following commits:

8717e13 [Tom White] Fix tests to run under Java 8.
35ea670 [Tom White] PARQUET-121. Allow Parquet to build with Java 8.


Project: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/commit/f1da5e92
Tree: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/tree/f1da5e92
Diff: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/diff/f1da5e92

Branch: refs/heads/master
Commit: f1da5e927ed18aeec1610bec67f88facd6a470e1
Parents: a29815a
Author: Tom White <tom@cloudera.com>
Authored: Mon Nov 3 14:11:03 2014 +0000
Committer: Tom White <tom@cloudera.com>
Committed: Mon Nov 3 14:11:03 2014 +0000

----------------------------------------------------------------------
 .../java/parquet/pig/convert/MapConverter.java  |  3 +-
 .../java/parquet/pig/TestParquetStorer.java     | 20 ++++++++--
 .../parquet/pig/TestPigSchemaConverter.java     |  7 +++-
 parquet-scrooge/pom.xml                         |  5 ++-
 .../thrift/TestParquetWriteProtocol.java        | 39 +++++++++++++++++++-
 pom.xml                                         |  6 ++-
 6 files changed, 69 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/f1da5e92/parquet-pig/src/main/java/parquet/pig/convert/MapConverter.java
----------------------------------------------------------------------
diff --git a/parquet-pig/src/main/java/parquet/pig/convert/MapConverter.java b/parquet-pig/src/main/java/parquet/pig/convert/MapConverter.java
index 2827239..81220e1 100644
--- a/parquet-pig/src/main/java/parquet/pig/convert/MapConverter.java
+++ b/parquet-pig/src/main/java/parquet/pig/convert/MapConverter.java
@@ -20,6 +20,7 @@ import java.util.AbstractSet;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -78,7 +79,7 @@ final class MapConverter extends GroupConverter {
 
   @Override
   public void end() {
-    parent.add(new HashMap<String, Object>(buffer));
+    parent.add(new LinkedHashMap<String, Object>(buffer));
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/f1da5e92/parquet-pig/src/test/java/parquet/pig/TestParquetStorer.java
----------------------------------------------------------------------
diff --git a/parquet-pig/src/test/java/parquet/pig/TestParquetStorer.java b/parquet-pig/src/test/java/parquet/pig/TestParquetStorer.java
index 99cd6a2..9d32226 100644
--- a/parquet-pig/src/test/java/parquet/pig/TestParquetStorer.java
+++ b/parquet-pig/src/test/java/parquet/pig/TestParquetStorer.java
@@ -121,16 +121,28 @@ public class TestParquetStorer {
 
     final Schema schema = data.getSchema("out");
     assertEquals(2, schema.size());
-    assertEquals("a", schema.getField(0).alias);
-    assertEquals("b", schema.getField(1).alias);
+    // union could be in either order
+    int ai;
+    int bi;
+    if ("a".equals(schema.getField(0).alias)) {
+      ai = 0;
+      bi = 1;
+      assertEquals("a", schema.getField(0).alias);
+      assertEquals("b", schema.getField(1).alias);
+    } else {
+      ai = 1;
+      bi = 0;
+      assertEquals("b", schema.getField(0).alias);
+      assertEquals("a", schema.getField(1).alias);
+    }
 
     assertEquals(rows * 2, result.size());
 
     int a = 0;
     int b = 0;
     for (Tuple tuple : result) {
-      String fa = (String) tuple.get(0);
-      String fb = (String) tuple.get(1);
+      String fa = (String) tuple.get(ai);
+      String fb = (String) tuple.get(bi);
       if (fa != null) {
         assertEquals("a" + a, fa);
         ++a;

http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/f1da5e92/parquet-pig/src/test/java/parquet/pig/TestPigSchemaConverter.java
----------------------------------------------------------------------
diff --git a/parquet-pig/src/test/java/parquet/pig/TestPigSchemaConverter.java b/parquet-pig/src/test/java/parquet/pig/TestPigSchemaConverter.java
index e95150f..8615602 100644
--- a/parquet-pig/src/test/java/parquet/pig/TestPigSchemaConverter.java
+++ b/parquet-pig/src/test/java/parquet/pig/TestPigSchemaConverter.java
@@ -22,11 +22,14 @@ import static parquet.pig.TupleReadSupport.getPigSchemaFromMultipleFiles;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
 import java.util.Map;
 import java.util.Set;
 
 import org.apache.pig.impl.logicalLayer.schema.Schema;
 import org.apache.pig.impl.util.Utils;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import parquet.schema.MessageType;
@@ -214,8 +217,8 @@ public class TestPigSchemaConverter {
 
   @Test
   public void testSchemaEvolution() {
-    Map<String, Set<String>> map = new HashMap<String, Set<String>>();
-    map.put("pig.schema", new HashSet<String>(Arrays.asList(
+    Map<String, Set<String>> map = new LinkedHashMap<String, Set<String>>();
+    map.put("pig.schema", new LinkedHashSet<String>(Arrays.asList(
         "a:int, b:int, c:int, d:int, e:int, f:int",
         "aa:int, aaa:int, b:int, c:int, ee:int")));
     Schema result = getPigSchemaFromMultipleFiles(new MessageType("empty"), map);

http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/f1da5e92/parquet-scrooge/pom.xml
----------------------------------------------------------------------
diff --git a/parquet-scrooge/pom.xml b/parquet-scrooge/pom.xml
index 341a804..035a0ac 100644
--- a/parquet-scrooge/pom.xml
+++ b/parquet-scrooge/pom.xml
@@ -128,7 +128,7 @@
         <plugin>
             <groupId>net.alchim31.maven</groupId>
             <artifactId>scala-maven-plugin</artifactId>
-            <version>3.0.1</version>
+            <version>3.1.6</version>
             <executions>
                 <execution>
                     <id>scala-compile-first</id>
@@ -146,6 +146,9 @@
                     </goals>
                 </execution>
             </executions>
+            <configuration>
+              <skip>${scala.maven.test.skip}</skip>
+            </configuration>
         </plugin>
 
         <plugin>

http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/f1da5e92/parquet-thrift/src/test/java/parquet/thrift/TestParquetWriteProtocol.java
----------------------------------------------------------------------
diff --git a/parquet-thrift/src/test/java/parquet/thrift/TestParquetWriteProtocol.java b/parquet-thrift/src/test/java/parquet/thrift/TestParquetWriteProtocol.java
index 4c211cd..bea5baf 100644
--- a/parquet-thrift/src/test/java/parquet/thrift/TestParquetWriteProtocol.java
+++ b/parquet-thrift/src/test/java/parquet/thrift/TestParquetWriteProtocol.java
@@ -27,6 +27,7 @@ import java.util.Map;
 import java.util.Set;
 import java.util.TreeMap;
 
+import org.junit.ComparisonFailure;
 import thrift.test.OneOfEach;
 
 import org.apache.pig.data.Tuple;
@@ -95,12 +96,48 @@ public class TestParquetWriteProtocol {
          "endField(names, 1)",
         "endMessage()"
     };
+    String[] expectationsAlt = {
+        "startMessage()",
+         "startField(name, 0)",
+          "addBinary(map_name)",
+         "endField(name, 0)",
+         "startField(names, 1)",
+          "startGroup()",
+           "startField(map, 0)",
+            "startGroup()",
+             "startField(key, 0)",
+              "addBinary(foo2)",
+             "endField(key, 0)",
+             "startField(value, 1)",
+              "addBinary(bar2)",
+             "endField(value, 1)",
+            "endGroup()",
+            "startGroup()",
+             "startField(key, 0)",
+              "addBinary(foo)",
+             "endField(key, 0)",
+             "startField(value, 1)",
+              "addBinary(bar)",
+             "endField(value, 1)",
+            "endGroup()",
+           "endField(map, 0)",
+          "endGroup()",
+         "endField(names, 1)",
+        "endMessage()"
+    };
 
     final Map<String, String> map = new TreeMap<String, String>();
     map.put("foo", "bar");
     map.put("foo2", "bar2");
     TestMap testMap = new TestMap("map_name", map);
-    validatePig(expectations, testMap);
+    try {
+      validatePig(expectations, testMap);
+    } catch (ComparisonFailure e) {
+      // This can happen despite using a stable TreeMap, since ThriftToPig#toPigMap
+      // in com.twitter.elephantbird.pig.util creates a HashMap.
+      // So we test with the map elements in reverse order
+      validatePig(expectationsAlt, testMap);
+    }
     validateThrift(expectations, testMap);
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/f1da5e92/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 1293a5d..95bacea 100644
--- a/pom.xml
+++ b/pom.xml
@@ -75,8 +75,9 @@
   </pluginRepositories>
 
   <properties>
+    <targetJavaVersion>1.6</targetJavaVersion>
     <maven.compiler.source>1.6</maven.compiler.source>
-    <maven.compiler.target>1.6</maven.compiler.target>
+    <maven.compiler.target>${targetJavaVersion}</maven.compiler.target>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <github.global.server>github</github.global.server>
     <maven-jar-plugin.version>2.4</maven-jar-plugin.version>
@@ -93,6 +94,7 @@
     <scala.version>2.10.4</scala.version>
     <!-- scala.binary.version is used for projects that fetch dependencies that are in
scala -->
     <scala.binary.version>2.10</scala.binary.version>
+    <scala.maven.test.skip>false</scala.maven.test.skip>
     <pig.version>0.11.1</pig.version>
     <pig.classifier></pig.classifier>
   </properties>
@@ -318,7 +320,7 @@
         <configuration>
           <argLine>-XX:MaxPermSize=256m</argLine>
           <source>1.6</source>
-          <target>1.6</target>
+          <target>${targetJavaVersion}</target>
           <encoding>UTF-8</encoding>
         </configuration>
       </plugin>


Mime
View raw message