parquet-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject git commit: PARQUET-89: Add hadoop-2 test profile for Travis CI.
Date Mon, 22 Sep 2014 22:07:18 GMT
Repository: incubator-parquet-mr
Updated Branches:
  refs/heads/master 3dc223cc8 -> 0eb963795


PARQUET-89: Add hadoop-2 test profile for Travis CI.

This also fixes problems that prevented hadoop-2 from passing:
* Dynamically resolve counter methods in parquet-hadoop
* Parameterize pig version with hadoop-2 support
* Update pig test for hadoop-2 change (no nulls allowed)
* Update parquet-hive to depend on hadoop-client

The travis config will now run each test profile in a different run.

Author: Ryan Blue <rblue@cloudera.com>

Closes #55 from rdblue/PARQUET-89-add-test-profiles and squashes the following commits:

006c6d8 [Ryan Blue] PARQUET-89: Add hadoop-2 test profile for travis CI.


Project: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/commit/0eb96379
Tree: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/tree/0eb96379
Diff: http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/diff/0eb96379

Branch: refs/heads/master
Commit: 0eb96379518db4f84f7a95b651c6dc9a639cc9ac
Parents: 3dc223c
Author: Ryan Blue <rblue@cloudera.com>
Authored: Mon Sep 22 15:07:04 2014 -0700
Committer: Ryan Blue <rblue@cloudera.com>
Committed: Mon Sep 22 15:07:04 2014 -0700

----------------------------------------------------------------------
 .travis.yml                                     |   7 +
 .../hadoop/example/TestInputOutputFormat.java   |  29 +++-
 .../parquet-hive-0.10-binding/pom.xml           |   2 +-
 .../parquet-hive-0.12-binding/pom.xml           |   2 +-
 .../parquet-hive-binding-factory/pom.xml        |   2 +-
 .../parquet-hive-binding-interface/pom.xml      |   2 +-
 .../parquet-hive-storage-handler/pom.xml        |   2 +-
 parquet-pig/pom.xml                             |   3 +-
 .../java/parquet/pig/summary/TestSummary.java   |  13 +-
 parquet-test-hadoop2/pom.xml                    |  61 --------
 .../parquet/hadoop2/TestInputOutputFormat.java  | 139 -------------------
 parquet-thrift/pom.xml                          |   3 +-
 pom.xml                                         |  20 ++-
 13 files changed, 61 insertions(+), 224 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/0eb96379/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index e1acf30..a75ac3a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -22,3 +22,10 @@ before_install:
   - ./configure --disable-gen-erl --disable-gen-hs --without-ruby --without-haskell --without-erlang
   - sudo make install
   - cd ..
+
+env:
+  - HADOOP_PROFILE=default
+  - HADOOP_PROFILE=hadoop-2
+
+install: mvn install -DskipTests=true -Dmaven.javadoc.skip=true -Dsource.skip=true
+script: mvn test -P $HADOOP_PROFILE

http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/0eb96379/parquet-hadoop/src/test/java/parquet/hadoop/example/TestInputOutputFormat.java
----------------------------------------------------------------------
diff --git a/parquet-hadoop/src/test/java/parquet/hadoop/example/TestInputOutputFormat.java
b/parquet-hadoop/src/test/java/parquet/hadoop/example/TestInputOutputFormat.java
index e65f03c..6dd472b 100644
--- a/parquet-hadoop/src/test/java/parquet/hadoop/example/TestInputOutputFormat.java
+++ b/parquet-hadoop/src/test/java/parquet/hadoop/example/TestInputOutputFormat.java
@@ -24,6 +24,7 @@ import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileReader;
 import java.io.IOException;
+import java.lang.reflect.Method;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
@@ -34,6 +35,8 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.CounterGroup;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
@@ -239,13 +242,25 @@ public class TestInputOutputFormat {
     runMapReduceJob(CompressionCodecName.GZIP);
   }
 
+  private static long value(Job job, String groupName, String name) throws Exception {
+    // getGroup moved to AbstractCounters
+    Method getGroup = org.apache.hadoop.mapreduce.Counters.class.getMethod("getGroup", String.class);
+    // CounterGroup changed to an interface
+    Method findCounter = org.apache.hadoop.mapreduce.CounterGroup.class.getMethod("findCounter",
String.class);
+    // Counter changed to an interface
+    Method getValue = org.apache.hadoop.mapreduce.Counter.class.getMethod("getValue");
+    CounterGroup group = (CounterGroup) getGroup.invoke(job.getCounters(), groupName);
+    Counter counter = (Counter) findCounter.invoke(group, name);
+    return (Long) getValue.invoke(counter);
+  }
+
   @Test
   public void testReadWriteWithCounter() throws Exception {
     runMapReduceJob(CompressionCodecName.GZIP);
-    assertTrue(readJob.getCounters().getGroup("parquet").findCounter("bytesread").getValue()
> 0L);
-    assertTrue(readJob.getCounters().getGroup("parquet").findCounter("bytestotal").getValue()
> 0L);
-    assertTrue(readJob.getCounters().getGroup("parquet").findCounter("bytesread").getValue()
-            == readJob.getCounters().getGroup("parquet").findCounter("bytestotal").getValue());
+    assertTrue(value(readJob, "parquet", "bytesread") > 0L);
+    assertTrue(value(readJob, "parquet", "bytestotal") > 0L);
+    assertTrue(value(readJob, "parquet", "bytesread")
+            == value(readJob, "parquet", "bytestotal"));
     //not testing the time read counter since it could be zero due to the size of data is
too small
   }
 
@@ -255,9 +270,9 @@ public class TestInputOutputFormat {
     conf.set("parquet.benchmark.bytes.total", "false");
     conf.set("parquet.benchmark.bytes.read", "false");
     runMapReduceJob(CompressionCodecName.GZIP);
-    assertTrue(readJob.getCounters().getGroup("parquet").findCounter("bytesread").getValue()
== 0L);
-    assertTrue(readJob.getCounters().getGroup("parquet").findCounter("bytestotal").getValue()
== 0L);
-    assertTrue(readJob.getCounters().getGroup("parquet").findCounter("timeread").getValue()
== 0L);
+    assertTrue(value(readJob, "parquet", "bytesread") == 0L);
+    assertTrue(value(readJob, "parquet", "bytestotal") == 0L);
+    assertTrue(value(readJob, "parquet", "timeread") == 0L);
   }
 
   private void waitForJob(Job job) throws InterruptedException, IOException {

http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/0eb96379/parquet-hive/parquet-hive-binding/parquet-hive-0.10-binding/pom.xml
----------------------------------------------------------------------
diff --git a/parquet-hive/parquet-hive-binding/parquet-hive-0.10-binding/pom.xml b/parquet-hive/parquet-hive-binding/parquet-hive-0.10-binding/pom.xml
index 7a7cee2..96546bd 100644
--- a/parquet-hive/parquet-hive-binding/parquet-hive-0.10-binding/pom.xml
+++ b/parquet-hive/parquet-hive-binding/parquet-hive-0.10-binding/pom.xml
@@ -29,7 +29,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-core</artifactId>
+      <artifactId>hadoop-client</artifactId>
       <version>${hadoop.version}</version>
       <scope>provided</scope>
     </dependency>

http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/0eb96379/parquet-hive/parquet-hive-binding/parquet-hive-0.12-binding/pom.xml
----------------------------------------------------------------------
diff --git a/parquet-hive/parquet-hive-binding/parquet-hive-0.12-binding/pom.xml b/parquet-hive/parquet-hive-binding/parquet-hive-0.12-binding/pom.xml
index 7eecebd..8f4be2d 100644
--- a/parquet-hive/parquet-hive-binding/parquet-hive-0.12-binding/pom.xml
+++ b/parquet-hive/parquet-hive-binding/parquet-hive-0.12-binding/pom.xml
@@ -29,7 +29,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-core</artifactId>
+      <artifactId>hadoop-client</artifactId>
       <version>${hadoop.version}</version>
       <scope>provided</scope>
     </dependency>

http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/0eb96379/parquet-hive/parquet-hive-binding/parquet-hive-binding-factory/pom.xml
----------------------------------------------------------------------
diff --git a/parquet-hive/parquet-hive-binding/parquet-hive-binding-factory/pom.xml b/parquet-hive/parquet-hive-binding/parquet-hive-binding-factory/pom.xml
index 69dfbc4..9ccebb1 100644
--- a/parquet-hive/parquet-hive-binding/parquet-hive-binding-factory/pom.xml
+++ b/parquet-hive/parquet-hive-binding/parquet-hive-binding-factory/pom.xml
@@ -35,7 +35,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-core</artifactId>
+      <artifactId>hadoop-client</artifactId>
       <version>${hadoop.version}</version>
       <scope>test</scope>
     </dependency>

http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/0eb96379/parquet-hive/parquet-hive-binding/parquet-hive-binding-interface/pom.xml
----------------------------------------------------------------------
diff --git a/parquet-hive/parquet-hive-binding/parquet-hive-binding-interface/pom.xml b/parquet-hive/parquet-hive-binding/parquet-hive-binding-interface/pom.xml
index 723732a..b3720dd 100644
--- a/parquet-hive/parquet-hive-binding/parquet-hive-binding-interface/pom.xml
+++ b/parquet-hive/parquet-hive-binding/parquet-hive-binding-interface/pom.xml
@@ -15,7 +15,7 @@
   <dependencies>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-core</artifactId>
+      <artifactId>hadoop-client</artifactId>
       <version>${hadoop.version}</version>
       <scope>provided</scope>
     </dependency>

http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/0eb96379/parquet-hive/parquet-hive-storage-handler/pom.xml
----------------------------------------------------------------------
diff --git a/parquet-hive/parquet-hive-storage-handler/pom.xml b/parquet-hive/parquet-hive-storage-handler/pom.xml
index 0527006..71b779f 100644
--- a/parquet-hive/parquet-hive-storage-handler/pom.xml
+++ b/parquet-hive/parquet-hive-storage-handler/pom.xml
@@ -42,7 +42,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-core</artifactId>
+      <artifactId>hadoop-client</artifactId>
       <version>${hadoop.version}</version>
       <scope>provided</scope>
     </dependency>

http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/0eb96379/parquet-pig/pom.xml
----------------------------------------------------------------------
diff --git a/parquet-pig/pom.xml b/parquet-pig/pom.xml
index 47a08ff..b93d4e1 100644
--- a/parquet-pig/pom.xml
+++ b/parquet-pig/pom.xml
@@ -36,7 +36,8 @@
     <dependency>
       <groupId>org.apache.pig</groupId>
       <artifactId>pig</artifactId>
-      <version>0.11.1</version>
+      <version>${pig.version}</version>
+      <classifier>${pig.classifier}</classifier>
       <scope>provided</scope>
     </dependency>
     <dependency>

http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/0eb96379/parquet-pig/src/test/java/parquet/pig/summary/TestSummary.java
----------------------------------------------------------------------
diff --git a/parquet-pig/src/test/java/parquet/pig/summary/TestSummary.java b/parquet-pig/src/test/java/parquet/pig/summary/TestSummary.java
index 6e7a990..f8442e3 100644
--- a/parquet-pig/src/test/java/parquet/pig/summary/TestSummary.java
+++ b/parquet-pig/src/test/java/parquet/pig/summary/TestSummary.java
@@ -36,11 +36,6 @@ import org.codehaus.jackson.JsonParseException;
 import org.codehaus.jackson.map.JsonMappingException;
 import org.junit.Test;
 
-import parquet.pig.summary.MapSummaryData;
-import parquet.pig.summary.Summary;
-import parquet.pig.summary.SummaryData;
-import parquet.pig.summary.TupleSummaryData;
-
 public class TestSummary {
 
   private static final TupleFactory tf = TupleFactory.getInstance();
@@ -50,9 +45,9 @@ public class TestSummary {
       t(b(t(1l), t(2l, m("foo", "bar")), t(3))),
       t(b(t(1l), t(1l), t(3, "blah"))),
       t(b(t(1l), t(2l), t(2, "bloh"))),
-      t(b(t(1l), null, t(2, "bloh"))),
-      t(b(t("foo"), null, t(2, "bloh"))),
-      t(b(t(b(t("bar"))), null, t(2, "bloh"))),
+      t(b(t(1l), t(2, "bloh"))),
+      t(b(t("foo"), t(2, "bloh"))),
+      t(b(t(b(t("bar"))), t(2, "bloh"))),
       t(b(t(b(t("bar"))), t(1l, m("foo", "bar", "baz", "buz")), t(2, "bloh"))),
       t(),
       t(null, null)
@@ -115,7 +110,7 @@ public class TestSummary {
     assertEquals(9 * factor, s.getCount());
     assertEquals(1 * factor, s.getFields().get(0).getNull().longValue());
     assertEquals(7 * factor, s.getFields().get(0).getBag().getCount());
-    assertEquals(15 * factor,
+    assertEquals(18 * factor,
         s.getFields().get(0).getBag().getContent().getTuple().getFields().get(0).getCount());
     MapSummaryData map =
         s.getFields().get(0).getBag().getContent().getTuple().getFields().get(1).getMap();

http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/0eb96379/parquet-test-hadoop2/pom.xml
----------------------------------------------------------------------
diff --git a/parquet-test-hadoop2/pom.xml b/parquet-test-hadoop2/pom.xml
deleted file mode 100644
index b980eaf..0000000
--- a/parquet-test-hadoop2/pom.xml
+++ /dev/null
@@ -1,61 +0,0 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-  <parent>
-    <groupId>com.twitter</groupId>
-    <artifactId>parquet</artifactId>
-    <relativePath>../pom.xml</relativePath>
-    <version>1.6.0rc3-SNAPSHOT</version>
-  </parent>
-
-  <modelVersion>4.0.0</modelVersion>
-
-  <artifactId>parquet-test-hadoop2</artifactId>
-  <packaging>jar</packaging>
-
-  <name>Parquet Hadoop 2 tests</name>
-  <url>https://github.com/Parquet/parquet-mr</url>
-
-  <properties>
-  </properties>
-
-  <dependencies>
-    <dependency>
-      <groupId>com.twitter</groupId>
-      <artifactId>parquet-column</artifactId>
-      <version>${project.version}</version>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>com.twitter</groupId>
-      <artifactId>parquet-hadoop</artifactId>
-      <version>${project.version}</version>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>com.twitter</groupId>
-      <artifactId>parquet-hadoop</artifactId>
-      <version>${project.version}</version>
-      <type>test-jar</type>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-client</artifactId>
-      <version>2.0.3-alpha</version>
-      <scope>test</scope>
-    </dependency>
-  </dependencies>
-
-  <build>
-    <plugins>
-      <plugin>
-        <artifactId>maven-enforcer-plugin</artifactId>
-      </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-jar-plugin</artifactId>
-      </plugin>
-    </plugins>
-  </build>
-
-
-</project>

http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/0eb96379/parquet-test-hadoop2/src/test/java/parquet/hadoop2/TestInputOutputFormat.java
----------------------------------------------------------------------
diff --git a/parquet-test-hadoop2/src/test/java/parquet/hadoop2/TestInputOutputFormat.java
b/parquet-test-hadoop2/src/test/java/parquet/hadoop2/TestInputOutputFormat.java
deleted file mode 100644
index 002044d..0000000
--- a/parquet-test-hadoop2/src/test/java/parquet/hadoop2/TestInputOutputFormat.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/**
- * Copyright 2012 Twitter, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package parquet.hadoop2;
-
-import static java.lang.Thread.sleep;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
-import org.junit.Test;
-
-import parquet.Log;
-import parquet.example.data.Group;
-import parquet.example.data.simple.SimpleGroupFactory;
-import parquet.hadoop.example.ExampleInputFormat;
-import parquet.hadoop.example.ExampleOutputFormat;
-import parquet.hadoop.example.GroupWriteSupport;
-import parquet.hadoop.metadata.CompressionCodecName;
-import parquet.schema.MessageTypeParser;
-
-public class TestInputOutputFormat {
-  private static final Log LOG = Log.getLog(TestInputOutputFormat.class);
-
-  public static class MyMapper extends Mapper<LongWritable, Text, Void, Group> {
-    private SimpleGroupFactory factory;
-    protected void setup(org.apache.hadoop.mapreduce.Mapper<LongWritable,Text,Void,Group>.Context
context) throws java.io.IOException ,InterruptedException {
-      factory = new SimpleGroupFactory(GroupWriteSupport.getSchema(context.getConfiguration()));
-    };
-    protected void map(LongWritable key, Text value, Mapper<LongWritable,Text,Void,Group>.Context
context) throws java.io.IOException ,InterruptedException {
-      Group group = factory.newGroup()
-          .append("line", (int)key.get())
-          .append("content", value.toString());
-      context.write(null, group);
-    }
-  }
-
-  public static class MyMapper2 extends Mapper<Void, Group, LongWritable, Text> {
-    protected void map(Void key, Group value, Mapper<Void,Group,LongWritable,Text>.Context
context) throws IOException ,InterruptedException {
-      context.write(new LongWritable(value.getInteger("line", 0)), new Text(value.getString("content",
0)));
-    }
-
-  }
-
-  @Test
-  public void testReadWrite() throws IOException, ClassNotFoundException, InterruptedException
{
-    final Configuration conf = new Configuration();
-    final Path inputPath = new Path("src/test/java/parquet/hadoop2/TestInputOutputFormat.java");
-    final Path parquetPath = new Path("target/test/hadoop2/example/TestInputOutputFormat/parquet");
-    final Path outputPath = new Path("target/test/hadoop2/example/TestInputOutputFormat/out");
-    final FileSystem fileSystem = parquetPath.getFileSystem(conf);
-    fileSystem.delete(parquetPath, true);
-    fileSystem.delete(outputPath, true);
-    {
-      final Job job = new Job(conf, "write");
-      TextInputFormat.addInputPath(job, inputPath);
-      job.setInputFormatClass(TextInputFormat.class);
-      job.setNumReduceTasks(0);
-      ExampleOutputFormat.setCompression(job, CompressionCodecName.GZIP);
-      ExampleOutputFormat.setOutputPath(job, parquetPath);
-      job.setOutputFormatClass(ExampleOutputFormat.class);
-      job.setMapperClass(TestInputOutputFormat.MyMapper.class);
-      ExampleOutputFormat.setSchema(
-          job,
-          MessageTypeParser.parseMessageType(
-              "message example {\n" +
-              "required int32 line;\n" +
-              "required binary content;\n" +
-              "}"));
-      job.submit();
-      waitForJob(job);
-    }
-    {
-      final Job job = new Job(conf, "read");
-      job.setInputFormatClass(ExampleInputFormat.class);
-      ExampleInputFormat.setInputPaths(job, parquetPath);
-      job.setOutputFormatClass(TextOutputFormat.class);
-      TextOutputFormat.setOutputPath(job, outputPath);
-      job.setMapperClass(TestInputOutputFormat.MyMapper2.class);
-      job.setNumReduceTasks(0);
-      job.submit();
-      waitForJob(job);
-    }
-
-    final BufferedReader in = new BufferedReader(new FileReader(new File(inputPath.toString())));
-    final BufferedReader out = new BufferedReader(new FileReader(new File(outputPath.toString(),
"part-m-00000")));
-    String lineIn;
-    String lineOut = null;
-    int lineNumber = 0;
-    while ((lineIn = in.readLine()) != null && (lineOut = out.readLine()) != null)
{
-      ++ lineNumber;
-      lineOut = lineOut.substring(lineOut.indexOf("\t") + 1);
-      assertEquals("line " + lineNumber, lineIn, lineOut);
-    }
-    assertNull("line " + lineNumber, lineIn);
-    assertNull("line " + lineNumber, out.readLine());
-    assertTrue(lineNumber > 0);
-    in.close();
-    out.close();
-  }
-
-  private void waitForJob(Job job) throws InterruptedException, IOException {
-    while (!job.isComplete()) {
-      LOG.debug("waiting for job " + job.getJobName());
-      sleep(100);
-    }
-    LOG.info("status for job " + job.getJobName() + ": " + (job.isSuccessful() ? "SUCCESS"
: "FAILURE"));
-    if (!job.isSuccessful()) {
-      throw new RuntimeException("job failed " + job.getJobName());
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/0eb96379/parquet-thrift/pom.xml
----------------------------------------------------------------------
diff --git a/parquet-thrift/pom.xml b/parquet-thrift/pom.xml
index 8935d6b..08ba975 100644
--- a/parquet-thrift/pom.xml
+++ b/parquet-thrift/pom.xml
@@ -81,7 +81,8 @@
     <dependency>
       <groupId>org.apache.pig</groupId>
       <artifactId>pig</artifactId>
-      <version>0.11.1</version>
+      <version>${pig.version}</version>
+      <classifier>${pig.classifier}</classifier>
       <scope>provided</scope>
     </dependency>
     <dependency>

http://git-wip-us.apache.org/repos/asf/incubator-parquet-mr/blob/0eb96379/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e14884b..1293a5d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -93,6 +93,8 @@
     <scala.version>2.10.4</scala.version>
     <!-- scala.binary.version is used for projects that fetch dependencies that are in
scala -->
     <scala.binary.version>2.10</scala.binary.version>
+    <pig.version>0.11.1</pig.version>
+    <pig.classifier></pig.classifier>
   </properties>
 
   <modules>
@@ -110,7 +112,6 @@
     <module>parquet-scala</module>
     <module>parquet-scrooge</module>
     <module>parquet-thrift</module>
-    <module>parquet-test-hadoop2</module>
     <module>parquet-hadoop-bundle</module>
     <module>parquet-hive</module>
     <module>parquet-hive-bundle</module>
@@ -459,6 +460,23 @@
         </plugins>
       </build>
     </profile>
+
+    <profile>
+      <id>hadoop-2</id>
+      <activation>
+        <property>
+          <name>hadoop.profile</name>
+          <value>hadoop2</value>
+        </property>
+      </activation>
+      <properties>
+        <!-- test hadoop-1 with the same jars that were produced for default profile -->
+        <maven.main.skip>true</maven.main.skip>
+        <hadoop.version>2.3.0</hadoop.version>
+        <pig.version>0.13.0</pig.version>
+        <pig.classifier>h2</pig.classifier>
+      </properties>
+    </profile>
   </profiles>
 
 </project>


Mime
View raw message