gora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lewi...@apache.org
Subject [1/3] git commit: GORA-346 Create shim layer to support multiple hadoop versions
Date Mon, 08 Sep 2014 20:14:59 GMT
Repository: gora
Updated Branches:
  refs/heads/master 90d1cc0af -> 74d78cbe5


GORA-346 Create shim layer to support multiple hadoop versions


Project: http://git-wip-us.apache.org/repos/asf/gora/repo
Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/b51cdeea
Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/b51cdeea
Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/b51cdeea

Branch: refs/heads/master
Commit: b51cdeeadc1e79474acd250e51a77645ad676ce6
Parents: 90d1cc0
Author: Lewis John McGibbney <lewis.j.mcgibbney@jpl.nasa.gov>
Authored: Sun Sep 7 15:49:09 2014 -0700
Committer: Lewis John McGibbney <lewis.j.mcgibbney@jpl.nasa.gov>
Committed: Sun Sep 7 15:49:09 2014 -0700

----------------------------------------------------------------------
 gora-accumulo/pom.xml                           |   4 +-
 gora-cassandra/pom.xml                          |   6 +-
 gora-core/pom.xml                               |  34 +++++--
 .../gora/mapreduce/GoraMapReduceUtils.java      |  10 +-
 .../org/apache/gora/util/ReflectionUtils.java   |   3 +-
 gora-shims-distribution/pom.xml                 |  79 +++++++++++++++
 gora-shims-hadoop/pom.xml                       |  38 +++++++
 .../apache/gora/shims/hadoop/HadoopShim.java    |  55 ++++++++++
 .../gora/shims/hadoop/HadoopShimFactory.java    | 100 +++++++++++++++++++
 .../gora/shims/hadoop/HadoopShimTest.java       |  37 +++++++
 gora-shims-hadoop1/pom.xml                      |  49 +++++++++
 .../apache/gora/shims/hadoop1/HadoopShim1.java  |  50 ++++++++++
 .../gora/shims/hadoop1/TestHadoopShim1.java     |  18 ++++
 gora-shims-hadoop2/pom.xml                      |  43 ++++++++
 .../apache/gora/shims/hadoop2/HadoopShim2.java  |  56 +++++++++++
 .../gora/shims/hadoop2/TestHadoopShim2.java     |  18 ++++
 gora-tutorial/pom.xml                           |   7 +-
 pom.xml                                         |  29 +++++-
 18 files changed, 613 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-accumulo/pom.xml
----------------------------------------------------------------------
diff --git a/gora-accumulo/pom.xml b/gora-accumulo/pom.xml
index b588a7d..3613298 100644
--- a/gora-accumulo/pom.xml
+++ b/gora-accumulo/pom.xml
@@ -155,7 +155,9 @@
 
     <dependency>
       <groupId>org.apache.hadoop</groupId>
-      <artifactId>hadoop-test</artifactId>
+      <artifactId>hadoop-common</artifactId>
+      <version>${hadoop-2.version}</version>
+      <classifier>tests</classifier>
     </dependency>
 
     <dependency>

http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-cassandra/pom.xml
----------------------------------------------------------------------
diff --git a/gora-cassandra/pom.xml b/gora-cassandra/pom.xml
index 9852562..8e140be 100644
--- a/gora-cassandra/pom.xml
+++ b/gora-cassandra/pom.xml
@@ -183,8 +183,10 @@
         </dependency>
 
         <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-test</artifactId>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-common</artifactId>
+          <version>${hadoop-2.version}</version>
+          <classifier>tests</classifier>
         </dependency>
         
     </dependencies>

http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-core/pom.xml
----------------------------------------------------------------------
diff --git a/gora-core/pom.xml b/gora-core/pom.xml
index c6ef288..1aac38e 100644
--- a/gora-core/pom.xml
+++ b/gora-core/pom.xml
@@ -103,10 +103,32 @@
     <dependencies>
         <!-- Hadoop Dependencies -->
         <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-core</artifactId>
+            <groupId>org.apache.gora</groupId>
+            <artifactId>gora-shims-distribution</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-common</artifactId>
+          <version>${hadoop-2.version}</version>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-common</artifactId>
+          <version>${hadoop-2.version}</version>
+          <classifier>tests</classifier>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+          <version>${hadoop-2.version}</version>
+          <classifier>tests</classifier>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-mapreduce-client-core</artifactId>
+          <version>${hadoop-2.version}</version>
         </dependency>
-        
         <dependency>
             <groupId>org.apache.cxf</groupId>
             <artifactId>cxf-rt-frontend-jaxrs</artifactId>
@@ -156,12 +178,6 @@
             <artifactId>slf4j-simple</artifactId>
             <scope>provided</scope>
         </dependency>
-
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-test</artifactId>
-            <scope>test</scope>
-        </dependency>
         
     </dependencies>
 

http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-core/src/main/java/org/apache/gora/mapreduce/GoraMapReduceUtils.java
----------------------------------------------------------------------
diff --git a/gora-core/src/main/java/org/apache/gora/mapreduce/GoraMapReduceUtils.java b/gora-core/src/main/java/org/apache/gora/mapreduce/GoraMapReduceUtils.java
index ac90157..69545de 100644
--- a/gora-core/src/main/java/org/apache/gora/mapreduce/GoraMapReduceUtils.java
+++ b/gora-core/src/main/java/org/apache/gora/mapreduce/GoraMapReduceUtils.java
@@ -21,6 +21,8 @@ package org.apache.gora.mapreduce;
 import java.io.IOException;
 import java.util.List;
 
+import org.apache.gora.shims.hadoop.HadoopShim;
+import org.apache.gora.shims.hadoop.HadoopShimFactory;
 import org.apache.gora.util.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
@@ -36,6 +38,8 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
  */
 public class GoraMapReduceUtils {
 
+  private static HadoopShim hadoopShim = HadoopShimFactory.INSTANCE().getHadoopShim();
+
   public static class HelperInputFormat<K,V> extends FileInputFormat<K, V> {
     @Override
     public RecordReader<K, V> createRecordReader(InputSplit arg0,
@@ -75,11 +79,11 @@ public class GoraMapReduceUtils {
     throws IOException {
     
     if(inputPath != null) {
-      Job job = new Job(conf);
+      Job job = hadoopShim.createJob(conf);
       FileInputFormat.addInputPath(job, new Path(inputPath));
-      return new JobContext(job.getConfiguration(), null);
+      return hadoopShim.createJobContext(job.getConfiguration());
     } 
     
-    return new JobContext(conf, null);
+    return hadoopShim.createJobContext(conf);
   }
 }

http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-core/src/main/java/org/apache/gora/util/ReflectionUtils.java
----------------------------------------------------------------------
diff --git a/gora-core/src/main/java/org/apache/gora/util/ReflectionUtils.java b/gora-core/src/main/java/org/apache/gora/util/ReflectionUtils.java
index 00f94e4..a43dd64 100644
--- a/gora-core/src/main/java/org/apache/gora/util/ReflectionUtils.java
+++ b/gora-core/src/main/java/org/apache/gora/util/ReflectionUtils.java
@@ -20,8 +20,6 @@ package org.apache.gora.util;
 
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-
 import org.apache.avro.specific.SpecificRecordBuilderBase;
 import org.apache.gora.persistency.Persistent;
 
@@ -105,6 +103,7 @@ public class ReflectionUtils {
     return clazz.getField(fieldName).get(null);
   }
   
+  @SuppressWarnings("unchecked")
   public static <T extends Persistent> SpecificRecordBuilderBase<T> classBuilder(Class<T>
clazz) throws SecurityException
     , NoSuchMethodException, IllegalArgumentException, IllegalAccessException, InvocationTargetException
{
     return (SpecificRecordBuilderBase<T>) clazz.getMethod("newBuilder").invoke(null);

http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-shims-distribution/pom.xml
----------------------------------------------------------------------
diff --git a/gora-shims-distribution/pom.xml b/gora-shims-distribution/pom.xml
new file mode 100644
index 0000000..acff69a
--- /dev/null
+++ b/gora-shims-distribution/pom.xml
@@ -0,0 +1,79 @@
+     <!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor

+	license agreements. See the NOTICE file distributed with this work for additional 
+	information regarding copyright ownership. The ASF licenses this file to 
+	You under the Apache License, Version 2.0 (the "License"); you may not use 
+	this file except in compliance with the License. You may obtain a copy of 
+	the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required 
+	by applicable law or agreed to in writing, software distributed under the 
+	License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS 
+	OF ANY KIND, either express or implied. See the License for the specific 
+	language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+	<packaging>bundle</packaging>
+	<parent>
+		<groupId>org.apache.gora</groupId>
+		<artifactId>gora</artifactId>
+		<version>0.5-SNAPSHOT</version>
+	</parent>
+	<artifactId>gora-shims-distribution</artifactId>
+
+	<name>Apache Gora :: Shims Distribution</name>
+	<licenses>
+		<license>
+			<name>The Apache Software License, Version 2.0</name>
+			<url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+		</license>
+	</licenses>
+
+	<organization>
+		<name>The Apache Software Foundation</name>
+		<url>http://www.apache.org/</url>
+	</organization>
+
+	<build>
+		<plugins>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-shade-plugin</artifactId>
+				<version>2.3</version>
+				<executions>
+					<execution>
+						<phase>package</phase>
+						<goals>
+							<goal>shade</goal>
+						</goals>
+						<configuration>
+							<artifactSet>
+								<includes>
+									<include>${project.groupId}:gora-shims-hadoop</include>
+									<include>${project.groupId}:gora-shims-hadoop1</include>
+									<include>${project.groupId}:gora-shims-hadoop2</include>
+								</includes>
+							</artifactSet>
+						</configuration>
+					</execution>
+				</executions>
+			</plugin>
+		</plugins>
+	</build>
+
+	<dependencies>
+		<dependency>
+			<groupId>org.apache.gora</groupId>
+			<artifactId>gora-shims-hadoop1</artifactId>
+			<version>${project.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.gora</groupId>
+			<artifactId>gora-shims-hadoop2</artifactId>
+			<version>${project.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.gora</groupId>
+			<artifactId>gora-shims-hadoop</artifactId>
+			<version>${project.version}</version>
+		</dependency>
+	</dependencies>
+</project>

http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-shims-hadoop/pom.xml
----------------------------------------------------------------------
diff --git a/gora-shims-hadoop/pom.xml b/gora-shims-hadoop/pom.xml
new file mode 100644
index 0000000..a9f10ad
--- /dev/null
+++ b/gora-shims-hadoop/pom.xml
@@ -0,0 +1,38 @@
+     <!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor

+	license agreements. See the NOTICE file distributed with this work for additional 
+	information regarding copyright ownership. The ASF licenses this file to 
+	You under the Apache License, Version 2.0 (the "License"); you may not use 
+	this file except in compliance with the License. You may obtain a copy of 
+	the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required 
+	by applicable law or agreed to in writing, software distributed under the 
+	License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS 
+	OF ANY KIND, either express or implied. See the License for the specific 
+	language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+
+	<parent>
+		<groupId>org.apache.gora</groupId>
+		<artifactId>gora</artifactId>
+		<version>0.5-SNAPSHOT</version>
+	</parent>
+	<artifactId>gora-shims-hadoop</artifactId>
+	<packaging>bundle</packaging>
+
+	<name>Apache Gora :: Shims Hadoop</name>
+
+	<dependencies>
+		<dependency>
+			<groupId>junit</groupId>
+			<artifactId>junit</artifactId>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.hadoop</groupId>
+			<artifactId>hadoop-core</artifactId>
+			<version>${hadoop-1.version}</version>
+			<optional>true</optional>
+		</dependency>
+	</dependencies>
+</project>

http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-shims-hadoop/src/main/java/org/apache/gora/shims/hadoop/HadoopShim.java
----------------------------------------------------------------------
diff --git a/gora-shims-hadoop/src/main/java/org/apache/gora/shims/hadoop/HadoopShim.java
b/gora-shims-hadoop/src/main/java/org/apache/gora/shims/hadoop/HadoopShim.java
new file mode 100644
index 0000000..78d746a
--- /dev/null
+++ b/gora-shims-hadoop/src/main/java/org/apache/gora/shims/hadoop/HadoopShim.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.shims.hadoop;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+
+/**
+ * Provides version independent utility methods for interacting with Hadoop.
+ */
+public abstract class HadoopShim {
+
+  public HadoopShim() {
+    super();
+  }
+
+  /**
+   * Create a new Hadoop {@link Job}.
+   * 
+   * @param configuration
+   *            The job's configuration.
+   * @return A newly created instance backed by the provided configuration.
+   * @throws IOException
+   */
+  public abstract Job createJob(Configuration configuration)
+      throws IOException;
+
+  /**
+   * Create a new {@link JobContext} object.
+   * 
+   * @param configuration
+   *            The configuration for this job context.
+   * @return A newly created instance.
+   */
+  public abstract JobContext createJobContext(Configuration configuration);
+
+}

http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-shims-hadoop/src/main/java/org/apache/gora/shims/hadoop/HadoopShimFactory.java
----------------------------------------------------------------------
diff --git a/gora-shims-hadoop/src/main/java/org/apache/gora/shims/hadoop/HadoopShimFactory.java
b/gora-shims-hadoop/src/main/java/org/apache/gora/shims/hadoop/HadoopShimFactory.java
new file mode 100644
index 0000000..161f62e
--- /dev/null
+++ b/gora-shims-hadoop/src/main/java/org/apache/gora/shims/hadoop/HadoopShimFactory.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.shims.hadoop;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.util.VersionInfo;
+
+/**
+ * Factory to create {@link HadoopShim} objects.
+ */
+public class HadoopShimFactory {
+
+  /**
+   * Hadoop shim version mapping.
+   */
+  private static final Map<String, String> HADOOP_VERSION_TO_IMPL_MAP = new HashMap<String,
String>();
+
+  static {
+    HADOOP_VERSION_TO_IMPL_MAP.put("1",
+        "org.apache.gora.shims.hadoop1.HadoopShim1");
+    HADOOP_VERSION_TO_IMPL_MAP.put("2",
+        "org.apache.gora.shims.hadoop2.HadoopShim2");
+  }
+
+  // package private
+  static class Singleton {
+    public static final HadoopShimFactory INSTANCE = new HadoopShimFactory();
+  };
+
+  /**
+   * Access the {@link HadoopShimFactory} singleton.
+   * 
+   * @return the shared instance of {@link HadoopShimFactory}.
+   */
+  public static HadoopShimFactory INSTANCE() {
+    return Singleton.INSTANCE;
+  }
+
+  /**
+   * Get the Hadoop shim for the Hadoop version on the class path. In case it
+   * fails to obtain an appropriate shim (i.e. unsupported Hadoop version), it
+   * throws a {@link RuntimeException}.
+   * 
+   * Note that this method is potentially costly.
+   * 
+   * @return A newly created instance of a {@link HadoopShim}.
+   */
+  public HadoopShim getHadoopShim() {
+    String version = getMajorVersion();
+    String className = HADOOP_VERSION_TO_IMPL_MAP.get(version);
+
+    try {
+      Class<?> class1 = Class.forName(className);
+      return HadoopShim.class.cast(class1.newInstance());
+    } catch (Exception e) {
+      throw new RuntimeException(
+          "Could not load Hadoop shim for version " + version
+          + ", className=" + className, e);
+    }
+  }
+
+  /**
+   * Get the Hadoop major version number.
+   * 
+   * @return The major version number of Hadoop.
+   */
+  public String getMajorVersion() {
+    String vers = VersionInfo.getVersion();
+
+    String[] parts = vers.split("\\.");
+    if (parts.length < 2) {
+      throw new RuntimeException("Unable to parse Hadoop version: "
+          + vers + " (expected X.Y.* format)");
+    }
+    return parts[0];
+
+  }
+
+  // package private
+  HadoopShimFactory() {
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-shims-hadoop/src/test/java/org/apache/gora/shims/hadoop/HadoopShimTest.java
----------------------------------------------------------------------
diff --git a/gora-shims-hadoop/src/test/java/org/apache/gora/shims/hadoop/HadoopShimTest.java
b/gora-shims-hadoop/src/test/java/org/apache/gora/shims/hadoop/HadoopShimTest.java
new file mode 100644
index 0000000..762d124
--- /dev/null
+++ b/gora-shims-hadoop/src/test/java/org/apache/gora/shims/hadoop/HadoopShimTest.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.shims.hadoop;
+
+import org.junit.Test;
+import static org.junit.Assert.*;
+
+public class HadoopShimTest {
+
+  @Test
+  public void testGetMajorVersion() {
+    String version = HadoopShimFactory.INSTANCE().getMajorVersion();
+    assertNotNull(version);
+    assertTrue(version.length() > 0);
+  }
+
+  @Test
+  public void testInstance() {
+    assertNotNull(HadoopShimFactory.INSTANCE());
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-shims-hadoop1/pom.xml
----------------------------------------------------------------------
diff --git a/gora-shims-hadoop1/pom.xml b/gora-shims-hadoop1/pom.xml
new file mode 100644
index 0000000..c798f18
--- /dev/null
+++ b/gora-shims-hadoop1/pom.xml
@@ -0,0 +1,49 @@
+     <!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor

+	license agreements. See the NOTICE file distributed with this work for additional 
+	information regarding copyright ownership. The ASF licenses this file to 
+	You under the Apache License, Version 2.0 (the "License"); you may not use 
+	this file except in compliance with the License. You may obtain a copy of 
+	the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required 
+	by applicable law or agreed to in writing, software distributed under the 
+	License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS 
+	OF ANY KIND, either express or implied. See the License for the specific 
+	language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+
+	<parent>
+		<groupId>org.apache.gora</groupId>
+		<artifactId>gora</artifactId>
+		<version>0.5-SNAPSHOT</version>
+	</parent>
+	<artifactId>gora-shims-hadoop1</artifactId>
+	<packaging>bundle</packaging>
+
+	<name>Apache Gora :: Shims Hadoop 1.x</name>
+
+	<dependencies>
+		<dependency>
+			<groupId>junit</groupId>
+			<artifactId>junit</artifactId>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.hadoop</groupId>
+			<artifactId>hadoop-core</artifactId>
+			<version>${hadoop-1.version}</version>
+			<optional>true</optional>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.hadoop</groupId>
+			<artifactId>hadoop-test</artifactId>
+			<version>${hadoop-1.version}</version>
+			<optional>true</optional>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.gora</groupId>
+			<artifactId>gora-shims-hadoop</artifactId>
+			<version>${project.version}</version>
+		</dependency>
+	</dependencies>
+</project>

http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-shims-hadoop1/src/main/java/org/apache/gora/shims/hadoop1/HadoopShim1.java
----------------------------------------------------------------------
diff --git a/gora-shims-hadoop1/src/main/java/org/apache/gora/shims/hadoop1/HadoopShim1.java
b/gora-shims-hadoop1/src/main/java/org/apache/gora/shims/hadoop1/HadoopShim1.java
new file mode 100644
index 0000000..9728af6
--- /dev/null
+++ b/gora-shims-hadoop1/src/main/java/org/apache/gora/shims/hadoop1/HadoopShim1.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.shims.hadoop1;
+
+import java.io.IOException;
+
+import org.apache.gora.shims.hadoop.HadoopShim;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+
+/**
+ * Provides shim implementation for Hadoop 1.x.
+ */
+public class HadoopShim1 extends HadoopShim {
+
+  public HadoopShim1() {
+    super();
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  public Job createJob(Configuration configuration) throws IOException {
+    return new Job(configuration);
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  public JobContext createJobContext(Configuration configuration) {
+    return new JobContext(configuration, null);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-shims-hadoop1/src/test/java/org/apache/gora/shims/hadoop1/TestHadoopShim1.java
----------------------------------------------------------------------
diff --git a/gora-shims-hadoop1/src/test/java/org/apache/gora/shims/hadoop1/TestHadoopShim1.java
b/gora-shims-hadoop1/src/test/java/org/apache/gora/shims/hadoop1/TestHadoopShim1.java
new file mode 100644
index 0000000..f0f1533
--- /dev/null
+++ b/gora-shims-hadoop1/src/test/java/org/apache/gora/shims/hadoop1/TestHadoopShim1.java
@@ -0,0 +1,18 @@
+package org.apache.gora.shims.hadoop1;
+
+import org.apache.gora.shims.hadoop.HadoopShim;
+import org.apache.gora.shims.hadoop.HadoopShimFactory;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class TestHadoopShim1 {
+
+    @Test
+    public void testCorrectVersion() {
+        HadoopShim shim = HadoopShimFactory.INSTANCE().getHadoopShim();
+
+        assertSame(shim.getClass(), HadoopShim1.class);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-shims-hadoop2/pom.xml
----------------------------------------------------------------------
diff --git a/gora-shims-hadoop2/pom.xml b/gora-shims-hadoop2/pom.xml
new file mode 100644
index 0000000..1ff9776
--- /dev/null
+++ b/gora-shims-hadoop2/pom.xml
@@ -0,0 +1,43 @@
+     <!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor

+	license agreements. See the NOTICE file distributed with this work for additional 
+	information regarding copyright ownership. The ASF licenses this file to 
+	You under the Apache License, Version 2.0 (the "License"); you may not use 
+	this file except in compliance with the License. You may obtain a copy of 
+	the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required 
+	by applicable law or agreed to in writing, software distributed under the 
+	License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS 
+	OF ANY KIND, either express or implied. See the License for the specific 
+	language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+
+	<parent>
+		<groupId>org.apache.gora</groupId>
+		<artifactId>gora</artifactId>
+		<version>0.5-SNAPSHOT</version>
+	</parent>
+	<artifactId>gora-shims-hadoop2</artifactId>
+	<packaging>bundle</packaging>
+
+	<name>Apache Gora :: Shims Hadoop 2.x</name>
+
+	<dependencies>
+		<dependency>
+			<groupId>junit</groupId>
+			<artifactId>junit</artifactId>
+			<scope>test</scope>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.hadoop</groupId>
+			<artifactId>hadoop-client</artifactId>
+			<version>${hadoop-2.version}</version>
+			<optional>true</optional>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.gora</groupId>
+			<artifactId>gora-shims-hadoop</artifactId>
+			<version>${project.version}</version>
+		</dependency>
+	</dependencies>
+</project>

http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-shims-hadoop2/src/main/java/org/apache/gora/shims/hadoop2/HadoopShim2.java
----------------------------------------------------------------------
diff --git a/gora-shims-hadoop2/src/main/java/org/apache/gora/shims/hadoop2/HadoopShim2.java
b/gora-shims-hadoop2/src/main/java/org/apache/gora/shims/hadoop2/HadoopShim2.java
new file mode 100644
index 0000000..527ff36
--- /dev/null
+++ b/gora-shims-hadoop2/src/main/java/org/apache/gora/shims/hadoop2/HadoopShim2.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gora.shims.hadoop2;
+
+import java.io.IOException;
+
+import org.apache.gora.shims.hadoop.HadoopShim;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.task.JobContextImpl;
+
+/**
+ * Shim for Hadoop 2.x.
+ */
+public class HadoopShim2 extends HadoopShim {
+
+  public HadoopShim2() {
+    super();
+  }
+
+  /**
+   * {@inheritDoc}
+   * 
+   * Use the Hadoop 2.x way of creating a {@link Job} object.
+   */
+  public Job createJob(Configuration configuration) throws IOException {
+    Job instance = Job.getInstance(configuration);
+    return instance;
+  }
+
+  /**
+   * {@inheritDoc}
+   * 
+   * Use the Hadoop 2.x way of creating a {@link JobContext} object.
+   */
+  public JobContext createJobContext(Configuration configuration) {
+    return new JobContextImpl(configuration, null);
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-shims-hadoop2/src/test/java/org/apache/gora/shims/hadoop2/TestHadoopShim2.java
----------------------------------------------------------------------
diff --git a/gora-shims-hadoop2/src/test/java/org/apache/gora/shims/hadoop2/TestHadoopShim2.java
b/gora-shims-hadoop2/src/test/java/org/apache/gora/shims/hadoop2/TestHadoopShim2.java
new file mode 100644
index 0000000..e97004d
--- /dev/null
+++ b/gora-shims-hadoop2/src/test/java/org/apache/gora/shims/hadoop2/TestHadoopShim2.java
@@ -0,0 +1,18 @@
+package org.apache.gora.shims.hadoop2;
+
+import org.apache.gora.shims.hadoop.HadoopShim;
+import org.apache.gora.shims.hadoop.HadoopShimFactory;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+public class TestHadoopShim2 {
+
+    @Test
+    public void testCorrectVersion() {
+        HadoopShim shim = HadoopShimFactory.INSTANCE().getHadoopShim();
+
+        assertSame(shim.getClass(), HadoopShim2.class);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/gora-tutorial/pom.xml
----------------------------------------------------------------------
diff --git a/gora-tutorial/pom.xml b/gora-tutorial/pom.xml
index 3461fba..cf97905 100644
--- a/gora-tutorial/pom.xml
+++ b/gora-tutorial/pom.xml
@@ -117,9 +117,10 @@
         </dependency-->
 
 	<!-- Hadoop Dependencies -->
-	<dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-core</artifactId>
+        <dependency>
+          <groupId>org.apache.hadoop</groupId>
+          <artifactId>hadoop-common</artifactId>
+          <version>${hadoop-2.version}</version>
         </dependency>
         
         <dependency>

http://git-wip-us.apache.org/repos/asf/gora/blob/b51cdeea/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 3362329..c5ed7d8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -573,12 +573,17 @@
 		<module>gora-core</module>
 		<module>gora-accumulo</module>
 		<module>gora-cassandra</module>
-                <module>gora-goraci</module>
+		<module>gora-goraci</module>
+
 		<module>gora-hbase</module>
 		<!-- module>gora-lucene</module -->
 		<!--module>gora-dynamodb</module -->
 		<!--module>gora-sql</module -->
 		<module>gora-mongodb</module>
+		<module>gora-shims-hadoop</module>
+		<module>gora-shims-hadoop1</module>
+		<module>gora-shims-hadoop2</module>
+		<module>gora-shims-distribution</module>
 		<module>gora-solr</module>
 		<module>gora-tutorial</module>
 		<module>sources-dist</module>
@@ -591,7 +596,8 @@
 		<avro.version>1.7.6</avro.version>
 		<jackson.version>1.6.9</jackson.version>
 		<!-- Hadoop Dependencies -->
-		<hadoop.version>1.0.1</hadoop.version>
+		<hadoop-1.version>1.0.1</hadoop-1.version>
+		<hadoop-2.version>2.4.0</hadoop-2.version>
 		<hadoop.test.version>1.0.1</hadoop.test.version>
 		<hbase.version>0.94.14</hbase.version>
 		<cxf-rt-frontend-jaxrs.version>2.5.2</cxf-rt-frontend-jaxrs.version>
@@ -766,7 +772,7 @@
 			<dependency>
 				<groupId>org.apache.hadoop</groupId>
 				<artifactId>hadoop-core</artifactId>
-				<version>${hadoop.version}</version>
+				<version>${hadoop-1.version}</version>
 				<exclusions>
 					<!-- jackson is conflicting with the Avro dep -->
 					<exclusion>
@@ -795,6 +801,23 @@
 					</exclusion>
 				</exclusions>
 			</dependency>
+			<dependency>
+				<groupId>org.apache.hadoop</groupId>
+				<artifactId>hadoop-common</artifactId>
+				<version>${hadoop-2.version}</version>
+				<classifier>tests</classifier>
+			</dependency>
+			<dependency>
+				<groupId>org.apache.hadoop</groupId>
+				<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+				<version>${hadoop-2.version}</version>
+				<classifier>tests</classifier>
+			</dependency>
+			<dependency>
+				<groupId>org.apache.hadoop</groupId>
+				<artifactId>hadoop-mapreduce-client-core</artifactId>
+				<version>${hadoop-2.version}</version>
+			</dependency>
 
 			<dependency>
 				<groupId>org.apache.cxf</groupId>


Mime
View raw message