tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hit...@apache.org
Subject tez git commit: TEZ-2977. Make HadoopShim selection be overridable for distro-specific implementations. (hitesh)
Date Sat, 12 Dec 2015 00:35:11 GMT
Repository: tez
Updated Branches:
  refs/heads/master 7b16b77aa -> 7940facf6


TEZ-2977. Make HadoopShim selection be overridable for distro-specific implementations. (hitesh)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/7940facf
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/7940facf
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/7940facf

Branch: refs/heads/master
Commit: 7940facf676087eab0301d6ada96d0b6576ca693
Parents: 7b16b77
Author: Hitesh Shah <hitesh@apache.org>
Authored: Fri Dec 11 16:34:16 2015 -0800
Committer: Hitesh Shah <hitesh@apache.org>
Committed: Fri Dec 11 16:34:55 2015 -0800

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 hadoop-shim-impls/hadoop-shim-2.4/pom.xml       |  6 ++
 .../shim/TestHadoop23_24ShimProvider.java       | 44 ++++++++++++++
 hadoop-shim-impls/hadoop-shim-2.6/pom.xml       |  6 ++
 .../shim/TestHadoop25_26_27ShimProvider.java    | 42 +++++++++++++
 hadoop-shim-impls/hadoop-shim-2.8/pom.xml       |  6 ++
 .../hadoop/shim/TestHadoopShim28Provider.java   | 42 +++++++++++++
 .../tez/hadoop/shim/HadoopShimsLoader.java      | 63 +++++++++++++++++++-
 .../tez/hadoop/shim/DummyShimProvider.java      | 30 ++++++++++
 .../tez/hadoop/shim/TestHadoopShimsLoader.java  | 54 +++++++++++++++++
 hadoop-shim/src/test/resources/log4j.properties | 19 ++++++
 pom.xml                                         |  6 ++
 tez-dag/findbugs-exclude.xml                    |  1 +
 .../org/apache/tez/dag/app/DAGAppMaster.java    |  6 +-
 14 files changed, 322 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index b19c4be..1e1537f 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -10,6 +10,7 @@ INCOMPATIBLE CHANGES
   TEZ-604. Revert temporary changes made in TEZ-603 to kill the provided tez session, if
running a MapReduce job.
 
 ALL CHANGES:
+  TEZ-2977. Make HadoopShim selection be overridable for distro-specific implementations.
   TEZ-2472. Change slf4j version to 1.7.10.
   TEZ-2920. org.apache.tez.client.TestTezClient.testStopRetriesUntilTimeout is flaky.
   TEZ-2824. Add javadocs for Vertex.setConf and DAG.setConf.

http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim-impls/hadoop-shim-2.4/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.4/pom.xml b/hadoop-shim-impls/hadoop-shim-2.4/pom.xml
index 095c2b1..3ffe4e2 100644
--- a/hadoop-shim-impls/hadoop-shim-2.4/pom.xml
+++ b/hadoop-shim-impls/hadoop-shim-2.4/pom.xml
@@ -38,6 +38,12 @@
       <artifactId>hadoop-shim</artifactId>
     </dependency>
     <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>hadoop-shim</artifactId>
+      <scope>test</scope>
+      <type>test-jar</type>
+    </dependency>
+    <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-yarn-api</artifactId>
     </dependency>

http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java
b/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java
index 9bc4435..871fb67 100644
--- a/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java
+++ b/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java
@@ -18,6 +18,8 @@
 
 package org.apache.tez.hadoop.shim;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.hadoop.shim.DummyShimProvider.DummyShim;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -33,6 +35,48 @@ public class TestHadoop23_24ShimProvider {
     Assert.assertNull(provider.createHadoopShim("foo", 3, 3));
     Assert.assertNotNull(provider.createHadoopShim("foo", 2, 3));
     Assert.assertNotNull(provider.createHadoopShim("foo", 2, 4));
+
+    Assert.assertEquals(HadoopShim24.class,
+        provider.createHadoopShim("foo", 2, 3).getClass());
+
+  }
+
+  @Test
+  public void testLoaderOverride() {
+    Configuration conf = new Configuration(false);
+    // Set shim and version to ensure that hadoop version from the build does not create
+    // a mismatch
+    conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS,
+        HadoopShim23_24Provider.class.getName());
+    conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.4.0");
+    HadoopShimsLoader loader = new HadoopShimsLoader(conf, true);
+    HadoopShim shim = loader.getHadoopShim();
+    Assert.assertNotNull(shim);
+    Assert.assertEquals(HadoopShim24.class, shim.getClass());
+  }
+
+  @Test
+  public void testInvalidVersion() {
+    Configuration conf = new Configuration(false);
+    // Set incompatible version so that shim in this module does not match
+    conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.6.0");
+    HadoopShimsLoader loader = new HadoopShimsLoader(conf, true);
+    HadoopShim shim = loader.getHadoopShim();
+    Assert.assertNotNull(shim);
+    Assert.assertEquals(DefaultHadoopShim.class, shim.getClass());
+  }
+
+  @Test
+  public void testLoaderOverrideInvalidVersion() {
+    Configuration conf = new Configuration(false);
+    // Set incompatible version so that override shim does not return a valid shim
+    conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS,
+        HadoopShim23_24Provider.class.getName());
+    conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.1.0");
+    HadoopShimsLoader loader = new HadoopShimsLoader(conf, true);
+    HadoopShim shim = loader.getHadoopShim();
+    Assert.assertNotNull(shim);
+    Assert.assertEquals(DefaultHadoopShim.class, shim.getClass());
   }
 
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim-impls/hadoop-shim-2.6/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.6/pom.xml b/hadoop-shim-impls/hadoop-shim-2.6/pom.xml
index 17f4711..8346c37 100644
--- a/hadoop-shim-impls/hadoop-shim-2.6/pom.xml
+++ b/hadoop-shim-impls/hadoop-shim-2.6/pom.xml
@@ -38,6 +38,12 @@
       <artifactId>hadoop-shim</artifactId>
     </dependency>
     <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>hadoop-shim</artifactId>
+      <scope>test</scope>
+      <type>test-jar</type>
+    </dependency>
+    <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-yarn-api</artifactId>
     </dependency>

http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java
b/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java
index 0e6d997..13faf37 100644
--- a/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java
+++ b/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java
@@ -18,6 +18,7 @@
 
 package org.apache.tez.hadoop.shim;
 
+import org.apache.hadoop.conf.Configuration;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -34,6 +35,47 @@ public class TestHadoop25_26_27ShimProvider {
     Assert.assertNotNull(provider.createHadoopShim("foo", 2, 5));
     Assert.assertNotNull(provider.createHadoopShim("foo", 2, 6));
     Assert.assertNotNull(provider.createHadoopShim("foo", 2, 7));
+
+    Assert.assertEquals(HadoopShim26.class,
+        provider.createHadoopShim("foo", 2, 7).getClass());
+  }
+
+  @Test
+  public void testLoaderOverride() {
+    Configuration conf = new Configuration(false);
+    // Set shim and version to ensure that hadoop version from the build does not create
+    // a mismatch
+    conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS,
+        HadoopShim25_26_27Provider.class.getName());
+    conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.7.0");
+    HadoopShimsLoader loader = new HadoopShimsLoader(conf, true);
+    HadoopShim shim = loader.getHadoopShim();
+    Assert.assertNotNull(shim);
+    Assert.assertEquals(HadoopShim26.class, shim.getClass());
+  }
+
+  @Test
+  public void testInvalidVersion() {
+    Configuration conf = new Configuration(false);
+    // Set incompatible version so that shim in this module does not match
+    conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.9.0");
+    HadoopShimsLoader loader = new HadoopShimsLoader(conf, true);
+    HadoopShim shim = loader.getHadoopShim();
+    Assert.assertNotNull(shim);
+    Assert.assertEquals(DefaultHadoopShim.class, shim.getClass());
+  }
+
+  @Test
+  public void testLoaderOverrideInvalidVersion() {
+    Configuration conf = new Configuration(false);
+    // Set incompatible version so that override shim does not return a valid shim
+    conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS,
+        HadoopShim25_26_27Provider.class.getName());
+    conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.1.0");
+    HadoopShimsLoader loader = new HadoopShimsLoader(conf, true);
+    HadoopShim shim = loader.getHadoopShim();
+    Assert.assertNotNull(shim);
+    Assert.assertEquals(DefaultHadoopShim.class, shim.getClass());
   }
 
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim-impls/hadoop-shim-2.8/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.8/pom.xml b/hadoop-shim-impls/hadoop-shim-2.8/pom.xml
index 18d22c5..41928fe 100644
--- a/hadoop-shim-impls/hadoop-shim-2.8/pom.xml
+++ b/hadoop-shim-impls/hadoop-shim-2.8/pom.xml
@@ -38,6 +38,12 @@
       <artifactId>hadoop-shim</artifactId>
     </dependency>
     <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>hadoop-shim</artifactId>
+      <scope>test</scope>
+      <type>test-jar</type>
+    </dependency>
+    <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-yarn-api</artifactId>
     </dependency>

http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java
b/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java
index df92627..353d590 100644
--- a/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java
+++ b/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java
@@ -18,6 +18,7 @@
 
 package org.apache.tez.hadoop.shim;
 
+import org.apache.hadoop.conf.Configuration;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -33,6 +34,47 @@ public class TestHadoopShim28Provider {
     Assert.assertNull(provider.createHadoopShim("foo", 3, 3));
     Assert.assertNotNull(provider.createHadoopShim("foo", 2, 8));
     Assert.assertNotNull(provider.createHadoopShim("foo", 2, 111));
+
+    Assert.assertEquals(HadoopShim28.class,
+        provider.createHadoopShim("foo", 2, 9).getClass());
+  }
+
+  @Test
+  public void testLoaderOverride() {
+    Configuration conf = new Configuration(false);
+    // Set shim and version to ensure that hadoop version from the build does not create
+    // a mismatch
+    conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS,
+        HadoopShim28Provider.class.getName());
+    conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.8.0");
+    HadoopShimsLoader loader = new HadoopShimsLoader(conf, true);
+    HadoopShim shim = loader.getHadoopShim();
+    Assert.assertNotNull(shim);
+    Assert.assertEquals(HadoopShim28.class, shim.getClass());
+  }
+
+  @Test
+  public void testInvalidVersion() {
+    Configuration conf = new Configuration(false);
+    // Set incompatible version so that shim in this module does not match
+    conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.2.0");
+    HadoopShimsLoader loader = new HadoopShimsLoader(conf, true);
+    HadoopShim shim = loader.getHadoopShim();
+    Assert.assertNotNull(shim);
+    Assert.assertEquals(DefaultHadoopShim.class, shim.getClass());
+  }
+
+  @Test
+  public void testLoaderOverrideInvalidVersion() {
+    Configuration conf = new Configuration(false);
+    // Set incompatible version so that override shim does not return a valid shim
+    conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS,
+        HadoopShim28Provider.class.getName());
+    conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE, "2.1.0");
+    HadoopShimsLoader loader = new HadoopShimsLoader(conf, true);
+    HadoopShim shim = loader.getHadoopShim();
+    Assert.assertNotNull(shim);
+    Assert.assertEquals(DefaultHadoopShim.class, shim.getClass());
   }
 
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java
index fc2ec81..9d9dc3f 100644
--- a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java
+++ b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java
@@ -22,10 +22,13 @@ import java.util.ServiceLoader;
 import java.util.StringTokenizer;
 
 import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.util.VersionInfo;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.common.annotations.VisibleForTesting;
+
 @Private
 public class HadoopShimsLoader {
 
@@ -37,17 +40,57 @@ public class HadoopShimsLoader {
   private final HadoopShim currentShim;
   private final HadoopShimProvider currentShimProvider;
 
-  public HadoopShimsLoader() {
+  public static final String TEZ_HADOOP_SHIM_PROVIDER_CLASS =
+      "tez.hadoop.shim.provider.class";
+
+  // A way to override the hadoop version for testing
+  @Private
+  @VisibleForTesting
+  static final String TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE =
+      "tez.hadoop.shim.hadoop.version.override";
+
+  public HadoopShimsLoader(Configuration conf) {
+    this(conf, false);
+  }
+
+  @VisibleForTesting
+  @SuppressWarnings("unchecked")
+  HadoopShimsLoader(Configuration conf, boolean useReflection) {
+    String overrideProviderClassStr = conf.get(TEZ_HADOOP_SHIM_PROVIDER_CLASS);
+    if (overrideProviderClassStr != null && !overrideProviderClassStr.isEmpty())
{
+      LOG.info("HadoopShim Selection is overridden, using Configured Provider="
+          + overrideProviderClassStr);
+    } else {
+      overrideProviderClassStr = null;
+    }
+
     String versionStr = VersionInfo.getVersion();
+    String overrideVersionStr = conf.get(TEZ_HADOOP_SHIM_HADOOP_VERSION_OVERRIDE);
+    if (overrideVersionStr != null && !overrideVersionStr.isEmpty()) {
+      LOG.warn("Using overridden hadoop version instead of actual version"
+          + ", realVersion=" + versionStr
+          + ", overrideVersion=" + overrideVersionStr);
+      versionStr = overrideVersionStr;
+    }
+
     Version version = new Version(versionStr);
     HadoopShim selectedShim = null;
     HadoopShimProvider selectedShimProvider = null;
+
     LOG.info("Trying to locate HadoopShimProvider for "
         + "hadoopVersion=" + versionStr
         + ", majorVersion=" + version.majorVersion
         + ", minorVersion=" + version.minorVersion);
     synchronized (shimLoader) {
       for (HadoopShimProvider provider : shimLoader) {
+        if (overrideProviderClassStr != null
+            && !provider.getClass().getName().equals(overrideProviderClassStr)) {
+          LOG.debug("Skipping HadoopShimProvider : "
+              + provider.getClass().getName()
+              + " as config provided to override selection");
+          continue;
+        }
+
         LOG.debug("Trying HadoopShimProvider : "
             + provider.getClass().getName());
         HadoopShim hadoopShim = null;
@@ -70,6 +113,21 @@ public class HadoopShimsLoader {
               + " due to error: ", e);
         }
       }
+      if (selectedShim == null && useReflection && overrideProviderClassStr
!= null) {
+        try {
+          LOG.debug("Using Reflection to create HadoopShim from provider class="
+              + overrideProviderClassStr);
+          Class<HadoopShimProvider> clazz = (Class<HadoopShimProvider>)Class.forName(
+              overrideProviderClassStr, true, Thread.currentThread().getContextClassLoader());
+          selectedShimProvider = clazz.newInstance();
+          selectedShim = selectedShimProvider.createHadoopShim(versionStr,
+              version.majorVersion, version.minorVersion);
+        } catch (Exception e) {
+          throw new RuntimeException("Unable to create HadoopShim from provider class: "
+              + overrideProviderClassStr, e);
+        }
+      }
+
       if (selectedShim == null) {
         currentShim = new DefaultHadoopShim();
         currentShimProvider = null;
@@ -80,7 +138,8 @@ public class HadoopShimsLoader {
     }
     LOG.info("Picked HadoopShim " + currentShim.getClass().getName()
         + ", providerName="
-        + (currentShimProvider != null ? currentShimProvider.getClass().getName() : "null"
)
+        + (currentShimProvider != null ? currentShimProvider.getClass().getName() : "null")
+        + ", overrideProviderViaConfig=" + overrideProviderClassStr
         + ", hadoopVersion=" + versionStr
         + ", majorVersion=" + version.majorVersion
         + ", minorVersion=" + version.minorVersion);

http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim/src/test/java/org/apache/tez/hadoop/shim/DummyShimProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-shim/src/test/java/org/apache/tez/hadoop/shim/DummyShimProvider.java b/hadoop-shim/src/test/java/org/apache/tez/hadoop/shim/DummyShimProvider.java
new file mode 100644
index 0000000..7b72c7a
--- /dev/null
+++ b/hadoop-shim/src/test/java/org/apache/tez/hadoop/shim/DummyShimProvider.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+public class DummyShimProvider extends HadoopShimProvider {
+
+  public static class DummyShim extends HadoopShim {
+  }
+
+  @Override
+  public HadoopShim createHadoopShim(String hadoopVersion, int majorVersion, int minorVersion)
{
+    return new DummyShim();
+  }
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShimsLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-shim/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShimsLoader.java
b/hadoop-shim/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShimsLoader.java
new file mode 100644
index 0000000..b0215ed
--- /dev/null
+++ b/hadoop-shim/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShimsLoader.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.hadoop.shim.DummyShimProvider.DummyShim;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestHadoopShimsLoader {
+
+  @Test
+  public void testBasicLoader() {
+    HadoopShimsLoader loader = new HadoopShimsLoader(new Configuration(false));
+    HadoopShim shim = loader.getHadoopShim();
+    Assert.assertNotNull(shim);
+    Assert.assertEquals(DefaultHadoopShim.class, shim.getClass());
+  }
+
+  @Test
+  public void testLoaderOverride() {
+    Configuration conf = new Configuration(false);
+    conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS, DummyShimProvider.class.getName());
+    HadoopShimsLoader loader = new HadoopShimsLoader(conf, true);
+    HadoopShim shim = loader.getHadoopShim();
+    Assert.assertNotNull(shim);
+    Assert.assertEquals(DummyShim.class, shim.getClass());
+  }
+
+  @Test(expected = RuntimeException.class)
+  public void testInvalidOverride() {
+    Configuration conf = new Configuration(false);
+    conf.set(HadoopShimsLoader.TEZ_HADOOP_SHIM_PROVIDER_CLASS, "org.apache.tez.foo");
+    HadoopShimsLoader loader = new HadoopShimsLoader(conf, true);
+    HadoopShim shim = loader.getHadoopShim();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/hadoop-shim/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-shim/src/test/resources/log4j.properties b/hadoop-shim/src/test/resources/log4j.properties
new file mode 100644
index 0000000..63353bd
--- /dev/null
+++ b/hadoop-shim/src/test/resources/log4j.properties
@@ -0,0 +1,19 @@
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=debug,stdout
+log4j.threshhold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) -
%m%n

http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ac449e2..7759de5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -108,6 +108,12 @@
       </dependency>
       <dependency>
         <groupId>org.apache.tez</groupId>
+        <artifactId>hadoop-shim</artifactId>
+        <version>${project.version}</version>
+        <type>test-jar</type>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.tez</groupId>
         <artifactId>tez-api</artifactId>
         <version>${project.version}</version>
       </dependency>

http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/tez-dag/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/tez-dag/findbugs-exclude.xml b/tez-dag/findbugs-exclude.xml
index 2122c5e..43046d9 100644
--- a/tez-dag/findbugs-exclude.xml
+++ b/tez-dag/findbugs-exclude.xml
@@ -161,6 +161,7 @@
       <Field name="nodes"/>
       <Field name="recoveryEnabled"/>
       <Field name="isLocal"/>
+      <Field name="hadoopShim"/>
     </Or>
     <Bug pattern="IS2_INCONSISTENT_SYNC"/>
   </Match>

http://git-wip-us.apache.org/repos/asf/tez/blob/7940facf/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index 05261ba..5cbdab4 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -240,7 +240,7 @@ public class DAGAppMaster extends AbstractService {
   private final String[] localDirs;
   private final String[] logDirs;
   private final AMPluginDescriptorProto amPluginDescriptorProto;
-  private final HadoopShim hadoopShim;
+  private HadoopShim hadoopShim;
   private ContainerSignatureMatcher containerSignatureMatcher;
   private AMContainerMap containers;
   private AMNodeTracker nodes;
@@ -360,7 +360,6 @@ public class DAGAppMaster extends AbstractService {
 
     this.containerLogs = getRunningLogURL(this.nmHost + ":" + this.nmHttpPort,
         this.containerID.toString(), this.appMasterUgi.getShortUserName());
-    this.hadoopShim = new HadoopShimsLoader().getHadoopShim();
 
     LOG.info("Created DAGAppMaster for application " + applicationAttemptId
         + ", versionInfo=" + dagVersionInfo.toString());
@@ -417,6 +416,9 @@ public class DAGAppMaster extends AbstractService {
 
     this.amConf = conf;
     initResourceCalculatorPlugins();
+    this.hadoopShim = new HadoopShimsLoader(this.amConf).getHadoopShim();
+
+
     this.isLocal = conf.getBoolean(TezConfiguration.TEZ_LOCAL_MODE,
         TezConfiguration.TEZ_LOCAL_MODE_DEFAULT);
 


Mime
View raw message