tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hit...@apache.org
Subject tez git commit: TEZ_2928. Framework for Hadoop shims. (hitesh)
Date Sat, 14 Nov 2015 00:28:10 GMT
Repository: tez
Updated Branches:
  refs/heads/master 24ba80f81 -> 40e864d14


TEZ_2928. Framework for Hadoop shims. (hitesh)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/40e864d1
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/40e864d1
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/40e864d1

Branch: refs/heads/master
Commit: 40e864d145458ce94588b90f065204d80ffa1cc7
Parents: 24ba80f
Author: Hitesh Shah <hitesh@apache.org>
Authored: Fri Nov 13 16:27:47 2015 -0800
Committer: Hitesh Shah <hitesh@apache.org>
Committed: Fri Nov 13 16:27:47 2015 -0800

----------------------------------------------------------------------
 BUILDING.txt                                    |   3 +
 CHANGES.txt                                     |   1 +
 hadoop-shim-impls/findbugs-exclude.xml          |  16 +++
 .../hadoop-shim-2.4/findbugs-exclude.xml        |  16 +++
 hadoop-shim-impls/hadoop-shim-2.4/pom.xml       |  50 ++++++++
 .../hadoop/shim/HadoopShim23_24Provider.java    |  33 +++++
 .../apache/tez/hadoop/shim/HadoopShim24.java    |  45 +++++++
 ...rg.apache.tez.hadoop.shim.HadoopShimProvider |  14 +++
 .../shim/TestHadoop23_24ShimProvider.java       |  38 ++++++
 .../hadoop-shim-2.6/findbugs-exclude.xml        |  16 +++
 hadoop-shim-impls/hadoop-shim-2.6/pom.xml       |  50 ++++++++
 .../hadoop/shim/HadoopShim25_26_27Provider.java |  33 +++++
 .../apache/tez/hadoop/shim/HadoopShim26.java    |  52 ++++++++
 ...rg.apache.tez.hadoop.shim.HadoopShimProvider |  14 +++
 .../shim/TestHadoop25_26_27ShimProvider.java    |  39 ++++++
 .../hadoop-shim-2.8/findbugs-exclude.xml        |  16 +++
 hadoop-shim-impls/hadoop-shim-2.8/pom.xml       |  50 ++++++++
 .../apache/tez/hadoop/shim/HadoopShim28.java    |  52 ++++++++
 .../tez/hadoop/shim/HadoopShim28Provider.java   |  30 +++++
 ...rg.apache.tez.hadoop.shim.HadoopShimProvider |  14 +++
 .../hadoop/shim/TestHadoopShim28Provider.java   |  38 ++++++
 hadoop-shim-impls/pom.xml                       |  58 +++++++++
 hadoop-shim/findbugs-exclude.xml                |  16 +++
 hadoop-shim/pom.xml                             |  46 +++++++
 .../tez/hadoop/shim/DefaultHadoopShim.java      |  28 +++++
 .../org/apache/tez/hadoop/shim/HadoopShim.java  |  60 +++++++++
 .../tez/hadoop/shim/HadoopShimProvider.java     |  38 ++++++
 .../tez/hadoop/shim/HadoopShimsLoader.java      | 126 +++++++++++++++++++
 pom.xml                                         |   7 ++
 tez-dag/pom.xml                                 |   4 +
 .../java/org/apache/tez/dag/app/AppContext.java |   3 +
 .../org/apache/tez/dag/app/DAGAppMaster.java    |  10 ++
 32 files changed, 1016 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/BUILDING.txt
----------------------------------------------------------------------
diff --git a/BUILDING.txt b/BUILDING.txt
index 7e4d39a..241b7a8 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -89,6 +89,9 @@ For recent versions of Hadoop, you can bundle AWS-S3 (2.6.0+) or Azure (2.7.0+)
 
  $ mvn package -Dhadoop.version=${X} -Paws -Pazure 
 
+Tez also has some shims to provide version-specific implementations for various APIs.
+For more details, please refer to https://cwiki.apache.org/confluence/display/TEZ/HadoopShims
+
 For building on Windows with the hadoop24 profile enabled:
 
 Tez has a couple of symlinks in its codebase. Windows is a bit difficult with

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 7233b03..399c6d5 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
   TEZ-2679. Admin forms of launch env settings
 
 ALL CHANGES:
+  TEZ-2928. Framework for Hadoop shims.
   TEZ-2935. Add MR slow start translation for ShuffleVertexManager
   TEZ-2918. Make progress notifications in IOs
   TEZ-2940. Invalid shuffle max slow start setting causes vertex to hang indefinitely

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/findbugs-exclude.xml b/hadoop-shim-impls/findbugs-exclude.xml
new file mode 100644
index 0000000..5b11308
--- /dev/null
+++ b/hadoop-shim-impls/findbugs-exclude.xml
@@ -0,0 +1,16 @@
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<FindBugsFilter>
+
+</FindBugsFilter>

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.4/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.4/findbugs-exclude.xml b/hadoop-shim-impls/hadoop-shim-2.4/findbugs-exclude.xml
new file mode 100644
index 0000000..5b11308
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.4/findbugs-exclude.xml
@@ -0,0 +1,16 @@
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<FindBugsFilter>
+
+</FindBugsFilter>

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.4/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.4/pom.xml b/hadoop-shim-impls/hadoop-shim-2.4/pom.xml
new file mode 100644
index 0000000..095c2b1
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.4/pom.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <parent>
+    <artifactId>hadoop-shim-impls</artifactId>
+    <groupId>org.apache.tez</groupId>
+    <version>0.8.2-SNAPSHOT</version>
+  </parent>
+  <modelVersion>4.0.0</modelVersion>
+  <artifactId>hadoop-shim-2.4</artifactId>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+    </dependency>
+    <dependency>
+     <groupId>junit</groupId>
+     <artifactId>junit</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>hadoop-shim</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+    </dependency>
+  </dependencies>
+
+</project>

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim23_24Provider.java
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim23_24Provider.java b/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim23_24Provider.java
new file mode 100644
index 0000000..d94b5bf
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim23_24Provider.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+
+@Private
+public class HadoopShim23_24Provider extends HadoopShimProvider {
+
+  @Override
+  public HadoopShim createHadoopShim(String hadoopVersion, int majorVersion, int minorVersion) {
+    if (majorVersion == 2 && (minorVersion == 3 || minorVersion == 4)) {
+      return new org.apache.tez.hadoop.shim.HadoopShim24();
+    }
+    return null;
+  }
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim24.java
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim24.java b/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim24.java
new file mode 100644
index 0000000..3f7e56e
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.4/src/main/java/org/apache/tez/hadoop/shim/HadoopShim24.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+
+@Private
+public class HadoopShim24 extends HadoopShim {
+
+  @Override
+  public void setHadoopCallerContext(String context) {
+    // Not supported
+  }
+
+  @Override
+  public void clearHadoopCallerContext() {
+    // Not supported
+  }
+
+  @Override
+  public Set<String> getSupportedResourceTypes(RegisterApplicationMasterResponse response) {
+    // Not supported
+    return null;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.4/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.4/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider b/hadoop-shim-impls/hadoop-shim-2.4/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider
new file mode 100644
index 0000000..a675e20
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.4/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider
@@ -0,0 +1,14 @@
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+org.apache.tez.hadoop.shim.HadoopShim23_24Provider

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java b/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java
new file mode 100644
index 0000000..9bc4435
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.4/src/test/java/org/apache/tez/hadoop/shim/TestHadoop23_24ShimProvider.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestHadoop23_24ShimProvider {
+
+  @Test
+  public void testShimProvider() {
+    HadoopShim23_24Provider provider = new HadoopShim23_24Provider();
+    Assert.assertNull(provider.createHadoopShim("foo", 2, 2));
+    Assert.assertNull(provider.createHadoopShim("foo", 2, 1));
+    Assert.assertNull(provider.createHadoopShim("foo", 2, 5));
+    Assert.assertNull(provider.createHadoopShim("foo", 2, 6));
+    Assert.assertNull(provider.createHadoopShim("foo", 3, 3));
+    Assert.assertNotNull(provider.createHadoopShim("foo", 2, 3));
+    Assert.assertNotNull(provider.createHadoopShim("foo", 2, 4));
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.6/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.6/findbugs-exclude.xml b/hadoop-shim-impls/hadoop-shim-2.6/findbugs-exclude.xml
new file mode 100644
index 0000000..5b11308
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.6/findbugs-exclude.xml
@@ -0,0 +1,16 @@
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<FindBugsFilter>
+
+</FindBugsFilter>

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.6/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.6/pom.xml b/hadoop-shim-impls/hadoop-shim-2.6/pom.xml
new file mode 100644
index 0000000..17f4711
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.6/pom.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <parent>
+    <artifactId>hadoop-shim-impls</artifactId>
+    <groupId>org.apache.tez</groupId>
+    <version>0.8.2-SNAPSHOT</version>
+  </parent>
+  <modelVersion>4.0.0</modelVersion>
+  <artifactId>hadoop-shim-2.6</artifactId>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+    </dependency>
+    <dependency>
+     <groupId>junit</groupId>
+     <artifactId>junit</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>hadoop-shim</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+    </dependency>
+  </dependencies>
+
+</project>

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim25_26_27Provider.java
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim25_26_27Provider.java b/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim25_26_27Provider.java
new file mode 100644
index 0000000..ab013e9
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim25_26_27Provider.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+
+@Private
+public class HadoopShim25_26_27Provider extends HadoopShimProvider {
+
+  @Override
+  public HadoopShim createHadoopShim(String hadoopVersion, int majorVersion, int minorVersion) {
+    if (majorVersion == 2 && (minorVersion == 5 || minorVersion == 6 || minorVersion == 7)) {
+      return new HadoopShim26();
+    }
+    return null;
+  }
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim26.java
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim26.java b/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim26.java
new file mode 100644
index 0000000..c1622dc
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.6/src/main/java/org/apache/tez/hadoop/shim/HadoopShim26.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
+
+@Private
+public class HadoopShim26 extends HadoopShim {
+
+  @Override
+  public void setHadoopCallerContext(String context) {
+    // Not supported
+  }
+
+  @Override
+  public void clearHadoopCallerContext() {
+    // Not supported
+  }
+
+  @Override
+  public Set<String> getSupportedResourceTypes(RegisterApplicationMasterResponse response) {
+    EnumSet<SchedulerResourceTypes> supportedResourceTypes = response.getSchedulerResourceTypes();
+    Set<String> supportedTypes = new HashSet<String>();
+    for (SchedulerResourceTypes resourceType : supportedResourceTypes) {
+      supportedTypes.add(resourceType.name());
+    }
+    return supportedTypes;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.6/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.6/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider b/hadoop-shim-impls/hadoop-shim-2.6/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider
new file mode 100644
index 0000000..4226f18
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.6/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider
@@ -0,0 +1,14 @@
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+org.apache.tez.hadoop.shim.HadoopShim25_26_27Provider

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java b/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java
new file mode 100644
index 0000000..0e6d997
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.6/src/test/java/org/apache/tez/hadoop/shim/TestHadoop25_26_27ShimProvider.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestHadoop25_26_27ShimProvider {
+
+  @Test
+  public void testShimProvider() {
+    HadoopShim25_26_27Provider provider = new HadoopShim25_26_27Provider();
+    Assert.assertNull(provider.createHadoopShim("foo", 2, 2));
+    Assert.assertNull(provider.createHadoopShim("foo", 2, 1));
+    Assert.assertNull(provider.createHadoopShim("foo", 2, 3));
+    Assert.assertNull(provider.createHadoopShim("foo", 2, 8));
+    Assert.assertNull(provider.createHadoopShim("foo", 3, 3));
+    Assert.assertNotNull(provider.createHadoopShim("foo", 2, 5));
+    Assert.assertNotNull(provider.createHadoopShim("foo", 2, 6));
+    Assert.assertNotNull(provider.createHadoopShim("foo", 2, 7));
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.8/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.8/findbugs-exclude.xml b/hadoop-shim-impls/hadoop-shim-2.8/findbugs-exclude.xml
new file mode 100644
index 0000000..5b11308
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.8/findbugs-exclude.xml
@@ -0,0 +1,16 @@
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<FindBugsFilter>
+
+</FindBugsFilter>

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.8/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.8/pom.xml b/hadoop-shim-impls/hadoop-shim-2.8/pom.xml
new file mode 100644
index 0000000..18d22c5
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.8/pom.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <parent>
+    <artifactId>hadoop-shim-impls</artifactId>
+    <groupId>org.apache.tez</groupId>
+    <version>0.8.2-SNAPSHOT</version>
+  </parent>
+  <modelVersion>4.0.0</modelVersion>
+  <artifactId>hadoop-shim-2.8</artifactId>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+    </dependency>
+    <dependency>
+     <groupId>junit</groupId>
+     <artifactId>junit</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
+      <artifactId>hadoop-shim</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+    </dependency>
+  </dependencies>
+
+</project>

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28.java
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28.java b/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28.java
new file mode 100644
index 0000000..0c599e4
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.hadoop.ipc.CallerContext;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
+
+public class HadoopShim28 extends HadoopShim {
+
+  final static CallerContext nullCallerContext = new CallerContext.Builder("").build();
+
+  @Override
+  public void setHadoopCallerContext(String context) {
+    CallerContext.setCurrent(new CallerContext.Builder(context).build());
+  }
+
+  @Override
+  public void clearHadoopCallerContext() {
+    CallerContext.setCurrent(nullCallerContext);
+  }
+
+  @Override
+  public Set<String> getSupportedResourceTypes(RegisterApplicationMasterResponse response) {
+    EnumSet<SchedulerResourceTypes> supportedResourceTypes = response.getSchedulerResourceTypes();
+    Set<String> supportedTypes = new HashSet<String>();
+    for (SchedulerResourceTypes resourceType : supportedResourceTypes) {
+      supportedTypes.add(resourceType.name());
+    }
+    return supportedTypes;
+  }
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28Provider.java
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28Provider.java b/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28Provider.java
new file mode 100644
index 0000000..d03a538
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.8/src/main/java/org/apache/tez/hadoop/shim/HadoopShim28Provider.java
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+public class HadoopShim28Provider extends HadoopShimProvider {
+
+  @Override
+  public HadoopShim createHadoopShim(String hadoopVersion, int majorVersion, int minorVersion) {
+    if (majorVersion == 2 && minorVersion >= 8) {
+      return new HadoopShim28();
+    }
+    return null;
+  }
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.8/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.8/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider b/hadoop-shim-impls/hadoop-shim-2.8/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider
new file mode 100644
index 0000000..24fe49d
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.8/src/main/resources/META-INF/services/org.apache.tez.hadoop.shim.HadoopShimProvider
@@ -0,0 +1,14 @@
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+org.apache.tez.hadoop.shim.HadoopShim28Provider

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java b/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java
new file mode 100644
index 0000000..df92627
--- /dev/null
+++ b/hadoop-shim-impls/hadoop-shim-2.8/src/test/java/org/apache/tez/hadoop/shim/TestHadoopShim28Provider.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestHadoopShim28Provider {
+
+  @Test
+  public void testShimProvider() {
+    HadoopShim28Provider provider = new HadoopShim28Provider();
+    Assert.assertNull(provider.createHadoopShim("foo", 2, 2));
+    Assert.assertNull(provider.createHadoopShim("foo", 2, 7));
+    Assert.assertNull(provider.createHadoopShim("foo", 2, 5));
+    Assert.assertNull(provider.createHadoopShim("foo", 2, 6));
+    Assert.assertNull(provider.createHadoopShim("foo", 3, 3));
+    Assert.assertNotNull(provider.createHadoopShim("foo", 2, 8));
+    Assert.assertNotNull(provider.createHadoopShim("foo", 2, 111));
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim-impls/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-shim-impls/pom.xml b/hadoop-shim-impls/pom.xml
new file mode 100644
index 0000000..6c8f7d2
--- /dev/null
+++ b/hadoop-shim-impls/pom.xml
@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <artifactId>tez</artifactId>
+    <groupId>org.apache.tez</groupId>
+    <version>0.8.2-SNAPSHOT</version>
+  </parent>
+  <artifactId>hadoop-shim-impls</artifactId>
+  <packaging>pom</packaging>
+
+  <profiles>
+    <profile>
+      <id>hadoop24</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <modules>
+        <module>hadoop-shim-2.4</module>
+      </modules>
+    </profile>
+    <profile>
+      <id>hadoop26</id>
+      <activation>
+        <activeByDefault>true</activeByDefault>
+      </activation>
+      <modules>
+        <module>hadoop-shim-2.6</module>
+      </modules>
+    </profile>
+    <profile>
+      <id>hadoop28</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <modules>
+        <module>hadoop-shim-2.8</module>
+      </modules>
+    </profile>
+  </profiles>
+
+</project>

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-shim/findbugs-exclude.xml b/hadoop-shim/findbugs-exclude.xml
new file mode 100644
index 0000000..5b11308
--- /dev/null
+++ b/hadoop-shim/findbugs-exclude.xml
@@ -0,0 +1,16 @@
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+<FindBugsFilter>
+
+</FindBugsFilter>

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-shim/pom.xml b/hadoop-shim/pom.xml
new file mode 100644
index 0000000..1f70e6f
--- /dev/null
+++ b/hadoop-shim/pom.xml
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License. See accompanying LICENSE file.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+     <artifactId>tez</artifactId>
+     <groupId>org.apache.tez</groupId>
+     <version>0.8.2-SNAPSHOT</version>
+  </parent>
+  <artifactId>hadoop-shim</artifactId>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+    </dependency>
+    <dependency>
+     <groupId>junit</groupId>
+     <artifactId>junit</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-api</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-common</artifactId>
+    </dependency>
+  </dependencies>
+
+</project>

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/DefaultHadoopShim.java
----------------------------------------------------------------------
diff --git a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/DefaultHadoopShim.java b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/DefaultHadoopShim.java
new file mode 100644
index 0000000..c68777c
--- /dev/null
+++ b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/DefaultHadoopShim.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+
+/**
+ * Default Hadoop Shim. For now, it mimics HadoopShim.
+ */
+@Private
+public class DefaultHadoopShim extends HadoopShim {
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShim.java
----------------------------------------------------------------------
diff --git a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShim.java b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShim.java
new file mode 100644
index 0000000..7314c2d
--- /dev/null
+++ b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShim.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+
+import com.google.common.collect.Sets;
+
+@Public
+@Unstable
+public abstract class HadoopShim {
+
+  /**
+   * Set up Hadoop Caller Context
+   * @param context Context to be set
+   */
+  public void setHadoopCallerContext(String context) {
+    // Nothing to do
+  }
+
+  /**
+   * Clear the Hadoop Caller Context
+   */
+  public void clearHadoopCallerContext() {
+    // Nothing to do
+  }
+
+  public static String CPU_RESOURCE = "CPU";
+  public static String MEMORY_RESOURCE = "MEMORY";
+
+  /**
+   * Extract supported Resource types from the RM's response when the AM registers
+   * @param response ApplicationMasterResponse from RM after registering
+   * @return Set of Resource types that are supported
+   */
+  public Set<String> getSupportedResourceTypes(RegisterApplicationMasterResponse response) {
+    return null;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimProvider.java b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimProvider.java
new file mode 100644
index 0000000..728408c
--- /dev/null
+++ b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimProvider.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+@Public
+@Unstable
+public abstract class HadoopShimProvider {
+
+  /**
+   * Provides a HadoopShim if the version matches the implementation of the Shim
+   * @param hadoopVersion Full version string of hadoop-common's version
+   * @param majorVersion Major version
+   * @param minorVersion Minor version
+   * @return HadoopShim object if there is a match else null
+   */
+  public abstract HadoopShim createHadoopShim(String hadoopVersion, int majorVersion,
+      int minorVersion);
+
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java
new file mode 100644
index 0000000..fc2ec81
--- /dev/null
+++ b/hadoop-shim/src/main/java/org/apache/tez/hadoop/shim/HadoopShimsLoader.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.hadoop.shim;
+
+import java.util.ServiceLoader;
+import java.util.StringTokenizer;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.util.VersionInfo;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Private
+public class HadoopShimsLoader {
+
+  private static final Logger LOG = LoggerFactory.getLogger(HadoopShimsLoader.class);
+
+  private static ServiceLoader<HadoopShimProvider> shimLoader =
+      ServiceLoader.load(HadoopShimProvider.class);
+
+  private final HadoopShim currentShim;
+  private final HadoopShimProvider currentShimProvider;
+
+  public HadoopShimsLoader() {
+    String versionStr = VersionInfo.getVersion();
+    Version version = new Version(versionStr);
+    HadoopShim selectedShim = null;
+    HadoopShimProvider selectedShimProvider = null;
+    LOG.info("Trying to locate HadoopShimProvider for "
+        + "hadoopVersion=" + versionStr
+        + ", majorVersion=" + version.majorVersion
+        + ", minorVersion=" + version.minorVersion);
+    synchronized (shimLoader) {
+      for (HadoopShimProvider provider : shimLoader) {
+        LOG.debug("Trying HadoopShimProvider : "
+            + provider.getClass().getName());
+        HadoopShim hadoopShim = null;
+        try {
+          hadoopShim = provider.createHadoopShim(versionStr,
+              version.majorVersion, version.minorVersion);
+
+          if (hadoopShim != null) {
+            selectedShim = hadoopShim;
+            selectedShimProvider = provider;
+            break;
+          } else {
+            if (LOG.isDebugEnabled()) {
+              LOG.debug("Cannot pick " + provider.getClass().getName()
+                  + " as the HadoopShimProvider - returned null hadoop shim");
+            }
+          }
+        } catch (Exception e) {
+          LOG.info("Failed to use " + provider.getClass().getName()
+              + " due to error: ", e);
+        }
+      }
+      if (selectedShim == null) {
+        currentShim = new DefaultHadoopShim();
+        currentShimProvider = null;
+      } else {
+        currentShim = selectedShim;
+        currentShimProvider = selectedShimProvider;
+      }
+    }
+    LOG.info("Picked HadoopShim " + currentShim.getClass().getName()
+        + ", providerName="
+        + (currentShimProvider != null ? currentShimProvider.getClass().getName() : "null" )
+        + ", hadoopVersion=" + versionStr
+        + ", majorVersion=" + version.majorVersion
+        + ", minorVersion=" + version.minorVersion);
+
+  }
+
+  public HadoopShim getHadoopShim() {
+    return currentShim;
+  }
+
+  private static class Version {
+
+    int majorVersion = -1;
+    int minorVersion = -1;
+
+    public Version(String versionString) {
+      int index = 0;
+      StringTokenizer tokenizer = new StringTokenizer(versionString, ".-", true);
+      while (tokenizer.hasMoreTokens()) {
+        String token = tokenizer.nextToken();
+        if (token.equals("-")) {
+          break;
+        }
+        if (token.equals(".")) {
+          continue;
+        }
+        try {
+          int i = Integer.parseInt(token);
+          if (index == 0) {
+            majorVersion = i;
+          } else if (index == 1) {
+            minorVersion = i;
+            break;
+          }
+          ++index;
+        } catch (NumberFormatException nfe) {
+          break;
+        }
+      }
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index b8b8a22..a17f714 100644
--- a/pom.xml
+++ b/pom.xml
@@ -102,6 +102,11 @@
     <dependencies>
       <dependency>
         <groupId>org.apache.tez</groupId>
+        <artifactId>hadoop-shim</artifactId>
+        <version>${project.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.tez</groupId>
         <artifactId>tez-api</artifactId>
         <version>${project.version}</version>
       </dependency>
@@ -679,6 +684,7 @@
   </dependencyManagement>
 
   <modules>
+    <module>hadoop-shim</module>
     <module>tez-api</module>
     <module>tez-common</module>
     <module>tez-runtime-library</module>
@@ -691,6 +697,7 @@
     <module>tez-ui</module>
     <module>tez-plugins</module>
     <module>tez-tools</module>
+    <module>hadoop-shim-impls</module>
     <module>tez-dist</module>
     <module>docs</module>
   </modules>

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/tez-dag/pom.xml
----------------------------------------------------------------------
diff --git a/tez-dag/pom.xml b/tez-dag/pom.xml
index adda218..7ca92db 100644
--- a/tez-dag/pom.xml
+++ b/tez-dag/pom.xml
@@ -35,6 +35,10 @@
   <dependencies>
     <dependency>
       <groupId>org.apache.tez</groupId>
+      <artifactId>hadoop-shim</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.tez</groupId>
       <artifactId>tez-api</artifactId>
     </dependency>
     <dependency>

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
index 657267b..68453b1 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/AppContext.java
@@ -37,6 +37,7 @@ import org.apache.tez.dag.app.rm.node.AMNodeTracker;
 import org.apache.tez.common.security.ACLManager;
 import org.apache.tez.dag.history.HistoryEventHandler;
 import org.apache.tez.dag.records.TezDAGID;
+import org.apache.tez.hadoop.shim.HadoopShim;
 
 import com.google.common.util.concurrent.ListeningExecutorService;
 
@@ -123,4 +124,6 @@ public interface AppContext {
   public String getTaskSchedulerName(int schedulerId);
   public String getContainerLauncherName(int launcherId);
 
+  public HadoopShim getHadoopShim();
+
 }

http://git-wip-us.apache.org/repos/asf/tez/blob/40e864d1/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
index f4c57e3..9c09f95 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java
@@ -174,6 +174,8 @@ import org.apache.tez.dag.records.TezVertexID;
 import org.apache.tez.dag.utils.Graph;
 import org.apache.tez.dag.utils.RelocalizationUtils;
 import org.apache.tez.dag.utils.Simple2LevelVersionComparator;
+import org.apache.tez.hadoop.shim.HadoopShim;
+import org.apache.tez.hadoop.shim.HadoopShimsLoader;
 import org.apache.tez.util.TezMxBeanResourceCalculator;
 import org.codehaus.jettison.json.JSONException;
 import org.slf4j.Logger;
@@ -234,6 +236,7 @@ public class DAGAppMaster extends AbstractService {
   private final String[] localDirs;
   private final String[] logDirs;
   private final AMPluginDescriptorProto amPluginDescriptorProto;
+  private final HadoopShim hadoopShim;
   private ContainerSignatureMatcher containerSignatureMatcher;
   private AMContainerMap containers;
   private AMNodeTracker nodes;
@@ -353,9 +356,11 @@ public class DAGAppMaster extends AbstractService {
 
     this.containerLogs = getRunningLogURL(this.nmHost + ":" + this.nmHttpPort,
         this.containerID.toString(), this.appMasterUgi.getShortUserName());
+    this.hadoopShim = new HadoopShimsLoader().getHadoopShim();
 
     LOG.info("Created DAGAppMaster for application " + applicationAttemptId
         + ", versionInfo=" + dagVersionInfo.toString());
+
   }
 
   // Pull this WebAppUtils function into Tez until YARN-4186
@@ -1596,6 +1601,11 @@ public class DAGAppMaster extends AbstractService {
     }
 
     @Override
+    public HadoopShim getHadoopShim() {
+      return hadoopShim;
+    }
+
+    @Override
     public Map<ApplicationAccessType, String> getApplicationACLs() {
       if (getServiceState() != STATE.STARTED) {
         throw new TezUncheckedException(


Mime
View raw message