hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j..@apache.org
Subject svn commit: r1021549 - in /hadoop/hive/trunk: ./ common/src/java/org/apache/hadoop/hive/conf/ contrib/ hbase-handler/ ql/ ql/src/java/org/apache/hadoop/hive/ql/ shims/ shims/src/0.17/java/org/apache/hadoop/hive/shims/ shims/src/0.18/java/org/apache/had...
Date Mon, 11 Oct 2010 23:41:25 GMT
Author: jvs
Date: Mon Oct 11 23:41:24 2010
New Revision: 1021549

URL: http://svn.apache.org/viewvc?rev=1021549&view=rev
Log:
HIVE-1264. Make Hive work with Hadoop security
(Todd Lipcon via jvs)


Added:
    hadoop/hive/trunk/shims/src/0.20S/
    hadoop/hive/trunk/shims/src/0.20S/java/
    hadoop/hive/trunk/shims/src/0.20S/java/org/
    hadoop/hive/trunk/shims/src/0.20S/java/org/apache/
    hadoop/hive/trunk/shims/src/0.20S/java/org/apache/hadoop/
    hadoop/hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/
    hadoop/hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/
    hadoop/hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
    hadoop/hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Jetty20SShims.java
Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/build-common.xml
    hadoop/hive/trunk/build.properties
    hadoop/hive/trunk/build.xml
    hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    hadoop/hive/trunk/contrib/build.xml
    hadoop/hive/trunk/hbase-handler/build.xml
    hadoop/hive/trunk/ql/build.xml
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
    hadoop/hive/trunk/shims/build.xml
    hadoop/hive/trunk/shims/ivy.xml
    hadoop/hive/trunk/shims/src/0.17/java/org/apache/hadoop/hive/shims/Hadoop17Shims.java
    hadoop/hive/trunk/shims/src/0.18/java/org/apache/hadoop/hive/shims/Hadoop18Shims.java
    hadoop/hive/trunk/shims/src/0.19/java/org/apache/hadoop/hive/shims/Hadoop19Shims.java
    hadoop/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
    hadoop/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
    hadoop/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/ShimLoader.java

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=1021549&r1=1021548&r2=1021549&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Mon Oct 11 23:41:24 2010
@@ -181,6 +181,9 @@ Trunk -  Unreleased
     HIVE-1697. Migration scripts should increase size of PARAM_VALUE in
     PARTITION_PARAMS (Paul Yang via namit)
 
+    HIVE-1264. Make Hive work with Hadoop security
+    (Todd Lipcon via jvs)
+
   OPTIMIZATIONS
 
   BUG FIXES

Modified: hadoop/hive/trunk/build-common.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/build-common.xml?rev=1021549&r1=1021548&r2=1021549&view=diff
==============================================================================
--- hadoop/hive/trunk/build-common.xml (original)
+++ hadoop/hive/trunk/build-common.xml Mon Oct 11 23:41:24 2010
@@ -218,8 +218,10 @@
 
   <!-- the normal classpath -->
   <path id="common-classpath">
-    <pathelement location="${hadoop.jar}"/>
-    <pathelement location="${hadoop.tools.jar}"/>
+    <pathelement location="${hadoop.oldstyle-name.jar}"/>
+    <pathelement location="${hadoop.oldstyle-name.tools.jar}"/>
+    <pathelement location="${hadoop.newstyle-name.jar}"/>
+    <pathelement location="${hadoop.newstyle-name.tools.jar}"/>
     <pathelement location="${build.dir.hive}/classes"/>
     <fileset dir="${build.dir.hive}" includes="*/*.jar"/>
     <fileset dir="${hive.root}/lib" includes="*.jar"/>

Modified: hadoop/hive/trunk/build.properties
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/build.properties?rev=1021549&r1=1021548&r2=1021549&view=diff
==============================================================================
--- hadoop/hive/trunk/build.properties (original)
+++ hadoop/hive/trunk/build.properties Mon Oct 11 23:41:24 2010
@@ -20,9 +20,15 @@ build.dir.hadoop=${build.dir.hive}/hadoo
 hadoop.version.ant-internal=${hadoop.version}
 hadoop.root.default=${build.dir.hadoop}/hadoop-${hadoop.version.ant-internal}
 hadoop.root=${hadoop.root.default}
-hadoop.jar=${hadoop.root}/hadoop-${hadoop.version.ant-internal}-core.jar
-hadoop.tools.jar=${hadoop.root}/hadoop-${hadoop.version.ant-internal}-tools.jar
-hadoop.test.jar=${hadoop.root}/hadoop-${hadoop.version.ant-internal}-test.jar
+# Newer versions of Hadoop name the jar as hadoop-{core,test}-VERSION instead of hadoop-VERSION-{core,test}
+# We will add both styles to the classpath and it will pick up whichever is there
+hadoop.oldstyle-name.jar=${hadoop.root}/hadoop-${hadoop.version.ant-internal}-core.jar
+hadoop.oldstyle-name.tools.jar=${hadoop.root}/hadoop-${hadoop.version.ant-internal}-tools.jar
+hadoop.oldstyle-name.test.jar=${hadoop.root}/hadoop-${hadoop.version.ant-internal}-test.jar
+hadoop.newstyle-name.jar=${hadoop.root}/hadoop-core-${hadoop.version.ant-internal}.jar
+hadoop.newstyle-name.test.jar=${hadoop.root}/hadoop-test-${hadoop.version.ant-internal}.jar
+hadoop.newstyle-name.tools.jar=${hadoop.root}/hadoop-tools-${hadoop.version.ant-internal}.jar
+
 jetty.test.jar=${hadoop.root}/lib/jetty-5.1.4.jar
 servlet.test.jar=${hadoop.root}/lib/servlet-api.jar
 jasper.test.jar=${hadoop.root}/lib/jetty-ext/jasper-runtime.jar
@@ -34,3 +40,8 @@ common.jar=${hadoop.root}/lib/commons-ht
 # Data nucleus repository - needed for jdo2-api-2.3-ec.jar download
 #
 datanucleus.repo=http://www.datanucleus.org/downloads/maven2
+
+# URLs pointing to a built tarball of a secure hadoop release
+hadoop.security.url=http://mirror.facebook.net/facebook/hive-deps/hadoop/core/hadoop-0.20.3-CDH3-SNAPSHOT/hadoop-0.20.3-CDH3-SNAPSHOT.tar.gz
+hadoop.security.version=0.20.3-CDH3-SNAPSHOT
+

Modified: hadoop/hive/trunk/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/build.xml?rev=1021549&r1=1021548&r2=1021549&view=diff
==============================================================================
--- hadoop/hive/trunk/build.xml (original)
+++ hadoop/hive/trunk/build.xml Mon Oct 11 23:41:24 2010
@@ -112,7 +112,8 @@
 
   <!-- the normal classpath -->
   <path id="common-classpath">
-    <pathelement location="${hadoop.jar}"/>
+    <pathelement location="${hadoop.oldstyle-name.jar}"/>
+    <pathelement location="${hadoop.newstyle-name.jar}"/>
     <pathelement location="${build.dir.hive}/classes"/>
     <fileset dir="${hive.root}" includes="hive-*.jar"/>
     <fileset dir="${hive.root}/lib" includes="*.jar"/>

Modified: hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1021549&r1=1021548&r2=1021549&view=diff
==============================================================================
--- hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Mon Oct 11
23:41:24 2010
@@ -591,13 +591,11 @@ public class HiveConf extends Configurat
    */
   public String getUser() throws IOException {
     try {
-      UserGroupInformation ugi = UserGroupInformation.readFrom(this);
-      if (ugi == null) {
-        ugi = UserGroupInformation.login(this);
-      }
+      UserGroupInformation ugi = ShimLoader.getHadoopShims()
+        .getUGIForConf(this);
       return ugi.getUserName();
-    } catch (LoginException e) {
-      throw (IOException) new IOException().initCause(e);
+    } catch (LoginException le) {
+      throw new IOException(le);
     }
   }
 

Modified: hadoop/hive/trunk/contrib/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/contrib/build.xml?rev=1021549&r1=1021548&r2=1021549&view=diff
==============================================================================
--- hadoop/hive/trunk/contrib/build.xml (original)
+++ hadoop/hive/trunk/contrib/build.xml Mon Oct 11 23:41:24 2010
@@ -36,7 +36,6 @@
     <pathelement location="${test.src.data.dir}/conf"/>
     <pathelement location="${hive.conf.dir}"/>
     <pathelement location="${hive.root}/cli/lib/jline-0.9.94.jar"/>
-    <pathelement location="${hadoop.test.jar}"/>
     <pathelement location="${jetty.test.jar}"/>
     <pathelement location="${servlet.test.jar}"/>
     <pathelement location="${jasper.test.jar}"/>

Modified: hadoop/hive/trunk/hbase-handler/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/hbase-handler/build.xml?rev=1021549&r1=1021548&r2=1021549&view=diff
==============================================================================
--- hadoop/hive/trunk/hbase-handler/build.xml (original)
+++ hadoop/hive/trunk/hbase-handler/build.xml Mon Oct 11 23:41:24 2010
@@ -37,7 +37,8 @@
     <pathelement location="${test.src.data.dir}/conf"/>
     <pathelement location="${hive.conf.dir}"/>
     <pathelement location="${hive.root}/cli/lib/jline-0.9.94.jar"/>
-    <pathelement location="${hadoop.test.jar}"/>
+    <pathelement location="${hadoop.oldstyle-name.test.jar}"/>
+    <pathelement location="${hadoop.newstyle-name.test.jar}"/>
     <pathelement location="${jetty.test.jar}"/>
     <pathelement location="${servlet.test.jar}"/>
     <pathelement location="${jasper.test.jar}"/>

Modified: hadoop/hive/trunk/ql/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/build.xml?rev=1021549&r1=1021548&r2=1021549&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/build.xml (original)
+++ hadoop/hive/trunk/ql/build.xml Mon Oct 11 23:41:24 2010
@@ -40,7 +40,8 @@
     <pathelement location="${test.src.data.dir}/conf"/>
     <pathelement location="${hive.conf.dir}"/>
     <pathelement location="${hive.root}/cli/lib/jline-0.9.94.jar"/>
-    <pathelement location="${hadoop.test.jar}"/>
+    <pathelement location="${hadoop.oldstyle-name.test.jar}"/>
+    <pathelement location="${hadoop.newstyle-name.test.jar}"/>
     <pathelement location="${jetty.test.jar}"/>
     <pathelement location="${servlet.test.jar}"/>
     <pathelement location="${jasper.test.jar}"/>

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=1021549&r1=1021548&r2=1021549&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Mon Oct 11 23:41:24
2010
@@ -85,10 +85,10 @@ import org.apache.hadoop.hive.ql.process
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
 import org.apache.hadoop.hive.serde2.ByteStream;
+import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.mapred.ClusterStatus;
 import org.apache.hadoop.mapred.JobClient;
 import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.security.UnixUserGroupInformation;
 import org.apache.hadoop.util.ReflectionUtils;
 
 public class Driver implements CommandProcessor {
@@ -285,21 +285,11 @@ public class Driver implements CommandPr
    */
   public Driver(HiveConf conf) {
     this.conf = conf;
-    try {
-      UnixUserGroupInformation.login(conf, true);
-    } catch (Exception e) {
-      LOG.warn("Ignoring " + e.getMessage());
-    }
   }
 
   public Driver() {
     if (SessionState.get() != null) {
       conf = SessionState.get().getConf();
-      try {
-        UnixUserGroupInformation.login(conf, true);
-      } catch (Exception e) {
-        LOG.warn("Ignoring " + e.getMessage());
-      }
     }
   }
 
@@ -739,8 +729,7 @@ public class Driver implements CommandPr
       // Get all the pre execution hooks and execute them.
       for (PreExecute peh : getPreExecHooks()) {
         peh.run(SessionState.get(), plan.getInputs(), plan.getOutputs(),
-            UnixUserGroupInformation.readFromConf(conf,
-                UnixUserGroupInformation.UGI_PROPERTY_NAME));
+                ShimLoader.getHadoopShims().getUGIForConf(conf));
       }
 
       int jobs = Utilities.getMRTasks(plan.getRootTasks()).size();
@@ -822,8 +811,7 @@ public class Driver implements CommandPr
       for (PostExecute peh : getPostExecHooks()) {
         peh.run(SessionState.get(), plan.getInputs(), plan.getOutputs(),
             (SessionState.get() != null ? SessionState.get().getLineageState().getLineageInfo()
: null),
-            UnixUserGroupInformation.readFromConf(conf,
-                UnixUserGroupInformation.UGI_PROPERTY_NAME));
+                ShimLoader.getHadoopShims().getUGIForConf(conf));
       }
 
       if (SessionState.get() != null) {

Modified: hadoop/hive/trunk/shims/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/shims/build.xml?rev=1021549&r1=1021548&r2=1021549&view=diff
==============================================================================
--- hadoop/hive/trunk/shims/build.xml (original)
+++ hadoop/hive/trunk/shims/build.xml Mon Oct 11 23:41:24 2010
@@ -26,9 +26,12 @@ to call at top-level: ant deploy-contrib
   <import file="../build-common.xml"/>
 
   <path id="classpath">
-    <pathelement location="${hadoop.jar}"/>
-    <pathelement location="${hadoop.tools.jar}"/>
-    <pathelement location="${hadoop.test.jar}"/>
+    <pathelement location="${hadoop.oldstyle-name.jar}"/>
+    <pathelement location="${hadoop.oldstyle-name.tools.jar}"/>
+    <pathelement location="${hadoop.oldstyle-name.test.jar}"/>
+    <pathelement location="${hadoop.newstyle-name.jar}"/>
+    <pathelement location="${hadoop.newstyle-name.test.jar}"/>
+    <pathelement location="${hadoop.newstyle-name.tools.jar}"/>
     <fileset dir="${hadoop.root}/lib">
       <include name="**/*.jar" />
       <exclude name="**/excluded/" />
@@ -66,6 +69,10 @@ to call at top-level: ant deploy-contrib
     <antcall target="build_shims" inheritRefs="false" inheritAll="false">
       <param name="hadoop.version.ant-internal" value="0.20.0" />
     </antcall>
+    <antcall target="build_shims" inheritRefs="false" inheritAll="false">
+      <param name="hadoop.version.ant-internal" value="${hadoop.security.version}" />
+      <param name="hadoop.version.ant-internal.prefix" value="0.20S" />
+    </antcall>
     <getversionpref property="hadoop.version.ant-internal.prefix" input="${hadoop.version}"
/>
     <javac
      encoding="${build.encoding}"

Modified: hadoop/hive/trunk/shims/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/shims/ivy.xml?rev=1021549&r1=1021548&r2=1021549&view=diff
==============================================================================
--- hadoop/hive/trunk/shims/ivy.xml (original)
+++ hadoop/hive/trunk/shims/ivy.xml Mon Oct 11 23:41:24 2010
@@ -13,6 +13,10 @@
         <dependency org="hadoop" name="core" rev="0.20.0">
           <artifact name="hadoop" type="source" ext="tar.gz"/>
         </dependency> 
+        <dependency org="hadoop" name="core" rev="${hadoop.security.version}">
+          <artifact name="hadoop" type="source" ext="tar.gz"
+            url="${hadoop.security.url}" />
+        </dependency> 
         <conflict manager="all" />
     </dependencies>
 </ivy-module>

Modified: hadoop/hive/trunk/shims/src/0.17/java/org/apache/hadoop/hive/shims/Hadoop17Shims.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/shims/src/0.17/java/org/apache/hadoop/hive/shims/Hadoop17Shims.java?rev=1021549&r1=1021548&r2=1021549&view=diff
==============================================================================
--- hadoop/hive/trunk/shims/src/0.17/java/org/apache/hadoop/hive/shims/Hadoop17Shims.java
(original)
+++ hadoop/hive/trunk/shims/src/0.17/java/org/apache/hadoop/hive/shims/Hadoop17Shims.java
Mon Oct 11 23:41:24 2010
@@ -29,6 +29,9 @@ import org.apache.hadoop.mapred.JobConf;
 import org.apache.hadoop.mapred.RunningJob;
 import org.apache.hadoop.mapred.TaskCompletionEvent;
 import org.apache.hadoop.mapred.lib.NullOutputFormat;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UnixUserGroupInformation;
+import javax.security.auth.login.LoginException;
 
 import java.io.IOException;
 
@@ -134,4 +137,13 @@ public class Hadoop17Shims implements Ha
     throw new RuntimeException("Not implemented in this Hadoop version");
   }
 
+  @Override
+  public UserGroupInformation getUGIForConf(Configuration conf) throws LoginException {
+    UserGroupInformation ugi =
+      UnixUserGroupInformation.readFromConf(conf, UnixUserGroupInformation.UGI_PROPERTY_NAME);
+    if(ugi == null) {
+      ugi = UserGroupInformation.login(conf);
+    }
+    return ugi;
+  }
 }

Modified: hadoop/hive/trunk/shims/src/0.18/java/org/apache/hadoop/hive/shims/Hadoop18Shims.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/shims/src/0.18/java/org/apache/hadoop/hive/shims/Hadoop18Shims.java?rev=1021549&r1=1021548&r2=1021549&view=diff
==============================================================================
--- hadoop/hive/trunk/shims/src/0.18/java/org/apache/hadoop/hive/shims/Hadoop18Shims.java
(original)
+++ hadoop/hive/trunk/shims/src/0.18/java/org/apache/hadoop/hive/shims/Hadoop18Shims.java
Mon Oct 11 23:41:24 2010
@@ -31,6 +31,9 @@ import org.apache.hadoop.mapred.TaskID;
 import org.apache.hadoop.mapred.TaskAttemptID;
 import org.apache.hadoop.mapred.TaskCompletionEvent;
 import org.apache.hadoop.mapred.lib.NullOutputFormat;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UnixUserGroupInformation;
+import javax.security.auth.login.LoginException;
 
 import java.io.IOException;
 
@@ -137,4 +140,14 @@ public class Hadoop18Shims implements Ha
   public void setNullOutputFormat(JobConf conf) {
     conf.setOutputFormat(NullOutputFormat.class);
   }
+
+  @Override
+  public UserGroupInformation getUGIForConf(Configuration conf) throws LoginException {
+    UserGroupInformation ugi =
+      UnixUserGroupInformation.readFromConf(conf, UnixUserGroupInformation.UGI_PROPERTY_NAME);
+    if(ugi == null) {
+      ugi = UserGroupInformation.login(conf);
+    }
+    return ugi;
+  }
 }

Modified: hadoop/hive/trunk/shims/src/0.19/java/org/apache/hadoop/hive/shims/Hadoop19Shims.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/shims/src/0.19/java/org/apache/hadoop/hive/shims/Hadoop19Shims.java?rev=1021549&r1=1021548&r2=1021549&view=diff
==============================================================================
--- hadoop/hive/trunk/shims/src/0.19/java/org/apache/hadoop/hive/shims/Hadoop19Shims.java
(original)
+++ hadoop/hive/trunk/shims/src/0.19/java/org/apache/hadoop/hive/shims/Hadoop19Shims.java
Mon Oct 11 23:41:24 2010
@@ -37,6 +37,9 @@ import org.apache.hadoop.mapred.OutputCo
 import org.apache.hadoop.mapred.TaskAttemptContext;
 import org.apache.hadoop.mapred.JobContext;
 import org.apache.hadoop.mapred.lib.NullOutputFormat;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UnixUserGroupInformation;
+import javax.security.auth.login.LoginException;
 
 import java.io.IOException;
 import java.lang.reflect.Constructor;
@@ -512,4 +515,15 @@ public class Hadoop19Shims implements Ha
     // but can be backported. So we disable setup/cleanup in all versions >= 0.19
     conf.setBoolean("mapred.committer.job.setup.cleanup.needed", false);
   }
+
+
+  @Override
+  public UserGroupInformation getUGIForConf(Configuration conf) throws LoginException {
+    UserGroupInformation ugi =
+      UnixUserGroupInformation.readFromConf(conf, UnixUserGroupInformation.UGI_PROPERTY_NAME);
+    if(ugi == null) {
+      ugi = UserGroupInformation.login(conf);
+    }
+    return ugi;
+  }
 }

Modified: hadoop/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=1021549&r1=1021548&r2=1021549&view=diff
==============================================================================
--- hadoop/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
(original)
+++ hadoop/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
Mon Oct 11 23:41:24 2010
@@ -23,6 +23,7 @@ import java.io.IOException;
 import java.lang.reflect.Constructor;
 import java.util.ArrayList;
 import java.util.List;
+import javax.security.auth.login.LoginException;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileStatus;
@@ -48,6 +49,8 @@ import org.apache.hadoop.mapred.TaskID;
 import org.apache.hadoop.mapred.lib.CombineFileInputFormat;
 import org.apache.hadoop.mapred.lib.CombineFileSplit;
 import org.apache.hadoop.mapred.lib.NullOutputFormat;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UnixUserGroupInformation;
 import org.apache.hadoop.tools.HadoopArchives;
 import org.apache.hadoop.util.ToolRunner;
 
@@ -436,4 +439,14 @@ public class Hadoop20Shims implements Ha
     // but can be backported. So we disable setup/cleanup in all versions >= 0.19
     conf.setBoolean("mapred.committer.job.setup.cleanup.needed", false);
   }
+
+  @Override
+  public UserGroupInformation getUGIForConf(Configuration conf) throws LoginException {
+    UserGroupInformation ugi =
+      UnixUserGroupInformation.readFromConf(conf, UnixUserGroupInformation.UGI_PROPERTY_NAME);
+    if(ugi == null) {
+      ugi = UserGroupInformation.login(conf);
+    }
+    return ugi;
+  }
 }

Added: hadoop/hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java?rev=1021549&view=auto
==============================================================================
--- hadoop/hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
(added)
+++ hadoop/hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
Mon Oct 11 23:41:24 2010
@@ -0,0 +1,446 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.shims;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.util.ArrayList;
+import java.util.List;
+import javax.security.auth.login.LoginException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.JobContext;
+import org.apache.hadoop.mapred.JobStatus;
+import org.apache.hadoop.mapred.OutputCommitter;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reporter;
+import org.apache.hadoop.mapred.RunningJob;
+import org.apache.hadoop.mapred.TaskAttemptContext;
+import org.apache.hadoop.mapred.TaskCompletionEvent;
+import org.apache.hadoop.mapred.TaskID;
+import org.apache.hadoop.mapred.lib.CombineFileInputFormat;
+import org.apache.hadoop.mapred.lib.CombineFileSplit;
+import org.apache.hadoop.mapred.lib.NullOutputFormat;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.tools.HadoopArchives;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * Implemention of shims against Hadoop 0.20.0.
+ */
+public class Hadoop20SShims implements HadoopShims {
+  public boolean usesJobShell() {
+    return false;
+  }
+
+  public boolean fileSystemDeleteOnExit(FileSystem fs, Path path)
+      throws IOException {
+
+    return fs.deleteOnExit(path);
+  }
+
+  public void inputFormatValidateInput(InputFormat fmt, JobConf conf)
+      throws IOException {
+    // gone in 0.18+
+  }
+
+  public boolean isJobPreparing(RunningJob job) throws IOException {
+    return job.getJobState() == JobStatus.PREP;
+  }
+  /**
+   * Workaround for hadoop-17 - jobclient only looks at commandlineconfig.
+   */
+  public void setTmpFiles(String prop, String files) {
+    // gone in 20+
+  }
+
+  public HadoopShims.MiniDFSShim getMiniDfs(Configuration conf,
+      int numDataNodes,
+      boolean format,
+      String[] racks) throws IOException {
+    return new MiniDFSShim(new MiniDFSCluster(conf, numDataNodes, format, racks));
+  }
+
+  /**
+   * MiniDFSShim.
+   *
+   */
+  public class MiniDFSShim implements HadoopShims.MiniDFSShim {
+    private final MiniDFSCluster cluster;
+
+    public MiniDFSShim(MiniDFSCluster cluster) {
+      this.cluster = cluster;
+    }
+
+    public FileSystem getFileSystem() throws IOException {
+      return cluster.getFileSystem();
+    }
+
+    public void shutdown() {
+      cluster.shutdown();
+    }
+  }
+
+  /**
+   * We define this function here to make the code compatible between
+   * hadoop 0.17 and hadoop 0.20.
+   *
+   * Hive binary that compiled Text.compareTo(Text) with hadoop 0.20 won't
+   * work with hadoop 0.17 because in hadoop 0.20, Text.compareTo(Text) is
+   * implemented in org.apache.hadoop.io.BinaryComparable, and Java compiler
+   * references that class, which is not available in hadoop 0.17.
+   */
+  public int compareText(Text a, Text b) {
+    return a.compareTo(b);
+  }
+
+  @Override
+  public long getAccessTime(FileStatus file) {
+    return file.getAccessTime();
+  }
+
+  public HadoopShims.CombineFileInputFormatShim getCombineFileInputFormat() {
+    return new CombineFileInputFormatShim() {
+      @Override
+      public RecordReader getRecordReader(InputSplit split,
+          JobConf job, Reporter reporter) throws IOException {
+        throw new IOException("CombineFileInputFormat.getRecordReader not needed.");
+      }
+    };
+  }
+
+  public static class InputSplitShim extends CombineFileSplit implements HadoopShims.InputSplitShim
{
+    public InputSplitShim() {
+      super();
+    }
+
+    public InputSplitShim(CombineFileSplit old) throws IOException {
+      super(old);
+    }
+  }
+
+  public static class CombineHiveKey implements WritableComparable {
+    Object key;
+
+    public CombineHiveKey(Object key) {
+      this.key = key;
+    }
+
+    public Object getKey() {
+      return key;
+    }
+
+    public void setKey(Object key) {
+      this.key = key;
+    }
+
+    public void write(DataOutput out) throws IOException {
+      throw new IOException("Method not supported");
+    }
+
+    public void readFields(DataInput in) throws IOException {
+      throw new IOException("Method not supported");
+    }
+
+    public int compareTo(Object w) {
+      assert false;
+      return 0;
+    }
+  }
+
+  /* This class should be replaced with org.apache.hadoop.mapred.lib.CombineFileRecordReader
class, once
+   * https://issues.apache.org/jira/browse/MAPREDUCE-955 is fixed. This code should be removed
- it is a copy
+   * of org.apache.hadoop.mapred.lib.CombineFileRecordReader
+   */
+  public static class CombineFileRecordReader<K, V> implements RecordReader<K, V>
{
+
+    static final Class[] constructorSignature = new Class[] {
+        InputSplit.class,
+        Configuration.class,
+        Reporter.class,
+        Integer.class
+        };
+
+    protected CombineFileSplit split;
+    protected JobConf jc;
+    protected Reporter reporter;
+    protected Class<RecordReader<K, V>> rrClass;
+    protected Constructor<RecordReader<K, V>> rrConstructor;
+    protected FileSystem fs;
+
+    protected int idx;
+    protected long progress;
+    protected RecordReader<K, V> curReader;
+
+    public boolean next(K key, V value) throws IOException {
+
+      while ((curReader == null) || !curReader.next((K)((CombineHiveKey)key).getKey(), value))
{
+        if (!initNextRecordReader(key)) {
+          return false;
+        }
+      }
+      return true;
+    }
+
+    public K createKey() {
+      K newKey = curReader.createKey();
+      return (K)(new CombineHiveKey(newKey));
+    }
+
+    public V createValue() {
+      return curReader.createValue();
+    }
+
+    /**
+     * Return the amount of data processed.
+     */
+    public long getPos() throws IOException {
+      return progress;
+    }
+
+    public void close() throws IOException {
+      if (curReader != null) {
+        curReader.close();
+        curReader = null;
+      }
+    }
+
+    /**
+     * Return progress based on the amount of data processed so far.
+     */
+    public float getProgress() throws IOException {
+      return Math.min(1.0f, progress / (float) (split.getLength()));
+    }
+
+    /**
+     * A generic RecordReader that can hand out different recordReaders
+     * for each chunk in the CombineFileSplit.
+     */
+    public CombineFileRecordReader(JobConf job, CombineFileSplit split,
+        Reporter reporter,
+        Class<RecordReader<K, V>> rrClass)
+        throws IOException {
+      this.split = split;
+      this.jc = job;
+      this.rrClass = rrClass;
+      this.reporter = reporter;
+      this.idx = 0;
+      this.curReader = null;
+      this.progress = 0;
+
+      try {
+        rrConstructor = rrClass.getDeclaredConstructor(constructorSignature);
+        rrConstructor.setAccessible(true);
+      } catch (Exception e) {
+        throw new RuntimeException(rrClass.getName() +
+            " does not have valid constructor", e);
+      }
+      initNextRecordReader(null);
+    }
+
+    /**
+     * Get the record reader for the next chunk in this CombineFileSplit.
+     */
+    protected boolean initNextRecordReader(K key) throws IOException {
+
+      if (curReader != null) {
+        curReader.close();
+        curReader = null;
+        if (idx > 0) {
+          progress += split.getLength(idx - 1); // done processing so far
+        }
+      }
+
+      // if all chunks have been processed, nothing more to do.
+      if (idx == split.getNumPaths()) {
+        return false;
+      }
+
+      // get a record reader for the idx-th chunk
+      try {
+        curReader = rrConstructor.newInstance(new Object[]
+            {split, jc, reporter, Integer.valueOf(idx)});
+
+        // change the key if need be
+        if (key != null) {
+          K newKey = curReader.createKey();
+          ((CombineHiveKey)key).setKey(newKey);
+        }
+
+        // setup some helper config variables.
+        jc.set("map.input.file", split.getPath(idx).toString());
+        jc.setLong("map.input.start", split.getOffset(idx));
+        jc.setLong("map.input.length", split.getLength(idx));
+      } catch (Exception e) {
+        throw new RuntimeException(e);
+      }
+      idx++;
+      return true;
+    }
+  }
+
+  public abstract static class CombineFileInputFormatShim<K, V> extends
+      CombineFileInputFormat<K, V>
+      implements HadoopShims.CombineFileInputFormatShim<K, V> {
+
+    public Path[] getInputPathsShim(JobConf conf) {
+      try {
+        return FileInputFormat.getInputPaths(conf);
+      } catch (Exception e) {
+        throw new RuntimeException(e);
+      }
+    }
+
+    @Override
+    public void createPool(JobConf conf, PathFilter... filters) {
+      super.createPool(conf, filters);
+    }
+
+    @Override
+    public InputSplitShim[] getSplits(JobConf job, int numSplits) throws IOException {
+      long minSize = job.getLong("mapred.min.split.size", 0);
+
+      // For backward compatibility, let the above parameter be used
+      if (job.getLong("mapred.min.split.size.per.node", 0) == 0) {
+        super.setMinSplitSizeNode(minSize);
+      }
+
+      if (job.getLong("mapred.min.split.size.per.rack", 0) == 0) {
+        super.setMinSplitSizeRack(minSize);
+      }
+
+      if (job.getLong("mapred.max.split.size", 0) == 0) {
+        super.setMaxSplitSize(minSize);
+      }
+
+      CombineFileSplit[] splits = (CombineFileSplit[]) super.getSplits(job, numSplits);
+
+      InputSplitShim[] isplits = new InputSplitShim[splits.length];
+      for (int pos = 0; pos < splits.length; pos++) {
+        isplits[pos] = new InputSplitShim(splits[pos]);
+      }
+
+      return isplits;
+    }
+
+    public InputSplitShim getInputSplitShim() throws IOException {
+      return new InputSplitShim();
+    }
+
+    public RecordReader getRecordReader(JobConf job, HadoopShims.InputSplitShim split,
+        Reporter reporter,
+        Class<RecordReader<K, V>> rrClass)
+        throws IOException {
+      CombineFileSplit cfSplit = (CombineFileSplit) split;
+      return new CombineFileRecordReader(job, cfSplit, reporter, rrClass);
+    }
+
+  }
+
+  public String getInputFormatClassName() {
+    return "org.apache.hadoop.hive.ql.io.CombineHiveInputFormat";
+  }
+
+  String[] ret = new String[2];
+
+  @Override
+  public String[] getTaskJobIDs(TaskCompletionEvent t) {
+    TaskID tid = t.getTaskAttemptId().getTaskID();
+    ret[0] = tid.toString();
+    ret[1] = tid.getJobID().toString();
+    return ret;
+  }
+
+  public void setFloatConf(Configuration conf, String varName, float val) {
+    conf.setFloat(varName, val);
+  }
+
+  @Override
+  public int createHadoopArchive(Configuration conf, Path sourceDir, Path destDir,
+      String archiveName) throws Exception {
+
+    HadoopArchives har = new HadoopArchives(conf);
+    List<String> args = new ArrayList<String>();
+
+    if (conf.get("hive.archive.har.parentdir.settable") == null) {
+      throw new RuntimeException("hive.archive.har.parentdir.settable is not set");
+    }
+    boolean parentSettable =
+      conf.getBoolean("hive.archive.har.parentdir.settable", false);
+
+    if (parentSettable) {
+      args.add("-archiveName");
+      args.add(archiveName);
+      args.add("-p");
+      args.add(sourceDir.toString());
+      args.add(destDir.toString());
+    } else {
+      args.add("-archiveName");
+      args.add(archiveName);
+      args.add(sourceDir.toString());
+      args.add(destDir.toString());
+    }
+
+    return ToolRunner.run(har, args.toArray(new String[0]));
+  }
+
+  public static class NullOutputCommitter extends OutputCommitter {
+    @Override
+    public void setupJob(JobContext jobContext) { }
+    @Override
+    public void cleanupJob(JobContext jobContext) { }
+
+    @Override
+    public void setupTask(TaskAttemptContext taskContext) { }
+    @Override
+    public boolean needsTaskCommit(TaskAttemptContext taskContext) {
+      return false;
+    }
+    @Override
+    public void commitTask(TaskAttemptContext taskContext) { }
+    @Override
+    public void abortTask(TaskAttemptContext taskContext) { }
+  }
+
+  public void setNullOutputFormat(JobConf conf) {
+    conf.setOutputFormat(NullOutputFormat.class);
+    conf.setOutputCommitter(Hadoop20Shims.NullOutputCommitter.class);
+
+    // option to bypass job setup and cleanup was introduced in hadoop-21 (MAPREDUCE-463)
+    // but can be backported. So we disable setup/cleanup in all versions >= 0.19
+    conf.setBoolean("mapred.committer.job.setup.cleanup.needed", false);
+  }
+
+  @Override
+  public UserGroupInformation getUGIForConf(Configuration conf) throws IOException {
+    return UserGroupInformation.getCurrentUser();
+  }
+}

Added: hadoop/hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Jetty20SShims.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Jetty20SShims.java?rev=1021549&view=auto
==============================================================================
--- hadoop/hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Jetty20SShims.java
(added)
+++ hadoop/hive/trunk/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Jetty20SShims.java
Mon Oct 11 23:41:24 2010
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.shims;
+
+
+import org.mortbay.jetty.bio.SocketConnector;
+import org.mortbay.jetty.handler.RequestLogHandler;
+import org.mortbay.jetty.webapp.WebAppContext;
+
+import java.io.IOException;
+
+public class Jetty20SShims implements JettyShims {
+  public Server startServer(String listen, int port) throws IOException {
+    Server s = new Server();
+    s.setupListenerHostPort(listen, port);
+    return s;
+  }
+
+  private static class Server extends org.mortbay.jetty.Server implements JettyShims.Server
{
+    public void addWar(String war, String contextPath) {
+      WebAppContext wac = new WebAppContext();
+      wac.setContextPath(contextPath);
+      wac.setWar(war);
+      RequestLogHandler rlh = new RequestLogHandler();
+      rlh.setHandler(wac);
+      this.addHandler(rlh);
+   }
+
+    public void setupListenerHostPort(String listen, int port)
+      throws IOException {
+
+      SocketConnector connector  = new SocketConnector();
+      connector.setPort(port);
+      connector.setHost(listen);
+      this.addConnector(connector);
+    }
+  }
+}

Modified: hadoop/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1021549&r1=1021548&r2=1021549&view=diff
==============================================================================
--- hadoop/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
(original)
+++ hadoop/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
Mon Oct 11 23:41:24 2010
@@ -34,6 +34,11 @@ import org.apache.hadoop.mapred.RecordRe
 import org.apache.hadoop.mapred.Reporter;
 import org.apache.hadoop.mapred.RunningJob;
 import org.apache.hadoop.mapred.TaskCompletionEvent;
+import org.apache.hadoop.security.UserGroupInformation;
+import javax.security.auth.login.LoginException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 /**
  * In order to be compatible with multiple versions of Hadoop, all parts
@@ -45,6 +50,8 @@ import org.apache.hadoop.mapred.TaskComp
  */
 public interface HadoopShims {
 
+  static final Log LOG = LogFactory.getLog(HadoopShims.class);
+
   /**
    * Return true if the current version of Hadoop uses the JobShell for
    * command line interpretation.
@@ -144,6 +151,14 @@ public interface HadoopShims {
   void setNullOutputFormat(JobConf conf);
 
   /**
+   * Get the UGI that the given job configuration will run as.
+   *
+   * In secure versions of Hadoop, this simply returns the current
+   * access control context's user, ignoring the configuration.
+   */
+  public UserGroupInformation getUGIForConf(Configuration conf) throws LoginException, IOException;
+
+  /**
    * InputSplitShim.
    *
    */

Modified: hadoop/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/ShimLoader.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/ShimLoader.java?rev=1021549&r1=1021548&r2=1021549&view=diff
==============================================================================
--- hadoop/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/ShimLoader.java (original)
+++ hadoop/hive/trunk/shims/src/common/java/org/apache/hadoop/hive/shims/ShimLoader.java Mon
Oct 11 23:41:24 2010
@@ -41,6 +41,7 @@ public abstract class ShimLoader {
     HADOOP_SHIM_CLASSES.put("0.18", "org.apache.hadoop.hive.shims.Hadoop18Shims");
     HADOOP_SHIM_CLASSES.put("0.19", "org.apache.hadoop.hive.shims.Hadoop19Shims");
     HADOOP_SHIM_CLASSES.put("0.20", "org.apache.hadoop.hive.shims.Hadoop20Shims");
+    HADOOP_SHIM_CLASSES.put("0.20S", "org.apache.hadoop.hive.shims.Hadoop20SShims");
   }
 
   /**
@@ -55,6 +56,7 @@ public abstract class ShimLoader {
     JETTY_SHIM_CLASSES.put("0.18", "org.apache.hadoop.hive.shims.Jetty18Shims");
     JETTY_SHIM_CLASSES.put("0.19", "org.apache.hadoop.hive.shims.Jetty19Shims");
     JETTY_SHIM_CLASSES.put("0.20", "org.apache.hadoop.hive.shims.Jetty20Shims");
+    JETTY_SHIM_CLASSES.put("0.20S", "org.apache.hadoop.hive.shims.Jetty20SShims");
   }
 
   /**
@@ -105,7 +107,16 @@ public abstract class ShimLoader {
       throw new RuntimeException("Illegal Hadoop Version: " + vers +
           " (expected A.B.* format)");
     }
-    return parts[0] + "." + parts[1];
+    String majorVersion = parts[0] + "." + parts[1];
+
+    // If we are running a security release, we won't have UnixUserGroupInformation
+    // (removed by HADOOP-6299 when switching to JAAS for Login)
+    try {
+      Class.forName("org.apache.hadoop.security.UnixUserGroupInformation");
+    } catch (ClassNotFoundException cnf) {
+      majorVersion += "S";
+    }
+    return majorVersion;
   }
 
   private ShimLoader() {



Mime
View raw message