hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amareshw...@apache.org
Subject svn commit: r1233260 [2/2] - in /hive/branches/branch-0.8-r2: common/src/java/org/apache/hadoop/hive/conf/ conf/ metastore/if/ metastore/src/gen/thrift/gen-cpp/ metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ metastore/src/g...
Date Thu, 19 Jan 2012 09:19:39 GMT
Modified: hive/branches/branch-0.8-r2/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.8-r2/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1233260&r1=1233259&r2=1233260&view=diff
==============================================================================
--- hive/branches/branch-0.8-r2/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
(original)
+++ hive/branches/branch-0.8-r2/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
Thu Jan 19 09:19:38 2012
@@ -104,6 +104,7 @@ import org.apache.hadoop.hive.serde2.Ser
 import org.apache.hadoop.hive.serde2.SerDeUtils;
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge;
+import org.apache.hadoop.hive.thrift.TUGIContainingTransport;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.StringUtils;
@@ -3581,6 +3582,13 @@ public class HiveMetaStore extends Thrif
 
       return ret;
     }
+
+    @Override
+    public List<String> set_ugi(String username, List<String> groupNames) throws
MetaException,
+      TException {
+      Collections.addAll(groupNames, username);
+      return groupNames;
+  }
   }
 
 
@@ -3742,19 +3750,28 @@ public class HiveMetaStore extends Thrif
       TProcessor processor;
       TTransportFactory transFactory;
       if (useSasl) {
+        // we are in secure mode.
          saslServer = bridge.createServer(
            conf.getVar(HiveConf.ConfVars.METASTORE_KERBEROS_KEYTAB_FILE),
            conf.getVar(HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL));
-
         // start delegation token manager
         saslServer.startDelegationTokenSecretManager(conf);
         transFactory = saslServer.createTransportFactory();
-        processor = saslServer.wrapProcessor(new ThriftHiveMetastore.Processor(
+        processor = saslServer.wrapProcessor(new ThriftHiveMetastore.Processor<HMSHandler>(
             new HMSHandler("new db based metaserver", conf)));
+        LOG.info("Starting DB backed MetaStore Server in Secure Mode");
       } else {
-        processor = new ThriftHiveMetastore.Processor(
-            new HMSHandler("new db based metaserver", conf));
+        // we are in unsecure mode.
+        HMSHandler handler = new HMSHandler("new db based metaserver", conf);
+        if (conf.getBoolVar(ConfVars.METASTORE_EXECUTE_SET_UGI)){
+          transFactory = new TUGIContainingTransport.Factory();
+          processor = new TUGIBasedProcessor<HMSHandler>(handler);
+          LOG.info("Starting DB backed MetaStore Server with SetUGI enabled");
+        } else{
         transFactory = new TTransportFactory();
+          processor  = new ThriftHiveMetastore.Processor<HMSHandler>(handler);
+          LOG.info("Starting DB backed MetaStore Server");
+      }
       }
 
       TThreadPoolServer.Args args = new TThreadPoolServer.Args(serverTransport)

Modified: hive/branches/branch-0.8-r2/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.8-r2/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java?rev=1233260&r1=1233259&r2=1233260&view=diff
==============================================================================
--- hive/branches/branch-0.8-r2/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
(original)
+++ hive/branches/branch-0.8-r2/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
Thu Jan 19 09:19:38 2012
@@ -28,10 +28,13 @@ import java.lang.reflect.Proxy;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 
+import javax.security.auth.login.LoginException;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.conf.HiveConf;
@@ -60,8 +63,10 @@ import org.apache.hadoop.hive.metastore.
 import org.apache.hadoop.hive.metastore.api.UnknownDBException;
 import org.apache.hadoop.hive.metastore.api.UnknownPartitionException;
 import org.apache.hadoop.hive.metastore.api.UnknownTableException;
+import org.apache.hadoop.hive.shims.HadoopShims;
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge;
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.thrift.TException;
 import org.apache.thrift.protocol.TBinaryProtocol;
 import org.apache.thrift.protocol.TProtocol;
@@ -219,7 +224,8 @@ public class HiveMetaStoreClient impleme
       ((TSocket)transport).setTimeout(1000 * conf.getIntVar(ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT));
 
       // Wrap thrift connection with SASL if enabled.
-      boolean useSasl = conf.getBoolVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL);
+      HadoopShims shim = ShimLoader.getHadoopShims();
+      boolean useSasl = conf.getBoolVar(ConfVars.METASTORE_USE_THRIFT_SASL);
       if (useSasl) {
         try {
           HadoopThriftAuthBridge.Client authBridge =
@@ -232,7 +238,7 @@ public class HiveMetaStoreClient impleme
           // submission.
           String tokenSig = conf.get("hive.metastore.token.signature");
           // tokenSig could be null
-          tokenStrForm = ShimLoader.getHadoopShims().getTokenStrForm(tokenSig);
+          tokenStrForm = shim.getTokenStrForm(tokenSig);
 
           if(tokenStrForm != null) {
             // authenticate using delegation tokens via the "DIGEST" mechanism
@@ -264,6 +270,21 @@ public class HiveMetaStoreClient impleme
           LOG.warn("Failed to connect to the MetaStore Server...");
         }
       }
+     if (!useSasl && conf.getBoolVar(ConfVars.METASTORE_EXECUTE_SET_UGI)){
+       // Call set_ugi, only in unsecure mode.
+       try {
+         UserGroupInformation ugi = shim.getUGIForConf(conf);
+         client.set_ugi(ugi.getUserName(), Arrays.asList(ugi.getGroupNames()));
+       } catch (LoginException e) {
+         LOG.warn("Failed to do login. set_ugi() is not successful, Continuing without it.",
e);
+       } catch (IOException e) {
+         LOG.warn("Failed to find ugi of client set_ugi() is not successful, " +
+            "Continuing without it.", e);
+       } catch (TException e) {
+         LOG.warn("set_ugi() not successful, Likely cause: new client talking to old server.
" +
+         		"Continuing without it.", e);
+    }
+     }
     }
     if (!isConnected) {
       throw new MetaException("Could not connect to the MetaStore server!");

Modified: hive/branches/branch-0.8-r2/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.8-r2/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java?rev=1233260&r1=1233259&r2=1233260&view=diff
==============================================================================
--- hive/branches/branch-0.8-r2/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java
(original)
+++ hive/branches/branch-0.8-r2/metastore/src/test/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java
Thu Jan 19 09:19:38 2012
@@ -23,7 +23,7 @@ import org.apache.hadoop.hive.conf.HiveC
 
 
 public class TestRemoteHiveMetaStore extends TestHiveMetaStore {
-  private static final String METASTORE_PORT = "29083";
+  protected static final String METASTORE_PORT = "29083";
   private static boolean isServerStarted = false;
 
   public TestRemoteHiveMetaStore() {
@@ -62,14 +62,14 @@ public class TestRemoteHiveMetaStore ext
     // Wait a little bit for the metastore to start. Should probably have
     // a better way of detecting if the metastore has started?
     Thread.sleep(5000);
+    // This is default case with setugi off for both client and server
+    createClient(false);
+  }
 
-    // hive.metastore.local should be defined in HiveConf
+  protected void createClient(boolean setugi) throws Exception {
     hiveConf.setBoolVar(ConfVars.METASTORE_MODE, false);
     hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + METASTORE_PORT);
-    hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTRETRIES, 3);
-    hiveConf.setIntVar(ConfVars.METASTORE_CLIENT_CONNECT_RETRY_DELAY, 60);
-
+    hiveConf.setBoolVar(ConfVars.METASTORE_EXECUTE_SET_UGI,setugi);
     client = new HiveMetaStoreClient(hiveConf);
   }
-
 }

Modified: hive/branches/branch-0.8-r2/shims/ivy.xml
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.8-r2/shims/ivy.xml?rev=1233260&r1=1233259&r2=1233260&view=diff
==============================================================================
--- hive/branches/branch-0.8-r2/shims/ivy.xml (original)
+++ hive/branches/branch-0.8-r2/shims/ivy.xml Thu Jan 19 09:19:38 2012
@@ -43,6 +43,8 @@
                 transitive="false"/>
     <dependency org="commons-logging" name="commons-logging-api" rev="${commons-logging-api.version}"
                 transitive="false"/>
+    <dependency org="com.google.guava" name="guava" rev="${guava.version}"
+                transitive="false"/>
     <conflict manager="all" />
   </dependencies>
 </ivy-module>

Modified: hive/branches/branch-0.8-r2/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.8-r2/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=1233260&r1=1233259&r2=1233260&view=diff
==============================================================================
--- hive/branches/branch-0.8-r2/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
(original)
+++ hive/branches/branch-0.8-r2/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
Thu Jan 19 09:19:38 2012
@@ -21,9 +21,12 @@ import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
 import java.lang.reflect.Constructor;
+import java.security.PrivilegedActionException;
+import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
 import java.util.List;
 
+import javax.security.auth.Subject;
 import javax.security.auth.login.LoginException;
 
 import org.apache.hadoop.conf.Configuration;
@@ -32,7 +35,6 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hive.io.HiveIOExceptionHandlerChain;
 import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.ClusterStatus;
@@ -54,6 +56,7 @@ import org.apache.hadoop.mapred.lib.Comb
 import org.apache.hadoop.mapred.lib.NullOutputFormat;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UnixUserGroupInformation;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.tools.HadoopArchives;
@@ -510,6 +513,21 @@ public class Hadoop20Shims implements Ha
   }
 
   @Override
+  public void doAs(UserGroupInformation ugi, PrivilegedExceptionAction<Void> pvea)
throws
+    IOException, InterruptedException {
+    try {
+      Subject.doAs(SecurityUtil.getSubject(ugi),pvea);
+    } catch (PrivilegedActionException e) {
+      throw new IOException(e);
+    }
+  }
+
+  @Override
+  public UserGroupInformation createRemoteUser(String userName, List<String> groupNames)
{
+    return new UnixUserGroupInformation(userName, groupNames.toArray(new String[0]));
+  }
+
+  @Override
   public JobTrackerState getJobTrackerState(ClusterStatus clusterStatus) throws Exception
{
     JobTrackerState state;
     switch (clusterStatus.getJobTrackerState()) {

Modified: hive/branches/branch-0.8-r2/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.8-r2/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java?rev=1233260&r1=1233259&r2=1233260&view=diff
==============================================================================
--- hive/branches/branch-0.8-r2/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
(original)
+++ hive/branches/branch-0.8-r2/shims/src/0.20S/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
Thu Jan 19 09:19:38 2012
@@ -21,6 +21,7 @@ import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
 import java.lang.reflect.Constructor;
+import java.security.PrivilegedExceptionAction;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -30,14 +31,9 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.PathFilter;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hive.io.HiveIOExceptionHandlerChain;
 import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil;
-import org.apache.hadoop.hive.shims.Hadoop20Shims;
-import org.apache.hadoop.hive.shims.HadoopShims;
-import org.apache.hadoop.hive.shims.Hadoop20Shims.InputSplitShim;
 import org.apache.hadoop.hive.thrift.DelegationTokenSelector;
 import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.WritableComparable;
 import org.apache.hadoop.mapred.ClusterStatus;
 import org.apache.hadoop.mapred.FileInputFormat;
 import org.apache.hadoop.mapred.InputFormat;
@@ -517,6 +513,16 @@ public class Hadoop20SShims implements H
   }
   
   @Override
+  public void doAs(UserGroupInformation ugi, PrivilegedExceptionAction<Void> pvea)
throws IOException, InterruptedException {
+    ugi.doAs(pvea);
+  }
+
+  @Override
+  public UserGroupInformation createRemoteUser(String userName, List<String> groupNames)
{
+    return UserGroupInformation.createRemoteUser(userName);
+  }
+
+  @Override
   public JobTrackerState getJobTrackerState(ClusterStatus clusterStatus) throws Exception
{
     JobTrackerState state;
     switch (clusterStatus.getJobTrackerState()) {

Modified: hive/branches/branch-0.8-r2/shims/src/0.20S/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.8-r2/shims/src/0.20S/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java?rev=1233260&r1=1233259&r2=1233260&view=diff
==============================================================================
--- hive/branches/branch-0.8-r2/shims/src/0.20S/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java
(original)
+++ hive/branches/branch-0.8-r2/shims/src/0.20S/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java
Thu Jan 19 09:19:38 2012
@@ -39,6 +39,7 @@ import org.apache.commons.lang.StringUti
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.thrift.client.TUGIAssumingTransport;
 import org.apache.hadoop.security.SaslRpcServer;
 import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
 import org.apache.hadoop.security.SecurityUtil;
@@ -189,131 +190,7 @@ import org.apache.thrift.transport.TTran
         return new String(Base64.encodeBase64(password)).toCharArray();
        }
      }
-    /**
-      * The Thrift SASL transports call Sasl.createSaslServer and Sasl.createSaslClient
-      * inside open(). So, we need to assume the correct UGI when the transport is opened
-      * so that the SASL mechanisms have access to the right principal. This transport
-      * wraps the Sasl transports to set up the right UGI context for open().
-      *
-      * This is used on the client side, where the API explicitly opens a transport to
-      * the server.
-      */
-     private static class TUGIAssumingTransport extends TFilterTransport {
-       private final UserGroupInformation ugi;
-
-       public TUGIAssumingTransport(TTransport wrapped, UserGroupInformation ugi) {
-         super(wrapped);
-         this.ugi = ugi;
-       }
-
-       @Override
-       public void open() throws TTransportException {
-         try {
-           ugi.doAs(new PrivilegedExceptionAction<Void>() {
-             public Void run() {
-               try {
-                 wrapped.open();
-               } catch (TTransportException tte) {
-                 // Wrap the transport exception in an RTE, since UGI.doAs() then goes
-                 // and unwraps this for us out of the doAs block. We then unwrap one
-                 // more time in our catch clause to get back the TTE. (ugh)
-                 throw new RuntimeException(tte);
-               }
-               return null;
-             }
-           });
-         } catch (IOException ioe) {
-           assert false : "Never thrown!";
-           throw new RuntimeException("Received an ioe we never threw!", ioe);
-         } catch (InterruptedException ie) {
-           assert false : "We never expect to see an InterruptedException thrown in this
block";
-           throw new RuntimeException("Received an ie we never threw!", ie);
-         } catch (RuntimeException rte) {
-           if (rte.getCause() instanceof TTransportException) {
-             throw (TTransportException)rte.getCause();
-           } else {
-             throw rte;
-           }
-         }
        }
-     }
-    /**
-      * Transport that simply wraps another transport.
-      * This is the equivalent of FilterInputStream for Thrift transports.
-      */
-     private static class TFilterTransport extends TTransport {
-       protected final TTransport wrapped;
-
-       public TFilterTransport(TTransport wrapped) {
-         this.wrapped = wrapped;
-       }
-
-       @Override
-       public void open() throws TTransportException {
-         wrapped.open();
-       }
-
-       @Override
-       public boolean isOpen() {
-         return wrapped.isOpen();
-       }
-
-       @Override
-       public boolean peek() {
-         return wrapped.peek();
-       }
-
-       @Override
-       public void close() {
-         wrapped.close();
-       }
-
-       @Override
-       public int read(byte[] buf, int off, int len) throws TTransportException {
-         return wrapped.read(buf, off, len);
-       }
-
-       @Override
-       public int readAll(byte[] buf, int off, int len) throws TTransportException {
-         return wrapped.readAll(buf, off, len);
-       }
-
-       @Override
-       public void write(byte[] buf) throws TTransportException {
-         wrapped.write(buf);
-       }
-
-       @Override
-       public void write(byte[] buf, int off, int len) throws TTransportException {
-         wrapped.write(buf, off, len);
-       }
-
-       @Override
-       public void flush() throws TTransportException {
-         wrapped.flush();
-       }
-
-       @Override
-       public byte[] getBuffer() {
-         return wrapped.getBuffer();
-       }
-
-       @Override
-       public int getBufferPosition() {
-         return wrapped.getBufferPosition();
-       }
-
-       @Override
-       public int getBytesRemainingInBuffer() {
-         return wrapped.getBytesRemainingInBuffer();
-       }
-
-       @Override
-       public void consumeBuffer(int len) {
-         wrapped.consumeBuffer(len);
-       }
-     }
-   }
 
    public static class Server extends HadoopThriftAuthBridge.Server {
      final UserGroupInformation realUgi;

Modified: hive/branches/branch-0.8-r2/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.8-r2/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1233260&r1=1233259&r2=1233260&view=diff
==============================================================================
--- hive/branches/branch-0.8-r2/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
(original)
+++ hive/branches/branch-0.8-r2/shims/src/common/java/org/apache/hadoop/hive/shims/HadoopShims.java
Thu Jan 19 09:19:38 2012
@@ -20,6 +20,8 @@ package org.apache.hadoop.hive.shims;
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import java.util.List;
 
 import javax.security.auth.login.LoginException;
 
@@ -164,6 +166,24 @@ public interface HadoopShims {
   public UserGroupInformation getUGIForConf(Configuration conf) throws LoginException, IOException;
 
   /**
+   * Used by metastore server to perform requested rpc in client context.
+   * @param ugi
+   * @param pvea
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  public void doAs(UserGroupInformation ugi, PrivilegedExceptionAction<Void> pvea)
throws
+    IOException, InterruptedException;
+
+  /**
+   * Used by metastore server to creates UGI object for a remote user.
+   * @param userName remote User Name
+   * @param groupNames group names associated with remote user name
+   * @return UGI created for the remote user.
+   */
+
+  public UserGroupInformation createRemoteUser(String userName, List<String> groupNames);
+  /**
    * Get the short name corresponding to the subject in the passed UGI
    *
    * In secure versions of Hadoop, this returns the short name (after



Mime
View raw message