hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From the...@apache.org
Subject svn commit: r1629778 - in /hive/branches/branch-0.14: itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/ metastore/src/java/org/apache/hadoop/hive/metastore/ shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/
Date Mon, 06 Oct 2014 22:19:48 GMT
Author: thejas
Date: Mon Oct  6 22:19:47 2014
New Revision: 1629778

URL: http://svn.apache.org/r1629778
Log:
HIVE-8310 : RetryingHMSHandler is not used when kerberos auth enabled (Thejas Nair, reviewed
by Ashutosh Chauhan)

Modified:
    hive/branches/branch-0.14/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestDBTokenStore.java
    hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
    hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
    hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInit.java
    hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java
    hive/branches/branch-0.14/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/DBTokenStore.java
    hive/branches/branch-0.14/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java

Modified: hive/branches/branch-0.14/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestDBTokenStore.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestDBTokenStore.java?rev=1629778&r1=1629777&r2=1629778&view=diff
==============================================================================
--- hive/branches/branch-0.14/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestDBTokenStore.java
(original)
+++ hive/branches/branch-0.14/itests/hive-unit/src/test/java/org/apache/hadoop/hive/thrift/TestDBTokenStore.java
Mon Oct  6 22:19:47 2014
@@ -37,7 +37,7 @@ public class TestDBTokenStore extends Te
   public void testDBTokenStore() throws TokenStoreException, MetaException, IOException {
 
     DelegationTokenStore ts = new DBTokenStore();
-    ts.setStore(new HMSHandler("Test handler"));
+    ts.setStore(new HMSHandler("Test handler").getMS());
     assertEquals(0, ts.getMasterKeys().length);
     assertEquals(false,ts.removeMasterKey(-1));
     try{

Modified: hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1629778&r1=1629777&r2=1629778&view=diff
==============================================================================
--- hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
(original)
+++ hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
Mon Oct  6 22:19:47 2014
@@ -5489,13 +5489,20 @@ public class HiveMetaStore extends Thrif
   }
 
 
-  public static IHMSHandler newHMSHandler(String name, HiveConf hiveConf) throws MetaException
{
-    return newHMSHandler(name, hiveConf, false);
+  public static IHMSHandler newRetryingHMSHandler(IHMSHandler baseHandler, HiveConf hiveConf)
+      throws MetaException {
+    return newRetryingHMSHandler(baseHandler, hiveConf, false);
+  }
+
+  public static IHMSHandler newRetryingHMSHandler(IHMSHandler baseHandler, HiveConf hiveConf,
+      boolean local) throws MetaException {
+    return RetryingHMSHandler.getProxy(hiveConf, baseHandler, local);
   }
 
-  public static IHMSHandler newHMSHandler(String name, HiveConf hiveConf, boolean local)
+  public static Iface newRetryingHMSHandler(String name, HiveConf conf, boolean local)
       throws MetaException {
-    return RetryingHMSHandler.getProxy(hiveConf, name, local);
+    HMSHandler baseHandler = new HiveMetaStore.HMSHandler(name, conf, false);
+    return RetryingHMSHandler.getProxy(conf, baseHandler, local);
   }
 
   /**
@@ -5704,6 +5711,9 @@ public class HiveMetaStore extends Thrif
 
       TProcessor processor;
       TTransportFactory transFactory;
+      HMSHandler baseHandler = new HiveMetaStore.HMSHandler("new db based metaserver", conf,
+          false);
+      IHMSHandler handler = newRetryingHMSHandler(baseHandler, conf);
       if (useSasl) {
         // we are in secure mode.
         if (useFramedTransport) {
@@ -5713,17 +5723,14 @@ public class HiveMetaStore extends Thrif
             conf.getVar(HiveConf.ConfVars.METASTORE_KERBEROS_KEYTAB_FILE),
             conf.getVar(HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL));
         // start delegation token manager
-        HMSHandler hmsHandler = new HMSHandler("new db based metaserver", conf);
-        saslServer.startDelegationTokenSecretManager(conf, hmsHandler);
+        saslServer.startDelegationTokenSecretManager(conf, baseHandler.getMS());
         transFactory = saslServer.createTransportFactory(
                 MetaStoreUtils.getMetaStoreSaslProperties(conf));
         processor = saslServer.wrapProcessor(
-          new ThriftHiveMetastore.Processor<HMSHandler>(hmsHandler));
+          new ThriftHiveMetastore.Processor<IHMSHandler>(handler));
         LOG.info("Starting DB backed MetaStore Server in Secure Mode");
       } else {
         // we are in unsecure mode.
-        IHMSHandler handler = newHMSHandler("new db based metaserver", conf);
-
         if (conf.getBoolVar(ConfVars.METASTORE_EXECUTE_SET_UGI)) {
           transFactory = useFramedTransport ?
               new ChainedTTransportFactory(new TFramedTransport.Factory(),

Modified: hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java?rev=1629778&r1=1629777&r2=1629778&view=diff
==============================================================================
--- hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
(original)
+++ hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
Mon Oct  6 22:19:47 2014
@@ -175,7 +175,7 @@ public class HiveMetaStoreClient impleme
     if (localMetaStore) {
       // instantiate the metastore server handler directly instead of connecting
       // through the network
-      client = HiveMetaStore.newHMSHandler("hive client", conf, true);
+      client = HiveMetaStore.newRetryingHMSHandler("hive client", conf, true);
       isConnected = true;
       snapshotActiveConf();
       return;

Modified: hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInit.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInit.java?rev=1629778&r1=1629777&r2=1629778&view=diff
==============================================================================
--- hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInit.java
(original)
+++ hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInit.java
Mon Oct  6 22:19:47 2014
@@ -41,25 +41,30 @@ public class MetaStoreInit {
   }
 
   /**
-   * Updates the connection URL in hiveConf using the hook
-   *
+   * Updates the connection URL in hiveConf using the hook (if a hook has been
+   * set using hive.metastore.ds.connection.url.hook property)
+   * @param originalConf - original configuration used to look up hook settings
+   * @param activeConf - the configuration file in use for looking up db url
+   * @param badUrl
+   * @param updateData - hook information
    * @return true if a new connection URL was loaded into the thread local
    *         configuration
+   * @throws MetaException
    */
-  static boolean updateConnectionURL(HiveConf hiveConf, Configuration conf,
+  static boolean updateConnectionURL(HiveConf originalConf, Configuration activeConf,
     String badUrl, MetaStoreInitData updateData)
       throws MetaException {
     String connectUrl = null;
-    String currentUrl = MetaStoreInit.getConnectionURL(conf);
+    String currentUrl = MetaStoreInit.getConnectionURL(activeConf);
     try {
       // We always call init because the hook name in the configuration could
       // have changed.
-      MetaStoreInit.initConnectionUrlHook(hiveConf, updateData);
+      MetaStoreInit.initConnectionUrlHook(originalConf, updateData);
       if (updateData.urlHook != null) {
         if (badUrl != null) {
           updateData.urlHook.notifyBadConnectionUrl(badUrl);
         }
-        connectUrl = updateData.urlHook.getJdoConnectionUrl(hiveConf);
+        connectUrl = updateData.urlHook.getJdoConnectionUrl(originalConf);
       }
     } catch (Exception e) {
       LOG.error("Exception while getting connection URL from the hook: " +
@@ -71,7 +76,7 @@ public class MetaStoreInit {
           String.format("Overriding %s with %s",
               HiveConf.ConfVars.METASTORECONNECTURLKEY.toString(),
               connectUrl));
-      conf.set(HiveConf.ConfVars.METASTORECONNECTURLKEY.toString(),
+      activeConf.set(HiveConf.ConfVars.METASTORECONNECTURLKEY.toString(),
           connectUrl);
       return true;
     }

Modified: hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java?rev=1629778&r1=1629777&r2=1629778&view=diff
==============================================================================
--- hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java
(original)
+++ hive/branches/branch-0.14/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java
Mon Oct  6 22:19:47 2014
@@ -41,34 +41,34 @@ public class RetryingHMSHandler implemen
 
   private static final Log LOG = LogFactory.getLog(RetryingHMSHandler.class);
 
-  private final IHMSHandler base;
+  private final IHMSHandler baseHandler;
   private final MetaStoreInit.MetaStoreInitData metaStoreInitData =
     new MetaStoreInit.MetaStoreInitData();
 
-  private final HiveConf hiveConf;            // base configuration
-  private final Configuration configuration;  // active configuration
+  private final HiveConf origConf;            // base configuration
+  private final Configuration activeConf;  // active configuration
 
-  private RetryingHMSHandler(HiveConf hiveConf, String name, boolean local) throws MetaException
{
-    this.hiveConf = hiveConf;
-    this.base = new HiveMetaStore.HMSHandler(name, hiveConf, false);
+  private RetryingHMSHandler(HiveConf hiveConf, IHMSHandler baseHandler, boolean local) throws
MetaException {
+    this.origConf = hiveConf;
+    this.baseHandler = baseHandler;
     if (local) {
-      base.setConf(hiveConf); // tests expect configuration changes applied directly to metastore
+      baseHandler.setConf(hiveConf); // tests expect configuration changes applied directly
to metastore
     }
-    configuration = base.getConf();
+    activeConf = baseHandler.getConf();
 
     // This has to be called before initializing the instance of HMSHandler
     // Using the hook on startup ensures that the hook always has priority
     // over settings in *.xml.  The thread local conf needs to be used because at this point
     // it has already been initialized using hiveConf.
-    MetaStoreInit.updateConnectionURL(hiveConf, getConf(), null, metaStoreInitData);
+    MetaStoreInit.updateConnectionURL(hiveConf, getActiveConf(), null, metaStoreInitData);
 
-    base.init();
+    baseHandler.init();
   }
 
-  public static IHMSHandler getProxy(HiveConf hiveConf, String name, boolean local)
+  public static IHMSHandler getProxy(HiveConf hiveConf, IHMSHandler baseHandler, boolean
local)
       throws MetaException {
 
-    RetryingHMSHandler handler = new RetryingHMSHandler(hiveConf, name, local);
+    RetryingHMSHandler handler = new RetryingHMSHandler(hiveConf, baseHandler, local);
 
     return (IHMSHandler) Proxy.newProxyInstance(
       RetryingHMSHandler.class.getClassLoader(),
@@ -79,15 +79,15 @@ public class RetryingHMSHandler implemen
   public Object invoke(final Object proxy, final Method method, final Object[] args) throws
Throwable {
 
     boolean gotNewConnectUrl = false;
-    boolean reloadConf = HiveConf.getBoolVar(hiveConf,
+    boolean reloadConf = HiveConf.getBoolVar(origConf,
         HiveConf.ConfVars.HMSHANDLERFORCERELOADCONF);
-    long retryInterval = HiveConf.getTimeVar(hiveConf,
+    long retryInterval = HiveConf.getTimeVar(origConf,
         HiveConf.ConfVars.HMSHANDLERINTERVAL, TimeUnit.MILLISECONDS);
-    int retryLimit = HiveConf.getIntVar(hiveConf,
+    int retryLimit = HiveConf.getIntVar(origConf,
         HiveConf.ConfVars.HMSHANDLERATTEMPTS);
 
     if (reloadConf) {
-      MetaStoreInit.updateConnectionURL(hiveConf, getConf(),
+      MetaStoreInit.updateConnectionURL(origConf, getActiveConf(),
         null, metaStoreInitData);
     }
 
@@ -96,9 +96,9 @@ public class RetryingHMSHandler implemen
     while (true) {
       try {
         if (reloadConf || gotNewConnectUrl) {
-          base.setConf(getConf());
+          baseHandler.setConf(getActiveConf());
         }
-        return method.invoke(base, args);
+        return method.invoke(baseHandler, args);
 
       } catch (javax.jdo.JDOException e) {
         caughtException = e;
@@ -158,13 +158,13 @@ public class RetryingHMSHandler implemen
       Thread.sleep(retryInterval);
       // If we have a connection error, the JDO connection URL hook might
       // provide us with a new URL to access the datastore.
-      String lastUrl = MetaStoreInit.getConnectionURL(getConf());
-      gotNewConnectUrl = MetaStoreInit.updateConnectionURL(hiveConf, getConf(),
+      String lastUrl = MetaStoreInit.getConnectionURL(getActiveConf());
+      gotNewConnectUrl = MetaStoreInit.updateConnectionURL(origConf, getActiveConf(),
         lastUrl, metaStoreInitData);
     }
   }
 
-  public Configuration getConf() {
-    return configuration;
+  public Configuration getActiveConf() {
+    return activeConf;
   }
 }

Modified: hive/branches/branch-0.14/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/DBTokenStore.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/DBTokenStore.java?rev=1629778&r1=1629777&r2=1629778&view=diff
==============================================================================
--- hive/branches/branch-0.14/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/DBTokenStore.java
(original)
+++ hive/branches/branch-0.14/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/DBTokenStore.java
Mon Oct  6 22:19:47 2014
@@ -108,18 +108,17 @@ public class DBTokenStore implements Del
     return delTokenIdents;
   }
 
-  private Object hmsHandler;
+  private Object rawStore;
 
   @Override
-  public void setStore(Object hms) throws TokenStoreException {
-    hmsHandler = hms;
+  public void setStore(Object rawStore) throws TokenStoreException {
+    this.rawStore = rawStore;
   }
 
   private Object invokeOnRawStore(String methName, Object[] params, Class<?> ... paramTypes)
       throws TokenStoreException{
 
     try {
-      Object rawStore = hmsHandler.getClass().getMethod("getMS").invoke(hmsHandler);
       return rawStore.getClass().getMethod(methName, paramTypes).invoke(rawStore, params);
     } catch (IllegalArgumentException e) {
         throw new TokenStoreException(e);

Modified: hive/branches/branch-0.14/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java?rev=1629778&r1=1629777&r2=1629778&view=diff
==============================================================================
--- hive/branches/branch-0.14/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java
(original)
+++ hive/branches/branch-0.14/shims/common-secure/src/main/java/org/apache/hadoop/hive/thrift/HadoopThriftAuthBridge20S.java
Mon Oct  6 22:19:47 2014
@@ -417,7 +417,7 @@ public class HadoopThriftAuthBridge20S e
     }
 
     @Override
-    public void startDelegationTokenSecretManager(Configuration conf, Object hms)
+    public void startDelegationTokenSecretManager(Configuration conf, Object rawStore)
         throws IOException{
       long secretKeyInterval =
           conf.getLong(DELEGATION_KEY_UPDATE_INTERVAL_KEY,
@@ -430,7 +430,7 @@ public class HadoopThriftAuthBridge20S e
               DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT);
 
       DelegationTokenStore dts = getTokenStore(conf);
-      dts.setStore(hms);
+      dts.setStore(rawStore);
       secretManager = new TokenStoreDelegationTokenSecretManager(secretKeyInterval,
           tokenMaxLifetime,
           tokenRenewInterval,



Mime
View raw message