hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aihu...@apache.org
Subject hive git commit: HIVE-13149: Remove some unnecessary HMS connections from HS2 (Reviewed by Szehon Ho, Chaoyu Tang)
Date Mon, 11 Apr 2016 19:00:24 GMT
Repository: hive
Updated Branches:
  refs/heads/master 1cb4ce871 -> 37e6e1bf5


HIVE-13149: Remove some unnecessary HMS connections from HS2 (Reviewed by Szehon Ho, Chaoyu
Tang)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/37e6e1bf
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/37e6e1bf
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/37e6e1bf

Branch: refs/heads/master
Commit: 37e6e1bf56d7d2fd557730380b147c745fc051ce
Parents: 1cb4ce8
Author: Aihua Xu <aihuaxu@apache.org>
Authored: Fri Mar 18 10:27:49 2016 -0400
Committer: Aihua Xu <aihuaxu@apache.org>
Committed: Mon Apr 11 14:58:59 2016 -0400

----------------------------------------------------------------------
 .../hadoop/hive/metastore/TestMetastoreVersion.java   |  7 ++++---
 .../metastore/hbase/TestHBaseMetastoreMetrics.java    |  4 +---
 .../org/apache/hadoop/hive/hbase/HBaseQTestUtil.java  | 10 +++++++++-
 .../org/apache/hadoop/hive/hbase/HBaseTestSetup.java  |  3 ---
 .../java/org/apache/hadoop/hive/ql/QTestUtil.java     | 14 ++++++++++----
 .../hadoop/hive/metastore/HiveMetaStoreClient.java    | 10 ++++++----
 .../apache/hadoop/hive/ql/session/SessionState.java   |  8 --------
 7 files changed, 30 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/37e6e1bf/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
index 53f0d0e..5ceb3d2 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetastoreVersion.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hive.metastore;
 
 import java.io.File;
 import java.lang.reflect.Field;
-import java.util.Random;
 
 import junit.framework.TestCase;
 
@@ -32,6 +31,7 @@ import org.apache.hive.common.util.HiveStringUtils;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.metastore.ObjectStore;
 import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.metadata.Hive;
 import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
 import org.apache.hadoop.hive.ql.session.SessionState;
 
@@ -96,8 +96,9 @@ public class TestMetastoreVersion extends TestCase {
     // session creation should fail since the schema didn't get created
     try {
       SessionState.start(new CliSessionState(hiveConf));
-      fail("Expected exception");
-    } catch (RuntimeException re) {
+      Hive.get(hiveConf).getMSC();
+      fail("An exception is expected since schema is not created.");
+    } catch (Exception re) {
       LOG.info("Exception in testVersionRestriction: " + re, re);
       String msg = HiveStringUtils.stringifyException(re);
       assertTrue("Expected 'Version information not found in metastore' in: " + msg, msg

http://git-wip-us.apache.org/repos/asf/hive/blob/37e6e1bf/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java
index 3ed88f2..aefafe0 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseMetastoreMetrics.java
@@ -41,8 +41,6 @@ import java.io.IOException;
  */
 public class TestHBaseMetastoreMetrics extends HBaseIntegrationTests {
 
-  private CodahaleMetrics metrics;
-
   @BeforeClass
   public static void startup() throws Exception {
     HBaseIntegrationTests.startMiniCluster();
@@ -66,7 +64,6 @@ public class TestHBaseMetastoreMetrics extends HBaseIntegrationTests {
     conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, MetricsReporting.JSON_FILE.name()
+ "," + MetricsReporting.JMX.name());
     SessionState.start(new CliSessionState(conf));
     driver = new Driver(conf);
-    metrics = (CodahaleMetrics) MetricsFactory.getInstance();
   }
 
   @Test
@@ -107,6 +104,7 @@ public class TestHBaseMetastoreMetrics extends HBaseIntegrationTests {
     driver.run("use default");
     driver.run("drop database tempdb cascade");
 
+    CodahaleMetrics metrics = (CodahaleMetrics) MetricsFactory.getInstance();
     String json = metrics.dumpJson();
     MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, MetricsConstant.CREATE_TOTAL_DATABASES,
2);
     MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.COUNTER, MetricsConstant.CREATE_TOTAL_TABLES,
7);

http://git-wip-us.apache.org/repos/asf/hive/blob/37e6e1bf/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
index 3ff5742..70c0b13 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
@@ -39,12 +39,14 @@ public class HBaseQTestUtil extends QTestUtil {
   /** A handle to this harness's cluster */
   private final HConnection conn;
 
+  private HBaseTestSetup setup;
+
   public HBaseQTestUtil(
     String outDir, String logDir, MiniClusterType miniMr, HBaseTestSetup setup,
     String initScript, String cleanupScript)
     throws Exception {
-
     super(outDir, logDir, miniMr, null, "0.20", initScript, cleanupScript, false, false);
+    this.setup = setup;
     setup.preTest(conf);
     this.conn = setup.getConnection();
     super.init();
@@ -69,6 +71,12 @@ public class HBaseQTestUtil extends QTestUtil {
   }
 
   @Override
+  protected void initConfFromSetup() throws Exception {
+    super.initConfFromSetup();
+    setup.preTest(conf);
+  }
+
+  @Override
   public void createSources(String tname) throws Exception {
     super.createSources(tname);
 

http://git-wip-us.apache.org/repos/asf/hive/blob/37e6e1bf/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java
index e6383dc..cee7158 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/hbase/HBaseTestSetup.java
@@ -22,9 +22,6 @@ import java.io.IOException;
 import java.net.ServerSocket;
 import java.util.Arrays;
 
-import junit.extensions.TestSetup;
-import junit.framework.Test;
-
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HColumnDescriptor;

http://git-wip-us.apache.org/repos/asf/hive/blob/37e6e1bf/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 8473436..2f109ab 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -510,6 +510,7 @@ public class QTestUtil {
       dfs.shutdown();
       dfs = null;
     }
+    Hive.closeCurrent();
   }
 
   public String readEntireFileIntoString(File queryFile) throws IOException {
@@ -731,8 +732,9 @@ public class QTestUtil {
       return;
     }
 
-    db.getConf().set("hive.metastore.filter.hook",
+    conf.set("hive.metastore.filter.hook",
         "org.apache.hadoop.hive.metastore.DefaultMetaStoreFilterHookImpl");
+    db = Hive.get(conf);
     // Delete any tables other than the source tables
     // and any databases other than the default database.
     for (String dbName : db.getAllDatabases()) {
@@ -800,16 +802,20 @@ public class QTestUtil {
       return;
     }
 
-    clearTablesCreatedDuringTests();
-    clearKeysCreatedInTests();
-
     // allocate and initialize a new conf since a test can
     // modify conf by using 'set' commands
     conf = new HiveConf(Driver.class);
     initConf();
+    initConfFromSetup();
+
     // renew the metastore since the cluster type is unencrypted
     db = Hive.get(conf);  // propagate new conf to meta store
 
+    clearTablesCreatedDuringTests();
+    clearKeysCreatedInTests();
+  }
+
+  protected void initConfFromSetup() throws Exception {
     setup.preTest(conf);
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/37e6e1bf/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index cdd12ab..64a26ac 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -178,7 +178,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient {
   private boolean isConnected = false;
   private URI metastoreUris[];
   private final HiveMetaHookLoader hookLoader;
-  protected final HiveConf conf;
+  protected final HiveConf conf;  // Keep a copy of HiveConf so if Session conf changes,
we may need to get a new HMS client.
   protected boolean fastpath = false;
   private String tokenStrForm;
   private final boolean localMetaStore;
@@ -205,8 +205,10 @@ public class HiveMetaStoreClient implements IMetaStoreClient {
     this.hookLoader = hookLoader;
     if (conf == null) {
       conf = new HiveConf(HiveMetaStoreClient.class);
+      this.conf = conf;
+    } else {
+      this.conf = new HiveConf(conf);
     }
-    this.conf = conf;
     filterHook = loadFilterHooks();
     fileMetadataBatchSize = HiveConf.getIntVar(
         conf, HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_OBJECTS_MAX);
@@ -221,10 +223,10 @@ public class HiveMetaStoreClient implements IMetaStoreClient {
       // instantiate the metastore server handler directly instead of connecting
       // through the network
       if (conf.getBoolVar(ConfVars.METASTORE_FASTPATH)) {
-        client = new HiveMetaStore.HMSHandler("hive client", conf, true);
+        client = new HiveMetaStore.HMSHandler("hive client", this.conf, true);
         fastpath = true;
       } else {
-        client = HiveMetaStore.newRetryingHMSHandler("hive client", conf, true);
+        client = HiveMetaStore.newRetryingHMSHandler("hive client", this.conf, true);
       }
       isConnected = true;
       snapshotActiveConf();

http://git-wip-us.apache.org/repos/asf/hive/blob/37e6e1bf/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
index 23b8a96..f7a0f31 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
@@ -546,10 +546,6 @@ public class SessionState {
     // Get the following out of the way when you start the session these take a
     // while and should be done when we start up.
     try {
-      // Hive object instance should be created with a copy of the conf object. If the conf
is
-      // shared with SessionState, other parts of the code might update the config, but
-      // Hive.get(HiveConf) would not recognize the case when it needs refreshing
-      Hive.get(new HiveConf(startSs.conf)).getMSC();
       UserGroupInformation sessionUGI = Utils.getUGI();
       FileSystem.get(startSs.conf);
 
@@ -575,10 +571,6 @@ public class SessionState {
       }
     } catch (RuntimeException e) {
       throw e;
-    } catch (Hive.SchemaException e) {
-      RuntimeException ex = new RuntimeException(e.getMessage());
-      ex.setStackTrace(new StackTraceElement[0]);
-      throw ex;
     } catch (Exception e) {
       // Catch-all due to some exec time dependencies on session state
       // that would cause ClassNoFoundException otherwise


Mime
View raw message