hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject hive git commit: HIVE-12170 normalize HBase metastore connection configuration (gates, reviewed by Sergey Shelukhin)
Date Wed, 21 Oct 2015 23:56:34 GMT
Repository: hive
Updated Branches:
  refs/heads/master 6293e97f0 -> 06282fc12


HIVE-12170 normalize HBase metastore connection configuration (gates, reviewed by Sergey Shelukhin)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/06282fc1
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/06282fc1
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/06282fc1

Branch: refs/heads/master
Commit: 06282fc121669acbaca1f31d112ccab89dbd71ad
Parents: 6293e97
Author: Alan Gates <gates@hortonworks.com>
Authored: Wed Oct 21 16:54:45 2015 -0700
Committer: Alan Gates <gates@hortonworks.com>
Committed: Wed Oct 21 16:54:45 2015 -0700

----------------------------------------------------------------------
 .../metastore/hbase/HBaseIntegrationTests.java  |  8 +-----
 .../hbase/TestHBaseStoreIntegration.java        | 12 +++++----
 .../hbase/TestStorageDescriptorSharing.java     | 20 +++++++--------
 .../metastore/hbase/HBaseStoreTestUtil.java     |  6 ++---
 .../hive/metastore/hbase/HBaseReadWrite.java    | 26 ++++++++++----------
 .../hive/metastore/hbase/HBaseSchemaTool.java   |  3 ++-
 .../hadoop/hive/metastore/hbase/HBaseStore.java |  5 +++-
 .../hadoop/hive/metastore/hbase/MockUtils.java  |  2 +-
 8 files changed, 41 insertions(+), 41 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/06282fc1/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/HBaseIntegrationTests.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/HBaseIntegrationTests.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/HBaseIntegrationTests.java
index 5b82579..3f5da4a 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/HBaseIntegrationTests.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/HBaseIntegrationTests.java
@@ -18,16 +18,11 @@
  */
 package org.apache.hadoop.hive.metastore.hbase;
 
-import co.cask.tephra.hbase10.coprocessor.TransactionProcessor;
-
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hive.cli.CliSessionState;
 import org.apache.hadoop.hive.conf.HiveConf;
@@ -38,7 +33,6 @@ import org.apache.hadoop.hive.ql.session.SessionState;
 
 import java.io.IOException;
 import java.util.HashMap;
-import java.util.List;
 import java.util.Map;
 
 /**
@@ -85,7 +79,7 @@ public class HBaseIntegrationTests {
     // This chicanery is necessary to make the driver work.  Hive tests need the pfile file
     // system, while the hbase one uses something else.  So first make sure we've configured
our
     // hbase connection, then get a new config file and populate it as desired.
-    HBaseReadWrite.getInstance(conf);
+    HBaseReadWrite.setConf(conf);
     conf = new HiveConf();
     conf.setVar(HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict");
     conf.setVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL,

http://git-wip-us.apache.org/repos/asf/hive/blob/06282fc1/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java
index 8b0b431..8750a05 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestHBaseStoreIntegration.java
@@ -837,7 +837,8 @@ public class TestHBaseStoreIntegration extends HBaseIntegrationTests {
     store.grantRole(role2, user1, PrincipalType.USER, "admin", PrincipalType.ROLE, false);
     store.grantRole(role1, user2, PrincipalType.USER, "bob", PrincipalType.USER, false);
 
-    roles = HBaseReadWrite.getInstance(conf).getUserRoles(user2);
+    HBaseReadWrite.setConf(conf);
+    roles = HBaseReadWrite.getInstance().getUserRoles(user2);
     Assert.assertEquals(2, roles.size());
     roleNames = roles.toArray(new String[roles.size()]);
     Arrays.sort(roleNames);
@@ -847,13 +848,13 @@ public class TestHBaseStoreIntegration extends HBaseIntegrationTests
{
 
     // user1 should still have both roles since she was granted into role1 specifically.
 user2
     // should only have role2 now since role2 was revoked from role1.
-    roles = HBaseReadWrite.getInstance(conf).getUserRoles(user1);
+    roles = HBaseReadWrite.getInstance().getUserRoles(user1);
     Assert.assertEquals(2, roles.size());
     roleNames = roles.toArray(new String[roles.size()]);
     Arrays.sort(roleNames);
     Assert.assertArrayEquals(new String[]{roleName1, roleName2}, roleNames);
 
-    roles = HBaseReadWrite.getInstance(conf).getUserRoles(user2);
+    roles = HBaseReadWrite.getInstance().getUserRoles(user2);
     Assert.assertEquals(1, roles.size());
     Assert.assertEquals(roleName1, roles.get(0));
   }
@@ -882,11 +883,12 @@ public class TestHBaseStoreIntegration extends HBaseIntegrationTests
{
 
     store.removeRole(roleName2);
 
-    roles = HBaseReadWrite.getInstance(conf).getUserRoles(user1);
+    HBaseReadWrite.setConf(conf);
+    roles = HBaseReadWrite.getInstance().getUserRoles(user1);
     Assert.assertEquals(1, roles.size());
     Assert.assertEquals(roleName1, roles.get(0));
 
-    roles = HBaseReadWrite.getInstance(conf).getUserRoles(user2);
+    roles = HBaseReadWrite.getInstance().getUserRoles(user2);
     Assert.assertEquals(1, roles.size());
     Assert.assertEquals(roleName1, roles.get(0));
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/06282fc1/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestStorageDescriptorSharing.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestStorageDescriptorSharing.java
b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestStorageDescriptorSharing.java
index decfa4a..fa7273e 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestStorageDescriptorSharing.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/hbase/TestStorageDescriptorSharing.java
@@ -97,7 +97,7 @@ public class TestStorageDescriptorSharing extends HBaseIntegrationTests
{
       Assert.assertEquals("file:/tmp/pc=" + val, p.getSd().getLocation());
     }
 
-    Assert.assertEquals(1, HBaseReadWrite.getInstance(conf).countStorageDescriptor());
+    Assert.assertEquals(1, HBaseReadWrite.getInstance().countStorageDescriptor());
 
     String tableName2 = "differentTable";
     sd = new StorageDescriptor(cols, "file:/tmp", "input2", "output", false, 0,
@@ -106,11 +106,11 @@ public class TestStorageDescriptorSharing extends HBaseIntegrationTests
{
         emptyParameters, null, null, null);
     store.createTable(table);
 
-    Assert.assertEquals(2, HBaseReadWrite.getInstance(conf).countStorageDescriptor());
+    Assert.assertEquals(2, HBaseReadWrite.getInstance().countStorageDescriptor());
 
     // Drop one of the partitions and make sure it doesn't drop the storage descriptor
     store.dropPartition(dbName, tableName, Arrays.asList(partVals.get(0)));
-    Assert.assertEquals(2, HBaseReadWrite.getInstance(conf).countStorageDescriptor());
+    Assert.assertEquals(2, HBaseReadWrite.getInstance().countStorageDescriptor());
 
     // Alter the second table in a few ways to make sure it changes it's descriptor properly
     table = store.getTable(dbName, tableName2);
@@ -119,7 +119,7 @@ public class TestStorageDescriptorSharing extends HBaseIntegrationTests
{
     // Alter the table without touching the storage descriptor
     table.setLastAccessTime(startTime + 1);
     store.alterTable(dbName, tableName2, table);
-    Assert.assertEquals(2, HBaseReadWrite.getInstance(conf).countStorageDescriptor());
+    Assert.assertEquals(2, HBaseReadWrite.getInstance().countStorageDescriptor());
     table = store.getTable(dbName, tableName2);
     byte[] alteredHash = HBaseUtils.hashStorageDescriptor(table.getSd(), md);
     Assert.assertArrayEquals(sdHash, alteredHash);
@@ -127,7 +127,7 @@ public class TestStorageDescriptorSharing extends HBaseIntegrationTests
{
     // Alter the table, changing the storage descriptor
     table.getSd().setOutputFormat("output_changed");
     store.alterTable(dbName, tableName2, table);
-    Assert.assertEquals(2, HBaseReadWrite.getInstance(conf).countStorageDescriptor());
+    Assert.assertEquals(2, HBaseReadWrite.getInstance().countStorageDescriptor());
     table = store.getTable(dbName, tableName2);
     alteredHash = HBaseUtils.hashStorageDescriptor(table.getSd(), md);
     Assert.assertFalse(Arrays.equals(sdHash, alteredHash));
@@ -137,7 +137,7 @@ public class TestStorageDescriptorSharing extends HBaseIntegrationTests
{
     sdHash = HBaseUtils.hashStorageDescriptor(part.getSd(), md);
     part.setLastAccessTime(part.getLastAccessTime() + 1);
     store.alterPartition(dbName, tableName, Arrays.asList(partVals.get(1)), part);
-    Assert.assertEquals(2, HBaseReadWrite.getInstance(conf).countStorageDescriptor());
+    Assert.assertEquals(2, HBaseReadWrite.getInstance().countStorageDescriptor());
     part = store.getPartition(dbName, tableName, Arrays.asList(partVals.get(1)));
     alteredHash = HBaseUtils.hashStorageDescriptor(part.getSd(), md);
     Assert.assertArrayEquals(sdHash, alteredHash);
@@ -145,7 +145,7 @@ public class TestStorageDescriptorSharing extends HBaseIntegrationTests
{
     // Alter the partition, changing the storage descriptor
     part.getSd().setOutputFormat("output_changed_some_more");
     store.alterPartition(dbName, tableName, Arrays.asList(partVals.get(1)), part);
-    Assert.assertEquals(3, HBaseReadWrite.getInstance(conf).countStorageDescriptor());
+    Assert.assertEquals(3, HBaseReadWrite.getInstance().countStorageDescriptor());
     part = store.getPartition(dbName, tableName, Arrays.asList(partVals.get(1)));
     alteredHash = HBaseUtils.hashStorageDescriptor(part.getSd(), md);
     Assert.assertFalse(Arrays.equals(sdHash, alteredHash));
@@ -161,7 +161,7 @@ public class TestStorageDescriptorSharing extends HBaseIntegrationTests
{
       listPartVals.add(Arrays.asList(pv));
     }
     store.alterPartitions(dbName, tableName, listPartVals, parts);
-    Assert.assertEquals(3, HBaseReadWrite.getInstance(conf).countStorageDescriptor());
+    Assert.assertEquals(3, HBaseReadWrite.getInstance().countStorageDescriptor());
     parts = store.getPartitions(dbName, tableName, -1);
     alteredHash = HBaseUtils.hashStorageDescriptor(parts.get(1).getSd(), md);
     Assert.assertArrayEquals(sdHash, alteredHash);
@@ -173,7 +173,7 @@ public class TestStorageDescriptorSharing extends HBaseIntegrationTests
{
       parts.get(i).getSd().setOutputFormat("yet_a_different_of");
     }
     store.alterPartitions(dbName, tableName, listPartVals, parts);
-    Assert.assertEquals(4, HBaseReadWrite.getInstance(conf).countStorageDescriptor());
+    Assert.assertEquals(4, HBaseReadWrite.getInstance().countStorageDescriptor());
     parts = store.getPartitions(dbName, tableName, -1);
     alteredHash = HBaseUtils.hashStorageDescriptor(parts.get(1).getSd(), md);
     Assert.assertFalse(Arrays.equals(sdHash, alteredHash));
@@ -184,7 +184,7 @@ public class TestStorageDescriptorSharing extends HBaseIntegrationTests
{
     store.dropTable(dbName, tableName);
     store.dropTable(dbName, tableName2);
 
-    Assert.assertEquals(0, HBaseReadWrite.getInstance(conf).countStorageDescriptor());
+    Assert.assertEquals(0, HBaseReadWrite.getInstance().countStorageDescriptor());
 
 
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/06282fc1/itests/util/src/main/java/org/apache/hadoop/hive/metastore/hbase/HBaseStoreTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/metastore/hbase/HBaseStoreTestUtil.java
b/itests/util/src/main/java/org/apache/hadoop/hive/metastore/hbase/HBaseStoreTestUtil.java
index 1f42007..21e8f7e 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/metastore/hbase/HBaseStoreTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/metastore/hbase/HBaseStoreTestUtil.java
@@ -18,14 +18,14 @@
  */
 package org.apache.hadoop.hive.metastore.hbase;
 
-import java.util.List;
-
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HTableDescriptor;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hive.conf.HiveConf;
 
+import java.util.List;
+
 public class HBaseStoreTestUtil {
   public static void initHBaseMetastore(HBaseAdmin admin, HiveConf conf) throws Exception
{
     for (String tableName : HBaseReadWrite.tableNames) {
@@ -39,7 +39,7 @@ public class HBaseStoreTestUtil {
     }
     admin.close();
     if (conf != null) {
-      HBaseReadWrite.getInstance(conf);
+      HBaseReadWrite.setConf(conf);
     }
   }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hive/blob/06282fc1/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
index 951c081..781f562 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseReadWrite.java
@@ -19,16 +19,14 @@
 package org.apache.hadoop.hive.metastore.hbase;
 
 import com.google.common.annotations.VisibleForTesting;
-
 import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HTableInterface;
@@ -41,7 +39,6 @@ import org.apache.hadoop.hbase.filter.CompareFilter;
 import org.apache.hadoop.hbase.filter.Filter;
 import org.apache.hadoop.hbase.filter.RegexStringComparator;
 import org.apache.hadoop.hbase.filter.RowFilter;
-import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MultiRequest;
 import org.apache.hadoop.hive.common.ObjectPair;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.api.AggrStats;
@@ -185,23 +182,26 @@ public class HBaseReadWrite {
   boolean entireRoleTableInCache;
 
   /**
-   * Get the instance of HBaseReadWrite for the current thread.  This is intended to be used
by
-   * {@link org.apache.hadoop.hive.metastore.hbase.HBaseStore} since it creates the thread
local
-   * version of this class.
+   * Set the configuration for all HBaseReadWrite instances.
    * @param configuration Configuration object
-   * @return thread's instance of HBaseReadWrite
    */
-  public static HBaseReadWrite getInstance(Configuration configuration) {
-    staticConf = configuration;
-    return self.get();
+  public static synchronized void setConf(Configuration configuration) {
+    if (staticConf == null) {
+      staticConf = configuration;
+    } else {
+      LOG.info("Attempt to set conf when it has already been set.");
+    }
   }
 
   /**
-   * Get the instance of HBaseReadWrite for the current thread.  This is inteded to be used
after
-   * the thread has been initialized.  Woe betide you if that's not the case.
+   * Get the instance of HBaseReadWrite for the current thread.  This can only be called
after
+   * {@link #setConf} has been called. Woe betide you if that's not the case.
    * @return thread's instance of HBaseReadWrite
    */
   static HBaseReadWrite getInstance() {
+    if (staticConf == null) {
+      throw new RuntimeException("Must set conf object before getting an instance");
+    }
     return self.get();
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/06282fc1/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java
b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java
index 1c407f1..8eb6116 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseSchemaTool.java
@@ -155,7 +155,8 @@ public class HBaseSchemaTool {
     roleName = rn;
     colNames = cn;
     hasStats = s;
-    hrw = HBaseReadWrite.getInstance(new Configuration());
+    HBaseReadWrite.setConf(new Configuration());
+    hrw = HBaseReadWrite.getInstance();
   }
 
   public void db() throws IOException, TException {

http://git-wip-us.apache.org/repos/asf/hive/blob/06282fc1/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
index 1ea25f5..09e57e5 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java
@@ -2254,7 +2254,10 @@ public class HBaseStore implements RawStore {
   }
 
   private HBaseReadWrite getHBase() {
-    if (hbase == null) hbase = HBaseReadWrite.getInstance(conf);
+    if (hbase == null) {
+      HBaseReadWrite.setConf(conf);
+      hbase = HBaseReadWrite.getInstance();
+    }
     return hbase;
   }
 

http://git-wip-us.apache.org/repos/asf/hive/blob/06282fc1/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/MockUtils.java
----------------------------------------------------------------------
diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/MockUtils.java b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/MockUtils.java
index 2198892..983129a 100644
--- a/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/MockUtils.java
+++ b/metastore/src/test/org/apache/hadoop/hive/metastore/hbase/MockUtils.java
@@ -203,7 +203,7 @@ public class MockUtils {
     Mockito.when(hconn.getHBaseTable(Mockito.anyString())).thenReturn(htable);
     HiveConf.setVar(conf, HiveConf.ConfVars.METASTORE_HBASE_CONNECTION_CLASS, HBaseReadWrite.TEST_CONN);
     HBaseReadWrite.setTestConnection(hconn);
-    HBaseReadWrite hbase = HBaseReadWrite.getInstance(conf);
+    HBaseReadWrite.setConf(conf);
     HBaseStore store = new HBaseStore();
     store.setConf(conf);
     return store;


Mime
View raw message