hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kevinwilf...@apache.org
Subject svn commit: r1422146 - in /hive/trunk: common/src/java/org/apache/hadoop/hive/conf/ conf/ metastore/src/java/org/apache/hadoop/hive/metastore/ metastore/src/test/org/apache/hadoop/hive/metastore/
Date Fri, 14 Dec 2012 23:28:08 GMT
Author: kevinwilfong
Date: Fri Dec 14 23:28:04 2012
New Revision: 1422146

URL: http://svn.apache.org/viewvc?rev=1422146&view=rev
Log:
HIVE-3766. Enable adding hooks to hive meta store init. (Jean Xu via kevinwilfong)

Added:
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitContext.java
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitListener.java
    hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyMetaStoreInitListener.java
    hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java
Modified:
    hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    hive/trunk/conf/hive-default.xml.template
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java

Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1422146&r1=1422145&r2=1422146&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Fri Dec 14 23:28:04
2012
@@ -125,6 +125,7 @@ public class HiveConf extends Configurat
       HiveConf.ConfVars.METASTORE_END_FUNCTION_LISTENERS,
       HiveConf.ConfVars.METASTORE_PART_INHERIT_TBL_PROPS,
       HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_TABLE_PARTITION_MAX,
+      HiveConf.ConfVars.METASTORE_INIT_HOOKS,
       HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS,
       HiveConf.ConfVars.HMSHANDLERATTEMPTS,
       HiveConf.ConfVars.HMSHANDLERINTERVAL,
@@ -311,6 +312,9 @@ public class HiveConf extends Configurat
     METASTORE_BATCH_RETRIEVE_MAX("hive.metastore.batch.retrieve.max", 300),
     METASTORE_BATCH_RETRIEVE_TABLE_PARTITION_MAX(
       "hive.metastore.batch.retrieve.table.partition.max", 1000),
+    // A comma separated list of hooks which implement MetaStoreInitListener and will be
run at
+    // the beginning of HMSHandler initialization
+    METASTORE_INIT_HOOKS("hive.metastore.init.hooks", ""),
     METASTORE_PRE_EVENT_LISTENERS("hive.metastore.pre.event.listeners", ""),
     METASTORE_EVENT_LISTENERS("hive.metastore.event.listeners", ""),
     // should we do checks against the storage (usually hdfs) for operations like drop_partition

Modified: hive/trunk/conf/hive-default.xml.template
URL: http://svn.apache.org/viewvc/hive/trunk/conf/hive-default.xml.template?rev=1422146&r1=1422145&r2=1422146&view=diff
==============================================================================
--- hive/trunk/conf/hive-default.xml.template (original)
+++ hive/trunk/conf/hive-default.xml.template Fri Dec 14 23:28:04 2012
@@ -688,6 +688,12 @@
 </property>
 
 <property>
+  <name>hive.metastore.init.hooks</name>
+  <value></value>
+  <description>A comma separated list of hooks to be invoked at the beginning of HMSHandler
initialization. Aninit hook is specified as the name of Java class which extends org.apache.hadoop.hive.metastore.MetaStoreInitListener.</description>
+</property>
+
+<property>
   <name>hive.client.stats.publishers</name>
   <value></value>
   <description>Comma-separated list of statistics publishers to be invoked on counters
on each job.  A client stats publisher is specified as the name of a Java class which implements
the org.apache.hadoop.hive.ql.stats.ClientStatsPublisher interface.</description>

Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1422146&r1=1422145&r2=1422146&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Fri
Dec 14 23:28:04 2012
@@ -294,6 +294,7 @@ public class HiveMetaStore extends Thrif
     private List<MetaStorePreEventListener> preListeners;
     private List<MetaStoreEventListener> listeners;
     private List<MetaStoreEndFunctionListener> endFunctionListeners;
+    private List<MetaStoreInitListener> initListeners;
 
     {
       classLoader = Thread.currentThread().getContextClassLoader();
@@ -306,6 +307,14 @@ public class HiveMetaStore extends Thrif
       rawStoreClassName = hiveConf.getVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL);
       checkForDefaultDb = hiveConf.getBoolean(
           "hive.metastore.checkForDefaultDb", true);
+      initListeners = MetaStoreUtils.getMetaStoreListeners(
+          MetaStoreInitListener.class, hiveConf,
+          hiveConf.getVar(HiveConf.ConfVars.METASTORE_INIT_HOOKS));
+      for (MetaStoreInitListener singleInitListener: initListeners) {
+          MetaStoreInitContext context = new MetaStoreInitContext();
+          singleInitListener.onInit(context);
+      }
+
       String alterHandlerName = hiveConf.get("hive.metastore.alter.impl",
           HiveAlterHandler.class.getName());
       alterHandler = (AlterHandler) ReflectionUtils.newInstance(MetaStoreUtils.getClass(

Added: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitContext.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitContext.java?rev=1422146&view=auto
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitContext.java
(added)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitContext.java
Fri Dec 14 23:28:04 2012
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+/**
+ * Base class which provides context to implementations of MetaStoreInitListener
+ */
+
+public class MetaStoreInitContext {
+
+}

Added: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitListener.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitListener.java?rev=1422146&view=auto
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitListener.java
(added)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreInitListener.java
Fri Dec 14 23:28:04 2012
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+
+/**
+ * This abstract class needs to be extended to  provide implementation of actions
+ * that needs to be performed when HMSHandler is initialized
+ */
+
+public abstract class MetaStoreInitListener implements Configurable {
+
+  private Configuration conf;
+
+  public MetaStoreInitListener(Configuration config){
+    this.conf = config;
+  }
+
+  public abstract void onInit(MetaStoreInitContext context) throws MetaException;
+
+  @Override
+  public Configuration getConf() {
+    return this.conf;
+  }
+
+  @Override
+  public void setConf(Configuration config) {
+    this.conf = config;
+  }
+}

Added: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyMetaStoreInitListener.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyMetaStoreInitListener.java?rev=1422146&view=auto
==============================================================================
--- hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyMetaStoreInitListener.java
(added)
+++ hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyMetaStoreInitListener.java
Fri Dec 14 23:28:04 2012
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.MetaStoreInitContext;
+import org.apache.hadoop.hive.metastore.MetaStoreInitListener;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+
+/*
+ * An implementation of MetaStoreInitListener to verify onInit is called when
+ * HMSHandler is initialized
+ */
+public class DummyMetaStoreInitListener extends MetaStoreInitListener{
+
+  public static boolean wasCalled = false;
+  public DummyMetaStoreInitListener(Configuration config) {
+    super(config);
+  }
+
+  @Override
+  public void onInit(MetaStoreInitContext context) throws MetaException {
+    wasCalled = true;
+  }
+}

Added: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java?rev=1422146&view=auto
==============================================================================
--- hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java
(added)
+++ hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java
Fri Dec 14 23:28:04 2012
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.hive.cli.CliSessionState;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
+import org.apache.hadoop.hive.ql.Driver;
+import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.shims.ShimLoader;
+
+/**
+ * TestMetaStoreInitListener. Test case for
+ * {@link org.apache.hadoop.hive.metastore.MetaStoreInitListener}
+ */
+public class TestMetaStoreInitListener extends TestCase {
+  private HiveConf hiveConf;
+  private HiveMetaStoreClient msc;
+  private Driver driver;
+
+  @Override
+  protected void setUp() throws Exception {
+
+    super.setUp();
+    System.setProperty("hive.metastore.init.hooks",
+        DummyMetaStoreInitListener.class.getName());
+    int port = MetaStoreUtils.findFreePort();
+    MetaStoreUtils.startMetaStore(port, ShimLoader.getHadoopThriftAuthBridge());
+    hiveConf = new HiveConf(this.getClass());
+    hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port);
+    hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
+    hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
+    hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, "");
+    hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false");
+    SessionState.start(new CliSessionState(hiveConf));
+    msc = new HiveMetaStoreClient(hiveConf, null);
+    driver = new Driver(hiveConf);
+  }
+
+  @Override
+  protected void tearDown() throws Exception {
+    super.tearDown();
+  }
+
+  public void testMetaStoreInitListener() throws Exception {
+    // DummyMataStoreInitListener's onInit will be called at HMSHandler
+    // initialization, and set this to true
+    assertTrue(DummyMetaStoreInitListener.wasCalled);
+  }
+
+}



Mime
View raw message