hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From xu...@apache.org
Subject svn commit: r1567721 - in /hive/trunk: common/src/java/org/apache/hadoop/hive/conf/ itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/ metastore/src/java/org/apache/hadoop/hive/metastore/
Date Wed, 12 Feb 2014 19:18:07 GMT
Author: xuefu
Date: Wed Feb 12 19:18:07 2014
New Revision: 1567721

URL: http://svn.apache.org/r1567721
Log:
HIVE-4996: unbalanced calls to openTransaction/commitTransaction (Szehon via Xuefu)

Added:
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStoreProxy.java
Removed:
    hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRawStoreTxn.java
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingRawStore.java
Modified:
    hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
    hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
    hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java

Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1567721&r1=1567720&r2=1567721&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Wed Feb 12 19:18:07
2014
@@ -94,8 +94,6 @@ public class HiveConf extends Configurat
       HiveConf.ConfVars.METASTOREPWD,
       HiveConf.ConfVars.METASTORECONNECTURLHOOK,
       HiveConf.ConfVars.METASTORECONNECTURLKEY,
-      HiveConf.ConfVars.METASTOREATTEMPTS,
-      HiveConf.ConfVars.METASTOREINTERVAL,
       HiveConf.ConfVars.METASTOREFORCERELOADCONF,
       HiveConf.ConfVars.METASTORESERVERMINTHREADS,
       HiveConf.ConfVars.METASTORESERVERMAXTHREADS,
@@ -273,10 +271,6 @@ public class HiveConf extends Configurat
     // Name of the connection url in the configuration
     METASTORECONNECTURLKEY("javax.jdo.option.ConnectionURL",
         "jdbc:derby:;databaseName=metastore_db;create=true"),
-    // Number of attempts to retry connecting after there is a JDO datastore err
-    METASTOREATTEMPTS("hive.metastore.ds.retry.attempts", 1),
-    // Number of miliseconds to wait between attepting
-    METASTOREINTERVAL("hive.metastore.ds.retry.interval", 1000),
     // Whether to force reloading of the metastore configuration (including
     // the connection URL, before the next metastore query that accesses the
     // datastore. Once reloaded, this value is reset to false. Used for
@@ -1258,8 +1252,14 @@ public class HiveConf extends Configurat
 
     if(this.get("hive.metastore.local", null) != null) {
       l4j.warn("DEPRECATED: Configuration property hive.metastore.local no longer has any
" +
-      		"effect. Make sure to provide a valid value for hive.metastore.uris if you are "
+
-      		"connecting to a remote metastore.");
+          "effect. Make sure to provide a valid value for hive.metastore.uris if you are
" +
+          "connecting to a remote metastore.");
+    }
+
+    if ((this.get("hive.metastore.ds.retry.attempts") != null) ||
+      this.get("hive.metastore.ds.retry.interval") != null) {
+        l4j.warn("DEPRECATED: hive.metastore.ds.retry.* no longer has any effect.  " +
+        "Use hive.hmshandler.retry.* instead");
     }
 
     // if the running class was loaded directly (through eclipse) rather than through a

Modified: hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java?rev=1567721&r1=1567720&r2=1567721&view=diff
==============================================================================
--- hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
(original)
+++ hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStore.java
Wed Feb 12 19:18:07 2014
@@ -2498,8 +2498,8 @@ public abstract class TestHiveMetaStore 
    */
   private void updateTableNameInDB(String oldTableName, String newTableName) throws SQLException
{
     String connectionStr = HiveConf.getVar(hiveConf, HiveConf.ConfVars.METASTORECONNECTURLKEY);
-    int interval= HiveConf.getIntVar(hiveConf, HiveConf.ConfVars.METASTOREINTERVAL);
-    int attempts = HiveConf.getIntVar(hiveConf, HiveConf.ConfVars.METASTOREATTEMPTS);
+    int interval= 1;
+    int attempts = 1;
 
 
     Utilities.SQLCommand<Void> execUpdate = new Utilities.SQLCommand<Void>()
{

Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1567721&r1=1567720&r2=1567721&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Wed
Feb 12 19:18:07 2014
@@ -441,7 +441,7 @@ public class HiveMetaStore extends Thrif
           + rawStoreClassName));
       Configuration conf = getConf();
 
-      return RetryingRawStore.getProxy(hiveConf, conf, rawStoreClassName, threadLocalId.get());
+      return RawStoreProxy.getProxy(hiveConf, conf, rawStoreClassName, threadLocalId.get());
     }
 
     private void createDefaultDB_core(RawStore ms) throws MetaException, InvalidObjectException
{

Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1567721&r1=1567720&r2=1567721&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Wed Feb
12 19:18:07 2014
@@ -39,6 +39,7 @@ import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
 
 import javax.jdo.JDODataStoreException;
+import javax.jdo.JDOEnhanceException;
 import javax.jdo.JDOHelper;
 import javax.jdo.JDOObjectNotFoundException;
 import javax.jdo.PersistenceManager;
@@ -362,7 +363,10 @@ public class ObjectStore implements RawS
       // currentTransaction is not active
       assert ((currentTransaction != null) && (currentTransaction.isActive()));
     }
-    return currentTransaction.isActive();
+
+    boolean result = currentTransaction.isActive();
+    debugLog("Open transaction: count = " + openTrasactionCalls + ", isActive = " + result);
+    return result;
   }
 
   /**
@@ -374,23 +378,31 @@ public class ObjectStore implements RawS
   @SuppressWarnings("nls")
   public boolean commitTransaction() {
     if (TXN_STATUS.ROLLBACK == transactionStatus) {
+      debugLog("Commit transaction: rollback");
       return false;
     }
     if (openTrasactionCalls <= 0) {
-      throw new RuntimeException("commitTransaction was called but openTransactionCalls =
"
+      RuntimeException e = new RuntimeException("commitTransaction was called but openTransactionCalls
= "
           + openTrasactionCalls + ". This probably indicates that there are unbalanced "
+
-              "calls to openTransaction/commitTransaction");
+          "calls to openTransaction/commitTransaction");
+      LOG.error(e);
+      throw e;
     }
     if (!currentTransaction.isActive()) {
-      throw new RuntimeException(
-          "Commit is called, but transaction is not active. Either there are"
-              + " mismatching open and close calls or rollback was called in the same trasaction");
+      RuntimeException e = new RuntimeException("commitTransaction was called but openTransactionCalls
= "
+          + openTrasactionCalls + ". This probably indicates that there are unbalanced "
+
+          "calls to openTransaction/commitTransaction");
+      LOG.error(e);
+      throw e;
     }
     openTrasactionCalls--;
+    debugLog("Commit transaction: count = " + openTrasactionCalls + ", isactive "+ currentTransaction.isActive());
+
     if ((openTrasactionCalls == 0) && currentTransaction.isActive()) {
       transactionStatus = TXN_STATUS.COMMITED;
       currentTransaction.commit();
     }
+
     return true;
   }
 
@@ -410,9 +422,11 @@ public class ObjectStore implements RawS
    */
   public void rollbackTransaction() {
     if (openTrasactionCalls < 1) {
+      debugLog("rolling back transaction: no open transactions: " + openTrasactionCalls);
       return;
     }
     openTrasactionCalls = 0;
+    debugLog("Rollback transaction, isActive: " + currentTransaction.isActive());
     if (currentTransaction.isActive()
         && transactionStatus != TXN_STATUS.ROLLBACK) {
       transactionStatus = TXN_STATUS.ROLLBACK;
@@ -6151,27 +6165,24 @@ public class ObjectStore implements RawS
     }
   }
 
-  /** Add this to code to debug lexer if needed. DebugTokenStream may also be added here.
*/
-  private void debugLexer(CommonTokenStream stream, FilterLexer lexer) {
-    try {
-      stream.fill();
-      List<?> tokens = stream.getTokens();
-      String report = "LEXER: tokens (" + ((tokens == null) ? "null" : tokens.size()) + "):
";
-      if (tokens != null) {
-        for (Object o : tokens) {
-          if (o == null || !(o instanceof Token)) {
-            report += "[not a token: " + o + "], ";
-          } else {
-            Token t = (Token)o;
-            report += "[at " + t.getCharPositionInLine() + ": "
-                + t.getType() + " " + t.getText() + "], ";
-          }
-        }
-      }
-      report += "; lexer error: " + lexer.errorMsg;
-      LOG.error(report);
-    } catch (Throwable t) {
-      LOG.error("LEXER: tokens (error)", t);
+
+  private void debugLog(String message) {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug(message + getCallStack());
+    }
+  }
+
+  private static final int stackLimit = 5;
+
+  private String getCallStack() {
+    StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace();
+    int thislimit = Math.min(stackLimit, stackTrace.length);
+    StringBuilder sb = new StringBuilder();
+    sb.append(" at:");
+    for (int i = 4; i < thislimit; i++) {
+      sb.append("\n\t");
+      sb.append(stackTrace[i].toString());
     }
+    return sb.toString();
   }
 }

Added: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStoreProxy.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStoreProxy.java?rev=1567721&view=auto
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStoreProxy.java (added)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStoreProxy.java Wed
Feb 12 19:18:07 2014
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.metastore;
+
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.util.List;
+
+import org.apache.commons.lang.ClassUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+import org.apache.hadoop.hive.common.classification.InterfaceStability;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.util.ReflectionUtils;
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class RawStoreProxy implements InvocationHandler {
+
+  private final RawStore base;
+  private final MetaStoreInit.MetaStoreInitData metaStoreInitData =
+    new MetaStoreInit.MetaStoreInitData();
+  private final int id;
+  private final HiveConf hiveConf;
+  private final Configuration conf; // thread local conf from HMS
+
+  protected RawStoreProxy(HiveConf hiveConf, Configuration conf,
+      Class<? extends RawStore> rawStoreClass, int id) throws MetaException {
+    this.conf = conf;
+    this.hiveConf = hiveConf;
+    this.id = id;
+
+    // This has to be called before initializing the instance of RawStore
+    init();
+
+    this.base = (RawStore) ReflectionUtils.newInstance(rawStoreClass, conf);
+  }
+
+  public static RawStore getProxy(HiveConf hiveConf, Configuration conf, String rawStoreClassName,
+      int id) throws MetaException {
+
+    Class<? extends RawStore> baseClass = (Class<? extends RawStore>) MetaStoreUtils.getClass(
+        rawStoreClassName);
+
+    RawStoreProxy handler = new RawStoreProxy(hiveConf, conf, baseClass, id);
+
+    // Look for interfaces on both the class and all base classes.
+    return (RawStore) Proxy.newProxyInstance(RawStoreProxy.class.getClassLoader(),
+        getAllInterfaces(baseClass), handler);
+  }
+
+  private static Class<?>[] getAllInterfaces(Class<?> baseClass) {
+    List interfaces = ClassUtils.getAllInterfaces(baseClass);
+    Class<?>[] result = new Class<?>[interfaces.size()];
+    int i = 0;
+    for (Object o : interfaces) {
+      result[i++] = (Class<?>)o;
+    }
+    return result;
+  }
+
+  private void init() throws MetaException {
+    // Using the hook on startup ensures that the hook always has priority
+    // over settings in *.xml.  The thread local conf needs to be used because at this point
+    // it has already been initialized using hiveConf.
+    MetaStoreInit.updateConnectionURL(hiveConf, getConf(), null, metaStoreInitData);
+  }
+
+  private void initMS() {
+    base.setConf(getConf());
+  }
+
+  @Override
+  public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+    Object ret = null;
+
+    boolean reloadConf = HiveConf.getBoolVar(hiveConf,
+        HiveConf.ConfVars.METASTOREFORCERELOADCONF);
+
+    if (reloadConf) {
+      MetaStoreInit.updateConnectionURL(hiveConf, getConf(), null, metaStoreInitData);
+      initMS();
+    }
+
+    try {
+      ret = method.invoke(base, args);
+    } catch (UndeclaredThrowableException e) {
+      throw e.getCause();
+    } catch (InvocationTargetException e) {
+      throw e.getCause();
+    }
+    return ret;
+  }
+
+  public Configuration getConf() {
+    return conf;
+  }
+
+}

Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java?rev=1567721&r1=1567720&r2=1567721&view=diff
==============================================================================
--- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java
(original)
+++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RetryingHMSHandler.java
Wed Feb 12 19:18:07 2014
@@ -45,13 +45,13 @@ public class RetryingHMSHandler implemen
     new MetaStoreInit.MetaStoreInitData();
   private final HiveConf hiveConf;
 
-  protected RetryingHMSHandler(HiveConf hiveConf, String name) throws MetaException {
+  protected RetryingHMSHandler(final HiveConf hiveConf, final String name) throws MetaException
{
     this.hiveConf = hiveConf;
 
     // This has to be called before initializing the instance of HMSHandler
     init();
 
-    this.base = (IHMSHandler) new HiveMetaStore.HMSHandler(name, hiveConf);
+    this.base = new HiveMetaStore.HMSHandler(name, hiveConf);
   }
 
   public static IHMSHandler getProxy(HiveConf hiveConf, String name) throws MetaException
{
@@ -75,8 +75,10 @@ public class RetryingHMSHandler implemen
     base.setConf(getConf());
   }
 
+
   @Override
-  public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+  public Object invoke(final Object proxy, final Method method, final Object[] args) throws
Throwable {
+
     Object ret = null;
 
     boolean gotNewConnectUrl = false;
@@ -144,7 +146,7 @@ public class RetryingHMSHandler implemen
       }
 
       if (retryCount >= retryLimit) {
-        LOG.error(ExceptionUtils.getStackTrace(caughtException));
+        LOG.error("HMSHandler Fatal error: " + ExceptionUtils.getStackTrace(caughtException));
         // Since returning exceptions with a nested "cause" can be a problem in
         // Thrift, we are stuffing the stack trace into the message itself.
         throw new MetaException(ExceptionUtils.getStackTrace(caughtException));
@@ -153,9 +155,10 @@ public class RetryingHMSHandler implemen
       assert (retryInterval >= 0);
       retryCount++;
       LOG.error(
-          String.format(
-              "JDO datastore error. Retrying HMSHandler " +
-                  "after %d ms (attempt %d of %d)", retryInterval, retryCount, retryLimit));
+        String.format(
+          "Retrying HMSHandler after %d ms (attempt %d of %d)", retryInterval, retryCount,
retryLimit) +
+          " with error: " + ExceptionUtils.getStackTrace(caughtException));
+
       Thread.sleep(retryInterval);
       // If we have a connection error, the JDO connection URL hook might
       // provide us with a new URL to access the datastore.



Mime
View raw message