hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gcha...@apache.org
Subject svn commit: r1394857 - in /hbase/branches/0.94/src: main/java/org/apache/hadoop/hbase/client/ main/java/org/apache/hadoop/hbase/ipc/ test/java/org/apache/hadoop/hbase/client/ test/java/org/apache/hadoop/hbase/ipc/
Date Fri, 05 Oct 2012 23:16:36 GMT
Author: gchanan
Date: Fri Oct  5 23:16:35 2012
New Revision: 1394857

URL: http://svn.apache.org/viewvc?rev=1394857&view=rev
Log:
HBASE-6920 On timeout connecting to master, client can get stuck and never make progress

Added:
    hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestClientTimeouts.java
    hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/ipc/RandomTimeoutRpcEngine.java
Modified:
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/RpcEngine.java
    hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=1394857&r1=1394856&r2=1394857&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
(original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
Fri Oct  5 23:16:35 2012
@@ -616,6 +616,23 @@ public class HConnectionManager {
       return this.conf;
     }
 
+    /**
+     * Log failure of getMaster attempt
+     * @return true if should retry
+     */
+    private boolean shouldRetryGetMaster(int tries, Exception e) {
+      if (tries == numRetries - 1) {
+        // This was our last chance - don't bother sleeping
+        LOG.info("getMaster attempt " + tries + " of " + numRetries +
+          " failed; no more retrying.", e);
+        return false;
+      }
+      LOG.info("getMaster attempt " + tries + " of " + numRetries +
+        " failed; retrying after sleep of " +
+        ConnectionUtils.getPauseTime(this.pause, tries), e);
+      return true;
+    }
+
     public HMasterInterface getMaster()
     throws MasterNotRunningException, ZooKeeperConnectionException {
       // TODO: REMOVE.  MOVE TO HBaseAdmin and redo as a Callable!!!
@@ -673,15 +690,9 @@ public class HConnectionManager {
             }
 
           } catch (IOException e) {
-            if (tries == numRetries - 1) {
-              // This was our last chance - don't bother sleeping
-              LOG.info("getMaster attempt " + tries + " of " + numRetries +
-                " failed; no more retrying.", e);
-              break;
-            }
-            LOG.info("getMaster attempt " + tries + " of " + numRetries +
-              " failed; retrying after sleep of " +
-              ConnectionUtils.getPauseTime(this.pause, tries), e);
+            if (!shouldRetryGetMaster(tries, e)) break;
+          } catch (UndeclaredThrowableException ute) {
+            if (!shouldRetryGetMaster(tries, ute)) break;
           }
 
           // Cannot connect to master or it is not running. Sleep & retry

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java?rev=1394857&r1=1394856&r2=1394857&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/HBaseRPC.java Fri Oct  5
23:16:35 2012
@@ -129,7 +129,7 @@ public class HBaseRPC {
   }
 
   // return the RpcEngine that handles a proxy object
-  private static synchronized RpcEngine getProxyEngine(Object proxy) {
+  static synchronized RpcEngine getProxyEngine(Object proxy) {
     return PROXY_ENGINES.get(proxy.getClass());
   }
 

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/RpcEngine.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/RpcEngine.java?rev=1394857&r1=1394856&r2=1394857&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/RpcEngine.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/RpcEngine.java Fri Oct 
5 23:16:35 2012
@@ -19,6 +19,7 @@
  */
 package org.apache.hadoop.hbase.ipc;
 
+import java.lang.reflect.InvocationHandler;
 import java.lang.reflect.Method;
 import java.io.IOException;
 import java.net.InetSocketAddress;
@@ -37,6 +38,13 @@ interface RpcEngine {
                   User ticket, Configuration conf,
                   SocketFactory factory, int rpcTimeout) throws IOException;
 
+  /** Construct a client-side proxy object, specifying an InvocationHandler for testing purposes
*/
+  VersionedProtocol getProxy(Class<? extends VersionedProtocol> protocol,
+                  long clientVersion, InetSocketAddress addr,
+                  User ticket, Configuration conf,
+                  SocketFactory factory, int rpcTimeout, InvocationHandler handler)
+  throws IOException;
+
   /** Stop this proxy. */
   void stopProxy(VersionedProtocol proxy);
 

Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java?rev=1394857&r1=1394856&r2=1394857&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/ipc/WritableRpcEngine.java Fri
Oct  5 23:16:35 2012
@@ -121,7 +121,7 @@ class WritableRpcEngine implements RpcEn
 
   protected final static ClientCache CLIENTS = new ClientCache();
 
-  private static class Invoker implements InvocationHandler {
+  static class Invoker implements InvocationHandler {
     private Class<? extends VersionedProtocol> protocol;
     private InetSocketAddress address;
     private User ticket;
@@ -172,13 +172,12 @@ class WritableRpcEngine implements RpcEn
   public VersionedProtocol getProxy(
       Class<? extends VersionedProtocol> protocol, long clientVersion,
       InetSocketAddress addr, User ticket,
-      Configuration conf, SocketFactory factory, int rpcTimeout)
+      Configuration conf, SocketFactory factory, int rpcTimeout, InvocationHandler handler)
     throws IOException {
 
       VersionedProtocol proxy =
           (VersionedProtocol) Proxy.newProxyInstance(
-              protocol.getClassLoader(), new Class[] { protocol },
-              new Invoker(protocol, addr, ticket, conf, factory, rpcTimeout));
+              protocol.getClassLoader(), new Class[] { protocol }, handler);
     if (proxy instanceof VersionedProtocol) {
       long serverVersion = ((VersionedProtocol)proxy)
         .getProtocolVersion(protocol.getName(), clientVersion);
@@ -190,6 +189,14 @@ class WritableRpcEngine implements RpcEn
     return proxy;
   }
 
+  public VersionedProtocol getProxy(Class<? extends VersionedProtocol> protocol,
+      long clientVersion, InetSocketAddress addr,
+      User ticket, Configuration conf,
+      SocketFactory factory, int rpcTimeout) throws IOException {
+    return getProxy(protocol, clientVersion, addr, ticket, conf, factory, rpcTimeout,
+      new Invoker(protocol, addr, ticket, conf, factory, rpcTimeout));
+  }
+
   /**
    * Stop this proxy and release its invoker's resource
    * @param proxy the proxy to be stopped

Added: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestClientTimeouts.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestClientTimeouts.java?rev=1394857&view=auto
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestClientTimeouts.java
(added)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/client/TestClientTimeouts.java
Fri Oct  5 23:16:35 2012
@@ -0,0 +1,89 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import static org.junit.Assert.*;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MasterNotRunningException;
+import org.apache.hadoop.hbase.MediumTests;
+import org.apache.hadoop.hbase.ipc.HMasterInterface;
+import org.apache.hadoop.hbase.ipc.RandomTimeoutRpcEngine;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(MediumTests.class)
+public class TestClientTimeouts {
+  final Log LOG = LogFactory.getLog(getClass());
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  protected static int SLAVES = 1;
+
+ /**
+   * @throws java.lang.Exception
+   */
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    Configuration conf = TEST_UTIL.getConfiguration();
+    RandomTimeoutRpcEngine.setProtocolEngine(conf, HMasterInterface.class);
+    TEST_UTIL.startMiniCluster(SLAVES);
+  }
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  /**
+   * Test that a client that fails an RPC to the master retries properly and
+   * doesn't throw any unexpected exceptions.
+   * @throws Exception
+   */
+  @Test
+  public void testClientGetMasterFailure() throws Exception {
+    long lastLimit = HConstants.DEFAULT_HBASE_CLIENT_PREFETCH_LIMIT;
+    HConnection lastConnection = null;
+    for (int i = 0; i < 5; ++i) {
+      // Ensure the HBaseAdmin uses a new connection by changing Configuration.
+      Configuration conf = HBaseConfiguration.create(TEST_UTIL.getConfiguration());
+      conf.setLong(HConstants.HBASE_CLIENT_PREFETCH_LIMIT, ++lastLimit);
+      try {
+        HBaseAdmin admin = new HBaseAdmin(conf);
+        HConnection connection = admin.getConnection();
+        assertFalse(connection == lastConnection);
+        // Ensure the RandomTimeoutRpcEngine is actually being used.
+        assertTrue(RandomTimeoutRpcEngine.isProxyForObject((admin.getMaster())));
+      } catch (MasterNotRunningException ex) {
+        // Since we are randomly throwing SocketTimeoutExceptions, it is possible to get
+        // a MasterNotRunningException.  We only care about other exceptions -- i.e.
+        // UndeclaredThrowableException.
+      }
+    }
+  }
+}

Added: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/ipc/RandomTimeoutRpcEngine.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/ipc/RandomTimeoutRpcEngine.java?rev=1394857&view=auto
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/ipc/RandomTimeoutRpcEngine.java
(added)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/ipc/RandomTimeoutRpcEngine.java
Fri Oct  5 23:16:35 2012
@@ -0,0 +1,77 @@
+/**
+ * Copyright The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.ipc;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.net.InetSocketAddress;
+import java.net.SocketTimeoutException;
+import java.util.Random;
+import org.apache.hadoop.hbase.ipc.VersionedProtocol;
+import javax.net.SocketFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.security.User;
+
+/**
+ * RpcEngine that random throws a SocketTimeoutEngine for testing.
+ * Make sure to call setProtocolEngine to have the client actually use the RpcEngine
+ * for a specific protocol
+ */
+public class RandomTimeoutRpcEngine extends WritableRpcEngine {
+
+  private static final Random RANDOM = new Random(System.currentTimeMillis());
+  public static double chanceOfTimeout = 0.3;
+
+  public VersionedProtocol getProxy(
+      Class<? extends VersionedProtocol> protocol, long clientVersion,
+      InetSocketAddress addr, User ticket,
+      Configuration conf, SocketFactory factory, int rpcTimeout) throws IOException {
+    RandomTimeoutInvoker handler =
+      new RandomTimeoutInvoker(protocol, addr, ticket, conf, factory, rpcTimeout);
+    return super.getProxy(protocol, clientVersion, addr, ticket, conf, factory, rpcTimeout,
handler);
+  }
+
+  // Call this in order to set this class to run as the RpcEngine for the given protocol
+  public static void setProtocolEngine(Configuration conf, Class protocol) {
+    HBaseRPC.setProtocolEngine(conf, protocol, RandomTimeoutRpcEngine.class);
+  }
+
+  public static boolean isProxyForObject(Object proxy) {
+    return HBaseRPC.getProxyEngine(proxy).getClass().equals(RandomTimeoutRpcEngine.class);
+  }
+
+  static class RandomTimeoutInvoker extends Invoker {
+
+    public RandomTimeoutInvoker(Class<? extends VersionedProtocol> protocol,
+        InetSocketAddress address, User ticket,
+        Configuration conf, SocketFactory factory, int rpcTimeout) {
+      super(protocol, address, ticket, conf, factory, rpcTimeout);
+    }
+
+    public Object invoke(Object proxy, Method method, Object[] args)
+    throws Throwable {
+      if (RANDOM.nextFloat() < chanceOfTimeout) {
+        throw new SocketTimeoutException("fake timeout");
+      }
+      return super.invoke(proxy, method, args);
+    }
+  }
+}



Mime
View raw message