hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From li...@apache.org
Subject svn commit: r1459472 - in /hbase/branches/0.89-fb/src: main/java/org/apache/hadoop/hbase/client/ test/java/org/apache/hadoop/hbase/client/
Date Thu, 21 Mar 2013 18:54:02 GMT
Author: liyin
Date: Thu Mar 21 18:54:02 2013
New Revision: 1459472

URL: http://svn.apache.org/r1459472
Log:
[HBASE-8083] Adds more information for HBase RetriesExhaustedExceptions

Author: rshroff

Summary:
The RetriesExhaustedException is the most common exception thrown to the
HBase client. Currently it just has a string which is either a text
message or set of exceptions converted into a text message.

For better diagnosability on the client side we need give more
information about these exceptions.

The change adds a new class HRegionFailureInfo which stores:
1. ServerName
2. RegionName
3. Exceptions encountered

This class is a part of RetriesExhaustedException and the clients can
use new methods to get more information about the exception.

Test Plan:
Added a few unit tests. Running MR unit test suite now.
Sample output https://phabricator.fb.com/P2010393

Reviewers: aaiyer, liyintang

Reviewed By: liyintang

CC: hbase-eng@

Differential Revision: https://phabricator.fb.com/D723010

Task ID: 2121042

Added:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HRegionFailureInfo.java
Modified:
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java
    hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
    hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnection.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnection.java?rev=1459472&r1=1459471&r2=1459472&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnection.java (original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnection.java Thu
Mar 21 18:54:02 2013
@@ -320,7 +320,8 @@ public interface HConnection extends Clo
    * @throws IOException
    */
   public List<Put> processListOfMultiPut(List<MultiPut> mputs,
-      final byte[] tableName, HBaseRPCOptions options) throws IOException;
+      final byte[] tableName, HBaseRPCOptions options,
+      Map<String, HRegionFailureInfo> failureInfo) throws IOException;
   
   /**
    * Delete the cached location

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=1459472&r1=1459471&r2=1459472&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HConnectionManager.java
Thu Mar 21 18:54:02 2013
@@ -38,8 +38,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
 import java.util.TreeSet;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
@@ -70,7 +68,6 @@ import org.apache.hadoop.hbase.MasterNot
 import org.apache.hadoop.hbase.NotServingRegionException;
 import org.apache.hadoop.hbase.RemoteExceptionHandler;
 import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.hadoop.hbase.client.HConnectionManager.TableServers.FailureInfo;
 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
 import org.apache.hadoop.hbase.ipc.HBaseRPC;
 import org.apache.hadoop.hbase.ipc.HBaseRPCOptions;
@@ -1528,9 +1525,15 @@ public class HConnectionManager {
                 prevLoc.getRegionInfo().getStartKey(),
                 prevLoc.getServerAddress());
           }
-          // do not retry if getting the location throws exception
-          callable.instantiateRegionLocation(false /* reload cache ? */);
 
+          try {
+            // do not retry if getting the location throws exception
+            callable.instantiateRegionLocation(false /* reload cache ? */);
+          } catch (IOException e) {
+            exceptions.add(e);
+            throw new RetriesExhaustedException(callable.getServerName(),
+                callable.getRegionName(), callable.getRow(), tries, exceptions);
+          }
           if (prevLoc.getServerAddress().
               equals(callable.location.getServerAddress())) {
             // Bail out of the retry loop if we have to wait too long
@@ -1978,7 +1981,8 @@ public class HConnectionManager {
      */
     private List<Mutation> collectResponsesForMutateFromAllRS(byte[] tableName,
         Map<HServerAddress, MultiAction> actionsByServer,
-        Map<HServerAddress, Future<MultiResponse>> futures)
+        Map<HServerAddress, Future<MultiResponse>> futures,
+        Map<String, HRegionFailureInfo> failureInfo)
       throws InterruptedException, IOException {
 
      List<Mutation> newWorkingList = null;
@@ -2004,11 +2008,11 @@ public class HConnectionManager {
        // If the response is null, we will add it to newWorkingList here.
        if (request.deletes != null) {
          newWorkingList = processMutationResponseFromOneRegionServer(tableName, address,
-             resp, request.deletes, newWorkingList, true);
+             resp, request.deletes, newWorkingList, true, failureInfo);
        }
        if (request.puts != null) {
          newWorkingList = processMutationResponseFromOneRegionServer(tableName, address,
-             resp, request.puts, newWorkingList, false);
+             resp, request.puts, newWorkingList, false, failureInfo);
        }
      }
       return newWorkingList;
@@ -2023,7 +2027,8 @@ public class HConnectionManager {
          Map<HServerAddress, MultiAction> actionsByServer,
          Map<HServerAddress, Future<MultiResponse>> futures,
          List<Get> orig_list,
-         Result[] results) throws IOException, InterruptedException {
+         Result[] results,
+         Map<String, HRegionFailureInfo> failureInfo) throws IOException, InterruptedException
{
 
        List<Get> newWorkingList = null;
        for (Entry<HServerAddress, Future<MultiResponse>> responsePerServer
@@ -2049,7 +2054,7 @@ public class HConnectionManager {
          }
 
          newWorkingList = processGetResponseFromOneRegionServer(tableName, address,
-             request, resp, orig_list, newWorkingList, results);
+             request, resp, orig_list, newWorkingList, results, failureInfo);
        }
        return newWorkingList;
      }
@@ -2060,7 +2065,8 @@ public class HConnectionManager {
         MultiResponse resp,
         Map<byte[], List<R>> map,
         List<Mutation> newWorkingList,
-        boolean isDelete) throws IOException {
+        boolean isDelete,
+        Map<String, HRegionFailureInfo> failureInfo) throws IOException {
         // If we got a response. Let us go through the responses from each region and
         // process the Puts and Deletes.
         for (Map.Entry<byte[], List<R>> e : map.entrySet()) {
@@ -2082,8 +2088,17 @@ public class HConnectionManager {
                   regionOps.size()));
             }
           } catch (Exception ex) {
+            String serverName = address.getHostname();
+            String regName = Bytes.toStringBinary(regionName);
             // If response is null, we will catch a NPE here.
             translateException(ex);
+
+            if (!failureInfo.containsKey(regName)) {
+              failureInfo.put(regName, new HRegionFailureInfo(regName));
+            }
+            failureInfo.get(regName).addException(ex);
+            failureInfo.get(regName).setServerName(serverName);
+
             if (newWorkingList == null)
               newWorkingList = new ArrayList<Mutation>();
 
@@ -2101,7 +2116,8 @@ public class HConnectionManager {
        MultiAction request,
        MultiResponse resp,
        List<Get> orig_list, List<Get> newWorkingList,
-       Result[] results) throws IOException {
+       Result[] results,
+       Map<String, HRegionFailureInfo> failureInfo) throws IOException {
 
        for (Map.Entry<byte[], List<Get>> e : request.gets.entrySet()) {
          byte[] regionName = e.getKey();
@@ -2120,9 +2136,19 @@ public class HConnectionManager {
          } catch (Exception ex) {
            // If response is null, we will catch a NPE here.
            translateException(ex);
+
            if (newWorkingList == null)
              newWorkingList = new ArrayList<Get>(orig_list.size());
 
+           String serverName = address.getHostname();
+           String regName = Bytes.toStringBinary(regionName);
+
+           if (!failureInfo.containsKey(regName)) {
+             failureInfo.put(regName, new HRegionFailureInfo(regName));
+           }
+           failureInfo.get(regName).addException(ex);
+           failureInfo.get(regName).setServerName(serverName);
+
            // Add the element to the correct position
            for(int i = 0; i < regionGets.size(); i++) {
              newWorkingList.add(origIndices.get(i), regionGets.get(i));
@@ -2148,6 +2174,8 @@ public class HConnectionManager {
       // lastServers.
       // Sort the puts based on the row key in order to optimize the row lock acquiring
       // in the server side.
+
+      Map<String, HRegionFailureInfo> failureInfo = new HashMap<String, HRegionFailureInfo>();
       List<Mutation> workingList = orig_list;
       Collections.sort(workingList);
 
@@ -2172,12 +2200,12 @@ public class HConnectionManager {
 
         // step 3: collect the failures and successes and prepare for retry
         workingList = collectResponsesForMutateFromAllRS(tableName,
-            actionsByServer, futures);
+            actionsByServer, futures, failureInfo);
       }
 
       if (workingList != null && !workingList.isEmpty()) {
         if (failures != null) failures.addAll(workingList);
-        throw new RetriesExhaustedException(
+        throw new RetriesExhaustedException(failureInfo,
             workingList.size() + "mutate operations remaining after "
             + numRetries + " retries");
       }
@@ -2190,6 +2218,8 @@ public class HConnectionManager {
         Result[] results, HBaseRPCOptions options)
     throws IOException, InterruptedException {
 
+      Map<String, HRegionFailureInfo> failureInfo =
+         new HashMap<String, HRegionFailureInfo>();
       // if results is not NULL
       // results must be the same size as list
       if (results != null && (results.length != orig_list.size())) {
@@ -2223,11 +2253,11 @@ public class HConnectionManager {
 
         // step 3: collect the failures and successes and prepare for retry
         workingList = collectResponsesForGetFromAllRS(tableName, actionsByServer,
-            futures, workingList, results);
+            futures, workingList, results, failureInfo);
       }
 
       if (workingList != null && !workingList.isEmpty()) {
-        throw new RetriesExhaustedException(
+        throw new RetriesExhaustedException(failureInfo,
             workingList.size() + " get operations remaining after "
             + numRetries + " retries");
       }
@@ -2499,7 +2529,8 @@ public class HConnectionManager {
     }
 
     public List<Put> processListOfMultiPut(List<MultiPut> multiPuts,
-      final byte[] givenTableName, HBaseRPCOptions options) throws IOException {
+      final byte[] givenTableName, HBaseRPCOptions options,
+      Map<String, HRegionFailureInfo> failedRegionsInfo) throws IOException {
       List<Put> failed = null;
 
       List<Future<MultiPutResponse>> futures =
@@ -2521,9 +2552,14 @@ public class HConnectionManager {
 
       RegionOverloadedException toThrow = null;
       long maxWaitTimeRequested = 0;
+
       for (int i = 0; i < futures.size(); i++ ) {
         Future<MultiPutResponse> future = futures.get(i);
         MultiPut request = multiPuts.get(i);
+        String serverName = request.address.getHostname();
+        String regionName = null;
+        HRegionFailureInfo regionFailure = null;
+
         MultiPutResponse resp = null;
         try {
           resp = future.get();
@@ -2531,6 +2567,19 @@ public class HConnectionManager {
           Thread.currentThread().interrupt();
           throw new InterruptedIOException(e.getMessage());
         } catch (ExecutionException ex) {
+
+          // Add entry for the all the regions involved in this operation.
+          for (Map.Entry<byte[], List<Put>> e : request.puts.entrySet()) {
+            regionName = Bytes.toStringBinary(e.getKey());
+            if (!failedRegionsInfo.containsKey(regionName)) {
+              regionFailure = new HRegionFailureInfo(regionName);
+              failedRegionsInfo.put(regionName, regionFailure);
+            } else {
+              regionFailure = failedRegionsInfo.get(regionName);
+            }
+            regionFailure.setServerName(serverName);
+            regionFailure.addException(ex);
+          }
           // retry, unless it is not to be retried.
           if (ex.getCause() instanceof DoNotRetryIOException) {
             throw (DoNotRetryIOException)ex.getCause();
@@ -2560,6 +2609,15 @@ public class HConnectionManager {
           if (resp != null)
             result = resp.getAnswer(region);
 
+          if (result == null || result != HConstants.MULTIPUT_SUCCESS) {
+            regionName = Bytes.toStringBinary(e.getKey());
+            if (!failedRegionsInfo.containsKey(regionName)) {
+              regionFailure = new HRegionFailureInfo(regionName);
+              failedRegionsInfo.put(regionName, regionFailure);
+            }
+            regionFailure.setServerName(serverName);
+            regionFailure.addException(new IOException("Put failed for " + regionName));
+          }
           if (result == null) {
             // failed
             LOG.debug("Failed all for region: " +
@@ -2570,11 +2628,11 @@ public class HConnectionManager {
             // and derive the tableName from the regionName.
             byte[] tableName = (givenTableName != null)? givenTableName
                 : HRegionInfo.parseRegionName(region)[0];
-
-
             deleteCachedLocation(tableName, lst.get(0).getRow(), request.address);
 
-            if (failed == null) failed = new ArrayList<Put>();
+            if (failed == null) {
+              failed = new ArrayList<Put>();
+            }
             failed.addAll(e.getValue());
           } else if (result != HConstants.MULTIPUT_SUCCESS) {
             // some failures
@@ -2621,7 +2679,8 @@ public class HConnectionManager {
       int tries;
       long serverRequestedWaitTime = 0;
       int serverRequestedRetries = 0;
-      
+      Map<String, HRegionFailureInfo> failedServersInfo =
+          new HashMap<String, HRegionFailureInfo>();
       // Sort the puts based on the row key in order to optimize the row lock acquiring
       // in the server side.
       Collections.sort(list);
@@ -2652,8 +2711,9 @@ public class HConnectionManager {
 
         List<Put> failed = null;
         List<MultiPut> multiPuts = this.splitPutsIntoMultiPuts(list, tableName, options);
+
         try {
-          failed = this.processListOfMultiPut(multiPuts, tableName, options);
+          failed = this.processListOfMultiPut(multiPuts, tableName, options, failedServersInfo);
         } catch (RegionOverloadedException ex) {
           roe = ex;
           serverRequestedWaitTime = roe.getBackoffTimeMillis();
@@ -2691,10 +2751,12 @@ public class HConnectionManager {
 
       // Get exhausted after the retries
       if (!list.isEmpty()) {
+
         // ran out of retries and didnt succeed everything!
-        throw new RetriesExhaustedException("Still had " + list.size() +
-            " puts left after retrying " +
-            tries + " times, in " + (System.currentTimeMillis() - callStartTime) + " ms.");
+        throw new RetriesExhaustedException(failedServersInfo,
+            "Still had " + list.size() + " puts left after retrying " +
+            tries + " times, in " + (System.currentTimeMillis() - callStartTime) +
+            " ms.");
       }
     }
 

Added: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HRegionFailureInfo.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HRegionFailureInfo.java?rev=1459472&view=auto
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HRegionFailureInfo.java
(added)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HRegionFailureInfo.java
Thu Mar 21 18:54:02 2013
@@ -0,0 +1,107 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.client;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.Serializable;
+import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Class to maintain the errors encountered while performing HBase operations at per region
level on the client side.
+ * It has information about:
+ * 1. region name
+ * 2. server name
+ * 3. exceptions
+ */
+
+public class HRegionFailureInfo implements Serializable {
+  private String serverName;
+  private String regionName;
+
+  // list of exceptions encountered at RegionServer level
+  private List<Throwable> exceptions = new ArrayList<Throwable>();
+
+  public HRegionFailureInfo(final String regionName) {
+    this.regionName = regionName;
+  }
+
+  public HRegionFailureInfo(final String serverName, final String regionName) {
+    this.regionName = regionName;
+    this.serverName = serverName;
+  }
+
+  public void setServerName(final String name) {
+    this.serverName = name;
+  }
+
+  public void addException(final Throwable ex) {
+    exceptions.add(ex);
+  }
+
+  public void addAllExceptions(final List<Throwable> ex) {
+    exceptions.addAll(ex);
+  }
+
+  public final String getServerName() {
+    return this.serverName;
+  }
+
+  public final String getRegionName() {
+    return this.regionName;
+  }
+
+  public final List<Throwable> getExceptions() {
+    return this.exceptions;
+  }
+
+  public String toString() {
+    StringBuilder buffer = new StringBuilder("");
+    buffer.append("Region: " + regionName + "\n");
+    buffer.append("Server: " + this.serverName + "\n");
+    buffer.append("Exceptions: \n");
+    for (Throwable e : exceptions) {
+      if (e == null) {
+        continue;
+      }
+      buffer.append(e.toString());
+
+      StringWriter errors = new StringWriter();
+      e.printStackTrace(new PrintWriter(errors));
+      buffer.append(errors.toString());
+      buffer.append("\n");
+
+      try {
+        errors.close();
+      } catch (IOException ex) {} // ignore
+    }
+
+    return buffer.toString();
+  }
+
+  public boolean equals(final HRegionFailureInfo h) {
+    return this.regionName.equals(h.getRegionName());
+  }
+}

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java?rev=1459472&r1=1459471&r2=1459472&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/HTableMultiplexer.java
Thu Mar 21 18:54:02 2013
@@ -500,8 +500,11 @@ public class HTableMultiplexer {
             
             // Process this multiput request
             List<Put> failed = null;
+            Map<String, HRegionFailureInfo> failureInfo =
+                new HashMap<String, HRegionFailureInfo>();
             try {
-              failed = connection.processListOfMultiPut(Arrays.asList(mput), null, options);
+              failed = connection.processListOfMultiPut(Arrays.asList(mput), null, options,
+                  failureInfo);
             } catch(PreemptiveFastFailException e) {
               // Client is not blocking on us. So, let us treat this
               // as a normal failure, and retry.

Modified: hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java?rev=1459472&r1=1459471&r2=1459472&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java
(original)
+++ hbase/branches/0.89-fb/src/main/java/org/apache/hadoop/hbase/client/RetriesExhaustedException.java
Thu Mar 21 18:54:02 2013
@@ -20,7 +20,10 @@ import org.apache.hadoop.hbase.util.Byte
 import java.io.IOException;
 import java.io.PrintWriter;
 import java.io.StringWriter;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
 /**
  * Exception thrown by HTable methods when an attempt to do something (like
@@ -28,10 +31,7 @@ import java.util.List;
  */
 public class RetriesExhaustedException extends IOException {
   private static final long serialVersionUID = 1876775844L;
-
-  public RetriesExhaustedException(final String msg) {
-    super(msg);
-  }
+  private Map<String, HRegionFailureInfo> failureInfo = null;
 
   /**
    * Create a new RetriesExhaustedException from the list of prior failures.
@@ -44,6 +44,21 @@ public class RetriesExhaustedException e
   public RetriesExhaustedException(String serverName, final byte [] regionName,
       final byte []  row, int numTries, List<Throwable> exceptions) {
     super(getMessage(serverName, regionName, row, numTries, exceptions));
+    failureInfo = new HashMap<String, HRegionFailureInfo>();
+    String regName = Bytes.toStringBinary(regionName);
+
+    if (!failureInfo.containsKey(regName)) {
+      failureInfo.put(regName, new HRegionFailureInfo(regName));
+    }
+
+    failureInfo.get(regName).setServerName(serverName);
+    this.failureInfo.get(regName).addAllExceptions(exceptions);
+  }
+
+  public RetriesExhaustedException(final Map<String, HRegionFailureInfo> failureInfo,
+      String msg) {
+    super(msg);
+    this.failureInfo = failureInfo;
   }
 
   private static String getMessage(String serverName, final byte [] regionName,
@@ -72,4 +87,16 @@ public class RetriesExhaustedException e
     }
     return buffer.toString();
   }
-}
\ No newline at end of file
+
+  public Set<String> getRegionNames() {
+    return this.failureInfo.keySet();
+  }
+
+  public HRegionFailureInfo getFailureInfoForRegion(String regionName) {
+    return this.failureInfo.get(regionName);
+  }
+
+  public Map<String, HRegionFailureInfo> getFailureInfo() {
+    return this.failureInfo;
+  }
+}

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java?rev=1459472&r1=1459471&r2=1459472&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestAdmin.java Thu
Mar 21 18:54:02 2013
@@ -269,21 +269,24 @@ public class TestAdmin {
     final byte [] value = Bytes.toBytes("value");
     final byte [] table = Bytes.toBytes("testDisableAndEnableTable");
     HTable ht = TEST_UTIL.createTable(table, HConstants.CATALOG_FAMILY);
+    
     Put put = new Put(row);
     put.add(HConstants.CATALOG_FAMILY, qualifier, value);
     ht.put(put);
-
+    
     this.admin.disableTable(table);
-
+    
     // Test that table is disabled
     Get get = new Get(row);
     get.addColumn(HConstants.CATALOG_FAMILY, qualifier);
+    
     boolean ok = false;
     try {
       ht.get(get);
-    } catch (RegionOfflineException e) {
+    } catch (RetriesExhaustedException e) {
       ok = true;
     }
+    
     // with online schema change it is possible to add column
     // without disabling the table
     assertEquals(true, ok);

Modified: hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java?rev=1459472&r1=1459471&r2=1459472&view=diff
==============================================================================
--- hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java (original)
+++ hbase/branches/0.89-fb/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java Thu Mar
21 18:54:02 2013
@@ -19,28 +19,42 @@
  */
 package org.apache.hadoop.hbase.client;
 
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.master.ServerManager;
 import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
 
 /**
  * This class is for testing HCM features
  */
 public class TestHCM {
+  private static final Log LOG = LogFactory.getLog(TestHCM.class);
 
   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private static final byte[] TABLE_NAME = Bytes.toBytes("test");
   private static final byte[] FAM_NAM = Bytes.toBytes("f");
   private static final byte[] ROW = Bytes.toBytes("bbb");
 
+  private static final int REGION_SERVERS = 10;
+
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
-    TEST_UTIL.startMiniCluster(1);
+    TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 3);
+    TEST_UTIL.getConfiguration().set("hbase.loadbalancer.impl",
+        "org.apache.hadoop.hbase.master.RegionManager$AssignmentLoadBalancer");
+    TEST_UTIL.startMiniCluster(REGION_SERVERS);
   }
 
   /**
@@ -50,6 +64,7 @@ public class TestHCM {
    */
   @Test
   public void testRegionCaching() throws Exception{
+
     HTable table = TEST_UTIL.createTable(TABLE_NAME, FAM_NAM);
     TEST_UTIL.createMultiRegions(table, FAM_NAM);
     Put put = new Put(ROW);
@@ -61,5 +76,116 @@ public class TestHCM {
     conn.deleteCachedLocation(TABLE_NAME, ROW, null);
     HRegionLocation rl = conn.getCachedLocation(TABLE_NAME, ROW);
     assertNull("What is this location?? " + rl, rl);
+
+  }
+
+  /**
+   * Test that on failures, HBase client reports the errors correctly.
+   * @throws Exception
+   */
+  @Test
+  public void testClientFailureReporting() throws Exception {
+
+    HTable table = TEST_UTIL.createTable(Bytes.toBytes("testClientFailureReporting"), FAM_NAM);
+    TEST_UTIL.createMultiRegions(table, FAM_NAM);
+
+    Put put = new Put(ROW);
+    put.add(FAM_NAM, ROW, ROW);
+    setupFailure(table);
+    try {
+      table.put(put);
+      assertNull("Put should not succeed");
+    } catch (IOException e) {
+      verifyException(e);
+    } finally {
+      ServerManager.clearRSBlacklistInTest();
+      waitForRegionsToGetReAssigned();
+    }
+  }
+
+  private void waitForRegionsToGetReAssigned() {
+    while(
+        TEST_UTIL.getHBaseCluster().getActiveMaster().
+          getRegionManager().areAllMetaRegionsOnline() == false) {}
+  }
+
+
+  /**
+   * Test that on failures, HBase client reports the errors correctly.
+   * @throws Exception
+   */
+  @Test
+  public void testClientFailureReportingGet() throws Exception {
+    HTable table = TEST_UTIL.createTable(Bytes.toBytes("testClientFailureReportingGet"),
FAM_NAM);
+    TEST_UTIL.createMultiRegions(table, FAM_NAM);
+
+
+    Put put = new Put(ROW);
+    put.add(FAM_NAM, ROW, ROW);
+    table.put(put);
+
+    setupFailure(table);
+    try {
+      Get get = new Get(ROW);
+      table.get(get);
+      assertNull("Get should not succeed");
+    } catch (IOException e) {
+      verifyException(e);
+    } finally {
+      ServerManager.clearRSBlacklistInTest();
+      waitForRegionsToGetReAssigned();
+    }
+  }
+
+  /**
+   * Test that on failures, HBase client reports the errors correctly.
+   * @throws Exception
+   */
+  @Test
+  public void testClientFailureReportingDelete() throws Exception {
+
+    HTable table = TEST_UTIL.createTable(Bytes.toBytes("testClientFailureReportingDelete"),
FAM_NAM);
+    TEST_UTIL.createMultiRegions(table, FAM_NAM);
+
+    Put put = new Put(ROW);
+    put.add(FAM_NAM, ROW, ROW);
+    table.put(put);
+
+    setupFailure(table);
+    try {
+      Delete delete = new Delete(ROW);
+      table.delete(delete);
+      assertNull("Delete should not succeed");
+    } catch (IOException e) {
+      verifyException(e);
+    } finally {
+      ServerManager.clearRSBlacklistInTest();
+      waitForRegionsToGetReAssigned();
+    }
+
+  }
+
+  private void setupFailure(HTable table) throws IOException {
+    HRegionLocation regLoc = table.getRegionLocation(ROW);
+    List<RegionServerThread> regionServers =
+        TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads();
+
+    // Add all region servers to the black list to prevent fail overs.
+    for(RegionServerThread r : regionServers) {
+      ServerManager.blacklistRSHostPortInTest(
+          r.getRegionServer().getHServerInfo().getHostnamePort());
+    }
+
+    // abort the region server
+    int srcRSIdx = TEST_UTIL.getHBaseCluster().
+        getServerWith(regLoc.getRegionInfo().getRegionName());
+    TEST_UTIL.getHBaseCluster().stopRegionServer(srcRSIdx);
+  }
+
+  private void verifyException(Exception e) {
+    assertTrue(e instanceof RetriesExhaustedException);
+    RetriesExhaustedException exp = (RetriesExhaustedException)(e);
+    assertTrue(exp.getRegionNames() != null);
+    LOG.info(exp.getFailureInfo().values());
   }
 }



Mime
View raw message