hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r1171646 - in /hbase/branches/0.90: CHANGES.txt src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckComparator.java
Date Fri, 16 Sep 2011 16:23:45 GMT
Author: stack
Date: Fri Sep 16 16:23:44 2011
New Revision: 1171646

URL: http://svn.apache.org/viewvc?rev=1171646&view=rev
Log:
HBASE-4322 [hbck] Update checkIntegrity/checkRegionChain to present more accurate region split
problem 

Added:
    hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckComparator.java
Modified:
    hbase/branches/0.90/CHANGES.txt
    hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
    hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java

Modified: hbase/branches/0.90/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/CHANGES.txt?rev=1171646&r1=1171645&r2=1171646&view=diff
==============================================================================
--- hbase/branches/0.90/CHANGES.txt (original)
+++ hbase/branches/0.90/CHANGES.txt Fri Sep 16 16:23:44 2011
@@ -45,6 +45,8 @@ Release 0.90.5 - Unreleased
    HBASE-4390  [replication] ReplicationSource's UncaughtExceptionHandler
                shouldn't join
    HBASE-4395  EnableTableHandler races with itself
+   HBASE-4322  [hbck] Update checkIntegrity/checkRegionChain to present more
+               accurate region split problem summary (Jon Hsieh)
 
   IMPROVEMENT
    HBASE-4205  Enhance HTable javadoc (Eric Charles)

Modified: hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java?rev=1171646&r1=1171645&r2=1171646&view=diff
==============================================================================
--- hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java (original)
+++ hbase/branches/0.90/src/main/java/org/apache/hadoop/hbase/util/HBaseFsck.java Fri Sep
16 16:23:44 2011
@@ -20,11 +20,20 @@
 package org.apache.hadoop.hbase.util;
 
 import java.io.IOException;
-import java.util.*;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
-import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -51,14 +60,15 @@ import org.apache.hadoop.hbase.client.Re
 import org.apache.hadoop.hbase.client.MetaScanner.MetaScannerVisitor;
 import org.apache.hadoop.hbase.ipc.HRegionInterface;
 import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.zookeeper.RootRegionTracker;
+import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
 import org.apache.hadoop.hbase.zookeeper.ZKTable;
 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
 import org.apache.zookeeper.KeeperException;
 
 import com.google.common.base.Joiner;
 import com.google.common.collect.Lists;
-
-import static org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
+import com.google.common.collect.Multimap;
 
 /**
  * Check consistency among the in-memory states of the master and the
@@ -504,7 +514,6 @@ public class HBaseFsck {
         modTInfo.addServer(server);
       }
 
-      //modTInfo.addEdge(hbi.metaEntry.getStartKey(), hbi.metaEntry.getEndKey());
       modTInfo.addRegionInfo(hbi);
 
       tablesInfo.put(tableName, modTInfo);
@@ -524,15 +533,35 @@ public class HBaseFsck {
     String tableName;
     TreeSet <HServerAddress> deployedOn;
 
-    List<HbckInfo> regions = new ArrayList<HbckInfo>();
+    final List<HbckInfo> backwards = new ArrayList<HbckInfo>();
+    final RegionSplitCalculator<HbckInfo> sc = new RegionSplitCalculator<HbckInfo>(cmp);
 
     TInfo(String name) {
       this.tableName = name;
       deployedOn = new TreeSet <HServerAddress>();
     }
 
-    public void addRegionInfo (HbckInfo r) {
-      regions.add(r);
+    public void addRegionInfo(HbckInfo hir) {
+      if (Bytes.equals(hir.getEndKey(), HConstants.EMPTY_END_ROW)) {
+        // end key is absolute end key, just add it.
+        sc.add(hir);
+        return;
+      }
+
+      // if not the absolute end key, check for cycle 
+      if (Bytes.compareTo(hir.getStartKey(), hir.getEndKey()) > 0) {
+        errors.reportError(
+            ERROR_CODE.REGION_CYCLE,
+            String.format("The endkey for this region comes before the "
+                + "startkey, startkey=%s, endkey=%s",
+                Bytes.toStringBinary(hir.getStartKey()),
+                Bytes.toStringBinary(hir.getEndKey())), this, hir);
+        backwards.add(hir);
+        return;
+      }
+
+      // main case, add to split calculator
+      sc.add(hir);
     }
 
     public void addServer(HServerAddress server) {
@@ -544,7 +573,7 @@ public class HBaseFsck {
     }
 
     public int getNumRegions() {
-      return regions.size();
+      return sc.getStarts().size();
     }
 
     /**
@@ -553,66 +582,59 @@ public class HBaseFsck {
      * @return false if there are errors
      */
     public boolean checkRegionChain() {
-      Collections.sort(regions);
-      HbckInfo last = null;
+      
       int originalErrorsCount = errors.getErrorList().size();
+      Multimap<byte[], HbckInfo> regions = sc.calcCoverage();
+      SortedSet<byte[]> splits = sc.getSplits();
 
-      for (HbckInfo r : regions) {
-        if (last == null) {
-          // This is the first region, check that the start key is empty
-          if (! Bytes.equals(r.metaEntry.getStartKey(), HConstants.EMPTY_BYTE_ARRAY)) {
+      byte[] prevKey = null;
+      for (byte[] key: splits) {
+        Collection<HbckInfo> ranges = regions.get(key);
+        if (prevKey == null && !Bytes.equals(key, HConstants.EMPTY_BYTE_ARRAY)) {
+          for (HbckInfo rng : ranges) {
             errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY,
                 "First region should start with an empty key.",
-                this, r);
-          }
-        } else {
-
-          // Check if endKey < startKey
-          // Previous implementation of this code checked for a cycle in the
-          // region chain.  A cycle would imply that the endKey comes before
-          // the startKey (i.e. endKey < startKey).
-          if (! Bytes.equals(r.metaEntry.getEndKey(), HConstants.EMPTY_BYTE_ARRAY)) {
-            // continue with this check if this is not the last region
-            int cmpRegionKeys = Bytes.compareTo(r.metaEntry.getStartKey(),
-                r.metaEntry.getEndKey());
-            if (cmpRegionKeys > 0) {
-              errors.reportError(ERROR_CODE.REGION_CYCLE,
-                  String.format("The endkey for this region comes before the "
-                      + "startkey, startkey=%s, endkey=%s",
-                      Bytes.toStringBinary(r.metaEntry.getStartKey()),
-                      Bytes.toStringBinary(r.metaEntry.getEndKey())),
-                  this, r, last);
-            }
+                this, rng);
           }
-
-          // Check if the startkeys are different
-          if (Bytes.equals(r.metaEntry.getStartKey(), last.metaEntry.getStartKey())) {
+        }
+        
+        // Check if the startkeys are different
+        if (ranges.size() > 1) {
+          ArrayList<HbckInfo> subRange = new ArrayList<HbckInfo>(ranges);
+          // this dumb and n^2 but this shouldn't happen often
+          for (HbckInfo r1 : ranges) {
+            subRange.remove(r1);
+            for (HbckInfo r2 : subRange) {
+              if (Bytes.compareTo(r1.getStartKey(), r2.getStartKey())==0) {
+            // dup start key
             errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
-                "Two regions have the same startkey: "
-                    + Bytes.toStringBinary(r.metaEntry.getStartKey()),
-                this, r, last);
-          } else {
-            // Check that the startkey is the same as the previous end key
-            int cmp = Bytes.compareTo(r.metaEntry.getStartKey(),
-                last.metaEntry.getEndKey());
-            if (cmp > 0) {
-              // hole
-              errors.reportError(ERROR_CODE.HOLE_IN_REGION_CHAIN,
-                  "There is a hole in the region chain.",
-                  this, r, last);
-            } else if (cmp < 0) {
-              // overlap
-              errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
-                  "There is an overlap in the region chain.",
-                  this, r, last);
+              "Multiple regions have the same startkey: "
+                  + Bytes.toStringBinary(key), this, r1);
+            errors.reportError(ERROR_CODE.DUPE_STARTKEYS,
+                "Multiple regions have the same startkey: "
+                    + Bytes.toStringBinary(key), this, r2);
+              } else {
+            // overlap
+            errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
+                "There is an overlap in the region chain.",
+                this, r1);
+              }
             }
           }
-
         }
-
-        last = r;
+        
+        if (ranges.size() == 0) {
+          byte[] holeStopKey = sc.getSplits().higher(key);
+          // if higher key is null we reached the top.
+          if (holeStopKey != null) {
+            // hole
+            errors.reportError(ERROR_CODE.HOLE_IN_REGION_CHAIN,
+                "There is a hole in the region chain between "
+                + Bytes.toString(key) + " and " + Bytes.toString(holeStopKey));
+          }
+        }
+        prevKey = key;
       }
-
       return errors.getErrorList().size() == originalErrorsCount;
     }
 
@@ -798,7 +820,7 @@ public class HBaseFsck {
   /**
    * Stores the entries scanned from META
    */
-  private static class MetaEntry extends HRegionInfo {
+  static class MetaEntry extends HRegionInfo {
     HServerAddress regionServer;   // server hosting this region
     long modTime;          // timestamp of most recent modification metadata
 
@@ -813,7 +835,7 @@ public class HBaseFsck {
   /**
    * Maintain information about a particular region.
    */
-  static class HbckInfo implements Comparable {
+  static class HbckInfo implements KeyRange {
     boolean onlyEdits = false;
     MetaEntry metaEntry = null;
     FileStatus foundRegionDir = null;
@@ -838,16 +860,53 @@ public class HBaseFsck {
     }
 
     @Override
-    public int compareTo(Object o) {
-      HbckInfo other = (HbckInfo) o;
-      int startComparison = Bytes.compareTo(this.metaEntry.getStartKey(), other.metaEntry.getStartKey());
-      if (startComparison != 0)
-        return startComparison;
-      else
-        return Bytes.compareTo(this.metaEntry.getEndKey(), other.metaEntry.getEndKey());
+    public byte[] getStartKey() {
+      return this.metaEntry.getStartKey();
+    }
+
+    @Override
+    public byte[] getEndKey() {
+      return this.metaEntry.getEndKey();
     }
   }
 
+  final static Comparator<HbckInfo> cmp = new Comparator<HbckInfo>() {
+    @Override
+    public int compare(HbckInfo l, HbckInfo r) {
+      if (l == r) {
+        // same instance
+        return 0;
+      }
+
+      int tableCompare = RegionSplitCalculator.BYTES_COMPARATOR.compare(
+          l.metaEntry.getTableDesc().getName(), r.metaEntry.getTableDesc().getName());
+      if (tableCompare != 0) {
+        return tableCompare;
+      }
+
+      int startComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
+          l.metaEntry.getStartKey(), r.metaEntry.getStartKey());
+      if (startComparison != 0) {
+        return startComparison;
+      }
+
+      // Special case for absolute endkey
+      byte[] endKey = r.metaEntry.getEndKey();
+      endKey = (endKey.length == 0) ? null : endKey;
+      byte[] endKey2 = l.metaEntry.getEndKey();
+      endKey2 = (endKey2.length == 0) ? null : endKey2;
+      int endComparison = RegionSplitCalculator.BYTES_COMPARATOR.compare(
+          endKey2,  endKey);
+
+      if (endComparison != 0) {
+        return endComparison;
+      }
+
+      // use modTime as tiebreaker.
+      return (int) (l.metaEntry.modTime - r.metaEntry.modTime);
+    }
+  };
+
   /**
    * Prints summary of all tables found on the system.
    */
@@ -1069,7 +1128,7 @@ public class HBaseFsck {
   }
 
   /**
-   * Contact hdfs and get all information about spcified table directory.
+   * Contact hdfs and get all information about specified table directory.
    */
   static class WorkItemHdfsDir implements Runnable {
     private HBaseFsck hbck;
@@ -1291,4 +1350,3 @@ public class HBaseFsck {
     Runtime.getRuntime().exit(code);
   }
 }
-

Modified: hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java?rev=1171646&r1=1171645&r2=1171646&view=diff
==============================================================================
--- hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java (original)
+++ hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsck.java Fri
Sep 16 16:23:44 2011
@@ -24,15 +24,21 @@ import static org.junit.Assert.assertEqu
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HServerAddress;
 import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.HBaseAdmin;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Put;
@@ -93,6 +99,49 @@ public class TestHBaseFsck {
     return hri;
   }
 
+  public void dumpMeta(HTableDescriptor htd) throws IOException {
+    List<byte[]> metaRows = TEST_UTIL.getMetaTableRows(htd.getName());
+    for (byte[] row : metaRows) {
+      LOG.info(Bytes.toString(row));
+    }
+  }
+
+  private void deleteRegion(Configuration conf, final HTableDescriptor htd, 
+      byte[] startKey, byte[] endKey) throws IOException {
+
+    LOG.info("Before delete:");
+    dumpMeta(htd);
+
+    Map<HRegionInfo, HServerAddress> hris = tbl.getRegionsInfo();
+    for (Entry<HRegionInfo, HServerAddress> e: hris.entrySet()) {
+      HRegionInfo hri = e.getKey();
+      HServerAddress hsa = e.getValue();
+      if (Bytes.compareTo(hri.getStartKey(), startKey) == 0 
+          && Bytes.compareTo(hri.getEndKey(), endKey) == 0) {
+
+        LOG.info("RegionName: " +hri.getRegionNameAsString());
+        byte[] deleteRow = hri.getRegionName();
+        TEST_UTIL.getHBaseAdmin().unassign(deleteRow, true);
+
+        LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString());
+        Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
+        FileSystem fs = rootDir.getFileSystem(conf);
+        Path p = new Path(rootDir + "/" + htd.getNameAsString(), hri.getEncodedName());
+        fs.delete(p, true);
+
+        HTable meta = new HTable(conf, HConstants.META_TABLE_NAME);
+        Delete delete = new Delete(deleteRow);
+        meta.delete(delete);
+      }
+      LOG.info(hri.toString() + hsa.toString());
+    }
+
+    TEST_UTIL.getMetaTableRows(htd.getName());
+    LOG.info("After delete:");
+    dumpMeta(htd);
+
+  }
+
   /**
    * Setup a clean table before we start mucking with it.
    * 
@@ -161,7 +210,8 @@ public class TestHBaseFsck {
           .waitForAssignment(hriDupe);
 
       assertErrors(doFsck(false),
-          new ERROR_CODE[] { ERROR_CODE.DUPE_STARTKEYS });
+          new ERROR_CODE[] { ERROR_CODE.DUPE_STARTKEYS,
+            ERROR_CODE.DUPE_STARTKEYS});
     } finally {
       deleteTable(table);
     }
@@ -201,10 +251,14 @@ public class TestHBaseFsck {
 
       // Mess it up by leaving a hole in the meta data
       HRegionInfo hriHole = createRegion(conf, tbl.getTableDescriptor(),
-          Bytes.toBytes("D"), Bytes.toBytes("E"));
+          Bytes.toBytes("D"), Bytes.toBytes(""));
       TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriHole);
       TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
           .waitForAssignment(hriHole);
+
+      TEST_UTIL.getHBaseAdmin().disableTable(table);
+      deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("C"), Bytes.toBytes(""));
+      TEST_UTIL.getHBaseAdmin().enableTable(table);
       assertErrors(doFsck(false),
           new ERROR_CODE[] { ERROR_CODE.HOLE_IN_REGION_CHAIN });
     } finally {

Added: hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckComparator.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckComparator.java?rev=1171646&view=auto
==============================================================================
--- hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckComparator.java
(added)
+++ hbase/branches/0.90/src/test/java/org/apache/hadoop/hbase/util/TestHBaseFsckComparator.java
Fri Sep 16 16:23:44 2011
@@ -0,0 +1,108 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.*;
+
+import org.apache.hadoop.hbase.HRegionInfo;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.util.HBaseFsck.HbckInfo;
+import org.apache.hadoop.hbase.util.HBaseFsck.MetaEntry;
+import org.junit.Test;
+
+/**
+ * Test the comparator used by Hbck.
+ */
+public class TestHBaseFsckComparator {
+
+  byte[] table = Bytes.toBytes("table1");
+  byte[] table2 = Bytes.toBytes("table2");
+  byte[] keyStart = Bytes.toBytes("");
+  byte[] keyA = Bytes.toBytes("A");
+  byte[] keyB = Bytes.toBytes("B");
+  byte[] keyC = Bytes.toBytes("C");
+  byte[] keyEnd = Bytes.toBytes("");
+
+  static HbckInfo genHbckInfo(byte[] table, byte[] start, byte[] end, int time) {
+    HTableDescriptor htd = mock(HTableDescriptor.class);
+    doReturn(table).when(htd).getName();
+
+    return new HbckInfo(new MetaEntry(new HRegionInfo(htd, start, end), null, null,
+        time));
+  }
+
+  @Test
+  public void testEquals() {
+    HbckInfo hi1 = genHbckInfo(table, keyA, keyB, 0);
+    HbckInfo hi2 = genHbckInfo(table, keyA, keyB, 0);
+    assertEquals(0, HBaseFsck.cmp.compare(hi1, hi2));
+    assertEquals(0, HBaseFsck.cmp.compare(hi2, hi1));
+  }
+
+  @Test
+  public void testEqualsInstance() {
+    HbckInfo hi1 = genHbckInfo(table, keyA, keyB, 0);
+    HbckInfo hi2 = hi1;
+    assertEquals(0, HBaseFsck.cmp.compare(hi1, hi2));
+    assertEquals(0, HBaseFsck.cmp.compare(hi2, hi1));
+  }
+
+  @Test
+  public void testDiffTable() {
+    HbckInfo hi1 = genHbckInfo(table, keyA, keyC, 0);
+    HbckInfo hi2 = genHbckInfo(table2, keyA, keyC, 0);
+    assertTrue(HBaseFsck.cmp.compare(hi1, hi2) < 0);
+    assertTrue(HBaseFsck.cmp.compare(hi2, hi1) > 0);
+  }
+
+  @Test
+  public void testDiffStartKey() {
+    HbckInfo hi1 = genHbckInfo(table, keyStart, keyC, 0);
+    HbckInfo hi2 = genHbckInfo(table, keyA, keyC, 0);
+    assertTrue(HBaseFsck.cmp.compare(hi1, hi2) < 0);
+    assertTrue(HBaseFsck.cmp.compare(hi2, hi1) > 0);
+  }
+
+  @Test
+  public void testDiffEndKey() {
+    HbckInfo hi1 = genHbckInfo(table, keyA, keyB, 0);
+    HbckInfo hi2 = genHbckInfo(table, keyA, keyC, 0);
+    assertTrue(HBaseFsck.cmp.compare(hi1, hi2) < 0);
+    assertTrue(HBaseFsck.cmp.compare(hi2, hi1) > 0);
+  }
+
+  @Test
+  public void testAbsEndKey() {
+    HbckInfo hi1 = genHbckInfo(table, keyA, keyC, 0);
+    HbckInfo hi2 = genHbckInfo(table, keyA, keyEnd, 0);
+    assertTrue(HBaseFsck.cmp.compare(hi1, hi2) < 0);
+    assertTrue(HBaseFsck.cmp.compare(hi2, hi1) > 0);
+  }
+
+  @Test
+  public void testTiebreaker() {
+    HbckInfo hi1 = genHbckInfo(table, keyA, keyC, 0);
+    HbckInfo hi2 = genHbckInfo(table, keyA, keyC, 1);
+    assertTrue(HBaseFsck.cmp.compare(hi1, hi2) < 0);
+    assertTrue(HBaseFsck.cmp.compare(hi2, hi1) > 0);
+  }
+}



Mime
View raw message