hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r690640 - in /hadoop/hbase/branches/0.2: ./ src/java/org/apache/hadoop/hbase/ src/java/org/apache/hadoop/hbase/regionserver/ src/test/org/apache/hadoop/hbase/
Date Sun, 31 Aug 2008 05:14:01 GMT
Author: stack
Date: Sat Aug 30 22:14:00 2008
New Revision: 690640

URL: http://svn.apache.org/viewvc?rev=690640&view=rev
Log:
HBASE-832 Problem with row keys beginnig with characters < than ',' and the region location
cache

Modified:
    hadoop/hbase/branches/0.2/CHANGES.txt
    hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/HConstants.java
    hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/HStoreKey.java
    hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
    hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java
    hadoop/hbase/branches/0.2/src/test/org/apache/hadoop/hbase/TestCompare.java

Modified: hadoop/hbase/branches/0.2/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.2/CHANGES.txt?rev=690640&r1=690639&r2=690640&view=diff
==============================================================================
--- hadoop/hbase/branches/0.2/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.2/CHANGES.txt Sat Aug 30 22:14:00 2008
@@ -49,6 +49,8 @@
                (Jean-Daniel Cryans via Stack)
    HBASE-855   compaction can return less versions then we should in some cases
                (Billy Pearson via Stack)
+   HBASE-832   Problem with row keys beginnig with characters < than ',' and
+               the region location cache
 
   IMPROVEMENTS
    HBASE-801  When a table haven't disable, shell could response in a "user

Modified: hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/HConstants.java?rev=690640&r1=690639&r2=690640&view=diff
==============================================================================
--- hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/HConstants.java Sat Aug 30
22:14:00 2008
@@ -132,7 +132,10 @@
   static final byte [] ROOT_TABLE_NAME = Bytes.toBytes("-ROOT-");
 
   /** The META table's name. */
-  static final byte [] META_TABLE_NAME = Bytes.toBytes(".META.");
+  static final byte [] META_TABLE_NAME = Bytes.toBytes(".META.");  
+
+  /** delimiter used between portions of a region name */
+  public static final int META_ROW_DELIMITER = ',';
 
   // Defines for the column names used in both ROOT and META HBase 'meta' tables.
   

Modified: hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/HStoreKey.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/HStoreKey.java?rev=690640&r1=690639&r2=690640&view=diff
==============================================================================
--- hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/HStoreKey.java (original)
+++ hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/HStoreKey.java Sat Aug 30 22:14:00
2008
@@ -40,6 +40,12 @@
   private byte [] column = HConstants.EMPTY_BYTE_ARRAY;
   private long timestamp = Long.MAX_VALUE;
 
+  /*
+   * regionInfo is only used as a hack to compare HSKs.
+   * It is not serialized.  See https://issues.apache.org/jira/browse/HBASE-832
+   */
+  private HRegionInfo regionInfo = null;
+
   /** Default constructor used in conjunction with Writable interface */
   public HStoreKey() {
     super();
@@ -47,8 +53,8 @@
   
   /**
    * Create an HStoreKey specifying only the row
-   * The column defaults to the empty string and the time stamp defaults to
-   * Long.MAX_VALUE
+   * The column defaults to the empty string, the time stamp defaults to
+   * Long.MAX_VALUE and the table defaults to empty string
    * 
    * @param row - row key
    */
@@ -58,8 +64,8 @@
 
   /**
    * Create an HStoreKey specifying only the row
-   * The column defaults to the empty string and the time stamp defaults to
-   * Long.MAX_VALUE
+   * The column defaults to the empty string, the time stamp defaults to
+   * Long.MAX_VALUE and the table defaults to empty string
    * 
    * @param row - row key
    */
@@ -69,7 +75,7 @@
 
   /**
    * Create an HStoreKey specifying the row and timestamp
-   * The column name defaults to the empty string
+   * The column and table names default to the empty string
    * 
    * @param row row key
    * @param timestamp timestamp value
@@ -80,29 +86,31 @@
 
   /**
    * Create an HStoreKey specifying the row and timestamp
-   * The column name defaults to the empty string
+   * The column and table names default to the empty string
    * 
    * @param row row key
    * @param timestamp timestamp value
    */
   public HStoreKey(final String row, long timestamp) {
-    this (row, "", timestamp);
+    this (row, "", timestamp, new HRegionInfo());
   }
 
   /**
    * Create an HStoreKey specifying the row and column names
    * The timestamp defaults to LATEST_TIMESTAMP
+   * and table name defaults to the empty string
    * 
    * @param row row key
    * @param column column key
    */
   public HStoreKey(final String row, final String column) {
-    this(row, column, HConstants.LATEST_TIMESTAMP);
+    this(row, column, HConstants.LATEST_TIMESTAMP, new HRegionInfo());
   }
 
   /**
    * Create an HStoreKey specifying the row and column names
    * The timestamp defaults to LATEST_TIMESTAMP
+   * and table name defaults to the empty string
    * 
    * @param row row key
    * @param column column key
@@ -110,6 +118,19 @@
   public HStoreKey(final byte [] row, final byte [] column) {
     this(row, column, HConstants.LATEST_TIMESTAMP);
   }
+  
+  /**
+   * Create an HStoreKey specifying the row, column names and table name
+   * The timestamp defaults to LATEST_TIMESTAMP
+   * 
+   * @param row row key
+   * @param column column key
+   * @param regionInfo region info
+   */
+  public HStoreKey(final byte [] row, 
+      final byte [] column, final HRegionInfo regionInfo) {
+    this(row, column, HConstants.LATEST_TIMESTAMP, regionInfo);
+  }
 
   /**
    * Create an HStoreKey specifying all the fields
@@ -118,13 +139,16 @@
    * @param row row key
    * @param column column key
    * @param timestamp timestamp value
+   * @param regionInfo region info
    */
-  public HStoreKey(final String row, final String column, long timestamp) {
-    this (Bytes.toBytes(row), Bytes.toBytes(column), timestamp);
+  public HStoreKey(final String row, 
+      final String column, long timestamp, final HRegionInfo regionInfo) {
+    this (Bytes.toBytes(row), Bytes.toBytes(column), 
+        timestamp, regionInfo);
   }
 
   /**
-   * Create an HStoreKey specifying all the fields
+   * Create an HStoreKey specifying all the fields with unspecified table
    * Does not make copies of the passed byte arrays. Presumes the passed 
    * arrays immutable.
    * @param row row key
@@ -132,10 +156,25 @@
    * @param timestamp timestamp value
    */
   public HStoreKey(final byte [] row, final byte [] column, long timestamp) {
+    this(row, column, timestamp, null);
+  }
+  
+  /**
+   * Create an HStoreKey specifying all the fields with specified table
+   * Does not make copies of the passed byte arrays. Presumes the passed 
+   * arrays immutable.
+   * @param row row key
+   * @param column column key
+   * @param timestamp timestamp value
+   * @param regionInfo region info
+   */
+  public HStoreKey(final byte [] row, 
+      final byte [] column, long timestamp, final HRegionInfo regionInfo) {
     // Make copies
     this.row = row;
     this.column = column;
     this.timestamp = timestamp;
+    this.regionInfo = regionInfo;
   }
   
   /** @return Approximate size in bytes of this key. */
@@ -205,6 +244,11 @@
     return this.timestamp;
   }
   
+  /** @return value of regioninfo */
+  public HRegionInfo getHRegionInfo() {
+    return this.regionInfo;
+  }
+  
   /**
    * Compares the row and column of two keys
    * @param other Key to compare against. Compares row and column.
@@ -274,7 +318,7 @@
   /** {@inheritDoc} */
   public int compareTo(Object o) {
     HStoreKey other = (HStoreKey)o;
-    int result = Bytes.compareTo(this.row, other.row);
+    int result = compareTwoRowKeys(this.regionInfo, this.row, other.row);
     if (result != 0) {
       return result;
     }
@@ -419,6 +463,66 @@
     return Bytes.add(hsk.getRow(), hsk.getColumn());
   }
   
+  /**
+   * Utility method to compare two row keys.
+   * This is required because of the meta delimiters.
+   * This is a hack.
+   * @param regioninfo
+   * @param rowA
+   * @param rowB
+   * @return value of the comparison
+   */
+  public static int compareTwoRowKeys(HRegionInfo regionInfo, 
+      byte[] rowA, byte[] rowB) {
+    if(regionInfo != null && (regionInfo.isMetaRegion() ||
+        regionInfo.isRootRegion())) {
+      byte[][] keysA = stripStartKeyMeta(rowA);
+      byte[][] KeysB = stripStartKeyMeta(rowB);
+      int rowCompare = Bytes.compareTo(keysA[0], KeysB[0]);
+      if(rowCompare == 0)
+        rowCompare = Bytes.compareTo(keysA[1], KeysB[1]);
+      return rowCompare;
+    } else {
+      return Bytes.compareTo(rowA, rowB);
+    }
+  }
+  
+  /**
+   * Utility method to check if two row keys are equal.
+   * This is required because of the meta delimiters
+   * This is a hack
+   * @param regioninfo
+   * @param rowA
+   * @param rowB
+   * @return if it's equal
+   */
+  public static boolean equalsTwoRowKeys(HRegionInfo regionInfo, 
+      byte[] rowA, byte[] rowB) {
+    return rowA == null && rowB == null? true:
+      rowA == null && rowB != null? false:
+        rowA != null && rowB == null? false:
+          rowA.length != rowB.length? false:
+        compareTwoRowKeys(regionInfo,rowA,rowB) == 0;
+  }
+  
+  private static byte[][] stripStartKeyMeta(byte[] rowKey) {
+    int offset = -1;
+    for (int i = rowKey.length - 1; i > 0; i--) {
+      if (rowKey[i] == HConstants.META_ROW_DELIMITER) {
+        offset = i;
+        break;
+      }
+    }
+    byte [] row = new byte[offset];
+    System.arraycopy(rowKey, 0, row, 0,offset);
+    byte [] timestamp = new byte[rowKey.length - offset - 1];
+    System.arraycopy(rowKey, offset+1, timestamp, 0,rowKey.length - offset - 1);
+    byte[][] elements = new byte[2][];
+    elements[0] = row;
+    elements[1] = timestamp;
+    return elements;
+  }
+  
   // Writable
 
   /** {@inheritDoc} */
@@ -434,4 +538,4 @@
     this.column = Bytes.readByteArray(in);
     this.timestamp = in.readLong();
   }
-}
\ No newline at end of file
+}

Modified: hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=690640&r1=690639&r2=690640&view=diff
==============================================================================
--- hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java Sat
Aug 30 22:14:00 2008
@@ -1258,13 +1258,14 @@
         // get the closest key
         byte [] closestKey = store.getRowKeyAtOrBefore(row);
         // if it happens to be an exact match, we can stop looping
-        if (Bytes.equals(row, closestKey)) {
+        if (HStoreKey.equalsTwoRowKeys(regionInfo,row, closestKey)) {
           key = new HStoreKey(closestKey);
           break;
         }
         // otherwise, we need to check if it's the max and move to the next
         if (closestKey != null 
-          && (key == null || Bytes.compareTo(closestKey, key.getRow()) > 0) )
{
+          && (key == null || HStoreKey.compareTwoRowKeys(
+              regionInfo,closestKey, key.getRow()) > 0) ) {
           key = new HStoreKey(closestKey);
         }
       }

Modified: hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java?rev=690640&r1=690639&r2=690640&view=diff
==============================================================================
--- hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java
(original)
+++ hadoop/hbase/branches/0.2/src/java/org/apache/hadoop/hbase/regionserver/Memcache.java
Sat Aug 30 22:14:00 2008
@@ -37,6 +37,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HRegionInfo;
 import org.apache.hadoop.hbase.HStoreKey;
 import org.apache.hadoop.hbase.io.Cell;
 import org.apache.hadoop.hbase.util.Bytes;
@@ -53,6 +54,8 @@
   private final Log LOG = LogFactory.getLog(this.getClass().getName());
   
   private final long ttl;
+  
+  private HRegionInfo regionInfo;
 
   // Note that since these structures are always accessed with a lock held,
   // so no additional synchronization is required.
@@ -72,14 +75,17 @@
    */
   public Memcache() {
     this.ttl = HConstants.FOREVER;
+    this.regionInfo = null;
   }
 
   /**
    * Constructor.
    * @param ttl The TTL for cache entries, in milliseconds.
+   * @param regionInfo The HRI for this cache 
    */
-  public Memcache(final long ttl) {
+  public Memcache(final long ttl, HRegionInfo regionInfo) {
     this.ttl = ttl;
+    this.regionInfo = regionInfo;
   }
 
   /*
@@ -383,7 +389,8 @@
     // the search key, or a range of values between the first candidate key
     // and the ultimate search key (or the end of the cache)
     if (!tailMap.isEmpty() &&
-        Bytes.compareTo(tailMap.firstKey().getRow(), search_key.getRow()) <= 0) {
+        HStoreKey.compareTwoRowKeys(regionInfo, 
+            tailMap.firstKey().getRow(), search_key.getRow()) <= 0) {
       Iterator<HStoreKey> key_iterator = tailMap.keySet().iterator();
 
       // Keep looking at cells as long as they are no greater than the 
@@ -391,9 +398,11 @@
       HStoreKey deletedOrExpiredRow = null;
       for (HStoreKey found_key = null; key_iterator.hasNext() &&
           (found_key == null ||
-            Bytes.compareTo(found_key.getRow(), row) <= 0);) {
+            HStoreKey.compareTwoRowKeys(regionInfo, 
+                found_key.getRow(), row) <= 0);) {
         found_key = key_iterator.next();
-        if (Bytes.compareTo(found_key.getRow(), row) <= 0) {
+        if (HStoreKey.compareTwoRowKeys(regionInfo, 
+            found_key.getRow(), row) <= 0) {
           if (HLogEdit.isDeleted(tailMap.get(found_key))) {
             HStore.handleDeleted(found_key, candidateKeys, deletes);
             if (deletedOrExpiredRow == null) {

Modified: hadoop/hbase/branches/0.2/src/test/org/apache/hadoop/hbase/TestCompare.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.2/src/test/org/apache/hadoop/hbase/TestCompare.java?rev=690640&r1=690639&r2=690640&view=diff
==============================================================================
--- hadoop/hbase/branches/0.2/src/test/org/apache/hadoop/hbase/TestCompare.java (original)
+++ hadoop/hbase/branches/0.2/src/test/org/apache/hadoop/hbase/TestCompare.java Sat Aug 30
22:14:00 2008
@@ -54,6 +54,43 @@
   }
   
   /**
+   * Tests cases where rows keys have characters below the ','.
+   * See HBASE-832
+   */
+  public void testHStoreKeyBorderCases() {
+    HRegionInfo info = new HRegionInfo(new HTableDescriptor("testtable"),
+        HConstants.EMPTY_BYTE_ARRAY,HConstants.EMPTY_BYTE_ARRAY);
+    HStoreKey rowA = new HStoreKey("testtable,www.hbase.org/,1234",
+        "", Long.MAX_VALUE, info);
+    HStoreKey rowB = new HStoreKey("testtable,www.hbase.org/%20,99999",
+        "", Long.MAX_VALUE, info);
+
+    assertTrue(rowA.compareTo(rowB) > 0);
+
+    rowA = new HStoreKey("testtable,www.hbase.org/,1234",
+        "", Long.MAX_VALUE, HRegionInfo.FIRST_META_REGIONINFO);
+    rowB = new HStoreKey("testtable,www.hbase.org/%20,99999",
+        "", Long.MAX_VALUE, HRegionInfo.FIRST_META_REGIONINFO);
+
+    assertTrue(rowA.compareTo(rowB) < 0);
+
+    rowA = new HStoreKey("testtable,,1234",
+        "", Long.MAX_VALUE, HRegionInfo.FIRST_META_REGIONINFO);
+    rowB = new HStoreKey("testtable,$www.hbase.org/,99999",
+        "", Long.MAX_VALUE, HRegionInfo.FIRST_META_REGIONINFO);
+
+    assertTrue(rowA.compareTo(rowB) < 0);
+
+    rowA = new HStoreKey(".META.,testtable,www.hbase.org/,1234,4321",
+        "", Long.MAX_VALUE, HRegionInfo.ROOT_REGIONINFO);
+    rowB = new HStoreKey(".META.,testtable,www.hbase.org/%20,99999,99999",
+        "", Long.MAX_VALUE, HRegionInfo.ROOT_REGIONINFO);
+
+    assertTrue(rowA.compareTo(rowB) > 0);
+  }
+
+  
+  /**
    * Sort of HRegionInfo.
    */
   public void testHRegionInfo() {



Mime
View raw message