hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r833194 - in /hadoop/hbase/trunk: ./ src/java/org/apache/hadoop/hbase/regionserver/ src/test/org/apache/hadoop/hbase/regionserver/
Date Thu, 05 Nov 2009 21:50:36 GMT
Author: stack
Date: Thu Nov  5 21:50:35 2009
New Revision: 833194

URL: http://svn.apache.org/viewvc?rev=833194&view=rev
Log:
HBASE-1949 KeyValue expiration by Time-to-Live during major compaction is broken

Modified:
    hadoop/hbase/trunk/CHANGES.txt
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/QueryMatcher.java
    hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
    hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
    hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java

Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=833194&r1=833193&r2=833194&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Thu Nov  5 21:50:35 2009
@@ -94,6 +94,8 @@
                when deleting a lot of values
    HBASE-1781  Weird behavior of WildcardColumnTracker.checkColumn(), 
                looks like recursive loop
+   HBASE-1949  KeyValue expiration by Time-to-Live during major compaction is
+               broken (Gary Helmling via Stack)
 
   IMPROVEMENTS
    HBASE-1760  Cleanup TODOs in HTable

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/QueryMatcher.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/QueryMatcher.java?rev=833194&r1=833193&r2=833194&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/QueryMatcher.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/QueryMatcher.java Thu
Nov  5 21:50:35 2009
@@ -231,8 +231,10 @@
      */
     long timestamp = Bytes.toLong(bytes, offset);
     if(isExpired(timestamp)) {
-      // reached the expired part, for scans, this indicates we're done.
-      return MatchCode.NEXT;  // done_row
+      /* KeyValue is expired, skip but don't early out since a non-expired
+       * kv could come next.
+       */
+      return MatchCode.SKIP;  // go to next kv
     }
     offset += Bytes.SIZEOF_LONG;
 

Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java?rev=833194&r1=833193&r2=833194&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
(original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
Thu Nov  5 21:50:35 2009
@@ -122,9 +122,8 @@
     
     long timestamp = kv.getTimestamp();
     if (isExpired(timestamp)) {
-      // done, the rest wil also be expired as well.
-      stickyNextRow = true;
-      return MatchCode.SEEK_NEXT_ROW;
+      // done, the rest of this column will also be expired as well.
+      return MatchCode.SEEK_NEXT_COL;
     }
 
     byte type = kv.getType();

Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java?rev=833194&r1=833193&r2=833194&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
(original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
Thu Nov  5 21:50:35 2009
@@ -27,6 +27,7 @@
 import org.apache.hadoop.hbase.HBaseTestCase;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueTestUtil;
 import org.apache.hadoop.hbase.KeyValue.KeyComparator;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.regionserver.QueryMatcher.MatchCode;
@@ -162,4 +163,103 @@
     }
   }
 
+  
+  /**
+   * Verify that {@link QueryMatcher} only skips expired KeyValue 
+   * instances and does not exit early from the row (skipping 
+   * later non-expired KeyValues).  This version mimics a Get with
+   * explicitly specified column qualifiers.
+   * 
+   * @throws IOException
+   */
+  public void testMatch_ExpiredExplicit()
+  throws IOException {
+    
+    long testTTL = 1000;
+    MatchCode [] expected = new MatchCode[] {
+        MatchCode.SKIP,
+        MatchCode.INCLUDE,
+        MatchCode.SKIP,
+        MatchCode.INCLUDE,
+        MatchCode.SKIP,
+        MatchCode.NEXT
+    };
+        
+    QueryMatcher qm = new QueryMatcher(get, fam2,
+        get.getFamilyMap().get(fam2), testTTL, rowComparator, 1);
+    
+    long now = System.currentTimeMillis();
+    KeyValue [] kvs = new KeyValue[] {
+        new KeyValue(row1, fam2, col1, now-100, data),
+        new KeyValue(row1, fam2, col2, now-50, data),
+        new KeyValue(row1, fam2, col3, now-5000, data),
+        new KeyValue(row1, fam2, col4, now-500, data),
+        new KeyValue(row1, fam2, col5, now-10000, data),
+        new KeyValue(row2, fam1, col1, now-10, data)        
+    };
+
+    List<MatchCode> actual = new ArrayList<MatchCode>(kvs.length);
+    for (KeyValue kv : kvs) {
+      actual.add( qm.match(kv) );
+    }
+    
+    assertEquals(expected.length, actual.size());
+    for (int i=0; i<expected.length; i++) {
+      if(PRINT){
+        System.out.println("expected "+expected[i]+ 
+            ", actual " +actual.get(i));
+      }
+      assertEquals(expected[i], actual.get(i));
+    }
+  }
+  
+  
+  /**
+   * Verify that {@link QueryMatcher} only skips expired KeyValue 
+   * instances and does not exit early from the row (skipping 
+   * later non-expired KeyValues).  This version mimics a Get with
+   * wildcard-inferred column qualifiers.
+   * 
+   * @throws IOException
+   */ 
+  public void testMatch_ExpiredWildcard()
+  throws IOException {
+    
+    long testTTL = 1000;
+    MatchCode [] expected = new MatchCode[] {
+        MatchCode.INCLUDE,
+        MatchCode.INCLUDE,
+        MatchCode.SKIP,
+        MatchCode.INCLUDE,
+        MatchCode.SKIP,
+        MatchCode.NEXT
+    };
+        
+    QueryMatcher qm = new QueryMatcher(get, fam2,
+        null, testTTL, rowComparator, 1);
+    
+    long now = System.currentTimeMillis();
+    KeyValue [] kvs = new KeyValue[] {
+        new KeyValue(row1, fam2, col1, now-100, data),
+        new KeyValue(row1, fam2, col2, now-50, data),
+        new KeyValue(row1, fam2, col3, now-5000, data),
+        new KeyValue(row1, fam2, col4, now-500, data),
+        new KeyValue(row1, fam2, col5, now-10000, data),
+        new KeyValue(row2, fam1, col1, now-10, data)        
+    };
+
+    List<MatchCode> actual = new ArrayList<MatchCode>(kvs.length);
+    for (KeyValue kv : kvs) {
+      actual.add( qm.match(kv) );
+    }
+    
+    assertEquals(expected.length, actual.size());
+    for (int i=0; i<expected.length; i++) {
+      if(PRINT){
+        System.out.println("expected "+expected[i]+ 
+            ", actual " +actual.get(i));
+      }
+      assertEquals(expected[i], actual.get(i));
+    }
+  }
 }

Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java?rev=833194&r1=833193&r2=833194&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
(original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStoreScanner.java
Thu Nov  5 21:50:35 2009
@@ -354,4 +354,46 @@
     results.clear();
     assertEquals(false, scan.next(results));
   }
+  
+  /**
+   * Test expiration of KeyValues in combination with a configured TTL for 
+   * a column family (as should be triggered in a major compaction).
+   */
+  public void testWildCardTtlScan() throws IOException {
+    long now = System.currentTimeMillis();
+    KeyValue [] kvs = new KeyValue[] {
+        KeyValueTestUtil.create("R1", "cf", "a", now-1000, KeyValue.Type.Put, "dont-care"),
+        KeyValueTestUtil.create("R1", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"),
+        KeyValueTestUtil.create("R1", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"),
+        KeyValueTestUtil.create("R1", "cf", "d", now-10000, KeyValue.Type.Put, "dont-care"),
+        KeyValueTestUtil.create("R2", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
+        KeyValueTestUtil.create("R2", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"),
+        KeyValueTestUtil.create("R2", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"),
+        KeyValueTestUtil.create("R2", "cf", "c", now-1000, KeyValue.Type.Put, "dont-care")
+    };
+    KeyValueScanner [] scanners = new KeyValueScanner[] {
+        new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)
+    };
+    Scan scan = new Scan();
+    scan.setMaxVersions(1);
+    StoreScanner scanner =
+      new StoreScanner(scan, CF, 500, KeyValue.COMPARATOR,
+          null, scanners);
+
+    List<KeyValue> results = new ArrayList<KeyValue>();
+    assertEquals(true, scanner.next(results));
+    assertEquals(2, results.size());
+    assertEquals(kvs[1], results.get(0));
+    assertEquals(kvs[2], results.get(1));
+    results.clear();
+    
+    assertEquals(true, scanner.next(results));
+    assertEquals(3, results.size());
+    assertEquals(kvs[4], results.get(0));
+    assertEquals(kvs[5], results.get(1));
+    assertEquals(kvs[6], results.get(2));
+    results.clear();
+    
+    assertEquals(false, scanner.next(results));
+  }
 }



Mime
View raw message