hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From la...@apache.org
Subject svn commit: r1576462 - in /hbase/branches/0.96: hbase-client/src/main/java/org/apache/hadoop/hbase/client/ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/ src/main/docbkx/
Date Tue, 11 Mar 2014 18:34:56 GMT
Author: larsh
Date: Tue Mar 11 18:34:56 2014
New Revision: 1576462

URL: http://svn.apache.org/r1576462
Log:
HBASE-9778 Add hint to ExplicitColumnTracker to avoid seeking

Modified:
    hbase/branches/0.96/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java
    hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
    hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
    hbase/branches/0.96/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
    hbase/branches/0.96/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
    hbase/branches/0.96/src/main/docbkx/performance.xml

Modified: hbase/branches/0.96/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java?rev=1576462&r1=1576461&r2=1576462&view=diff
==============================================================================
--- hbase/branches/0.96/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java
(original)
+++ hbase/branches/0.96/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Scan.java
Tue Mar 11 18:34:56 2014
@@ -85,6 +85,23 @@ public class Scan extends OperationWithA
   private static final String RAW_ATTR = "_raw_";
   private static final String ISOLATION_LEVEL = "_isolationlevel_";
 
+  /**
+   * EXPERT ONLY.
+   * An integer (not long) indicating to the scanner logic how many times we attempt to retrieve
the
+   * next KV before we schedule a reseek.
+   * The right value depends on the size of the average KV. A reseek is more efficient when
+   * it can skip 5-10 KVs or 512B-1KB, or when the next KV is likely found in another HFile
block.
+   * Setting this only has any effect when columns were added with
+   * {@link #addColumn(byte[], byte[])}
+   * <pre>{@code
+   * Scan s = new Scan(...);
+   * s.addColumn(...);
+   * s.setAttribute(Scan.HINT_LOOKAHEAD, Bytes.toBytes(2));
+   * }</pre>
+   * Default is 0 (always reseek).
+   */
+  public static final String HINT_LOOKAHEAD = "_look_ahead_";
+
   private byte [] startRow = HConstants.EMPTY_START_ROW;
   private byte [] stopRow  = HConstants.EMPTY_END_ROW;
   private int maxVersions = 1;

Modified: hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java?rev=1576462&r1=1576461&r2=1576462&view=diff
==============================================================================
--- hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
(original)
+++ hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
Tue Mar 11 18:34:56 2014
@@ -56,6 +56,10 @@ public class ExplicitColumnTracker imple
   private final int maxVersions;
   private final int minVersions;
 
+  // hint for the tracker about how many KVs we will attempt to search via next()
+  // before we schedule a (re)seek operation
+  private final int lookAhead; 
+
  /**
   * Contains the list of columns that the ExplicitColumnTracker is tracking.
   * Each ColumnCount instance also tracks how many versions of the requested
@@ -68,6 +72,7 @@ public class ExplicitColumnTracker imple
    * Used to eliminate duplicates. */
   private long latestTSOfCurrentColumn;
   private long oldestStamp;
+  private int skipCount;
 
   /**
    * Default constructor.
@@ -76,11 +81,14 @@ public class ExplicitColumnTracker imple
    * @param maxVersions maximum versions to return per column
    * @param oldestUnexpiredTS the oldest timestamp we are interested in,
    *  based on TTL 
+   * @param lookAhead number of KeyValues to look ahead via next before
+   *  (re)seeking
    */
   public ExplicitColumnTracker(NavigableSet<byte[]> columns, int minVersions,
-      int maxVersions, long oldestUnexpiredTS) {
+      int maxVersions, long oldestUnexpiredTS, int lookAhead) {
     this.maxVersions = maxVersions;
     this.minVersions = minVersions;
+    this.lookAhead = lookAhead;
     this.oldestStamp = oldestUnexpiredTS;
     this.columns = new ColumnCount[columns.size()];
     int i=0;
@@ -136,7 +144,8 @@ public class ExplicitColumnTracker imple
       if (ret > 0) {
         // The current KV is smaller than the column the ExplicitColumnTracker
         // is interested in, so seek to that column of interest.
-        return ScanQueryMatcher.MatchCode.SEEK_NEXT_COL;
+        return this.skipCount++ < this.lookAhead ? ScanQueryMatcher.MatchCode.SKIP
+            : ScanQueryMatcher.MatchCode.SEEK_NEXT_COL;
       }
 
       // The current KV is bigger than the column the ExplicitColumnTracker
@@ -145,6 +154,7 @@ public class ExplicitColumnTracker imple
       // column of interest, and check again.
       if (ret <= -1) {
         ++this.index;
+        this.skipCount = 0;
         if (done()) {
           // No more to match, do not include, done with this row.
           return ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW; // done_row
@@ -169,6 +179,7 @@ public class ExplicitColumnTracker imple
     if (count >= maxVersions || (count >= minVersions && isExpired(timestamp)))
{
       // Done with versions for this column
       ++this.index;
+      this.skipCount = 0;
       resetTS();
       if (done()) {
         // We have served all the requested columns.
@@ -187,6 +198,7 @@ public class ExplicitColumnTracker imple
   // Called between every row.
   public void reset() {
     this.index = 0;
+    this.skipCount = 0;
     this.column = this.columns[this.index];
     for(ColumnCount col : this.columns) {
       col.setCount(0);
@@ -226,6 +238,7 @@ public class ExplicitColumnTracker imple
       resetTS();
       if (compare <= 0) {
         ++this.index;
+        this.skipCount = 0;
         if (done()) {
           // Will not hit any more columns in this storefile
           this.column = null;

Modified: hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java?rev=1576462&r1=1576461&r2=1576462&view=diff
==============================================================================
--- hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
(original)
+++ hbase/branches/0.96/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
Tue Mar 11 18:34:56 2014
@@ -183,8 +183,9 @@ public class ScanQueryMatcher {
 
       // We can share the ExplicitColumnTracker, diff is we reset
       // between rows, not between storefiles.
-      this.columns = new ExplicitColumnTracker(columns,
-          scanInfo.getMinVersions(), maxVersions, oldestUnexpiredTS);
+      byte[] attr = scan.getAttribute(Scan.HINT_LOOKAHEAD);
+      this.columns = new ExplicitColumnTracker(columns, scanInfo.getMinVersions(), maxVersions,
+          oldestUnexpiredTS, attr == null ? 0 : Bytes.toInt(attr));
     }
   }
 

Modified: hbase/branches/0.96/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java?rev=1576462&r1=1576461&r2=1576462&view=diff
==============================================================================
--- hbase/branches/0.96/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
(original)
+++ hbase/branches/0.96/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestExplicitColumnTracker.java
Tue Mar 11 18:34:56 2014
@@ -44,9 +44,9 @@ public class TestExplicitColumnTracker e
   private void runTest(int maxVersions,
                        TreeSet<byte[]> trackColumns,
                        List<byte[]> scannerColumns,
-                       List<MatchCode> expected) throws IOException {
+                       List<MatchCode> expected, int lookAhead) throws IOException
{
     ColumnTracker exp = new ExplicitColumnTracker(
-      trackColumns, 0, maxVersions, Long.MIN_VALUE);
+      trackColumns, 0, maxVersions, Long.MIN_VALUE, lookAhead);
 
 
     //Initialize result
@@ -95,7 +95,7 @@ public class TestExplicitColumnTracker e
     scanner.add(col4);
     scanner.add(col5);
 
-    runTest(maxVersions, columns, scanner, expected);
+    runTest(maxVersions, columns, scanner, expected, 0);
   }
 
   public void testGet_MultiVersion() throws IOException{
@@ -150,9 +150,63 @@ public class TestExplicitColumnTracker e
     scanner.add(col5);
 
     //Initialize result
-    runTest(maxVersions, columns, scanner, expected);
+    runTest(maxVersions, columns, scanner, expected, 0);
   }
 
+  public void testGet_MultiVersionWithLookAhead() throws IOException{
+    if(PRINT){
+      System.out.println("\nMultiVersion");
+    }
+
+    //Create tracker
+    TreeSet<byte[]> columns = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
+    //Looking for every other
+    columns.add(col2);
+    columns.add(col4);
+
+    List<ScanQueryMatcher.MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>();
+    expected.add(ScanQueryMatcher.MatchCode.SKIP);
+    expected.add(ScanQueryMatcher.MatchCode.SKIP);
+    expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
+
+    expected.add(ScanQueryMatcher.MatchCode.INCLUDE);                   // col2; 1st version
+    expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL); // col2; 2nd version
+    expected.add(ScanQueryMatcher.MatchCode.SKIP);
+
+    expected.add(ScanQueryMatcher.MatchCode.SKIP);
+    expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
+    expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
+
+    expected.add(ScanQueryMatcher.MatchCode.INCLUDE);                   // col4; 1st version
+    expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW); // col4; 2nd version
+    expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW);
+
+    expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW);
+    expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW);
+    expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW);
+    int maxVersions = 2;
+
+    //Create "Scanner"
+    List<byte[]> scanner = new ArrayList<byte[]>();
+    scanner.add(col1);
+    scanner.add(col1);
+    scanner.add(col1);
+    scanner.add(col2);
+    scanner.add(col2);
+    scanner.add(col2);
+    scanner.add(col3);
+    scanner.add(col3);
+    scanner.add(col3);
+    scanner.add(col4);
+    scanner.add(col4);
+    scanner.add(col4);
+    scanner.add(col5);
+    scanner.add(col5);
+    scanner.add(col5);
+
+    //Initialize result
+    runTest(maxVersions, columns, scanner, expected, 2);
+  }
 
   /**
    * hbase-2259
@@ -165,7 +219,7 @@ public class TestExplicitColumnTracker e
     }
 
     ColumnTracker explicit = new ExplicitColumnTracker(columns, 0, maxVersions,
-        Long.MIN_VALUE);
+        Long.MIN_VALUE, 0);
     for (int i = 0; i < 100000; i+=2) {
       byte [] col = Bytes.toBytes("col"+i);
       ScanQueryMatcher.checkColumn(explicit, col, 0, col.length, 1, KeyValue.Type.Put.getCode(),
@@ -193,7 +247,7 @@ public class TestExplicitColumnTracker e
       new ScanQueryMatcher.MatchCode[] {
         ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
         ScanQueryMatcher.MatchCode.SEEK_NEXT_COL });
-    runTest(1, columns, scanner, expected);
+    runTest(1, columns, scanner, expected, 0);
   }
 
 }

Modified: hbase/branches/0.96/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java?rev=1576462&r1=1576461&r2=1576462&view=diff
==============================================================================
--- hbase/branches/0.96/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
(original)
+++ hbase/branches/0.96/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
Tue Mar 11 18:34:56 2014
@@ -89,22 +89,8 @@ public class TestQueryMatcher extends HB
 
   }
 
-  public void testMatch_ExplicitColumns()
-  throws IOException {
-    //Moving up from the Tracker by using Gets and List<KeyValue> instead
-    //of just byte []
-
-    //Expected result
-    List<MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>();
-    expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
-    expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
-    expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
-    expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
-    expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW);
-    expected.add(ScanQueryMatcher.MatchCode.DONE);
-
-    // 2,4,5
-    
+    private void _testMatch_ExplicitColumns(Scan scan, List<MatchCode> expected) throws
IOException {
+    // 2,4,5    
     ScanQueryMatcher qm = new ScanQueryMatcher(scan, new ScanInfo(fam2,
         0, 1, ttl, false, 0, rowComparator), get.getFamilyMap().get(fam2),
         EnvironmentEdgeManager.currentTimeMillis() - ttl);
@@ -136,6 +122,42 @@ public class TestQueryMatcher extends HB
     }
   }
 
+  public void testMatch_ExplicitColumns()
+  throws IOException {
+    //Moving up from the Tracker by using Gets and List<KeyValue> instead
+    //of just byte []
+
+    //Expected result
+    List<MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>();
+    expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
+    expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
+    expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
+    expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
+    expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW);
+    expected.add(ScanQueryMatcher.MatchCode.DONE);
+
+    _testMatch_ExplicitColumns(scan, expected);
+  }
+
+  public void testMatch_ExplicitColumnsWithLookAhead()
+  throws IOException {
+    //Moving up from the Tracker by using Gets and List<KeyValue> instead
+    //of just byte []
+
+    //Expected result
+    List<MatchCode> expected = new ArrayList<ScanQueryMatcher.MatchCode>();
+    expected.add(ScanQueryMatcher.MatchCode.SKIP);
+    expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
+    expected.add(ScanQueryMatcher.MatchCode.SKIP);
+    expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL);
+    expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW);
+    expected.add(ScanQueryMatcher.MatchCode.DONE);
+
+    Scan s = new Scan(scan);
+    s.setAttribute(Scan.HINT_LOOKAHEAD, Bytes.toBytes(2));
+    _testMatch_ExplicitColumns(s, expected);
+  }
+
 
   public void testMatch_Wildcard()
   throws IOException {

Modified: hbase/branches/0.96/src/main/docbkx/performance.xml
URL: http://svn.apache.org/viewvc/hbase/branches/0.96/src/main/docbkx/performance.xml?rev=1576462&r1=1576461&r2=1576462&view=diff
==============================================================================
Binary files - no diff available.



Mime
View raw message