hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From la...@apache.org
Subject svn commit: r1201101 - in /hbase/branches/0.92: ./ src/main/java/org/apache/hadoop/hbase/ src/main/java/org/apache/hadoop/hbase/filter/ src/main/java/org/apache/hadoop/hbase/regionserver/ src/test/java/org/apache/hadoop/hbase/
Date Fri, 11 Nov 2011 23:35:07 GMT
Author: larsh
Date: Fri Nov 11 23:35:06 2011
New Revision: 1201101

URL: http://svn.apache.org/viewvc?rev=1201101&view=rev
Log:
HBASE-3433  Remove the KV copy of every KV in Scan; introduced by HBASE-3232

Modified:
    hbase/branches/0.92/CHANGES.txt
    hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/KeyValue.java
    hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/Filter.java
    hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java
    hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
    hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java
    hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java
    hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java
    hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
    hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
    hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java

Modified: hbase/branches/0.92/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/CHANGES.txt?rev=1201101&r1=1201100&r2=1201101&view=diff
==============================================================================
--- hbase/branches/0.92/CHANGES.txt (original)
+++ hbase/branches/0.92/CHANGES.txt Fri Nov 11 23:35:06 2011
@@ -26,6 +26,7 @@ Release 0.92.0 - Unreleased
    HBASE-4374  Up default regions size from 256M to 1G
    HBASE-4648  Bytes.toBigDecimal() doesn't use offset (Brian Keller via Lars H)
    HBASE-4715  Remove stale broke .rb scripts from bin dir
+   HBASE-3433  Remove the KV copy of every KV in Scan; introduced by HBASE-3232 (Lars H)
 
   BUG FIXES
    HBASE-3280  YouAreDeadException being swallowed in HRS getMaster

Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/KeyValue.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/KeyValue.java?rev=1201101&r1=1201100&r2=1201101&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/KeyValue.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/KeyValue.java Fri Nov 11 23:35:06
2011
@@ -1230,15 +1230,11 @@ public class KeyValue implements Writabl
   }
 
   /**
-   * Converts this KeyValue to only contain the key portion (the value is
-   * changed to be null).  This method does a full copy of the backing byte
-   * array and does not modify the original byte array of this KeyValue.
-   * <p>
-   * This method is used by <code>KeyOnlyFilter</code> and is an advanced feature
of
-   * KeyValue, proceed with caution.
+   * Creates a new KeyValue that only contains the key portion (the value is
+   * set to be null).
    * @param lenAsVal replace value with the actual value length (false=empty)
    */
-  public void convertToKeyOnly(boolean lenAsVal) {
+  public KeyValue createKeyOnly(boolean lenAsVal) {
     // KV format:  <keylen:4><valuelen:4><key:keylen><value:valuelen>
     // Rebuild as: <keylen:4><0:4><key:keylen>
     int dataLen = lenAsVal? Bytes.SIZEOF_INT : 0;
@@ -1249,9 +1245,7 @@ public class KeyValue implements Writabl
     if (lenAsVal) {
       Bytes.putInt(newBuffer, newBuffer.length - dataLen, this.getValueLength());
     }
-    this.bytes = newBuffer;
-    this.offset = 0;
-    this.length = newBuffer.length;
+    return new KeyValue(newBuffer);
   }
 
   /**

Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/Filter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/Filter.java?rev=1201101&r1=1201100&r2=1201101&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/Filter.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/Filter.java Fri Nov 11
23:35:06 2011
@@ -87,7 +87,22 @@ public interface Filter extends Writable
    * @return code as described below
    * @see Filter.ReturnCode
    */
-  public ReturnCode filterKeyValue(KeyValue v);
+  public ReturnCode filterKeyValue(final KeyValue v);
+
+  /**
+   * Give the filter a chance to transform the passed KeyValue.
+   * If the KeyValue is changed a new KeyValue object must be returned.
+   * @see org.apache.hadoop.hbase.KeyValue#shallowCopy()
+   *
+   * The transformed KeyValue is what is eventually returned to the
+   * client. Most filters will return the passed KeyValue unchanged.
+   * @see org.apache.hadoop.hbase.filter.KeyOnlyFilter#transform(KeyValue)
+   * for an example of a transformation.
+   *
+   * @param v the KeyValue in question
+   * @return the changed KeyValue
+   */
+  public KeyValue transform(final KeyValue v);
 
   /**
    * Return codes for filterValue().
@@ -147,5 +162,5 @@ public interface Filter extends Writable
    * @return KeyValue which must be next seeked. return null if the filter is
    * not sure which key to seek to next.
    */
-  public KeyValue getNextKeyHint(KeyValue currentKV);
+  public KeyValue getNextKeyHint(final KeyValue currentKV);
 }

Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java?rev=1201101&r1=1201100&r2=1201101&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/FilterBase.java Fri Nov
11 23:35:06 2011
@@ -78,6 +78,16 @@ public abstract class FilterBase impleme
   }
 
   /**
+   * By default no transformation takes place
+   *
+   * @inheritDoc
+   */
+  @Override
+  public KeyValue transform(KeyValue v) {
+    return v;
+  }
+
+  /**
    * Filters that never filter by modifying the returned List of KeyValues can
    * inherit this implementation that does nothing.
    *
@@ -128,5 +138,5 @@ public abstract class FilterBase impleme
    */
   public static Filter createFilterFromArguments(ArrayList<byte []> filterArguments)
{
     throw new IllegalArgumentException("This method has not been implemented");
-}
+  }
 }

Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java?rev=1201101&r1=1201100&r2=1201101&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/FilterList.java Fri Nov
11 23:35:06 2011
@@ -181,6 +181,15 @@ public class FilterList implements Filte
   }
 
   @Override
+  public KeyValue transform(KeyValue v) {
+    KeyValue current = v;
+    for (Filter filter : filters) {
+      current = filter.transform(current);
+    }
+    return current;
+  }
+
+  @Override
   public ReturnCode filterKeyValue(KeyValue v) {
     ReturnCode rc = operator == Operator.MUST_PASS_ONE?
         ReturnCode.SKIP: ReturnCode.INCLUDE;

Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java?rev=1201101&r1=1201100&r2=1201101&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/KeyOnlyFilter.java Fri
Nov 11 23:35:06 2011
@@ -43,9 +43,8 @@ public class KeyOnlyFilter extends Filte
   public KeyOnlyFilter(boolean lenAsVal) { this.lenAsVal = lenAsVal; }
 
   @Override
-  public ReturnCode filterKeyValue(KeyValue kv) {
-    kv.convertToKeyOnly(this.lenAsVal);
-    return ReturnCode.INCLUDE;
+  public KeyValue transform(KeyValue kv) {
+    return kv.createKeyOnly(this.lenAsVal);
   }
 
   public static Filter createFilterFromArguments(ArrayList<byte []> filterArguments)
{

Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java?rev=1201101&r1=1201100&r2=1201101&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java (original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/SkipFilter.java Fri Nov
11 23:35:06 2011
@@ -76,6 +76,11 @@ public class SkipFilter extends FilterBa
     return c;
   }
 
+  @Override
+  public KeyValue transform(KeyValue v) {
+    return filter.transform(v);
+  }
+
   public boolean filterRow() {
     return filterRow;
   }

Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java?rev=1201101&r1=1201100&r2=1201101&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java
(original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/filter/WhileMatchFilter.java
Fri Nov 11 23:35:06 2011
@@ -75,6 +75,11 @@ public class WhileMatchFilter extends Fi
     return c;
   }
 
+  @Override
+  public KeyValue transform(KeyValue v) {
+    return filter.transform(v);
+  }
+
   public boolean filterRow() {
     boolean filterRow = this.filter.filterRow();
     changeFAR(filterRow);

Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java?rev=1201101&r1=1201100&r2=1201101&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
(original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
Fri Nov 11 23:35:06 2011
@@ -283,6 +283,14 @@ public class ScanQueryMatcher {
     return this.startKey;
   }
 
+  /**
+   *
+   * @return the Filter
+   */
+  Filter getFilter() {
+    return this.filter;
+  }
+
   public KeyValue getNextKeyHint(KeyValue kv) {
     if (filter == null) {
       return null;

Modified: hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java?rev=1201101&r1=1201100&r2=1201101&view=diff
==============================================================================
--- hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
(original)
+++ hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
Fri Nov 11 23:35:06 2011
@@ -25,6 +25,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.Filter;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -263,23 +264,22 @@ class StoreScanner implements KeyValueSc
         store != null ? store.getComparator() : null;
 
     LOOP: while((kv = this.heap.peek()) != null) {
-      // kv is no longer immutable due to KeyOnlyFilter! use copy for safety
-      KeyValue copyKv = kv.shallowCopy();
       // Check that the heap gives us KVs in an increasing order.
       if (prevKV != null && comparator != null
           && comparator.compare(prevKV, kv) > 0) {
         throw new IOException("Key " + prevKV + " followed by a " +
             "smaller key " + kv + " in cf " + store);
       }
-      prevKV = copyKv;
-      ScanQueryMatcher.MatchCode qcode = matcher.match(copyKv);
+      prevKV = kv;
+      ScanQueryMatcher.MatchCode qcode = matcher.match(kv);
 
       switch(qcode) {
         case INCLUDE:
         case INCLUDE_AND_SEEK_NEXT_ROW:
         case INCLUDE_AND_SEEK_NEXT_COL:
 
-          results.add(copyKv);
+          Filter f = matcher.getFilter();
+          results.add(f == null ? kv : f.transform(kv));
 
           if (qcode == ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW) {
             if (!matcher.moreRowsMayExistAfter(kv)) {

Modified: hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java?rev=1201101&r1=1201100&r2=1201101&view=diff
==============================================================================
--- hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java (original)
+++ hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java Fri Nov 11
23:35:06 2011
@@ -351,7 +351,7 @@ public class TestKeyValue extends TestCa
     assertKVLess(c, firstOnRowA, lastOnRowA);
   }
 
-  public void testConvertToKeyOnly() throws Exception {
+  public void testCreateKeyOnly() throws Exception {
     long ts = 1;
     byte [] value = Bytes.toBytes("a real value");
     byte [] evalue = new byte[0]; // empty value
@@ -359,9 +359,7 @@ public class TestKeyValue extends TestCa
     for (byte[] val : new byte[][]{value, evalue}) {
       for (boolean useLen : new boolean[]{false,true}) {
         KeyValue kv1 = new KeyValue(rowA, family, qualA, ts, val);
-        KeyValue kv1ko = kv1.clone();
-        assertTrue(kv1.equals(kv1ko));
-        kv1ko.convertToKeyOnly(useLen);
+        KeyValue kv1ko = kv1.createKeyOnly(useLen);
         // keys are still the same
         assertTrue(kv1.equals(kv1ko));
         // but values are not



Mime
View raw message