hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From st...@apache.org
Subject svn commit: r825242 - in /hadoop/hbase/branches/0.20: ./ src/java/org/apache/hadoop/hbase/filter/ src/java/org/apache/hadoop/hbase/regionserver/ src/test/org/apache/hadoop/hbase/ src/test/org/apache/hadoop/hbase/client/ src/test/org/apache/hadoop/hbase...
Date Wed, 14 Oct 2009 19:52:46 GMT
Author: stack
Date: Wed Oct 14 19:52:46 2009
New Revision: 825242

URL: http://svn.apache.org/viewvc?rev=825242&view=rev
Log:
HBASE-1906 FilterList of prefix and columnvalue not working properly with deletes and multiple
values

Modified:
    hadoop/hbase/branches/0.20/CHANGES.txt
    hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/CompareFilter.java
    hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/FilterList.java
    hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java
    hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
    hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/HBaseTestCase.java
    hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/client/TestClient.java
    hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java
    hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java

Modified: hadoop/hbase/branches/0.20/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/CHANGES.txt?rev=825242&r1=825241&r2=825242&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.20/CHANGES.txt Wed Oct 14 19:52:46 2009
@@ -5,6 +5,8 @@
 
   BUG FIXES
    HBASE-1905  Remove unused config. hbase.hstore.blockCache.blockSize
+   HBASE-1906  FilterList of prefix and columnvalue not working properly
+               with deletes and multiple values
 
   IMPROVEMENTS
    HBASE-1899  Use scanner caching in shell count

Modified: hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/CompareFilter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/CompareFilter.java?rev=825242&r1=825241&r2=825242&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/CompareFilter.java
(original)
+++ hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/CompareFilter.java
Wed Oct 14 19:52:46 2009
@@ -25,10 +25,8 @@
 import java.io.IOException;
 import java.util.Arrays;
 
-import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.io.HbaseObjectWritable;
-import org.apache.hadoop.io.ObjectWritable;
 
 /**
  * This is a generic filter to be used to filter by comparison.  It takes an 

Modified: hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/FilterList.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/FilterList.java?rev=825242&r1=825241&r2=825242&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/FilterList.java (original)
+++ hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/FilterList.java Wed
Oct 14 19:52:46 2009
@@ -33,13 +33,14 @@
 
 /**
  * Implementation of {@link Filter} that represents an ordered List of Filters
- * which will be evaluated with a specified boolean operator MUST_PASS_ALL 
- * (!AND) or MUST_PASS_ONE (!OR).  Since you can use Filter Lists as children
- * of Filter Lists, you can create a hierarchy of filters to be evaluated.
+ * which will be evaluated with a specified boolean operator {@link Operator#MUST_PASS_ALL}

+ * (<code>!AND</code>) or {@link Operator#MUST_PASS_ONE} (<code>!OR</code>).
+ * Since you can use Filter Lists as children of Filter Lists, you can create a
+ * hierarchy of filters to be evaluated.
+ * Defaults to {@link Operator#MUST_PASS_ALL}.
  * <p>TODO: Fix creation of Configuration on serialization and deserialization. 
  */
 public class FilterList implements Filter {
-
   /** set operator */
   public static enum Operator {
     /** !AND */
@@ -70,6 +71,15 @@
   }
 
   /**
+   * Constructor that takes an operator.
+   * 
+   * @param operator Operator to process filter set with.
+   */
+  public FilterList(final Operator operator) {
+    this.operator = operator;
+  }
+
+  /**
    * Constructor that takes a set of {@link Filter}s and an operator.
    * 
    * @param operator Operator to process filter set with.
@@ -115,19 +125,19 @@
 
   public boolean filterRowKey(byte[] rowKey, int offset, int length) {
     for (Filter filter : filters) {
-      if (operator == Operator.MUST_PASS_ALL) {
-        if (filter.filterAllRemaining()
-            || filter.filterRowKey(rowKey, offset, length)) {
+      if (this.operator == Operator.MUST_PASS_ALL) {
+        if (filter.filterAllRemaining() ||
+            filter.filterRowKey(rowKey, offset, length)) {
           return true;
         }
-      } else if (operator == Operator.MUST_PASS_ONE) {
-        if (!filter.filterAllRemaining()
-            && !filter.filterRowKey(rowKey, offset, length)) {
+      } else if (this.operator == Operator.MUST_PASS_ONE) {
+        if (!filter.filterAllRemaining() &&
+            !filter.filterRowKey(rowKey, offset, length)) {
           return false;
         }
       }
     }
-    return  operator == Operator.MUST_PASS_ONE;
+    return this.operator == Operator.MUST_PASS_ONE;
   }
 
   public boolean filterAllRemaining() {
@@ -179,8 +189,7 @@
   public boolean filterRow() {
     for (Filter filter : filters) {
       if (operator == Operator.MUST_PASS_ALL) {
-        if (filter.filterAllRemaining()
-            || filter.filterRow()) {
+        if (filter.filterAllRemaining() || filter.filterRow()) {
           return true;
         }
       } else if (operator == Operator.MUST_PASS_ONE) {

Modified: hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java?rev=825242&r1=825241&r2=825242&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java
(original)
+++ hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java
Wed Oct 14 19:52:46 2009
@@ -109,16 +109,19 @@
     this.compareOp = compareOp;
     this.comparator = comparator;
   }
-  
+
   public boolean filterRowKey(byte[] rowKey, int offset, int length) {
+    // We don't filter on the row key... we filter later on column value so
+    // always return false.
     return false;
   }
 
   public ReturnCode filterKeyValue(KeyValue keyValue) {
-    if(matchedColumn) {
+    // System.out.println("REMOVE KEY=" + keyValue.toString() + ", value=" + Bytes.toString(keyValue.getValue()));
+    if (this.matchedColumn) {
       // We already found and matched the single column, all keys now pass
       return ReturnCode.INCLUDE;
-    } else if(foundColumn) {
+    } else if (this.foundColumn) {
       // We found but did not match the single column, skip to next row
       return ReturnCode.NEXT_ROW;
     }
@@ -130,16 +133,18 @@
         keyValue.getValueOffset(), keyValue.getValueLength())) {
       return ReturnCode.NEXT_ROW;
     }
-    matchedColumn = true;
+    this.matchedColumn = true;
     return ReturnCode.INCLUDE;
   }
 
-  private boolean filterColumnValue(final byte[] data, final int offset,
+  private boolean filterColumnValue(final byte [] data, final int offset,
       final int length) {
-    int compareResult = comparator.compareTo(Arrays.copyOfRange(data, offset,
-        offset + length));
-
-    switch (compareOp) {
+    // TODO: Can this filter take a rawcomparator so don't have to make this
+    // byte array copy?
+    int compareResult =
+      this.comparator.compareTo(Arrays.copyOfRange(data, offset, offset + length));
+    LOG.debug("compareResult=" + compareResult + " " + Bytes.toString(data, offset, length));
+    switch (this.compareOp) {
     case LESS:
       return compareResult <= 0;
     case LESS_OR_EQUAL:
@@ -164,23 +169,23 @@
   public boolean filterRow() {
     // If column was found, return false if it was matched, true if it was not
     // If column not found, return true if we filter if missing, false if not
-    return foundColumn ? !matchedColumn : filterIfMissing;
+    return this.foundColumn? !this.matchedColumn: this.filterIfMissing;
   }
 
   public void reset() {
     foundColumn = false;
     matchedColumn = false;
   }
-  
+
   /**
    * Get whether entire row should be filtered if column is not found.
-   * @return filterIfMissing true if row should be skipped if column not found,
-   * false if row should be let through anyways
+   * @return true if row should be skipped if column not found, false if row
+   * should be let through anyways
    */
   public boolean getFilterIfMissing() {
     return filterIfMissing;
   }
-  
+
   /**
    * Set whether entire row should be filtered if column is not found.
    * <p>
@@ -201,12 +206,12 @@
     if(this.columnQualifier.length == 0) {
       this.columnQualifier = null;
     }
-    compareOp = CompareOp.valueOf(in.readUTF());
-    comparator = (WritableByteArrayComparable) HbaseObjectWritable.readObject(in,
-        null);
-    foundColumn = in.readBoolean();
-    matchedColumn = in.readBoolean();
-    filterIfMissing = in.readBoolean();
+    this.compareOp = CompareOp.valueOf(in.readUTF());
+    this.comparator =
+      (WritableByteArrayComparable)HbaseObjectWritable.readObject(in, null);
+    this.foundColumn = in.readBoolean();
+    this.matchedColumn = in.readBoolean();
+    this.filterIfMissing = in.readBoolean();
   }
 
   public void write(final DataOutput out) throws IOException {
@@ -219,4 +224,4 @@
     out.writeBoolean(matchedColumn);
     out.writeBoolean(filterIfMissing);
   }
-}
+}
\ No newline at end of file

Modified: hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=825242&r1=825241&r2=825242&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
(original)
+++ hadoop/hbase/branches/0.20/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
Wed Oct 14 19:52:46 2009
@@ -1740,58 +1740,58 @@
      * @throws IOException
      */
     private boolean nextInternal() throws IOException {
-      // This method should probably be reorganized a bit... has gotten messy
-      KeyValue kv;
-      byte[] currentRow = null;
+      byte [] currentRow = null;
       boolean filterCurrentRow = false;
       while (true) {
-        kv = this.storeHeap.peek();
-        if (kv == null) {
-          return false;
-        }
+        KeyValue kv = this.storeHeap.peek();
+        if (kv == null) return false;
         byte [] row = kv.getRow();
-        if (filterCurrentRow && Bytes.equals(currentRow, row)) {
-          // filter all columns until row changes
+        boolean samerow = Bytes.equals(currentRow, row);
+        if (samerow && filterCurrentRow) {
+          // Filter all columns until row changes
           this.storeHeap.next(results);
           results.clear();
           continue;
         }
-        // see if current row should be filtered based on row key
-        if ((filter != null && filter.filterRowKey(row, 0, row.length)) ||
-            (oldFilter != null && oldFilter.filterRowKey(row, 0, row.length))) {
-          if(!results.isEmpty() && !Bytes.equals(currentRow, row)) {
-            return true;
-          }
-          this.storeHeap.next(results);
-          results.clear();
-          resetFilters();
-          filterCurrentRow = true;
-          currentRow = row;
-          continue;
-        }
-        if(!Bytes.equals(currentRow, row)) {
+        if (!samerow) {
           // Continue on the next row:
           currentRow = row;
           filterCurrentRow = false;
           // See if we passed stopRow
-          if(stopRow != null &&
-              comparator.compareRows(stopRow, 0, stopRow.length, 
-                  currentRow, 0, currentRow.length) <= 0) {
+          if (this.stopRow != null &&
+              comparator.compareRows(this.stopRow, 0, this.stopRow.length, 
+                currentRow, 0, currentRow.length) <= 0) {
             return false;
           }
-          // if there are _no_ results or current row should be filtered
-          if (results.isEmpty() || filter != null && filter.filterRow()) {
-            // make sure results is empty
-            results.clear();
-            resetFilters();
-            continue;
-          }
-          return true;
+          if (hasResults()) return true;
+        }
+        // See if current row should be filtered based on row key
+        if (this.filter != null && this.filter.filterRowKey(row, 0, row.length))
{
+          resetFilters();
+          filterCurrentRow = true;
+          currentRow = row;
         }
         this.storeHeap.next(results);
       }
     }
 
+    /*
+     * Do we have results to return or should we continue.  Call when we get to
+     * the end of a row.  Does house cleaning -- clearing results and resetting
+     * filters -- if we are to continue.
+     * @return True if we should return else false if need to keep going.
+     */
+    private boolean hasResults() {
+      if (this.results.isEmpty() ||
+          this.filter != null && this.filter.filterRow()) {
+        // Make sure results is empty, reset filters
+        results.clear();
+        resetFilters();
+        return false;
+      }
+      return true;
+    }
+
     public void close() {
       storeHeap.close();
     }
@@ -2323,7 +2323,6 @@
       store.get(get, qualifiers, results);
 
       if (!results.isEmpty()) {
-        byte [] oldValue = results.get(0).getValue();
         KeyValue kv = results.get(0);
         byte [] buffer = kv.getBuffer();
         int valueOffset = kv.getValueOffset();
@@ -2514,4 +2513,4 @@
        if (bc != null) bc.shutdown();
      }
   }
-}
\ No newline at end of file
+}

Modified: hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/HBaseTestCase.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/HBaseTestCase.java?rev=825242&r1=825241&r2=825242&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/HBaseTestCase.java (original)
+++ hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/HBaseTestCase.java Wed Oct
14 19:52:46 2009
@@ -224,8 +224,7 @@
     if (startKeyBytes == null || startKeyBytes.length == 0) {
       startKeyBytes = START_KEY_BYTES;
     }
-    return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily),
-        null,
+    return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), null,
       startKeyBytes, endKey, -1);
   }
 

Modified: hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/client/TestClient.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/client/TestClient.java?rev=825242&r1=825241&r2=825242&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/client/TestClient.java (original)
+++ hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/client/TestClient.java Wed
Oct 14 19:52:46 2009
@@ -20,10 +20,6 @@
 
 package org.apache.hadoop.hbase.client;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Map;
-
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hbase.HBaseClusterTestCase;
@@ -35,14 +31,23 @@
 import org.apache.hadoop.hbase.KeyValue;
 import org.apache.hadoop.hbase.filter.BinaryComparator;
 import org.apache.hadoop.hbase.filter.CompareFilter;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
 import org.apache.hadoop.hbase.filter.Filter;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.PrefixFilter;
 import org.apache.hadoop.hbase.filter.QualifierFilter;
 import org.apache.hadoop.hbase.filter.RegexStringComparator;
 import org.apache.hadoop.hbase.filter.RowFilter;
+import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
 import org.apache.hadoop.hbase.filter.WhileMatchFilter;
-import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
 import org.apache.hadoop.hbase.util.Bytes;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.UUID;
+
 /**
  * Tests from client-side of a cluster.
  */
@@ -63,6 +68,135 @@
   }
 
   /**
+   * Test from client side of an involved filter against a multi family that
+   * involves deletes.
+   * 
+   * @throws Exception
+   */
+  public void testWeirdCacheBehaviour() throws Exception {
+    byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
+    byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"),
+        Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"),
+        Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
+    HTable ht = createTable(TABLE, FAMILIES);
+    String value = "this is the value";
+    String value2 = "this is some other value";
+    String keyPrefix1 = UUID.randomUUID().toString();
+    String keyPrefix2 = UUID.randomUUID().toString();
+    String keyPrefix3 = UUID.randomUUID().toString();
+    putRows(ht, 3, value, keyPrefix1);
+    putRows(ht, 3, value, keyPrefix2);
+    putRows(ht, 3, value, keyPrefix3);
+    ht.flushCommits();
+    putRows(ht, 3, value2, keyPrefix1);
+    putRows(ht, 3, value2, keyPrefix2);
+    putRows(ht, 3, value2, keyPrefix3);
+    HTable table = new HTable(conf, Bytes.toBytes("testWeirdCacheBehaviour"));
+    System.out.println("Checking values for key: " + keyPrefix1);
+    assertEquals("Got back incorrect number of rows from scan", 3,
+        getNumberOfRows(keyPrefix1, value2, table));
+    System.out.println("Checking values for key: " + keyPrefix2);
+    assertEquals("Got back incorrect number of rows from scan", 3,
+        getNumberOfRows(keyPrefix2, value2, table));
+    System.out.println("Checking values for key: " + keyPrefix3);
+    assertEquals("Got back incorrect number of rows from scan", 3,
+        getNumberOfRows(keyPrefix3, value2, table));
+    deleteColumns(ht, value2, keyPrefix1);
+    deleteColumns(ht, value2, keyPrefix2);
+    deleteColumns(ht, value2, keyPrefix3);
+    System.out.println("Starting important checks.....");
+    assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1,
+      0, getNumberOfRows(keyPrefix1, value2, table));
+    assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2,
+      0, getNumberOfRows(keyPrefix2, value2, table));
+    assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3,
+      0, getNumberOfRows(keyPrefix3, value2, table));
+    ht.setScannerCaching(0);
+    assertEquals("Got back incorrect number of rows from scan", 0,
+      getNumberOfRows(keyPrefix1, value2, table)); ht.setScannerCaching(100);
+    assertEquals("Got back incorrect number of rows from scan", 0,
+      getNumberOfRows(keyPrefix2, value2, table));
+  }
+
+  private void deleteColumns(HTable ht, String value, String keyPrefix)
+  throws IOException {
+    ResultScanner scanner = buildScanner(keyPrefix, value, ht);
+    Iterator<Result> it = scanner.iterator();
+    int count = 0;
+    while (it.hasNext()) {
+      Result result = it.next();
+      Delete delete = new Delete(result.getRow());
+      delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
+      ht.delete(delete);
+      count++;
+    }
+    assertEquals("Did not perform correct number of deletes", 3, count);
+  }
+
+  private int getNumberOfRows(String keyPrefix, String value, HTable ht)
+      throws Exception {
+    ResultScanner resultScanner = buildScanner(keyPrefix, value, ht);
+    Iterator<Result> scanner = resultScanner.iterator();
+    int numberOfResults = 0;
+    while (scanner.hasNext()) {
+      Result result = scanner.next();
+      System.out.println("Got back key: " + Bytes.toString(result.getRow()));
+      for (KeyValue kv : result.raw()) {
+        System.out.println("kv=" + kv.toString() + ", "
+            + Bytes.toString(kv.getValue()));
+      }
+      numberOfResults++;
+    }
+    return numberOfResults;
+  }
+
+  private ResultScanner buildScanner(String keyPrefix, String value, HTable ht)
+      throws IOException {
+    // OurFilterList allFilters = new OurFilterList();
+    FilterList allFilters = new FilterList(/* FilterList.Operator.MUST_PASS_ALL */);
+    allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
+    SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes
+        .toBytes("trans-tags"), Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes
+        .toBytes(value));
+    filter.setFilterIfMissing(true);
+    allFilters.addFilter(filter);
+
+    // allFilters.addFilter(new
+    // RowExcludingSingleColumnValueFilter(Bytes.toBytes("trans-tags"),
+    // Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value)));
+
+    Scan scan = new Scan();
+    scan.addFamily(Bytes.toBytes("trans-blob"));
+    scan.addFamily(Bytes.toBytes("trans-type"));
+    scan.addFamily(Bytes.toBytes("trans-date"));
+    scan.addFamily(Bytes.toBytes("trans-tags"));
+    scan.addFamily(Bytes.toBytes("trans-group"));
+    scan.setFilter(allFilters);
+
+    return ht.getScanner(scan);
+  }
+
+  private void putRows(HTable ht, int numRows, String value, String key)
+      throws IOException {
+    for (int i = 0; i < numRows; i++) {
+      String row = key + "_" + UUID.randomUUID().toString();
+      System.out.println(String.format("Saving row: %s, with value %s", row,
+          value));
+      Put put = new Put(Bytes.toBytes(row));
+      put.add(Bytes.toBytes("trans-blob"), null, Bytes
+          .toBytes("value for blob"));
+      put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
+      put.add(Bytes.toBytes("trans-date"), null, Bytes
+          .toBytes("20090921010101999"));
+      put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes
+          .toBytes(value));
+      put.add(Bytes.toBytes("trans-group"), null, Bytes
+          .toBytes("adhocTransactionGroupId"));
+      ht.put(put);
+    }
+  }
+
+  /**
    * Test filters when multiple regions.  It does counts.  Needs eye-balling of
    * logs to ensure that we're not scanning more regions that we're supposed to.
    * Related to the TestFilterAcrossRegions over in the o.a.h.h.filter package.
@@ -249,6 +383,7 @@
     scanner.close();
     System.out.println("Done.");
   }
+
   public void testFilters() throws Exception {
     byte [] TABLE = Bytes.toBytes("testFilters");
     HTable ht = createTable(TABLE, FAMILY);
@@ -2671,7 +2806,7 @@
   }
   
   private byte [][] makeN(byte [] base, int n) {
-    if(n > 256) {
+    if (n > 256) {
       return makeNBig(base, n);
     }
     byte [][] ret = new byte[n][];

Modified: hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java?rev=825242&r1=825241&r2=825242&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java
(original)
+++ hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java
Wed Oct 14 19:52:46 2009
@@ -40,6 +40,10 @@
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.client.Scan;
 import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.PrefixFilter;
+import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
 import org.apache.hadoop.hbase.regionserver.HRegion.RegionScanner;
 import org.apache.hadoop.hbase.util.Bytes;
 
@@ -79,6 +83,127 @@
   // individual code pieces in the HRegion. Putting files locally in
   // /tmp/testtable
   //////////////////////////////////////////////////////////////////////////////
+  
+
+  /**
+   * An involved filter test.  Has multiple column families and deletes in mix.
+   */
+  public void testWeirdCacheBehaviour() throws Exception {
+    byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour");
+    byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"),
+        Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"),
+        Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
+    initHRegion(TABLE, getName(), FAMILIES);
+    String value = "this is the value";
+    String value2 = "this is some other value";
+    String keyPrefix1 = "prefix1"; // UUID.randomUUID().toString();
+    String keyPrefix2 = "prefix2"; // UUID.randomUUID().toString();
+    String keyPrefix3 = "prefix3"; // UUID.randomUUID().toString();
+    putRows(this.region, 3, value, keyPrefix1);
+    putRows(this.region, 3, value, keyPrefix2);
+    putRows(this.region, 3, value, keyPrefix3);
+    // this.region.flushCommits();
+    putRows(this.region, 3, value2, keyPrefix1);
+    putRows(this.region, 3, value2, keyPrefix2);
+    putRows(this.region, 3, value2, keyPrefix3);
+    System.out.println("Checking values for key: " + keyPrefix1);
+    assertEquals("Got back incorrect number of rows from scan", 3,
+      getNumberOfRows(keyPrefix1, value2, this.region));
+    System.out.println("Checking values for key: " + keyPrefix2);
+    assertEquals("Got back incorrect number of rows from scan", 3,
+      getNumberOfRows(keyPrefix2, value2, this.region));
+    System.out.println("Checking values for key: " + keyPrefix3);
+    assertEquals("Got back incorrect number of rows from scan", 3,
+      getNumberOfRows(keyPrefix3, value2, this.region));
+    deleteColumns(this.region, value2, keyPrefix1);
+    deleteColumns(this.region, value2, keyPrefix2);
+    deleteColumns(this.region, value2, keyPrefix3);
+    System.out.println("Starting important checks.....");
+    assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1,
+      0, getNumberOfRows(keyPrefix1, value2, this.region));
+    assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2,
+      0, getNumberOfRows(keyPrefix2, value2, this.region));
+    assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3,
+      0, getNumberOfRows(keyPrefix3, value2, this.region));
+  }
+
+  private void deleteColumns(HRegion r, String value, String keyPrefix)
+  throws IOException {
+    InternalScanner scanner = buildScanner(keyPrefix, value, r);
+    int count = 0;
+    boolean more = false;
+    List<KeyValue> results = new ArrayList<KeyValue>();
+    do {
+      more = scanner.next(results);
+      if (results != null && !results.isEmpty())
+        count++;
+      else
+        break;
+      Delete delete = new Delete(results.get(0).getRow());
+      delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
+      r.delete(delete, null, false);
+      results.clear();
+    } while (more);
+    assertEquals("Did not perform correct number of deletes", 3, count);
+  }
+
+  private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception
{
+    InternalScanner resultScanner = buildScanner(keyPrefix, value, r);
+    int numberOfResults = 0;
+    List<KeyValue> results = new ArrayList<KeyValue>();
+    boolean more = false;
+    do {
+      more = resultScanner.next(results);
+      if (results != null && !results.isEmpty()) numberOfResults++;
+      else break;
+      for (KeyValue kv: results) {
+        System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(kv.getValue()));
+      }
+      results.clear();
+    } while(more);
+    return numberOfResults;
+  }
+
+  private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
+  throws IOException {
+    // Defaults FilterList.Operator.MUST_PASS_ALL.
+    FilterList allFilters = new FilterList();
+    allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
+    // Only return rows where this column value exists in the row.
+    SingleColumnValueFilter filter =
+      new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
+        Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
+    filter.setFilterIfMissing(true);
+    allFilters.addFilter(filter);
+    Scan scan = new Scan();
+    scan.addFamily(Bytes.toBytes("trans-blob"));
+    scan.addFamily(Bytes.toBytes("trans-type"));
+    scan.addFamily(Bytes.toBytes("trans-date"));
+    scan.addFamily(Bytes.toBytes("trans-tags"));
+    scan.addFamily(Bytes.toBytes("trans-group"));
+    scan.setFilter(allFilters);
+    return r.getScanner(scan);
+  }
+
+  private void putRows(HRegion r, int numRows, String value, String key)
+  throws IOException {
+    for (int i = 0; i < numRows; i++) {
+      String row = key + "_" + i/* UUID.randomUUID().toString() */;
+      System.out.println(String.format("Saving row: %s, with value %s", row,
+        value));
+      Put put = new Put(Bytes.toBytes(row));
+      put.add(Bytes.toBytes("trans-blob"), null,
+        Bytes.toBytes("value for blob"));
+      put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
+      put.add(Bytes.toBytes("trans-date"), null,
+        Bytes.toBytes("20090921010101999"));
+      put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"),
+        Bytes.toBytes(value));
+      put.add(Bytes.toBytes("trans-group"), null,
+        Bytes.toBytes("adhocTransactionGroupId"));
+      r.put(put);
+    }
+  }
 
   public void testFamilyWithAndWithoutColon() throws Exception {
     byte [] b = Bytes.toBytes(getName());
@@ -1764,7 +1889,8 @@
   }
   
   private void initHRegion (byte [] tableName, String callingMethod,
-      HBaseConfiguration conf, byte [] ... families) throws IOException{
+    HBaseConfiguration conf, byte [] ... families)
+  throws IOException{
     HTableDescriptor htd = new HTableDescriptor(tableName);
     for(byte [] family : families) {
       htd.addFamily(new HColumnDescriptor(family));

Modified: hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java?rev=825242&r1=825241&r2=825242&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java
(original)
+++ hadoop/hbase/branches/0.20/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java
Wed Oct 14 19:52:46 2009
@@ -180,9 +180,11 @@
     try {
       this.r = createNewHRegion(REGION_INFO.getTableDesc(), null, null);
       addContent(this.r, HConstants.CATALOG_FAMILY);
-      Filter newFilter = new PrefixFilter(Bytes.toBytes("ab"));
+      byte [] prefix = Bytes.toBytes("ab");
+      Filter newFilter = new PrefixFilter(prefix);
       Scan scan = new Scan();
       scan.setFilter(newFilter);
+      scan.setStartRow(prefix);
       rowPrefixFilter(scan);
       RowFilterInterface oldFilter = new PrefixRowFilter(Bytes.toBytes("ab"));
       scan = new Scan();



Mime
View raw message