Return-Path: Delivered-To: apmail-hadoop-hbase-commits-archive@minotaur.apache.org Received: (qmail 93727 invoked from network); 14 Oct 2009 19:38:10 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 14 Oct 2009 19:38:10 -0000 Received: (qmail 11436 invoked by uid 500); 14 Oct 2009 19:38:10 -0000 Delivered-To: apmail-hadoop-hbase-commits-archive@hadoop.apache.org Received: (qmail 11403 invoked by uid 500); 14 Oct 2009 19:38:09 -0000 Mailing-List: contact hbase-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hbase-dev@hadoop.apache.org Delivered-To: mailing list hbase-commits@hadoop.apache.org Received: (qmail 11394 invoked by uid 99); 14 Oct 2009 19:38:09 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 14 Oct 2009 19:38:09 +0000 X-ASF-Spam-Status: No, hits=-2.6 required=5.0 tests=AWL,BAYES_00 X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 14 Oct 2009 19:38:05 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id D633523888D0; Wed, 14 Oct 2009 19:37:45 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r825237 - in /hadoop/hbase/trunk: ./ src/java/org/apache/hadoop/hbase/filter/ src/java/org/apache/hadoop/hbase/regionserver/ src/test/org/apache/hadoop/hbase/ src/test/org/apache/hadoop/hbase/client/ src/test/org/apache/hadoop/hbase/regions... Date: Wed, 14 Oct 2009 19:37:45 -0000 To: hbase-commits@hadoop.apache.org From: stack@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20091014193745.D633523888D0@eris.apache.org> Author: stack Date: Wed Oct 14 19:37:44 2009 New Revision: 825237 URL: http://svn.apache.org/viewvc?rev=825237&view=rev Log: HBASE-1906 FilterList of prefix and columnvalue not working properly with deletes and multiple values Modified: hadoop/hbase/trunk/CHANGES.txt hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/filter/CompareFilter.java hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/filter/FilterList.java hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/HBaseTestCase.java hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/client/TestClient.java hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java Modified: hadoop/hbase/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=825237&r1=825236&r2=825237&view=diff ============================================================================== --- hadoop/hbase/trunk/CHANGES.txt (original) +++ hadoop/hbase/trunk/CHANGES.txt Wed Oct 14 19:37:44 2009 @@ -65,6 +65,8 @@ work HBASE-1889 ClassNotFoundException on trunk for REST HBASE-1905 Remove unused config. hbase.hstore.blockCache.blockSize + HBASE-1906 FilterList of prefix and columnvalue not working properly with + deletes and multiple values IMPROVEMENTS HBASE-1760 Cleanup TODOs in HTable Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/filter/CompareFilter.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/filter/CompareFilter.java?rev=825237&r1=825236&r2=825237&view=diff ============================================================================== --- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/filter/CompareFilter.java (original) +++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/filter/CompareFilter.java Wed Oct 14 19:37:44 2009 @@ -25,10 +25,8 @@ import java.io.IOException; import java.util.Arrays; -import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.io.HbaseObjectWritable; -import org.apache.hadoop.io.ObjectWritable; /** * This is a generic filter to be used to filter by comparison. It takes an Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/filter/FilterList.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/filter/FilterList.java?rev=825237&r1=825236&r2=825237&view=diff ============================================================================== --- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/filter/FilterList.java (original) +++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/filter/FilterList.java Wed Oct 14 19:37:44 2009 @@ -33,13 +33,14 @@ /** * Implementation of {@link Filter} that represents an ordered List of Filters - * which will be evaluated with a specified boolean operator MUST_PASS_ALL - * (!AND) or MUST_PASS_ONE (!OR). Since you can use Filter Lists as children - * of Filter Lists, you can create a hierarchy of filters to be evaluated. + * which will be evaluated with a specified boolean operator {@link Operator#MUST_PASS_ALL} + * (!AND) or {@link Operator#MUST_PASS_ONE} (!OR). + * Since you can use Filter Lists as children of Filter Lists, you can create a + * hierarchy of filters to be evaluated. + * Defaults to {@link Operator#MUST_PASS_ALL}. *

TODO: Fix creation of Configuration on serialization and deserialization. */ public class FilterList implements Filter { - /** set operator */ public static enum Operator { /** !AND */ @@ -70,6 +71,15 @@ } /** + * Constructor that takes an operator. + * + * @param operator Operator to process filter set with. + */ + public FilterList(final Operator operator) { + this.operator = operator; + } + + /** * Constructor that takes a set of {@link Filter}s and an operator. * * @param operator Operator to process filter set with. @@ -115,19 +125,19 @@ public boolean filterRowKey(byte[] rowKey, int offset, int length) { for (Filter filter : filters) { - if (operator == Operator.MUST_PASS_ALL) { - if (filter.filterAllRemaining() - || filter.filterRowKey(rowKey, offset, length)) { + if (this.operator == Operator.MUST_PASS_ALL) { + if (filter.filterAllRemaining() || + filter.filterRowKey(rowKey, offset, length)) { return true; } - } else if (operator == Operator.MUST_PASS_ONE) { - if (!filter.filterAllRemaining() - && !filter.filterRowKey(rowKey, offset, length)) { + } else if (this.operator == Operator.MUST_PASS_ONE) { + if (!filter.filterAllRemaining() && + !filter.filterRowKey(rowKey, offset, length)) { return false; } } } - return operator == Operator.MUST_PASS_ONE; + return this.operator == Operator.MUST_PASS_ONE; } public boolean filterAllRemaining() { @@ -179,8 +189,7 @@ public boolean filterRow() { for (Filter filter : filters) { if (operator == Operator.MUST_PASS_ALL) { - if (filter.filterAllRemaining() - || filter.filterRow()) { + if (filter.filterAllRemaining() || filter.filterRow()) { return true; } } else if (operator == Operator.MUST_PASS_ONE) { Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java?rev=825237&r1=825236&r2=825237&view=diff ============================================================================== --- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java (original) +++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/filter/SingleColumnValueFilter.java Wed Oct 14 19:37:44 2009 @@ -108,16 +108,19 @@ this.compareOp = compareOp; this.comparator = comparator; } - + public boolean filterRowKey(byte[] rowKey, int offset, int length) { + // We don't filter on the row key... we filter later on column value so + // always return false. return false; } public ReturnCode filterKeyValue(KeyValue keyValue) { - if(matchedColumn) { + // System.out.println("REMOVE KEY=" + keyValue.toString() + ", value=" + Bytes.toString(keyValue.getValue())); + if (this.matchedColumn) { // We already found and matched the single column, all keys now pass return ReturnCode.INCLUDE; - } else if(foundColumn) { + } else if (this.foundColumn) { // We found but did not match the single column, skip to next row return ReturnCode.NEXT_ROW; } @@ -129,16 +132,18 @@ keyValue.getValueOffset(), keyValue.getValueLength())) { return ReturnCode.NEXT_ROW; } - matchedColumn = true; + this.matchedColumn = true; return ReturnCode.INCLUDE; } - private boolean filterColumnValue(final byte[] data, final int offset, + private boolean filterColumnValue(final byte [] data, final int offset, final int length) { - int compareResult = comparator.compareTo(Arrays.copyOfRange(data, offset, - offset + length)); - - switch (compareOp) { + // TODO: Can this filter take a rawcomparator so don't have to make this + // byte array copy? + int compareResult = + this.comparator.compareTo(Arrays.copyOfRange(data, offset, offset + length)); + LOG.debug("compareResult=" + compareResult + " " + Bytes.toString(data, offset, length)); + switch (this.compareOp) { case LESS: return compareResult <= 0; case LESS_OR_EQUAL: @@ -163,23 +168,23 @@ public boolean filterRow() { // If column was found, return false if it was matched, true if it was not // If column not found, return true if we filter if missing, false if not - return foundColumn ? !matchedColumn : filterIfMissing; + return this.foundColumn? !this.matchedColumn: this.filterIfMissing; } public void reset() { foundColumn = false; matchedColumn = false; } - + /** * Get whether entire row should be filtered if column is not found. - * @return filterIfMissing true if row should be skipped if column not found, - * false if row should be let through anyways + * @return true if row should be skipped if column not found, false if row + * should be let through anyways */ public boolean getFilterIfMissing() { return filterIfMissing; } - + /** * Set whether entire row should be filtered if column is not found. *

@@ -200,12 +205,12 @@ if(this.columnQualifier.length == 0) { this.columnQualifier = null; } - compareOp = CompareOp.valueOf(in.readUTF()); - comparator = (WritableByteArrayComparable) HbaseObjectWritable.readObject(in, - null); - foundColumn = in.readBoolean(); - matchedColumn = in.readBoolean(); - filterIfMissing = in.readBoolean(); + this.compareOp = CompareOp.valueOf(in.readUTF()); + this.comparator = + (WritableByteArrayComparable)HbaseObjectWritable.readObject(in, null); + this.foundColumn = in.readBoolean(); + this.matchedColumn = in.readBoolean(); + this.filterIfMissing = in.readBoolean(); } public void write(final DataOutput out) throws IOException { @@ -218,4 +223,4 @@ out.writeBoolean(matchedColumn); out.writeBoolean(filterIfMissing); } -} +} \ No newline at end of file Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=825237&r1=825236&r2=825237&view=diff ============================================================================== --- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original) +++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java Wed Oct 14 19:37:44 2009 @@ -1179,8 +1179,6 @@ regionInfo.getTableDesc().getName(), kvs, (regionInfo.isMetaRegion() || regionInfo.isRootRegion()), now); } - - flush = isFlushSize(size); } finally { this.updatesLock.readLock().unlock(); @@ -1744,57 +1742,58 @@ * @throws IOException */ private boolean nextInternal() throws IOException { - // This method should probably be reorganized a bit... has gotten messy - KeyValue kv; - byte[] currentRow = null; + byte [] currentRow = null; boolean filterCurrentRow = false; while (true) { - kv = this.storeHeap.peek(); - if (kv == null) { - return false; - } + KeyValue kv = this.storeHeap.peek(); + if (kv == null) return false; byte [] row = kv.getRow(); - if (filterCurrentRow && Bytes.equals(currentRow, row)) { - // filter all columns until row changes - this.storeHeap.next(results); - results.clear(); + boolean samerow = Bytes.equals(currentRow, row); + if (samerow && filterCurrentRow) { + // Filter all columns until row changes + this.storeHeap.next(this.results); + this.results.clear(); continue; } - // see if current row should be filtered based on row key - if (filter != null && filter.filterRowKey(row, 0, row.length)) { - if(!results.isEmpty() && !Bytes.equals(currentRow, row)) { - return true; - } - this.storeHeap.next(results); - results.clear(); - resetFilters(); - filterCurrentRow = true; - currentRow = row; - continue; - } - if(!Bytes.equals(currentRow, row)) { + if (!samerow) { // Continue on the next row: currentRow = row; filterCurrentRow = false; // See if we passed stopRow - if(stopRow != null && - comparator.compareRows(stopRow, 0, stopRow.length, - currentRow, 0, currentRow.length) <= 0) { + if (this.stopRow != null && + comparator.compareRows(this.stopRow, 0, this.stopRow.length, + currentRow, 0, currentRow.length) <= 0) { return false; } - // if there are _no_ results or current row should be filtered - if (results.isEmpty() || filter != null && filter.filterRow()) { - // make sure results is empty - results.clear(); - resetFilters(); - continue; - } - return true; + if (hasResults()) return true; + } + // See if current row should be filtered based on row key + if (this.filter != null && this.filter.filterRowKey(row, 0, row.length)) { + resetFilters(); + filterCurrentRow = true; + currentRow = row; } this.storeHeap.next(results); } } + /* + * Do we have results to return or should we continue. Call when we get to + * the end of a row. Does house cleaning -- clearing results and resetting + * filters -- if we are to continue. + * @return True if we should return else false if need to keep going. + */ + private boolean hasResults() { + if (this.results.isEmpty() || + this.filter != null && this.filter.filterRow()) { + // Make sure results is empty, reset filters + results.clear(); + resetFilters(); + return false; + } + return true; + } + public void close() { storeHeap.close(); } @@ -2326,7 +2325,6 @@ store.get(get, qualifiers, results); if (!results.isEmpty()) { - byte [] oldValue = results.get(0).getValue(); KeyValue kv = results.get(0); byte [] buffer = kv.getBuffer(); int valueOffset = kv.getValueOffset(); Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/HBaseTestCase.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/HBaseTestCase.java?rev=825237&r1=825236&r2=825237&view=diff ============================================================================== --- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/HBaseTestCase.java (original) +++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/HBaseTestCase.java Wed Oct 14 19:37:44 2009 @@ -219,8 +219,7 @@ if (startKeyBytes == null || startKeyBytes.length == 0) { startKeyBytes = START_KEY_BYTES; } - return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), - null, + return addContent(new HRegionIncommon(r), Bytes.toString(columnFamily), null, startKeyBytes, endKey, -1); } Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/client/TestClient.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/client/TestClient.java?rev=825237&r1=825236&r2=825237&view=diff ============================================================================== --- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/client/TestClient.java (original) +++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/client/TestClient.java Wed Oct 14 19:37:44 2009 @@ -20,10 +20,6 @@ package org.apache.hadoop.hbase.client; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Map; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.HBaseClusterTestCase; @@ -35,14 +31,23 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.filter.BinaryComparator; import org.apache.hadoop.hbase.filter.CompareFilter; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.filter.Filter; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.PrefixFilter; import org.apache.hadoop.hbase.filter.QualifierFilter; import org.apache.hadoop.hbase.filter.RegexStringComparator; import org.apache.hadoop.hbase.filter.RowFilter; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; import org.apache.hadoop.hbase.filter.WhileMatchFilter; -import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.util.Bytes; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.Map; +import java.util.UUID; + /** * Tests from client-side of a cluster. */ @@ -63,6 +68,135 @@ } /** + * Test from client side of an involved filter against a multi family that + * involves deletes. + * + * @throws Exception + */ + public void testWeirdCacheBehaviour() throws Exception { + byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour"); + byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), + Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"), + Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") }; + HTable ht = createTable(TABLE, FAMILIES); + String value = "this is the value"; + String value2 = "this is some other value"; + String keyPrefix1 = UUID.randomUUID().toString(); + String keyPrefix2 = UUID.randomUUID().toString(); + String keyPrefix3 = UUID.randomUUID().toString(); + putRows(ht, 3, value, keyPrefix1); + putRows(ht, 3, value, keyPrefix2); + putRows(ht, 3, value, keyPrefix3); + ht.flushCommits(); + putRows(ht, 3, value2, keyPrefix1); + putRows(ht, 3, value2, keyPrefix2); + putRows(ht, 3, value2, keyPrefix3); + HTable table = new HTable(conf, Bytes.toBytes("testWeirdCacheBehaviour")); + System.out.println("Checking values for key: " + keyPrefix1); + assertEquals("Got back incorrect number of rows from scan", 3, + getNumberOfRows(keyPrefix1, value2, table)); + System.out.println("Checking values for key: " + keyPrefix2); + assertEquals("Got back incorrect number of rows from scan", 3, + getNumberOfRows(keyPrefix2, value2, table)); + System.out.println("Checking values for key: " + keyPrefix3); + assertEquals("Got back incorrect number of rows from scan", 3, + getNumberOfRows(keyPrefix3, value2, table)); + deleteColumns(ht, value2, keyPrefix1); + deleteColumns(ht, value2, keyPrefix2); + deleteColumns(ht, value2, keyPrefix3); + System.out.println("Starting important checks....."); + assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, + 0, getNumberOfRows(keyPrefix1, value2, table)); + assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, + 0, getNumberOfRows(keyPrefix2, value2, table)); + assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, + 0, getNumberOfRows(keyPrefix3, value2, table)); + ht.setScannerCaching(0); + assertEquals("Got back incorrect number of rows from scan", 0, + getNumberOfRows(keyPrefix1, value2, table)); ht.setScannerCaching(100); + assertEquals("Got back incorrect number of rows from scan", 0, + getNumberOfRows(keyPrefix2, value2, table)); + } + + private void deleteColumns(HTable ht, String value, String keyPrefix) + throws IOException { + ResultScanner scanner = buildScanner(keyPrefix, value, ht); + Iterator it = scanner.iterator(); + int count = 0; + while (it.hasNext()) { + Result result = it.next(); + Delete delete = new Delete(result.getRow()); + delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2")); + ht.delete(delete); + count++; + } + assertEquals("Did not perform correct number of deletes", 3, count); + } + + private int getNumberOfRows(String keyPrefix, String value, HTable ht) + throws Exception { + ResultScanner resultScanner = buildScanner(keyPrefix, value, ht); + Iterator scanner = resultScanner.iterator(); + int numberOfResults = 0; + while (scanner.hasNext()) { + Result result = scanner.next(); + System.out.println("Got back key: " + Bytes.toString(result.getRow())); + for (KeyValue kv : result.raw()) { + System.out.println("kv=" + kv.toString() + ", " + + Bytes.toString(kv.getValue())); + } + numberOfResults++; + } + return numberOfResults; + } + + private ResultScanner buildScanner(String keyPrefix, String value, HTable ht) + throws IOException { + // OurFilterList allFilters = new OurFilterList(); + FilterList allFilters = new FilterList(/* FilterList.Operator.MUST_PASS_ALL */); + allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix))); + SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes + .toBytes("trans-tags"), Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes + .toBytes(value)); + filter.setFilterIfMissing(true); + allFilters.addFilter(filter); + + // allFilters.addFilter(new + // RowExcludingSingleColumnValueFilter(Bytes.toBytes("trans-tags"), + // Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value))); + + Scan scan = new Scan(); + scan.addFamily(Bytes.toBytes("trans-blob")); + scan.addFamily(Bytes.toBytes("trans-type")); + scan.addFamily(Bytes.toBytes("trans-date")); + scan.addFamily(Bytes.toBytes("trans-tags")); + scan.addFamily(Bytes.toBytes("trans-group")); + scan.setFilter(allFilters); + + return ht.getScanner(scan); + } + + private void putRows(HTable ht, int numRows, String value, String key) + throws IOException { + for (int i = 0; i < numRows; i++) { + String row = key + "_" + UUID.randomUUID().toString(); + System.out.println(String.format("Saving row: %s, with value %s", row, + value)); + Put put = new Put(Bytes.toBytes(row)); + put.add(Bytes.toBytes("trans-blob"), null, Bytes + .toBytes("value for blob")); + put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement")); + put.add(Bytes.toBytes("trans-date"), null, Bytes + .toBytes("20090921010101999")); + put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes + .toBytes(value)); + put.add(Bytes.toBytes("trans-group"), null, Bytes + .toBytes("adhocTransactionGroupId")); + ht.put(put); + } + } + + /** * Test filters when multiple regions. It does counts. Needs eye-balling of * logs to ensure that we're not scanning more regions that we're supposed to. * Related to the TestFilterAcrossRegions over in the o.a.h.h.filter package. @@ -249,6 +383,7 @@ scanner.close(); System.out.println("Done."); } + public void testFilters() throws Exception { byte [] TABLE = Bytes.toBytes("testFilters"); HTable ht = createTable(TABLE, FAMILY); @@ -2671,7 +2806,7 @@ } private byte [][] makeN(byte [] base, int n) { - if(n > 256) { + if (n > 256) { return makeNBig(base, n); } byte [][] ret = new byte[n][]; Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java?rev=825237&r1=825236&r2=825237&view=diff ============================================================================== --- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java (original) +++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestHRegion.java Wed Oct 14 19:37:44 2009 @@ -40,6 +40,10 @@ import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.filter.ColumnCountGetFilter; +import org.apache.hadoop.hbase.filter.FilterList; +import org.apache.hadoop.hbase.filter.PrefixFilter; +import org.apache.hadoop.hbase.filter.SingleColumnValueFilter; +import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.regionserver.HRegion.RegionScanner; import org.apache.hadoop.hbase.util.Bytes; @@ -79,6 +83,127 @@ // individual code pieces in the HRegion. Putting files locally in // /tmp/testtable ////////////////////////////////////////////////////////////////////////////// + + + /** + * An involved filter test. Has multiple column families and deletes in mix. + */ + public void testWeirdCacheBehaviour() throws Exception { + byte[] TABLE = Bytes.toBytes("testWeirdCacheBehaviour"); + byte[][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"), + Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"), + Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") }; + initHRegion(TABLE, getName(), FAMILIES); + String value = "this is the value"; + String value2 = "this is some other value"; + String keyPrefix1 = "prefix1"; // UUID.randomUUID().toString(); + String keyPrefix2 = "prefix2"; // UUID.randomUUID().toString(); + String keyPrefix3 = "prefix3"; // UUID.randomUUID().toString(); + putRows(this.region, 3, value, keyPrefix1); + putRows(this.region, 3, value, keyPrefix2); + putRows(this.region, 3, value, keyPrefix3); + // this.region.flushCommits(); + putRows(this.region, 3, value2, keyPrefix1); + putRows(this.region, 3, value2, keyPrefix2); + putRows(this.region, 3, value2, keyPrefix3); + System.out.println("Checking values for key: " + keyPrefix1); + assertEquals("Got back incorrect number of rows from scan", 3, + getNumberOfRows(keyPrefix1, value2, this.region)); + System.out.println("Checking values for key: " + keyPrefix2); + assertEquals("Got back incorrect number of rows from scan", 3, + getNumberOfRows(keyPrefix2, value2, this.region)); + System.out.println("Checking values for key: " + keyPrefix3); + assertEquals("Got back incorrect number of rows from scan", 3, + getNumberOfRows(keyPrefix3, value2, this.region)); + deleteColumns(this.region, value2, keyPrefix1); + deleteColumns(this.region, value2, keyPrefix2); + deleteColumns(this.region, value2, keyPrefix3); + System.out.println("Starting important checks....."); + assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1, + 0, getNumberOfRows(keyPrefix1, value2, this.region)); + assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2, + 0, getNumberOfRows(keyPrefix2, value2, this.region)); + assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3, + 0, getNumberOfRows(keyPrefix3, value2, this.region)); + } + + private void deleteColumns(HRegion r, String value, String keyPrefix) + throws IOException { + InternalScanner scanner = buildScanner(keyPrefix, value, r); + int count = 0; + boolean more = false; + List results = new ArrayList(); + do { + more = scanner.next(results); + if (results != null && !results.isEmpty()) + count++; + else + break; + Delete delete = new Delete(results.get(0).getRow()); + delete.deleteColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2")); + r.delete(delete, null, false); + results.clear(); + } while (more); + assertEquals("Did not perform correct number of deletes", 3, count); + } + + private int getNumberOfRows(String keyPrefix, String value, HRegion r) throws Exception { + InternalScanner resultScanner = buildScanner(keyPrefix, value, r); + int numberOfResults = 0; + List results = new ArrayList(); + boolean more = false; + do { + more = resultScanner.next(results); + if (results != null && !results.isEmpty()) numberOfResults++; + else break; + for (KeyValue kv: results) { + System.out.println("kv=" + kv.toString() + ", " + Bytes.toString(kv.getValue())); + } + results.clear(); + } while(more); + return numberOfResults; + } + + private InternalScanner buildScanner(String keyPrefix, String value, HRegion r) + throws IOException { + // Defaults FilterList.Operator.MUST_PASS_ALL. + FilterList allFilters = new FilterList(); + allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix))); + // Only return rows where this column value exists in the row. + SingleColumnValueFilter filter = + new SingleColumnValueFilter(Bytes.toBytes("trans-tags"), + Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value)); + filter.setFilterIfMissing(true); + allFilters.addFilter(filter); + Scan scan = new Scan(); + scan.addFamily(Bytes.toBytes("trans-blob")); + scan.addFamily(Bytes.toBytes("trans-type")); + scan.addFamily(Bytes.toBytes("trans-date")); + scan.addFamily(Bytes.toBytes("trans-tags")); + scan.addFamily(Bytes.toBytes("trans-group")); + scan.setFilter(allFilters); + return r.getScanner(scan); + } + + private void putRows(HRegion r, int numRows, String value, String key) + throws IOException { + for (int i = 0; i < numRows; i++) { + String row = key + "_" + i/* UUID.randomUUID().toString() */; + System.out.println(String.format("Saving row: %s, with value %s", row, + value)); + Put put = new Put(Bytes.toBytes(row)); + put.add(Bytes.toBytes("trans-blob"), null, + Bytes.toBytes("value for blob")); + put.add(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement")); + put.add(Bytes.toBytes("trans-date"), null, + Bytes.toBytes("20090921010101999")); + put.add(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), + Bytes.toBytes(value)); + put.add(Bytes.toBytes("trans-group"), null, + Bytes.toBytes("adhocTransactionGroupId")); + r.put(put); + } + } public void testFamilyWithAndWithoutColon() throws Exception { byte [] b = Bytes.toBytes(getName()); @@ -1781,7 +1906,8 @@ } private void initHRegion (byte [] tableName, String callingMethod, - HBaseConfiguration conf, byte [] ... families) throws IOException{ + HBaseConfiguration conf, byte [] ... families) + throws IOException{ HTableDescriptor htd = new HTableDescriptor(tableName); for(byte [] family : families) { htd.addFamily(new HColumnDescriptor(family)); Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java?rev=825237&r1=825236&r2=825237&view=diff ============================================================================== --- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java (original) +++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestScanner.java Wed Oct 14 19:37:44 2009 @@ -176,9 +176,11 @@ try { this.r = createNewHRegion(REGION_INFO.getTableDesc(), null, null); addContent(this.r, HConstants.CATALOG_FAMILY); - Filter newFilter = new PrefixFilter(Bytes.toBytes("ab")); + byte [] prefix = Bytes.toBytes("ab"); + Filter newFilter = new PrefixFilter(prefix); Scan scan = new Scan(); scan.setFilter(newFilter); + scan.setStartRow(prefix); rowPrefixFilter(scan); byte[] stopRow = Bytes.toBytes("bbc");