hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From j...@apache.org
Subject svn commit: r640092 - in /hadoop/hbase/branches/0.1: ./ src/java/org/apache/hadoop/hbase/ src/java/org/apache/hadoop/hbase/filter/ src/test/org/apache/hadoop/hbase/filter/
Date Sat, 22 Mar 2008 22:06:53 GMT
Author: jimk
Date: Sat Mar 22 15:06:52 2008
New Revision: 640092

URL: http://svn.apache.org/viewvc?rev=640092&view=rev
Log:
HBASE-476   RegexpRowFilter behaves incorectly when there are multiple store files (Clint
Morgan via Jim Kellerman)
HBASE-527   RegexpRowFilter does not work when there are columns from multiple families (Clint
Morgan via Jim Kellerman)

Added:
    hadoop/hbase/branches/0.1/src/test/org/apache/hadoop/hbase/filter/TestRowFilterAfterWrite.java
    hadoop/hbase/branches/0.1/src/test/org/apache/hadoop/hbase/filter/TestRowFilterOnMultipleFamilies.java
Modified:
    hadoop/hbase/branches/0.1/CHANGES.txt
    hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HRegion.java
    hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HStore.java
    hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/InclusiveStopRowFilter.java
    hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/PageRowFilter.java
    hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/RegExpRowFilter.java
    hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/RowFilterInterface.java
    hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/RowFilterSet.java
    hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/StopRowFilter.java
    hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/WhileMatchRowFilter.java

Modified: hadoop/hbase/branches/0.1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.1/CHANGES.txt?rev=640092&r1=640091&r2=640092&view=diff
==============================================================================
--- hadoop/hbase/branches/0.1/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.1/CHANGES.txt Sat Mar 22 15:06:52 2008
@@ -62,6 +62,10 @@
    HBASE-497   RegionServer needs to recover if datanode goes down
    HBASE-456   Clearly state which ports need to be opened in order to run HBase
    HBASE-483   Merge tool won't merge two overlapping regions
+   HBASE-476   RegexpRowFilter behaves incorectly when there are multiple store
+               files (Clint Morgan via Jim Kellerman)
+   HBASE-527   RegexpRowFilter does not work when there are columns from 
+               multiple families (Clint Morgan via Jim Kellerman)
               
 Release 0.16.0
 

Modified: hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HRegion.java?rev=640092&r1=640091&r2=640092&view=diff
==============================================================================
--- hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HRegion.java (original)
+++ hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HRegion.java Sat Mar 22 15:06:52
2008
@@ -1717,12 +1717,14 @@
     private HInternalScannerInterface[] scanners;
     private TreeMap<Text, byte []>[] resultSets;
     private HStoreKey[] keys;
+    private RowFilterInterface filter;
 
     /** Create an HScanner with a handle on many HStores. */
     @SuppressWarnings("unchecked")
     HScanner(Text[] cols, Text firstRow, long timestamp, HStore[] stores,
         RowFilterInterface filter)
     throws IOException {
+      this.filter = filter;
       this.scanners = new HInternalScannerInterface[stores.length];
       try {
         for (int i = 0; i < stores.length; i++) {
@@ -1733,8 +1735,8 @@
           // At least WhileMatchRowFilter will mess up the scan if only
           // one shared across many rows. See HADOOP-2467.
           scanners[i] = stores[i].getScanner(timestamp, cols, firstRow,
-            (i > 0 && filter != null)?
-              (RowFilterInterface)WritableUtils.clone(filter, conf): filter);
+            filter != null ?
+              (RowFilterInterface)WritableUtils.clone(filter, conf) : filter);
         }
       } catch(IOException e) {
         for (int i = 0; i < this.scanners.length; i++) {
@@ -1835,6 +1837,12 @@
           }
         }
       }
+      
+      if (filter != null && filter.filterNotNull(results)) {
+          LOG.warn("Filter return true on assembled Results in hstore");
+          return moreToFollow == true && this.next(key, results);
+      }
+
       return moreToFollow;
     }
 

Modified: hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HStore.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HStore.java?rev=640092&r1=640091&r2=640092&view=diff
==============================================================================
--- hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HStore.java (original)
+++ hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/HStore.java Sat Mar 22 15:06:52
2008
@@ -30,8 +30,6 @@
 import java.util.SortedMap;
 import java.util.TreeMap;
 import java.util.Map.Entry;
-import java.util.SortedSet;
-import java.util.TreeSet;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.regex.Matcher;
@@ -226,8 +224,7 @@
      * preceeds it.
      */
     void getRowKeyAtOrBefore(final Text row, 
-      SortedMap<HStoreKey, Long> candidateKeys)
-    throws IOException{
+      SortedMap<HStoreKey, Long> candidateKeys) {
       this.lock.readLock().lock();
       
       try {
@@ -2527,11 +2524,6 @@
                   && !multipleMatchers
                   && (keys[i].getTimestamp() != chosenTimestamp)) {
                 break;
-              }
-
-              // Filter out null criteria columns that are not null
-              if (dataFilter != null) {
-                filtered = dataFilter.filterNotNull(resultSets[i]);
               }
 
               // NOTE: We used to do results.putAll(resultSets[i]);

Modified: hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/InclusiveStopRowFilter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/InclusiveStopRowFilter.java?rev=640092&r1=640091&r2=640092&view=diff
==============================================================================
--- hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/InclusiveStopRowFilter.java
(original)
+++ hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/InclusiveStopRowFilter.java
Sat Mar 22 15:06:52 2008
@@ -19,17 +19,10 @@
  */
 package org.apache.hadoop.hbase.filter;
 
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.TreeMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.io.Text;
 
 
-/*
+/**
  * Subclass of StopRowFilter that filters rows > the stop row,
  * making it include up to the last row but no further.
  */
@@ -49,6 +42,8 @@
     super(stopRowKey);
   }
   
+  /** {@inheritDoc} */
+  @Override
   public boolean filter(final Text rowKey) {
     if (rowKey == null) {
       if (this.stopRowKey == null) {

Modified: hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/PageRowFilter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/PageRowFilter.java?rev=640092&r1=640091&r2=640092&view=diff
==============================================================================
--- hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/PageRowFilter.java (original)
+++ hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/PageRowFilter.java Sat
Mar 22 15:06:52 2008
@@ -22,7 +22,7 @@
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
-import java.util.TreeMap;
+import java.util.SortedMap;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -137,7 +137,7 @@
    * {@inheritDoc}
    */
   public boolean filterNotNull(@SuppressWarnings("unused")
-      final TreeMap<Text, byte[]> columns) {
+      final SortedMap<Text, byte[]> columns) {
     return filterAllRemaining();
   }
 

Modified: hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/RegExpRowFilter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/RegExpRowFilter.java?rev=640092&r1=640091&r2=640092&view=diff
==============================================================================
--- hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/RegExpRowFilter.java
(original)
+++ hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/RegExpRowFilter.java
Sat Mar 22 15:06:52 2008
@@ -27,7 +27,7 @@
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
-import java.util.TreeMap;
+import java.util.SortedMap;
 import java.util.Map.Entry;
 import java.util.regex.Pattern;
 
@@ -196,7 +196,7 @@
    * 
    * {@inheritDoc}
    */
-  public boolean filterNotNull(final TreeMap<Text, byte[]> columns) {
+  public boolean filterNotNull(final SortedMap<Text, byte[]> columns) {
     for (Entry<Text, byte[]> col : columns.entrySet()) {
       if (nullColumns.contains(col.getKey())
           && !HLogEdit.isDeleted(col.getValue())) {
@@ -211,7 +211,7 @@
       if (!columns.containsKey(col)) {
         if (LOG.isDebugEnabled()) {
           LOG.debug("filterNotNull returning true for colKey: " + col + 
-            ", column not found in given TreeMap<Text, byte[]>.");
+            ", column not found in given SortedMap<Text, byte[]>.");
         }
         return true;
       }

Modified: hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/RowFilterInterface.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/RowFilterInterface.java?rev=640092&r1=640091&r2=640092&view=diff
==============================================================================
--- hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/RowFilterInterface.java
(original)
+++ hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/RowFilterInterface.java
Sat Mar 22 15:06:52 2008
@@ -19,7 +19,7 @@
  */
 package org.apache.hadoop.hbase.filter;
 
-import java.util.TreeMap;
+import java.util.SortedMap;
 
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Writable;
@@ -108,7 +108,7 @@
    * @param columns
    * @return true if null/non-null criteria not met.
    */
-  boolean filterNotNull(final TreeMap<Text, byte[]> columns);
+  boolean filterNotNull(final SortedMap<Text, byte[]> columns);
 
   /**
    * Validates that this filter applies only to a subset of the given columns.

Modified: hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/RowFilterSet.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/RowFilterSet.java?rev=640092&r1=640091&r2=640092&view=diff
==============================================================================
--- hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/RowFilterSet.java (original)
+++ hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/RowFilterSet.java Sat
Mar 22 15:06:52 2008
@@ -24,7 +24,7 @@
 import java.io.IOException;
 import java.util.HashSet;
 import java.util.Set;
-import java.util.TreeMap;
+import java.util.SortedMap;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -232,7 +232,7 @@
   }
 
   /** {@inheritDoc} */
-  public boolean filterNotNull(final TreeMap<Text, byte[]> columns) {
+  public boolean filterNotNull(final SortedMap<Text, byte[]> columns) {
     boolean resultFound = false;
     boolean result = operator == Operator.MUST_PASS_ONE;
     for (RowFilterInterface filter : filters) {

Modified: hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/StopRowFilter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/StopRowFilter.java?rev=640092&r1=640091&r2=640092&view=diff
==============================================================================
--- hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/StopRowFilter.java (original)
+++ hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/StopRowFilter.java Sat
Mar 22 15:06:52 2008
@@ -22,7 +22,7 @@
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
-import java.util.TreeMap;
+import java.util.SortedMap;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -132,7 +132,7 @@
    * @param columns
    */
   public boolean filterNotNull(@SuppressWarnings("unused")
-      final TreeMap<Text, byte[]> columns) {
+      final SortedMap<Text, byte[]> columns) {
     return filterAllRemaining();
   }
 

Modified: hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/WhileMatchRowFilter.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/WhileMatchRowFilter.java?rev=640092&r1=640091&r2=640092&view=diff
==============================================================================
--- hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/WhileMatchRowFilter.java
(original)
+++ hadoop/hbase/branches/0.1/src/java/org/apache/hadoop/hbase/filter/WhileMatchRowFilter.java
Sat Mar 22 15:06:52 2008
@@ -22,7 +22,7 @@
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
-import java.util.TreeMap;
+import java.util.SortedMap;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -31,7 +31,7 @@
 /**
  * WhileMatchRowFilter is a wrapper filter that filters everything after the 
  * first filtered row.  Once the nested filter returns true for either of it's 
- * filter(..) methods or filterNotNull(TreeMap<Text, byte[]>), this wrapper's 
+ * filter(..) methods or filterNotNull(SortedMap<Text, byte[]>), this wrapper's 
  * filterAllRemaining() will return true.  All filtering methods will 
  * thereafter defer to the result of filterAllRemaining().
  */
@@ -115,7 +115,7 @@
   }
   
   /** {@inheritDoc} */
-  public boolean filterNotNull(final TreeMap<Text, byte[]> columns) {
+  public boolean filterNotNull(final SortedMap<Text, byte[]> columns) {
     changeFAR(this.filter.filterNotNull(columns));
     boolean result = filterAllRemaining();
     if (LOG.isDebugEnabled()) {

Added: hadoop/hbase/branches/0.1/src/test/org/apache/hadoop/hbase/filter/TestRowFilterAfterWrite.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.1/src/test/org/apache/hadoop/hbase/filter/TestRowFilterAfterWrite.java?rev=640092&view=auto
==============================================================================
--- hadoop/hbase/branches/0.1/src/test/org/apache/hadoop/hbase/filter/TestRowFilterAfterWrite.java
(added)
+++ hadoop/hbase/branches/0.1/src/test/org/apache/hadoop/hbase/filter/TestRowFilterAfterWrite.java
Sat Mar 22 15:06:52 2008
@@ -0,0 +1,221 @@
+/**
+ * Copyright 2008 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.filter;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.TreeMap;
+
+import junit.framework.Assert;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HBaseClusterTestCase;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HScannerInterface;
+import org.apache.hadoop.hbase.HStoreKey;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.HBaseAdmin;
+import org.apache.hadoop.hbase.HTable;
+import org.apache.hadoop.io.Text;
+
+/** Test regexp filters HBASE-476 */
+public class TestRowFilterAfterWrite extends HBaseClusterTestCase {
+
+  @SuppressWarnings("hiding")
+  private static final Log LOG = LogFactory.getLog(TestRowFilterAfterWrite.class.getName());
+
+  static final String TABLE_NAME = "TestTable";
+  static final String FAMILY = "C:";
+  static final String COLUMN1 = FAMILY + "col1";
+  static final Text TEXT_COLUMN1 = new Text(COLUMN1);
+  static final String COLUMN2 = FAMILY + "col2";
+  static final Text TEXT_COLUMN2 = new Text(COLUMN2);
+
+  private static final Text[] columns = {
+    TEXT_COLUMN1, TEXT_COLUMN2
+  };
+
+  private static final int NUM_ROWS = 10;
+  private static final int VALUE_SIZE = 1000;
+  private static final byte[] VALUE = new byte[VALUE_SIZE];
+  private static final int COL_2_SIZE = 5;
+  private static final int KEY_SIZE = 9;
+  private static final int NUM_REWRITES = 10;
+  static {
+    Arrays.fill(VALUE, (byte) 'a');
+  }
+
+  /** constructor */
+  public TestRowFilterAfterWrite() {
+    super();
+
+    // Make sure the cache gets flushed so we get multiple stores
+    conf.setInt("hbase.hregion.memcache.flush.size", (NUM_ROWS * (VALUE_SIZE + COL_2_SIZE
+ KEY_SIZE)));
+    LOG.info("memcach flush : " + conf.get("hbase.hregion.memcache.flush.size"));
+    conf.setInt("hbase.regionserver.optionalcacheflushinterval", 100000000);
+    // Avoid compaction to keep multiple stores
+    conf.setInt("hbase.hstore.compactionThreshold", 10000);
+
+    // Make lease timeout longer, lease checks less frequent
+    conf.setInt("hbase.master.lease.period", 10 * 1000);
+    conf.setInt("hbase.master.lease.thread.wakefrequency", 5 * 1000);
+
+    // For debugging
+    conf.setInt("hbase.regionserver.lease.period", 20 * 60 * 1000);
+    conf.setInt("ipc.client.timeout", 20 * 60 * 1000);
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public void setUp() throws Exception {
+    // this.conf.set(HConstants.HBASE_DIR, "file:///opt/benchmark/hadoop/hbase");
+    this.conf.set(HConstants.MASTER_ADDRESS, "0.0.0.0:60100");
+    // Must call super.setup() after starting mini dfs cluster. Otherwise
+    // we get a local file system instead of hdfs
+
+    super.setUp();
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public void tearDown() throws Exception {
+    super.tearDown();
+  }
+
+  /**
+   * Test hbase mapreduce jobs against single region and multi-region tables.
+   * 
+   * @throws IOException
+   * @throws InterruptedException
+   */
+  public void testAfterWrite() throws IOException, InterruptedException {
+    singleTableTest();
+  }
+
+  /*
+   * Test against a single region. @throws IOException
+   */
+  private void singleTableTest() throws IOException, InterruptedException {
+    HTableDescriptor desc = new HTableDescriptor(TABLE_NAME);
+    desc.addFamily(new HColumnDescriptor(FAMILY));
+
+    // Create a table.
+    HBaseAdmin admin = new HBaseAdmin(this.conf);
+    admin.createTable(desc);
+
+    // insert some data into the test table
+    HTable table = new HTable(conf, new Text(TABLE_NAME));
+
+    for (int i = 0; i < NUM_ROWS; i++) {
+      long id = table.startUpdate(new Text("row_" + String.format("%1$05d", i)));
+
+      table.put(id, TEXT_COLUMN1, VALUE);
+      table.put(id, TEXT_COLUMN2, String.format("%1$05d", i).getBytes());
+      table.commit(id);
+    }
+
+    // LOG.info("Print table contents using scanner before map/reduce for " + TABLE_NAME);
+    // scanTable(TABLE_NAME, false);
+    // LOG.info("Print table contents using scanner+filter before map/reduce for " + TABLE_NAME);
+    // scanTableWithRowFilter(TABLE_NAME, false);
+
+    // Do some identity write operations on one column of the data.
+    for (int n = 0; n < NUM_REWRITES; n++) {
+      for (int i = 0; i < NUM_ROWS; i++) {
+        long id = table.startUpdate(new Text("row_" + String.format("%1$05d", i)));
+
+        table.put(id, TEXT_COLUMN2, String.format("%1$05d", i).getBytes());
+        table.commit(id);
+      }
+    }
+
+    // Wait for the flush to happen
+    LOG.info("Waiting, for flushes to complete");
+    Thread.sleep(5 * 1000);
+    // Wait for the flush to happen
+    LOG.info("Done. No flush should happen after this");
+
+    // Do another round so to populate the mem cache
+    for (int i = 0; i < NUM_ROWS; i++) {
+      long id = table.startUpdate(new Text("row_" + String.format("%1$05d", i)));
+
+      table.put(id, TEXT_COLUMN2, String.format("%1$05d", i).getBytes());
+      table.commit(id);
+    }
+
+    LOG.info("Print table contents using scanner after map/reduce for " + TABLE_NAME);
+    scanTable(TABLE_NAME, true);
+    LOG.info("Print table contents using scanner+filter after map/reduce for " + TABLE_NAME);
+    scanTableWithRowFilter(TABLE_NAME, true);
+  }
+
+  private void scanTable(final String tableName, final boolean printValues) throws IOException
{
+    HTable table = new HTable(conf, new Text(tableName));
+
+    HScannerInterface scanner = table.obtainScanner(columns, HConstants.EMPTY_START_ROW);
+    int numFound = doScan(scanner, printValues);
+    Assert.assertEquals(NUM_ROWS, numFound);
+  }
+
+  private void scanTableWithRowFilter(final String tableName, final boolean printValues)
throws IOException {
+    HTable table = new HTable(conf, new Text(tableName));
+    Map<Text, byte[]> columnMap = new HashMap<Text, byte[]>();
+    columnMap.put(TEXT_COLUMN1, VALUE);
+    RegExpRowFilter filter = new RegExpRowFilter(null, columnMap);
+    HScannerInterface scanner = table.obtainScanner(columns, HConstants.EMPTY_START_ROW,
filter);
+    int numFound = doScan(scanner, printValues);
+    Assert.assertEquals(NUM_ROWS, numFound);
+  }
+
+  private int doScan(final HScannerInterface scanner, final boolean printValues) throws IOException
{
+    {
+      int count = 0;
+
+      try {
+        HStoreKey key = new HStoreKey();
+        TreeMap<Text, byte[]> results = new TreeMap<Text, byte[]>();
+        while (scanner.next(key, results)) {
+          if (printValues) {
+            LOG.info("row: " + key.getRow());
+
+            for (Map.Entry<Text, byte[]> e : results.entrySet()) {
+              LOG.info(" column: " + e.getKey() + " value: "
+                  + new String(e.getValue(), HConstants.UTF8_ENCODING));
+            }
+          }
+          count++;
+        }
+
+      } finally {
+        scanner.close();
+      }
+      return count;
+    }
+  }
+}

Added: hadoop/hbase/branches/0.1/src/test/org/apache/hadoop/hbase/filter/TestRowFilterOnMultipleFamilies.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.1/src/test/org/apache/hadoop/hbase/filter/TestRowFilterOnMultipleFamilies.java?rev=640092&view=auto
==============================================================================
--- hadoop/hbase/branches/0.1/src/test/org/apache/hadoop/hbase/filter/TestRowFilterOnMultipleFamilies.java
(added)
+++ hadoop/hbase/branches/0.1/src/test/org/apache/hadoop/hbase/filter/TestRowFilterOnMultipleFamilies.java
Sat Mar 22 15:06:52 2008
@@ -0,0 +1,131 @@
+/**
+ * Copyright 2008 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.filter;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.TreeMap;
+
+import junit.framework.Assert;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HBaseClusterTestCase;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HScannerInterface;
+import org.apache.hadoop.hbase.HStoreKey;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.HBaseAdmin;
+import org.apache.hadoop.hbase.HTable;
+import org.apache.hadoop.io.Text;
+
+/**
+ * Test for regexp filters (HBASE-527)
+ */
+public class TestRowFilterOnMultipleFamilies extends HBaseClusterTestCase {
+  private static final Log LOG = LogFactory.getLog(TestRowFilterOnMultipleFamilies.class.getName());
+
+  static final String TABLE_NAME = "TestTable";
+  static final String COLUMN1 = "A:col1";
+  static final Text TEXT_COLUMN1 = new Text(COLUMN1);
+  static final String COLUMN2 = "B:col2";
+  static final Text TEXT_COLUMN2 = new Text(COLUMN2);
+
+  private static final Text[] columns = {
+    TEXT_COLUMN1, TEXT_COLUMN2
+  };
+
+  private static final int NUM_ROWS = 10;
+  private static final byte[] VALUE = "HELLO".getBytes();
+
+  /** @throws IOException */
+  public void testMultipleFamilies() throws IOException {
+    HTableDescriptor desc = new HTableDescriptor(TABLE_NAME);
+    desc.addFamily(new HColumnDescriptor("A:"));
+    desc.addFamily(new HColumnDescriptor("B:"));
+
+    // Create a table.
+    HBaseAdmin admin = new HBaseAdmin(this.conf);
+    admin.createTable(desc);
+
+    // insert some data into the test table
+    HTable table = new HTable(conf, new Text(TABLE_NAME));
+
+    for (int i = 0; i < NUM_ROWS; i++) {
+      long id = table.startUpdate(new Text("row_" + String.format("%1$05d", i)));
+      table.put(id, TEXT_COLUMN1, VALUE);
+      table.put(id, TEXT_COLUMN2, String.format("%1$05d", i).getBytes());
+      table.commit(id);
+    }
+
+    LOG.info("Print table contents using scanner before map/reduce for " + TABLE_NAME);
+    scanTable(TABLE_NAME, true);
+    LOG.info("Print table contents using scanner+filter before map/reduce for " + TABLE_NAME);
+    scanTableWithRowFilter(TABLE_NAME, true);
+  }
+
+  private void scanTable(final String tableName, final boolean printValues) throws IOException
{
+    HTable table = new HTable(conf, new Text(tableName));
+
+    HScannerInterface scanner = table.obtainScanner(columns, HConstants.EMPTY_START_ROW);
+    int numFound = doScan(scanner, printValues);
+    Assert.assertEquals(NUM_ROWS, numFound);
+  }
+
+  private void scanTableWithRowFilter(final String tableName, final boolean printValues)
throws IOException {
+    HTable table = new HTable(conf, new Text(tableName));
+    Map<Text, byte[]> columnMap = new HashMap<Text, byte[]>();
+    columnMap.put(TEXT_COLUMN1, VALUE);
+    RegExpRowFilter filter = new RegExpRowFilter(null, columnMap);
+    HScannerInterface scanner = table.obtainScanner(columns, HConstants.EMPTY_START_ROW,
filter);
+    int numFound = doScan(scanner, printValues);
+    Assert.assertEquals(NUM_ROWS, numFound);
+  }
+
+  private int doScan(final HScannerInterface scanner, final boolean printValues) throws IOException
{
+    {
+      int count = 0;
+
+      try {
+        HStoreKey key = new HStoreKey();
+        TreeMap<Text, byte[]> results = new TreeMap<Text, byte[]>();
+        while (scanner.next(key, results)) {
+          if (printValues) {
+            LOG.info("row: " + key.getRow());
+
+            for (Map.Entry<Text, byte[]> e : results.entrySet()) {
+              LOG.info(" column: " + e.getKey() + " value: "
+                  + new String(e.getValue(), HConstants.UTF8_ENCODING));
+            }
+          }
+          Assert.assertEquals(2, results.size());
+          count++;
+        }
+
+      } finally {
+        scanner.close();
+      }
+      return count;
+    }
+  }
+}



Mime
View raw message