hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nspiegelb...@apache.org
Subject svn commit: r1181548 - in /hbase/branches/0.89/src: main/java/org/apache/hadoop/hbase/ main/java/org/apache/hadoop/hbase/regionserver/ test/java/org/apache/hadoop/hbase/client/ test/java/org/apache/hadoop/hbase/regionserver/
Date Tue, 11 Oct 2011 02:18:51 GMT
Author: nspiegelberg
Date: Tue Oct 11 02:18:51 2011
New Revision: 1181548

URL: http://svn.apache.org/viewvc?rev=1181548&view=rev
Log:
Porting patch for HBASE-3416 to hbase-trunk.

Summary:
Don't call setRow when scanning over same row. The patch fixes one case, setRow
is also called from next. Fixed it there too.

Test Plan:
Unit test. When batch is used with scan, in some cases deletes were getting
ignored because of this bug.

Reviewed By: kannan
Reviewers: kannan, jgray
CC: , madhuvaidya, kannan
Differential Revision: 261692

Added:
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide2.java
Modified:
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/KeyValue.java
    hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
    hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java

Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/KeyValue.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/KeyValue.java?rev=1181548&r1=1181547&r2=1181548&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/KeyValue.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/KeyValue.java Tue Oct 11 02:18:51
2011
@@ -1066,6 +1066,15 @@ public class KeyValue implements Writabl
         this.bytes, o, l) == 0;
   }
 
+  public boolean matchingRow(final byte [] row) {
+    return matchingRow(row, 0, row.length);
+  }
+
+  public boolean matchingRow(final byte[] row, int offset, int length) {
+    return Bytes.compareTo(row, offset, length,
+        this.bytes, getRowOffset(), getRowLength()) == 0;
+  }
+
   /**
    * @param column Column minus its delimiter
    * @return True if column matches.

Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java?rev=1181548&r1=1181547&r2=1181548&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
(original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
Tue Oct 11 02:18:51 2011
@@ -235,7 +235,9 @@ class StoreScanner implements KeyValueSc
       return false;
     }
 
-    matcher.setRow(peeked.getRow());
+    if ((matcher.row == null) || !peeked.matchingRow(matcher.row)) {
+	matcher.setRow(peeked.getRow());
+    }
     KeyValue kv;
     List<KeyValue> results = new ArrayList<KeyValue>();
     LOOP: while((kv = this.heap.peek()) != null) {
@@ -361,10 +363,17 @@ class StoreScanner implements KeyValueSc
     // Combine all seeked scanners with a heap
     heap = new KeyValueHeap(scanners, store.comparator);
 
-    // Reset the state of the Query Matcher and set to top row
-    matcher.reset();
+    // Reset the state of the Query Matcher and set to top row.
+    // Only reset and call setRow if the row changes; avoids confusing the
+    // query matcher if scanning intra-row.
     KeyValue kv = heap.peek();
-    matcher.setRow((kv == null ? lastTopKey : kv).getRow());
+    if (kv == null) {
+	kv = lastTopKey;
+    }
+    if ((matcher.row == null) || !kv.matchingRow(matcher.row)) {
+	matcher.reset();
+	matcher.setRow(kv.getRow());
+    }
   }
 
   @Override

Added: hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide2.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide2.java?rev=1181548&view=auto
==============================================================================
--- hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide2.java
(added)
+++ hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide2.java
Tue Oct 11 02:18:51 2011
@@ -0,0 +1,172 @@
+package org.apache.hadoop.hbase.client;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueTestUtil;
+import org.apache.hadoop.hbase.filter.ColumnPrefixFilter;
+import org.apache.hadoop.hbase.filter.ColumnRangeFilter;
+import org.apache.hadoop.hbase.filter.PrefixFilter;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+
+
+public class TestFromClientSide2 {
+  final Log LOG = LogFactory.getLog(getClass());
+  private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+  private static byte [] ROW = Bytes.toBytes("testRow");
+  private static byte [] FAMILY = Bytes.toBytes("testFamily");
+  private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
+  private static byte [] VALUE = Bytes.toBytes("testValue");
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @BeforeClass
+  public static void setUpBeforeClass() throws Exception {
+    TEST_UTIL.startMiniCluster(3);
+  }
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @AfterClass
+  public static void tearDownAfterClass() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @Before
+  public void setUp() throws Exception {
+    // Nothing to do.
+  }
+
+  /**
+   * @throws java.lang.Exception
+   */
+  @After
+  public void tearDown() throws Exception {
+    // Nothing to do.
+  }
+
+
+  /**
+   * Test from client side for batch of scan
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testScanBatch() throws Exception {
+    byte [] TABLE = Bytes.toBytes("testScanBatch");
+    byte [][] QUALIFIERS = makeNAscii(QUALIFIER, 8);
+
+    HTable ht = TEST_UTIL.createTable(TABLE, FAMILY);
+
+    Put put;
+    Scan scan;
+    Delete delete;
+    Result result;
+    ResultScanner scanner;
+    boolean toLog = true;
+    List<KeyValue> kvListExp;
+
+    // table: row, family, c0:0, c1:1, ... , c7:7
+    put = new Put(ROW);
+    for (int i=0; i < QUALIFIERS.length; i++) {
+      KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[i], i, VALUE);
+      put.add(kv);
+    }
+    ht.put(put);
+
+    // table: row, family, c0:0, c1:1, ..., c6:2, c6:6 , c7:7
+    put = new Put(ROW);
+    KeyValue kv = new KeyValue(ROW, FAMILY, QUALIFIERS[6], 2, VALUE);
+    put.add(kv);
+    ht.put(put);
+
+    // delete upto ts: 3
+    delete = new Delete(ROW);
+    delete.deleteFamily(FAMILY, 3);
+    ht.delete(delete);
+
+    // without batch
+    scan = new Scan(ROW);
+    scan.setMaxVersions();
+    scanner = ht.getScanner(scan);
+
+    // c4:4, c5:5, c6:6, c7:7
+    kvListExp = new ArrayList<KeyValue>();
+    kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[4], 4, VALUE));
+    kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[5], 5, VALUE));
+    kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[6], 6, VALUE));
+    kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[7], 7, VALUE));
+    result = scanner.next();
+    verifyResult(result, kvListExp, toLog, "Testing first batch of scan");
+
+    // with batch
+    scan = new Scan(ROW);
+    scan.setMaxVersions();
+    scan.setBatch(2);
+    scanner = ht.getScanner(scan);
+
+    // First batch: c4:4, c5:5
+    kvListExp = new ArrayList<KeyValue>();
+    kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[4], 4, VALUE));
+    kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[5], 5, VALUE));
+    result = scanner.next();
+    verifyResult(result, kvListExp, toLog, "Testing first batch of scan");
+
+    // Second batch: c6:6, c7:7
+    kvListExp = new ArrayList<KeyValue>();
+    kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[6], 6, VALUE));
+    kvListExp.add(new KeyValue(ROW, FAMILY, QUALIFIERS[7], 7, VALUE));
+    result = scanner.next();
+    verifyResult(result, kvListExp, toLog, "Testing second batch of scan");
+
+  }
+
+  private void verifyResult(Result result, List<KeyValue> kvList, boolean toLog, String
msg) {
+    int i =0;
+
+    LOG.info(msg);
+    LOG.info("Exp cnt: " + kvList.size());
+    LOG.info("True cnt is: " + result.size());
+    assertEquals(kvList.size(), result.size());
+
+    for (KeyValue kv : result.sorted()) {
+      KeyValue kvExp = kvList.get(i++);
+      if (toLog) {
+	LOG.info("get kv is: " + kv.toString());
+	LOG.info("exp kv is: " + kvExp.toString());
+      }
+      assertTrue("Not equal", kvExp.equals(kv));
+    }
+
+  }
+
+  private byte [][] makeNAscii(byte [] base, int n) {
+    byte [][] ret = new byte[n][];
+    for(int i=0;i<n;i++) {
+      byte [] tail = Bytes.toBytes(Integer.toString(i));
+      ret[i] = Bytes.add(base, tail);
+    }
+    return ret;
+  }
+
+}

Modified: hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java?rev=1181548&r1=1181547&r2=1181548&view=diff
==============================================================================
--- hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java
(original)
+++ hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java
Tue Oct 11 02:18:51 2011
@@ -22,6 +22,7 @@ package org.apache.hadoop.hbase.regionse
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Random;
 
@@ -51,17 +52,17 @@ public class TestWideScanner extends HBa
     new HTableDescriptor("testwidescan");
   static {
     TESTTABLEDESC.addFamily(new HColumnDescriptor(A,
-      10,  // Ten is arbitrary number.  Keep versions to help debuggging.
+      100,  // Keep versions to help debuggging.
       Compression.Algorithm.NONE.getName(), false, true, 8 * 1024,
       HConstants.FOREVER, StoreFile.BloomType.NONE.toString(),
       HColumnDescriptor.DEFAULT_REPLICATION_SCOPE));
     TESTTABLEDESC.addFamily(new HColumnDescriptor(B,
-      10,  // Ten is arbitrary number.  Keep versions to help debuggging.
+      100,  // Keep versions to help debuggging.
       Compression.Algorithm.NONE.getName(), false, true, 8 * 1024,
       HConstants.FOREVER, StoreFile.BloomType.NONE.toString(),
       HColumnDescriptor.DEFAULT_REPLICATION_SCOPE));
     TESTTABLEDESC.addFamily(new HColumnDescriptor(C,
-      10,  // Ten is arbitrary number.  Keep versions to help debuggging.
+      100,  // Keep versions to help debuggging.
       Compression.Algorithm.NONE.getName(), false, true, 8 * 1024,
       HConstants.FOREVER, StoreFile.BloomType.NONE.toString(),
       HColumnDescriptor.DEFAULT_REPLICATION_SCOPE));
@@ -88,19 +89,23 @@ public class TestWideScanner extends HBa
     int count = 0;
     for (char c = 'a'; c <= 'c'; c++) {
       byte[] row = Bytes.toBytes("ab" + c);
-      int i;
-      for (i = 0; i < 2500; i++) {
+      int i, j;
+      long ts = System.currentTimeMillis();
+      for (i = 0; i < 100; i++) {
         byte[] b = Bytes.toBytes(String.format("%10d", i));
-        Put put = new Put(row);
-        put.add(COLUMNS[rng.nextInt(COLUMNS.length)], b, b);
-        region.put(put);
-        count++;
+        for (j = 0; j < 100; j++) {
+            Put put = new Put(row);
+            put.add(COLUMNS[rng.nextInt(COLUMNS.length)], b, ++ts, b);
+            region.put(put);
+            count++;
+        }
       }
     }
     return count;
   }
 
   public void testWideScanBatching() throws IOException {
+    final int batch = 256;
     try {
       this.r = createNewHRegion(REGION_INFO.getTableDesc(), null, null);
       int inserted = addWideContent(this.r);
@@ -109,7 +114,8 @@ public class TestWideScanner extends HBa
       scan.addFamily(A);
       scan.addFamily(B);
       scan.addFamily(C);
-      scan.setBatch(1000);
+      scan.setMaxVersions(100);
+      scan.setBatch(batch);
       InternalScanner s = r.getScanner(scan);
       int total = 0;
       int i = 0;
@@ -120,7 +126,7 @@ public class TestWideScanner extends HBa
         LOG.info("iteration #" + i + ", results.size=" + results.size());
 
         // assert that the result set is no larger than 1000
-        assertTrue(results.size() <= 1000);
+        assertTrue(results.size() <= batch);
 
         total += results.size();
 
@@ -133,11 +139,19 @@ public class TestWideScanner extends HBa
         }
 
         results.clear();
+
+        // trigger ChangedReadersObservers
+        Iterator<KeyValueScanner> scanners =
+            ((HRegion.RegionScanner)s).storeHeap.getHeap().iterator();
+        while (scanners.hasNext()) {
+            StoreScanner ss = (StoreScanner)scanners.next();
+            ss.updateReaders();
+        }
       } while (more);
 
       // assert that the scanner returned all values
       LOG.info("inserted " + inserted + ", scanned " + total);
-      assertTrue(total == inserted);
+      assertEquals(total, inserted);
 
       s.close();
     } finally {



Mime
View raw message