hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mberto...@apache.org
Subject [04/28] hbase git commit: HBASE-13381 Expand TestSizeFailures to include small scans (Josh Elser)
Date Fri, 10 Apr 2015 07:55:25 GMT
HBASE-13381 Expand TestSizeFailures to include small scans (Josh Elser)


Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3cd929ee
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3cd929ee
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3cd929ee

Branch: refs/heads/hbase-12439
Commit: 3cd929eea253afd632b6def7c24df434872a2d7d
Parents: 66f7bf4
Author: tedyu <yuzhihong@gmail.com>
Authored: Thu Apr 9 17:57:57 2015 -0700
Committer: tedyu <yuzhihong@gmail.com>
Committed: Thu Apr 9 17:57:57 2015 -0700

----------------------------------------------------------------------
 .../hadoop/hbase/client/TestSizeFailures.java   | 204 ++++++++++---------
 1 file changed, 109 insertions(+), 95 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hbase/blob/3cd929ee/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSizeFailures.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSizeFailures.java
b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSizeFailures.java
index a77c50c..a0b0f70 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSizeFailures.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSizeFailures.java
@@ -20,12 +20,10 @@ package org.apache.hadoop.hbase.client;
 
 import static org.junit.Assert.assertEquals;
 
-import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.TreeSet;
+import java.util.Map.Entry;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -41,16 +39,17 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.experimental.categories.Category;
 
+import com.google.common.collect.Maps;
+
 @Category(LargeTests.class)
 public class TestSizeFailures {
-  final Log LOG = LogFactory.getLog(getClass());
+  static final Log LOG = LogFactory.getLog(TestSizeFailures.class);
   protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
   private static byte [] FAMILY = Bytes.toBytes("testFamily");
   protected static int SLAVES = 1;
+  private static TableName TABLENAME;
+  private static final int NUM_ROWS = 1000 * 1000, NUM_COLS = 10;
 
-  /**
-   * @throws java.lang.Exception
-   */
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
     // Uncomment the following lines if more verbosity is needed for
@@ -61,11 +60,49 @@ public class TestSizeFailures {
     Configuration conf = TEST_UTIL.getConfiguration();
     conf.setBoolean("hbase.table.sanity.checks", true); // ignore sanity checks in the server
     TEST_UTIL.startMiniCluster(SLAVES);
+
+    // Write a bunch of data
+    TABLENAME = TableName.valueOf("testSizeFailures");
+    List<byte[]> qualifiers = new ArrayList<>();
+    for (int i = 1; i <= 10; i++) {
+      qualifiers.add(Bytes.toBytes(Integer.toString(i)));
+    }
+
+    HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
+    HTableDescriptor desc = new HTableDescriptor(TABLENAME);
+    desc.addFamily(hcd);
+    byte[][] splits = new byte[9][2];
+    for (int i = 1; i < 10; i++) {
+      int split = 48 + i;
+      splits[i - 1][0] = (byte) (split >>> 8);
+      splits[i - 1][0] = (byte) (split);
+    }
+    TEST_UTIL.getHBaseAdmin().createTable(desc, splits);
+    Connection conn = TEST_UTIL.getConnection();
+
+    try (Table table = conn.getTable(TABLENAME)) {
+      List<Put> puts = new LinkedList<>();
+      for (int i = 0; i < NUM_ROWS; i++) {
+        Put p = new Put(Bytes.toBytes(Integer.toString(i)));
+        for (int j = 0; j < NUM_COLS; j++) {
+          byte[] value = new byte[50];
+          Bytes.random(value);
+          p.addColumn(FAMILY, Bytes.toBytes(Integer.toString(j)), value);
+        }
+        puts.add(p);
+
+        if (puts.size() == 1000) {
+          table.batch(puts, new Object[1000]);
+          puts.clear();
+        }
+      }
+
+      if (puts.size() > 0) {
+        table.batch(puts, new Object[puts.size()]);
+      }
+    }
   }
 
-  /**
-   * @throws java.lang.Exception
-   */
   @AfterClass
   public static void tearDownAfterClass() throws Exception {
     TEST_UTIL.shutdownMiniCluster();
@@ -74,89 +111,66 @@ public class TestSizeFailures {
   /**
    * Basic client side validation of HBASE-13262
    */
-   @Test
-   public void testScannerSeesAllRecords() throws Exception {
-     final int NUM_ROWS = 1000 * 1000, NUM_COLS = 10;
-     final TableName TABLENAME = TableName.valueOf("testScannerSeesAllRecords");
-     List<byte[]> qualifiers = new ArrayList<>();
-     for (int i = 1; i <= 10; i++) {
-       qualifiers.add(Bytes.toBytes(Integer.toString(i)));
-     }
-
-     HColumnDescriptor hcd = new HColumnDescriptor(FAMILY);
-     HTableDescriptor desc = new HTableDescriptor(TABLENAME);
-     desc.addFamily(hcd);
-     byte[][] splits = new byte[9][2];
-     for (int i = 1; i < 10; i++) {
-       int split = 48 + i;
-       splits[i - 1][0] = (byte) (split >>> 8);
-       splits[i - 1][0] = (byte) (split);
-     }
-     TEST_UTIL.getHBaseAdmin().createTable(desc, splits);
-     Connection conn = TEST_UTIL.getConnection();
-
-     try (Table table = conn.getTable(TABLENAME)) {
-       List<Put> puts = new LinkedList<>();
-       for (int i = 0; i < NUM_ROWS; i++) {
-         Put p = new Put(Bytes.toBytes(Integer.toString(i)));
-         for (int j = 0; j < NUM_COLS; j++) {
-           byte[] value = new byte[50];
-           Bytes.random(value);
-           p.addColumn(FAMILY, Bytes.toBytes(Integer.toString(j)), value);
-         }
-         puts.add(p);
-
-         if (puts.size() == 1000) {
-           Object[] results = new Object[1000];
-           try {
-             table.batch(puts, results);
-           } catch (IOException e) {
-             LOG.error("Failed to write data", e);
-             LOG.debug("Errors: " +  Arrays.toString(results));
-           }
-
-           puts.clear();
-         }
-       }
-
-       if (puts.size() > 0) {
-         Object[] results = new Object[puts.size()];
-         try {
-           table.batch(puts, results);
-         } catch (IOException e) {
-           LOG.error("Failed to write data", e);
-           LOG.debug("Errors: " +  Arrays.toString(results));
-         }
-       }
-
-       // Flush the memstore to disk
-       TEST_UTIL.getHBaseAdmin().flush(TABLENAME);
-
-       TreeSet<Integer> rows = new TreeSet<>();
-       long rowsObserved = 0l;
-       long entriesObserved = 0l;
-       Scan s = new Scan();
-       s.addFamily(FAMILY);
-       s.setMaxResultSize(-1);
-       s.setBatch(-1);
-       s.setCaching(500);
-       ResultScanner scanner = table.getScanner(s);
-       // Read all the records in the table
-       for (Result result : scanner) {
-         rowsObserved++;
-         String row = new String(result.getRow());
-         rows.add(Integer.parseInt(row));
-         while (result.advance()) {
-           entriesObserved++;
-           // result.current();
-         }
-       }
-
-       // Verify that we see 1M rows and 10M cells
-       assertEquals(NUM_ROWS, rowsObserved);
-       assertEquals(NUM_ROWS * NUM_COLS, entriesObserved);
-     }
-
-     conn.close();
-   }
+  @Test
+  public void testScannerSeesAllRecords() throws Exception {
+    Connection conn = TEST_UTIL.getConnection();
+    try (Table table = conn.getTable(TABLENAME)) {
+      Scan s = new Scan();
+      s.addFamily(FAMILY);
+      s.setMaxResultSize(-1);
+      s.setBatch(-1);
+      s.setCaching(500);
+      Entry<Long,Long> entry = sumTable(table.getScanner(s));
+      long rowsObserved = entry.getKey();
+      long entriesObserved = entry.getValue();
+
+      // Verify that we see 1M rows and 10M cells
+      assertEquals(NUM_ROWS, rowsObserved);
+      assertEquals(NUM_ROWS * NUM_COLS, entriesObserved);
+    }
+  }
+
+  /**
+   * Basic client side validation of HBASE-13262
+   */
+  @Test
+  public void testSmallScannerSeesAllRecords() throws Exception {
+    Connection conn = TEST_UTIL.getConnection();
+    try (Table table = conn.getTable(TABLENAME)) {
+      Scan s = new Scan();
+      s.setSmall(true);
+      s.addFamily(FAMILY);
+      s.setMaxResultSize(-1);
+      s.setBatch(-1);
+      s.setCaching(500);
+      Entry<Long,Long> entry = sumTable(table.getScanner(s));
+      long rowsObserved = entry.getKey();
+      long entriesObserved = entry.getValue();
+
+      // Verify that we see 1M rows and 10M cells
+      assertEquals(NUM_ROWS, rowsObserved);
+      assertEquals(NUM_ROWS * NUM_COLS, entriesObserved);
+    }
+  }
+
+  /**
+   * Count the number of rows and the number of entries from a scanner
+   *
+   * @param scanner
+   *          The Scanner
+   * @return An entry where the first item is rows observed and the second is entries observed.
+   */
+  private Entry<Long,Long> sumTable(ResultScanner scanner) {
+    long rowsObserved = 0l;
+    long entriesObserved = 0l;
+
+    // Read all the records in the table
+    for (Result result : scanner) {
+      rowsObserved++;
+      while (result.advance()) {
+        entriesObserved++;
+      }
+    }
+    return Maps.immutableEntry(rowsObserved,entriesObserved);
+  }
 }


Mime
View raw message