hbase-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From apurt...@apache.org
Subject svn commit: r951682 - in /hbase/trunk/src: main/java/org/apache/hadoop/hbase/regionserver/HRegion.java main/resources/org/apache/hadoop/hbase/rest/XMLSchema.xsd test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java
Date Sat, 05 Jun 2010 08:54:33 GMT
Author: apurtell
Date: Sat Jun  5 08:54:32 2010
New Revision: 951682

URL: http://svn.apache.org/viewvc?rev=951682&view=rev
Log:
HBASE-1537 Intra-row scanning; apply limit over multiple families

Added:
    hbase/trunk/src/main/resources/org/apache/hadoop/hbase/rest/XMLSchema.xsd
Modified:
    hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
    hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java

Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=951682&r1=951681&r2=951682&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Sat Jun  5
08:54:32 2010
@@ -2065,7 +2065,7 @@ public class HRegion implements HConstan
         } else {
           byte [] nextRow;
           do {
-            this.storeHeap.next(results, limit);
+            this.storeHeap.next(results, limit - results.size());
             if (limit > 0 && results.size() == limit) {
               if (this.filter != null && filter.hasFilterRow()) throw new IncompatibleFilterException(
                   "Filter with filterRow(List<KeyValue>) incompatible with scan with
limit!");

Added: hbase/trunk/src/main/resources/org/apache/hadoop/hbase/rest/XMLSchema.xsd
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/resources/org/apache/hadoop/hbase/rest/XMLSchema.xsd?rev=951682&view=auto
==============================================================================
--- hbase/trunk/src/main/resources/org/apache/hadoop/hbase/rest/XMLSchema.xsd (added)
+++ hbase/trunk/src/main/resources/org/apache/hadoop/hbase/rest/XMLSchema.xsd Sat Jun  5 08:54:32
2010
@@ -0,0 +1,152 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<schema targetNamespace="ModelSchema" elementFormDefault="qualified" xmlns="http://www.w3.org/2001/XMLSchema"
xmlns:tns="ModelSchema">
+
+    <element name="Version" type="tns:Version"></element>
+    
+    <complexType name="Version">
+      <attribute name="REST" type="string"></attribute>
+      <attribute name="JVM" type="string"></attribute>
+      <attribute name="OS" type="string"></attribute>
+      <attribute name="Server" type="string"></attribute>
+      <attribute name="Jersey" type="string"></attribute>
+    </complexType>
+
+    <element name="TableList" type="tns:TableList"></element>
+    
+    <complexType name="TableList">
+        <sequence>
+            <element name="table" type="tns:Table" maxOccurs="unbounded" minOccurs="1"></element>
+        </sequence>
+    </complexType>
+
+    <complexType name="Table">
+        <sequence>
+            <element name="name" type="string"></element>
+        </sequence>
+    </complexType>
+
+    <element name="TableInfo" type="tns:TableInfo"></element>
+    
+    <complexType name="TableInfo">
+        <sequence>
+            <element name="region" type="tns:TableRegion" maxOccurs="unbounded" minOccurs="1"></element>
+        </sequence>
+        <attribute name="name" type="string"></attribute>
+    </complexType>
+
+    <complexType name="TableRegion">
+        <attribute name="name" type="string"></attribute>
+        <attribute name="id" type="int"></attribute>
+        <attribute name="startKey" type="base64Binary"></attribute>
+        <attribute name="endKey" type="base64Binary"></attribute>
+        <attribute name="location" type="string"></attribute>
+    </complexType>
+
+    <element name="TableSchema" type="tns:TableSchema"></element>
+    
+    <complexType name="TableSchema">
+        <sequence>
+            <element name="column" type="tns:ColumnSchema" maxOccurs="unbounded" minOccurs="1"></element>
+        </sequence>
+        <attribute name="name" type="string"></attribute>
+        <anyAttribute></anyAttribute>
+    </complexType>
+
+    <complexType name="ColumnSchema">
+        <attribute name="name" type="string"></attribute>
+        <anyAttribute></anyAttribute>
+    </complexType>
+
+    <element name="CellSet" type="tns:CellSet"></element>
+
+    <complexType name="CellSet">
+        <sequence>
+            <element name="row" type="tns:Row" maxOccurs="unbounded" minOccurs="1"></element>
+        </sequence>
+    </complexType>
+
+    <element name="Row" type="tns:Row"></element>
+
+    <complexType name="Row">
+        <sequence>
+            <element name="key" type="base64Binary"></element>
+            <element name="cell" type="tns:Cell" maxOccurs="unbounded" minOccurs="1"></element>
+        </sequence>
+    </complexType>
+ 
+    <element name="Cell" type="tns:Cell"></element>
+
+    <complexType name="Cell">
+        <sequence>
+            <element name="value" maxOccurs="1" minOccurs="1">
+                <simpleType><restriction base="base64Binary">
+                </simpleType>
+            </element>
+        </sequence>
+        <attribute name="column" type="base64Binary" />
+        <attribute name="timestamp" type="int" />
+    </complexType>
+
+    <element name="Scanner" type="tns:Scanner"></element>
+    
+    <complexType name="Scanner">
+        <sequence>
+            <element name="column" type="base64Binary" minOccurs="0" maxOccurs="unbounded"></element>
+        </sequence>
+        <sequence>
+            <element name="filter" type="string" minOccurs="0" maxOccurs="1"></element>
+        </sequence>
+        <attribute name="startRow" type="base64Binary"></attribute>
+        <attribute name="endRow" type="base64Binary"></attribute>
+        <attribute name="batch" type="int"></attribute>
+        <attribute name="startTime" type="int"></attribute>
+        <attribute name="endTime" type="int"></attribute>
+    </complexType>
+
+    <element name="StorageClusterVersion" type="tns:StorageClusterVersion" />
+
+    <complexType name="StorageClusterVersion">
+        <attribute name="version" type="string"></attribute>
+    </complexType>
+
+    <element name="StorageClusterStatus"
+        type="tns:StorageClusterStatus">
+    </element>
+    
+    <complexType name="StorageClusterStatus">
+        <sequence>
+            <element name="liveNode" type="tns:Node"
+                maxOccurs="unbounded" minOccurs="0">
+            </element>
+            <element name="deadNode" type="string" maxOccurs="unbounded"
+                minOccurs="0">
+            </element>
+        </sequence>
+        <attribute name="regions" type="int"></attribute>
+        <attribute name="requests" type="int"></attribute>
+        <attribute name="averageLoad" type="float"></attribute>
+    </complexType>
+
+    <complexType name="Node">
+        <sequence>
+            <element name="region" type="tns:Region"
+                maxOccurs="unbounded" minOccurs="0">
+            </element>
+        </sequence>
+        <attribute name="name" type="string"></attribute>
+        <attribute name="startCode" type="int"></attribute>
+        <attribute name="requests" type="int"></attribute>
+        <attribute name="heapSizeMB" type="int"></attribute>
+        <attribute name="maxHeapSizeMB" type="int"></attribute>
+    </complexType>
+
+    <complexType name="Region">
+        <attribute name="name" type="base64Binary"></attribute>
+        <attribute name="stores" type="int"></attribute>
+        <attribute name="storefiles" type="int"></attribute>
+        <attribute name="storefileSizeMB" type="int"></attribute>
+        <attribute name="memstoreSizeMB" type="int"></attribute>
+        <attribute name="storefileIndexSizeMB" type="int"></attribute>
+    </complexType>
+
+</schema>
\ No newline at end of file

Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java?rev=951682&r1=951681&r2=951682&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/regionserver/TestWideScanner.java Sat
Jun  5 08:54:32 2010
@@ -23,6 +23,7 @@ package org.apache.hadoop.hbase.regionse
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Random;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -41,25 +42,39 @@ import org.apache.hadoop.hdfs.MiniDFSClu
 public class TestWideScanner extends HBaseTestCase {
   private final Log LOG = LogFactory.getLog(this.getClass());
 
-  static final int BATCH = 1000;
-
-  private MiniDFSCluster cluster = null;
-  private HRegion r;
-
+  static final byte[] A = Bytes.toBytes("A");
+  static final byte[] B = Bytes.toBytes("B");
+  static final byte[] C = Bytes.toBytes("C");
+  static byte[][] COLUMNS = { A, B, C };
+  static final Random rng = new Random();
   static final HTableDescriptor TESTTABLEDESC =
     new HTableDescriptor("testwidescan");
   static {
-    TESTTABLEDESC.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY,
+    TESTTABLEDESC.addFamily(new HColumnDescriptor(A,
+      10,  // Ten is arbitrary number.  Keep versions to help debuggging.
+      Compression.Algorithm.NONE.getName(), false, true, 8 * 1024,
+      HConstants.FOREVER, StoreFile.BloomType.NONE.toString(),
+      HColumnDescriptor.DEFAULT_REPLICATION_SCOPE));
+    TESTTABLEDESC.addFamily(new HColumnDescriptor(B,
+      10,  // Ten is arbitrary number.  Keep versions to help debuggging.
+      Compression.Algorithm.NONE.getName(), false, true, 8 * 1024,
+      HConstants.FOREVER, StoreFile.BloomType.NONE.toString(),
+      HColumnDescriptor.DEFAULT_REPLICATION_SCOPE));
+    TESTTABLEDESC.addFamily(new HColumnDescriptor(C,
       10,  // Ten is arbitrary number.  Keep versions to help debuggging.
       Compression.Algorithm.NONE.getName(), false, true, 8 * 1024,
       HConstants.FOREVER, StoreFile.BloomType.NONE.toString(),
       HColumnDescriptor.DEFAULT_REPLICATION_SCOPE));
   }
+
   /** HRegionInfo for root region */
   public static final HRegionInfo REGION_INFO =
     new HRegionInfo(TESTTABLEDESC, HConstants.EMPTY_BYTE_ARRAY,
     HConstants.EMPTY_BYTE_ARRAY);
 
+  MiniDFSCluster cluster = null;
+  HRegion r;
+
   @Override
   public void setUp() throws Exception {
     cluster = new MiniDFSCluster(conf, 2, true, (String[])null);
@@ -69,30 +84,15 @@ public class TestWideScanner extends HBa
     super.setUp();
   }
 
-  private int addWideContent(HRegion region, byte[] family)
-      throws IOException {
+  private int addWideContent(HRegion region) throws IOException {
     int count = 0;
-    // add a few rows of 2500 columns (we'll use batch of 1000) to make things
-    // interesting
     for (char c = 'a'; c <= 'c'; c++) {
       byte[] row = Bytes.toBytes("ab" + c);
       int i;
       for (i = 0; i < 2500; i++) {
         byte[] b = Bytes.toBytes(String.format("%10d", i));
         Put put = new Put(row);
-        put.add(family, b, b);
-        region.put(put);
-        count++;
-      }
-    }
-    // add one row of 100,000 columns
-    {
-      byte[] row = Bytes.toBytes("abf");
-      int i;
-      for (i = 0; i < 100000; i++) {
-        byte[] b = Bytes.toBytes(String.format("%10d", i));
-        Put put = new Put(row);
-        put.add(family, b, b);
+        put.add(COLUMNS[rng.nextInt(COLUMNS.length)], b, b);
         region.put(put);
         count++;
       }
@@ -103,11 +103,13 @@ public class TestWideScanner extends HBa
   public void testWideScanBatching() throws IOException {
     try {
       this.r = createNewHRegion(REGION_INFO.getTableDesc(), null, null);
-      int inserted = addWideContent(this.r, HConstants.CATALOG_FAMILY);
+      int inserted = addWideContent(this.r);
       List<KeyValue> results = new ArrayList<KeyValue>();
       Scan scan = new Scan();
-      scan.addFamily(HConstants.CATALOG_FAMILY);
-      scan.setBatch(BATCH);
+      scan.addFamily(A);
+      scan.addFamily(B);
+      scan.addFamily(C);
+      scan.setBatch(1000);
       InternalScanner s = r.getScanner(scan);
       int total = 0;
       int i = 0;
@@ -117,8 +119,8 @@ public class TestWideScanner extends HBa
         i++;
         LOG.info("iteration #" + i + ", results.size=" + results.size());
 
-        // assert that the result set is no larger than BATCH
-        assertTrue(results.size() <= BATCH);
+        // assert that the result set is no larger than 1000
+        assertTrue(results.size() <= 1000);
 
         total += results.size();
 



Mime
View raw message