drill-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject drill git commit: DRILL-1917: Limit the number of results from HBasePStore.iterator() to MaxIteratorSize in BLOB_PERSISTENT mode
Date Thu, 01 Jan 2015 22:09:39 GMT
Repository: drill
Updated Branches:
  refs/heads/master 7b07e7bc0 -> 8afe10f20


DRILL-1917: Limit the number of results from HBasePStore.iterator() to MaxIteratorSize in
BLOB_PERSISTENT mode

*** Wishing everyone a Happy New Year ***


Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/8afe10f2
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/8afe10f2
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/8afe10f2

Branch: refs/heads/master
Commit: 8afe10f2037612c2fd9cf46fc20e81dd73ad6d24
Parents: 7b07e7b
Author: Aditya Kishore <adi@apache.org>
Authored: Thu Jan 1 00:00:00 2015 -0800
Committer: Aditya Kishore <adi@apache.org>
Committed: Thu Jan 1 00:00:00 2015 -0800

----------------------------------------------------------------------
 .../drill/exec/store/hbase/config/HBasePStore.java   | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/drill/blob/8afe10f2/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/config/HBasePStore.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/config/HBasePStore.java
b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/config/HBasePStore.java
index 59a3125..594b9f8 100644
--- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/config/HBasePStore.java
+++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/config/HBasePStore.java
@@ -28,6 +28,7 @@ import java.util.NoSuchElementException;
 import org.apache.drill.common.exceptions.DrillRuntimeException;
 import org.apache.drill.exec.store.sys.PStore;
 import org.apache.drill.exec.store.sys.PStoreConfig;
+import org.apache.drill.exec.store.sys.PStoreConfig.Mode;
 import org.apache.hadoop.hbase.client.Delete;
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HTableInterface;
@@ -147,12 +148,13 @@ public class HBasePStore<V> implements PStore<V> {
     private Result current = null;
     private Result last = null;
     private boolean done = false;
+    private int rowsRead = 0;
 
     Iter() {
       try {
         Scan scan = new Scan(tableNameStartKey, tableNameStopKey);
         scan.addColumn(FAMILY, QUALIFIER);
-        scan.setCaching(100);
+        scan.setCaching(config.getMaxIteratorSize() > 100 ? 100 : config.getMaxIteratorSize());
         scanner = table.getScanner(scan);
       } catch (IOException e) {
         throw new DrillRuntimeException("Caught error while creating HBase scanner for table:"
+ Bytes.toString(table.getTableName()), e);
@@ -161,16 +163,22 @@ public class HBasePStore<V> implements PStore<V> {
 
     @Override
     public boolean hasNext()  {
-      if (!done && current == null) {
+      if (config.getMode() == Mode.BLOB_PERSISTENT
+          && rowsRead >= config.getMaxIteratorSize()) {
+        done = true;
+      } else if (!done && current == null) {
         try {
           if ((current = scanner.next()) == null) {
             done = true;
-            scanner.close();
           }
         } catch (IOException e) {
           throw new DrillRuntimeException("Caught error while fetching rows from for table:"
+ Bytes.toString(table.getTableName()), e);
         }
       }
+
+      if (done && scanner != null) {
+        scanner.close();
+      }
       return (current != null);
     }
 
@@ -181,6 +189,7 @@ public class HBasePStore<V> implements PStore<V> {
       }
       last = current;
       current = null;
+      rowsRead++;
       return new DeferredEntry(last);
     }
 


Mime
View raw message