kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [13/50] [abbrv] incubator-kylin git commit: KYLIN-625, refactor aggregate scanner to decoration pattern
Date Sat, 21 Mar 2015 00:27:18 GMT
KYLIN-625, refactor aggregate scanner to decoration pattern


Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/73ad4d4b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/73ad4d4b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/73ad4d4b

Branch: refs/heads/streaming-localdict
Commit: 73ad4d4be50016c18cc80124199c2e97d3810a63
Parents: 5925320
Author: Li, Yang <yangli9@ebay.com>
Authored: Thu Mar 19 16:48:38 2015 +0800
Committer: Li, Yang <yangli9@ebay.com>
Committed: Thu Mar 19 16:48:38 2015 +0800

----------------------------------------------------------------------
 .../storage/gridtable/GTAggregateScanner.java     | 18 +++++++++---------
 .../apache/kylin/storage/gridtable/GTInfo.java    |  2 +-
 .../kylin/storage/gridtable/GTRawScanner.java     |  5 +++++
 .../apache/kylin/storage/gridtable/GridTable.java | 12 ++++++++----
 .../kylin/storage/gridtable/IGTScanner.java       |  2 ++
 5 files changed, 25 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/73ad4d4b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTAggregateScanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTAggregateScanner.java
b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTAggregateScanner.java
index 6cc82b4..ed38591 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTAggregateScanner.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTAggregateScanner.java
@@ -7,7 +7,6 @@ import java.util.Iterator;
 import java.util.Map.Entry;
 import java.util.SortedMap;
 
-import org.apache.kylin.metadata.filter.TupleFilter;
 import org.apache.kylin.metadata.measure.MeasureAggregator;
 
 import com.google.common.collect.Maps;
@@ -18,26 +17,27 @@ public class GTAggregateScanner implements IGTScanner {
     final BitSet dimensions;
     final BitSet metrics;
     final String[] metricsAggrFuncs;
-    final GTRawScanner rawScanner;
+    final IGTScanner rawScanner;
 
-    GTAggregateScanner(GTInfo info, IGTStore store, GTRecord pkStart, GTRecord pkEndExclusive,
BitSet dimensions, BitSet metrics, String[] metricsAggrFuncs, TupleFilter filterPushDown)
{
+    GTAggregateScanner(IGTScanner rawScanner, BitSet dimensions, BitSet metrics, String[]
metricsAggrFuncs) {
         if (dimensions.intersects(metrics))
             throw new IllegalStateException();
         if (metrics.cardinality() != metricsAggrFuncs.length)
             throw new IllegalStateException();
 
-        this.info = info;
+        this.info = rawScanner.getInfo();
         this.dimensions = dimensions;
         this.metrics = metrics;
         this.metricsAggrFuncs = metricsAggrFuncs;
-        
-        BitSet columns = new BitSet();
-        columns.or(dimensions);
-        columns.or(metrics);
-        this.rawScanner = new GTRawScanner(info, store, pkStart, pkEndExclusive, columns,
filterPushDown);
+        this.rawScanner = rawScanner;
     }
 
     @Override
+    public GTInfo getInfo() {
+        return info;
+    }
+    
+    @Override
     public int getScannedRowCount() {
         return rawScanner.getScannedRowCount();
     }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/73ad4d4b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
index 97cac87..04e720c 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTInfo.java
@@ -25,7 +25,7 @@ public class GTInfo {
     transient TblColRef[] colRefs;
 
     // grid info
-    BitSet primaryKey; // columns sorted and unique
+    BitSet primaryKey; // order by, uniqueness is not required
     BitSet[] colBlocks; // at least one column block
     BitSet colBlocksAll;
     int rowBlockSize; // 0: disable row block

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/73ad4d4b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
index 978e3cf..737cd41 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GTRawScanner.java
@@ -44,6 +44,11 @@ class GTRawScanner implements IGTScanner {
     }
 
     @Override
+    public GTInfo getInfo() {
+        return info;
+    }
+    
+    @Override
     public int getScannedRowCount() {
         return scannedRowCount;
     }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/73ad4d4b/storage/src/main/java/org/apache/kylin/storage/gridtable/GridTable.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/GridTable.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/GridTable.java
index d0304bc..2b49617 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/GridTable.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/GridTable.java
@@ -24,12 +24,12 @@ public class GridTable {
         assert shard < info.nShards;
         return new GTBuilder(info, shard, store);
     }
-    
+
     public GTBuilder append() {
         assert info.isShardingEnabled() == false;
         return append(-1);
     }
-    
+
     public GTBuilder append(int shard) {
         return new GTBuilder(info, shard, store, true);
     }
@@ -40,13 +40,17 @@ public class GridTable {
 
     public IGTScanner scanAndAggregate(GTRecord pkStart, GTRecord pkEndExclusive, BitSet
dimensions, //
             BitSet metrics, String[] metricsAggrFuncs, TupleFilter filterPushDown) throws
IOException {
-        return new GTAggregateScanner(info, store, pkStart, pkEndExclusive, dimensions, metrics,
metricsAggrFuncs, filterPushDown);
+        BitSet columns = new BitSet();
+        columns.or(dimensions);
+        columns.or(metrics);
+        IGTScanner rawScanner = scan(pkStart, pkEndExclusive, columns, filterPushDown);
+        return new GTAggregateScanner(rawScanner, dimensions, metrics, metricsAggrFuncs);
     }
 
     public GTInfo getInfo() {
         return info;
     }
-    
+
     public IGTStore getStore() {
         return store;
     }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/73ad4d4b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTScanner.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTScanner.java b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTScanner.java
index 8a8c79b..285a301 100644
--- a/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTScanner.java
+++ b/storage/src/main/java/org/apache/kylin/storage/gridtable/IGTScanner.java
@@ -4,6 +4,8 @@ import java.io.Closeable;
 
 public interface IGTScanner extends Iterable<GTRecord>, Closeable {
     
+    public GTInfo getInfo();
+    
     public int getScannedRowCount();
     
     public int getScannedRowBlockCount();


Mime
View raw message