kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject [1/2] incubator-kylin git commit: KYLIN-942 Code review
Date Mon, 23 Nov 2015 09:25:04 GMT
Repository: incubator-kylin
Updated Branches:
  refs/heads/2.x-staging b52c1bba8 -> e62e0b3ef


http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e62e0b3e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
index 4ced852..0c8c3bd 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
@@ -37,7 +37,6 @@ import com.google.common.collect.Lists;
 import com.google.common.collect.Range;
 import com.google.common.collect.Sets;
 
-@SuppressWarnings("unused")
 public class CubeStorageQuery implements ICachableStorageQuery {
 
     private static final Logger logger = LoggerFactory.getLogger(CubeStorageQuery.class);
@@ -46,30 +45,18 @@ public class CubeStorageQuery implements ICachableStorageQuery {
 
     private final CubeInstance cubeInstance;
     private final CubeDesc cubeDesc;
-    private Collection<TblColRef> topNColumns;
 
     public CubeStorageQuery(CubeInstance cube) {
         this.cubeInstance = cube;
         this.cubeDesc = cube.getDescriptor();
-        this.topNColumns = Lists.newArrayList();
-        for (MeasureDesc measureDesc : cubeDesc.getMeasures()) {
-            if (measureDesc.getFunction().isTopN()) {
-                List<TblColRef> colRefs = measureDesc.getFunction().getParameter().getColRefs();
-                topNColumns.add(colRefs.get(colRefs.size() - 1));
-            }
-        }
     }
 
     @Override
     public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo)
{
-        // check whether this is a TopN query;
-        checkAndRewriteTopN(context, sqlDigest, returnTupleInfo);
+        // check whether this is a TopN query
+        checkAndRewriteTopN(sqlDigest);
 
         Collection<TblColRef> groups = sqlDigest.groupbyColumns;
-        TblColRef topNCol = extractTopNCol(groups);
-        if (topNCol != null)
-            groups.remove(topNCol);
-
         TupleFilter filter = sqlDigest.filter;
 
         // build dimension & metrics
@@ -125,10 +112,22 @@ public class CubeStorageQuery implements ICachableStorageQuery {
         if (scanners.isEmpty())
             return ITupleIterator.EMPTY_TUPLE_ITERATOR;
 
+        return newSequentialCubeTupleIterator(scanners, cuboid, dimensionsD, metrics, returnTupleInfo,
context);
+    }
+
+    private ITupleIterator newSequentialCubeTupleIterator(List<CubeSegmentScanner>
scanners, Cuboid cuboid, Set<TblColRef> dimensionsD, Set<FunctionDesc> metrics,
TupleInfo returnTupleInfo, StorageContext context) {
+        TblColRef topNCol = null;
+        for (FunctionDesc func : metrics) {
+            if (func.isTopN()) {
+                topNCol = func.getTopNLiteralColumn();
+                break;
+            }
+        }
+
         if (topNCol != null)
             return new SequentialCubeTopNTupleIterator(scanners, cuboid, dimensionsD, topNCol,
metrics, returnTupleInfo, context);
-        
-        return new SequentialCubeTupleIterator(scanners, cuboid, dimensionsD, metrics, returnTupleInfo,
context);
+        else
+            return new SequentialCubeTupleIterator(scanners, cuboid, dimensionsD, metrics,
returnTupleInfo, context);
     }
 
     private void buildDimensionsAndMetrics(SQLDigest sqlDigest, Collection<TblColRef>
dimensions, Collection<FunctionDesc> metrics) {
@@ -144,11 +143,7 @@ public class CubeStorageQuery implements ICachableStorageQuery {
             if (sqlDigest.metricColumns.contains(column)) {
                 continue;
             }
-            
-            // skip topN display col
-            if (topNColumns.contains(column)) {
-                continue;
-            }
+
             dimensions.add(column);
         }
     }
@@ -398,48 +393,33 @@ public class CubeStorageQuery implements ICachableStorageQuery {
         return false;
     }
 
+    private void checkAndRewriteTopN(SQLDigest sqlDigest) {
+        FunctionDesc topnFunc = null;
+        TblColRef topnLiteralCol = null;
+        for (MeasureDesc measure : cubeDesc.getMeasures()) {
+            FunctionDesc func = measure.getFunction();
+            if (func.isTopN() && sqlDigest.groupbyColumns.contains(func.getTopNLiteralColumn()))
{
+                topnFunc = func;
+                topnLiteralCol = func.getTopNLiteralColumn();
+            }
+        }
 
-    private void checkAndRewriteTopN(StorageContext context, SQLDigest sqlDigest, TupleInfo
returnTupleInfo) {
-        Collection<TblColRef> groups = sqlDigest.groupbyColumns;
-        TblColRef topNDisplayCol = extractTopNCol(groups);
-        boolean hasTopN = topNDisplayCol != null;
-
-        if (hasTopN == false)
+        // if TopN is not involved
+        if (topnFunc == null)
             return;
 
         if (sqlDigest.aggregations.size() != 1) {
             throw new IllegalStateException("When query with topN, only one metrics is allowed.");
         }
 
-        FunctionDesc functionDesc = sqlDigest.aggregations.iterator().next();
-        if (functionDesc.isSum() == false) {
+        FunctionDesc origFunc = sqlDigest.aggregations.iterator().next();
+        if (origFunc.isSum() == false) {
             throw new IllegalStateException("When query with topN, only SUM function is allowed.");
         }
 
-        FunctionDesc rewriteFunction = null;
-        // replace the SUM to the TopN function
-        for (MeasureDesc measureDesc : cubeDesc.getMeasures()) {
-            if (measureDesc.getFunction().isCompatible(functionDesc) && topNDisplayCol.getName().equalsIgnoreCase(measureDesc.getFunction().getParameter().getDisplayColumn()))
{
-                rewriteFunction = measureDesc.getFunction();
-                break;
-            }
-        }
-
-        if (rewriteFunction == null) {
-            throw new IllegalStateException("Didn't find topN measure for " + functionDesc);
-        }
-
-        sqlDigest.aggregations = Lists.newArrayList(rewriteFunction);
-        logger.info("Rewrite function " + functionDesc + " to " + rewriteFunction);
-    }
-
-    private TblColRef extractTopNCol(Collection<TblColRef> colRefs) {
-        for (TblColRef colRef : colRefs) {
-            if (topNColumns.contains(colRef)) {
-                return colRef;
-            }
-        }
-
-        return null;
+        sqlDigest.aggregations = Lists.newArrayList(topnFunc);
+        sqlDigest.groupbyColumns.remove(topnLiteralCol);
+        sqlDigest.metricColumns.add(topnLiteralCol);
+        logger.info("Rewrite function " + origFunc + " to " + topnFunc);
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e62e0b3e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
index 99db123..2fd0b4f 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/ii/coprocessor/endpoint/EndpointTupleIterator.java
@@ -29,7 +29,6 @@ import java.util.Map;
 
 import javax.annotation.Nullable;
 
-import com.google.protobuf.ByteString;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.SerializationUtils;
 import org.apache.hadoop.hbase.client.HConnection;

http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/e62e0b3e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
index 7c1a19f..50c2fac 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/RowValueDecoder.java
@@ -140,4 +140,16 @@ public class RowValueDecoder implements Cloneable {
         return false;
     }
 
+    public static MeasureDesc findTopN(Collection<RowValueDecoder> rowValueDecoders)
{
+        for (RowValueDecoder decoder : rowValueDecoders) {
+            for (int i = decoder.projectionIndex.nextSetBit(0); i >= 0; i = decoder.projectionIndex.nextSetBit(i
+ 1)) {
+                MeasureDesc measure = decoder.measures[i];
+                FunctionDesc func = measure.getFunction();
+                if (func.isTopN())
+                    return measure;
+            }
+        }
+        return null;
+    }
+
 }


Mime
View raw message