kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From shaofeng...@apache.org
Subject [2/3] kylin git commit: KYLIN-1478 support non-dic encoding
Date Mon, 06 Jun 2016 10:30:36 GMT
KYLIN-1478 support non-dic encoding

Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/9113a467
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/9113a467
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/9113a467

Branch: refs/heads/master
Commit: 9113a4675430a0baffb8d133ba29465c5cd70b4a
Parents: a490224
Author: shaofengshi <shaofengshi@apache.org>
Authored: Mon Jun 6 17:37:11 2016 +0800
Committer: shaofengshi <shaofengshi@apache.org>
Committed: Mon Jun 6 18:28:56 2016 +0800

----------------------------------------------------------------------
 .../apache/kylin/cube/model/RowKeyColDesc.java  |  11 +-
 .../metadata/measure/TopNMeasureTypeTest.java   |  62 ++++++
 .../kylin/dimension/DimensionEncoding.java      |  13 ++
 .../apache/kylin/dimension/IntegerDimEnc.java   |  16 ++
 .../kylin/measure/topn/TopNMeasureType.java     | 202 +++++++++++++------
 .../kylin/metadata/model/FunctionDesc.java      |  14 +-
 .../kylin/metadata/model/MeasureDesc.java       |   2 +
 ...t_kylin_cube_without_slr_left_join_desc.json |   3 +-
 .../test_kylin_inner_join_model_desc.json       |  17 +-
 .../test_kylin_left_join_model_desc.json        |  16 +-
 .../test_case_data/sandbox/kylin_hive_conf.xml  |  12 +-
 11 files changed, 261 insertions(+), 107 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
index 7e03355..e1be041 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
@@ -21,6 +21,7 @@ package org.apache.kylin.cube.model;
 import org.apache.commons.lang.StringUtils;
 import org.apache.kylin.common.util.StringUtil;
 import org.apache.kylin.dimension.DictionaryDimEnc;
+import org.apache.kylin.dimension.DimensionEncoding;
 import org.apache.kylin.dimension.DimensionEncodingFactory;
 import org.apache.kylin.metadata.model.TblColRef;
 
@@ -57,13 +58,9 @@ public class RowKeyColDesc {
     public void init() {
         Preconditions.checkState(StringUtils.isNotEmpty(this.encoding));
 
-        String[] parts = this.encoding.split("\\s*[(),:]\\s*");
-        if (parts == null || parts.length == 0 || parts[0].isEmpty())
-            throw new IllegalArgumentException("Not supported row key col encoding: '" +
this.encoding + "'");
-
-        this.encodingName = parts[0];
-        this.encodingArgs = parts[parts.length - 1].isEmpty() //
-                ? StringUtil.subArray(parts, 1, parts.length - 1) : StringUtil.subArray(parts,
1, parts.length);
+        Object[] encodingConf = DimensionEncoding.parseEncodingConf(this.encoding);
+        this.encodingName = (String) encodingConf[0];
+        this.encodingArgs = (String[])encodingConf[1];
 
         if (!DimensionEncodingFactory.isVaildEncoding(this.encodingName))
             throw new IllegalArgumentException("Not supported row key col encoding: '" +
this.encoding + "'");

http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java
b/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java
new file mode 100644
index 0000000..3d3e7d0
--- /dev/null
+++ b/core-cube/src/test/java/org/apache/kylin/metadata/measure/TopNMeasureTypeTest.java
@@ -0,0 +1,62 @@
+package org.apache.kylin.metadata.measure;
+
+import static org.junit.Assert.*;
+
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.apache.kylin.cube.CubeDescManager;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.measure.MeasureTypeFactory;
+import org.apache.kylin.measure.topn.TopNMeasureType;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.List;
+
+/**
+ * Created by shishaofeng on 6/6/16.
+ */
+public class TopNMeasureTypeTest extends LocalFileMetadataTestCase {
+
+
+    @Before
+    public void setup() {
+        this.createTestMetadata();
+
+    }
+
+    @After
+    public void clear() {
+        this.cleanupTestMetadata();
+    }
+
+    @Test
+    public void test() {
+
+        CubeDesc desc = CubeDescManager.getInstance(this.getTestConfig()).getCubeDesc("test_kylin_cube_without_slr_left_join_desc");
+
+        MeasureDesc topSellerMeasure = null;
+
+        for (MeasureDesc measureDesc : desc.getMeasures()) {
+            if (measureDesc.getName().equals("TOP_SELLER")) {
+                topSellerMeasure = measureDesc;
+                break;
+            }
+        }
+        TopNMeasureType measureType = (TopNMeasureType) MeasureTypeFactory.create(topSellerMeasure.getFunction().getExpression(),
topSellerMeasure.getFunction().getReturnDataType());
+
+
+        topSellerMeasure.getFunction().getConfiguration().clear();
+        List<TblColRef> colsNeedDict = measureType.getColumnsNeedDictionary(topSellerMeasure.getFunction());
+
+        assertTrue(colsNeedDict != null && colsNeedDict.size() == 1);
+
+        TblColRef sellerColRef = topSellerMeasure.getFunction().getParameter().getColRefs().get(1);
+        topSellerMeasure.getFunction().getConfiguration().put(TopNMeasureType.CONFIG_ENCODING_PREFIX
+ sellerColRef.getName(), "int:6");
+        colsNeedDict = measureType.getColumnsNeedDictionary(topSellerMeasure.getFunction());
+
+        assertTrue(colsNeedDict.size() == 0);
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncoding.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncoding.java
b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncoding.java
index 5c32114..87dfc08 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncoding.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncoding.java
@@ -20,6 +20,7 @@ package org.apache.kylin.dimension;
 
 import java.io.Externalizable;
 
+import org.apache.kylin.common.util.StringUtil;
 import org.apache.kylin.metadata.datatype.DataTypeSerializer;
 
 /**
@@ -49,6 +50,18 @@ public abstract class DimensionEncoding implements Externalizable {
         return true;
     }
 
+    public static Object[] parseEncodingConf(String encoding) {
+        String[] parts = encoding.split("\\s*[(),:]\\s*");
+        if (parts == null || parts.length == 0 || parts[0].isEmpty())
+            throw new IllegalArgumentException("Not supported row key col encoding: '" +
encoding + "'");
+
+        final String encodingName = parts[0];
+        final String[] encodingArgs = parts[parts.length - 1].isEmpty() //
+                ? StringUtil.subArray(parts, 1, parts.length - 1) : StringUtil.subArray(parts,
1, parts.length);
+
+        return new Object[] {encodingName, encodingArgs};
+    }
+
     /** return the fixed length of encoded bytes */
     abstract public int getLengthOfEncoding();
     

http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
index 9701447..3f01948 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
@@ -173,4 +173,20 @@ public class IntegerDimEnc extends DimensionEncoding {
         fixedLen = in.readShort();
     }
 
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+
+        IntegerDimEnc that = (IntegerDimEnc) o;
+
+        return fixedLen == that.fixedLen;
+
+    }
+
+    @Override
+    public int hashCode() {
+        return fixedLen;
+    }
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
index 009edef..0ff5b3a 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/TopNMeasureType.java
@@ -18,24 +18,21 @@
 
 package org.apache.kylin.measure.topn;
 
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.BytesUtil;
+import java.util.*;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.kylin.common.util.*;
 import org.apache.kylin.common.util.Dictionary;
+import org.apache.kylin.dimension.DictionaryDimEnc;
+import org.apache.kylin.dimension.DimensionEncoding;
+import org.apache.kylin.dimension.DimensionEncodingFactory;
 import org.apache.kylin.measure.MeasureAggregator;
 import org.apache.kylin.measure.MeasureIngester;
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.measure.MeasureTypeFactory;
 import org.apache.kylin.metadata.datatype.DataType;
 import org.apache.kylin.metadata.datatype.DataTypeSerializer;
-import org.apache.kylin.metadata.model.FunctionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.model.*;
 import org.apache.kylin.metadata.realization.CapabilityResult.CapabilityInfluence;
 import org.apache.kylin.metadata.realization.SQLDigest;
 import org.apache.kylin.metadata.tuple.Tuple;
@@ -111,21 +108,38 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>>
{
     @Override
     public MeasureIngester<TopNCounter<ByteArray>> newIngester() {
         return new MeasureIngester<TopNCounter<ByteArray>>() {
+
+            private DimensionEncoding[] dimensionEncodings = null;
+            private List<TblColRef> literalCols = null;
+            private int keyLength = 0;
+
+            private DimensionEncoding[] newDimensionEncodings = null;
+            private int newKeyLength = 0;
+            private boolean needReEncode = true;
+
             @Override
             public TopNCounter<ByteArray> valueOf(String[] values, MeasureDesc measureDesc,
Map<TblColRef, Dictionary<String>> dictionaryMap) {
-                if (values.length != 2)
-                    throw new IllegalArgumentException();
-
                 double counter = values[0] == null ? 0 : Double.parseDouble(values[0]);
-                String literal = values[1];
 
-                // encode literal using dictionary
-                TblColRef literalCol = getTopNLiteralColumn(measureDesc.getFunction());
-                Dictionary<String> dictionary = dictionaryMap.get(literalCol);
-                int keyEncodedValue = dictionary.getIdFromValue(literal);
+                if (dimensionEncodings == null) {
+                    literalCols = getTopNLiteralColumn(measureDesc.getFunction());
+                    dimensionEncodings = getDimensionEncodings(measureDesc.getFunction(),
literalCols, dictionaryMap);
+                    for (DimensionEncoding encoding : dimensionEncodings) {
+                        keyLength += encoding.getLengthOfEncoding();
+                    }
 
-                ByteArray key = new ByteArray(dictionary.getSizeOfId());
-                BytesUtil.writeUnsigned(keyEncodedValue, key.array(), 0, dictionary.getSizeOfId());
+                    if (values.length != (literalCols.size() + 1)) {
+                        throw new IllegalArgumentException();
+                    }
+                }
+
+                final ByteArray key = new ByteArray(keyLength);
+                int offset = 0;
+                for (int i=0; i<dimensionEncodings.length; i++) {
+                    byte[] valueBytes = Bytes.toBytes(values[i+1]);
+                    dimensionEncodings[i].encode(valueBytes, valueBytes.length, key.array(),
offset);
+                    offset += dimensionEncodings[i].getLengthOfEncoding();
+                }
 
                 TopNCounter<ByteArray> topNCounter = new TopNCounter<ByteArray>(dataType.getPrecision()
* TopNCounter.EXTRA_SPACE_RATE);
                 topNCounter.offer(key, counter);
@@ -136,30 +150,51 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>>
{
             public TopNCounter<ByteArray> reEncodeDictionary(TopNCounter<ByteArray>
value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDicts, Map<TblColRef,
Dictionary<String>> newDicts) {
                 TopNCounter<ByteArray> topNCounter = value;
 
-                TblColRef colRef = getTopNLiteralColumn(measureDesc.getFunction());
-                Dictionary<String> sourceDict = oldDicts.get(colRef);
-                Dictionary<String> mergedDict = newDicts.get(colRef);
+                if (newDimensionEncodings == null) {
+                    literalCols = getTopNLiteralColumn(measureDesc.getFunction());
+                    dimensionEncodings = getDimensionEncodings(measureDesc.getFunction(),
literalCols, oldDicts);
+                    keyLength = 0;
+                    boolean hasDictEncoding = false;
+                    for (DimensionEncoding encoding : dimensionEncodings) {
+                        keyLength += encoding.getLengthOfEncoding();
+                        if (encoding instanceof DictionaryDimEnc) {
+                            hasDictEncoding = true;
+                        }
+                    }
+
+                    newDimensionEncodings = getDimensionEncodings(measureDesc.getFunction(),
literalCols, newDicts);
+                    newKeyLength = 0;
+                    for (DimensionEncoding encoding : newDimensionEncodings) {
+                        newKeyLength += encoding.getLengthOfEncoding();
+                    }
+
+                    needReEncode = hasDictEncoding;
+                }
+
+                if (needReEncode == false) {
+                    // no need re-encode
+                    return topNCounter;
+                }
 
                 int topNSize = topNCounter.size();
-                byte[] newIdBuf = new byte[topNSize * mergedDict.getSizeOfId()];
-                byte[] literal = new byte[sourceDict.getSizeOfValue()];
+                byte[] newIdBuf = new byte[topNSize * newKeyLength];
 
                 int bufOffset = 0;
                 for (Counter<ByteArray> c : topNCounter) {
-                    int oldId = BytesUtil.readUnsigned(c.getItem().array(), c.getItem().offset(),
c.getItem().length());
-                    int newId;
-                    int size = sourceDict.getValueBytesFromId(oldId, literal, 0);
-                    if (size < 0) {
-                        newId = mergedDict.nullId();
-                    } else {
-                        newId = mergedDict.getIdFromValueBytes(literal, 0, size);
+                    int offset = c.getItem().offset();
+                    int innerBuffOffset = 0;
+                    for (int i = 0; i < dimensionEncodings.length; i++) {
+                        String dimValue = dimensionEncodings[i].decode(c.getItem().array(),
offset, dimensionEncodings[i].getLengthOfEncoding());
+                        byte[] dimValueBytes = Bytes.toBytes(dimValue);
+                        newDimensionEncodings[i].encode(dimValueBytes, dimValueBytes.length,
newIdBuf, bufOffset + innerBuffOffset);
+                        innerBuffOffset += newDimensionEncodings[i].getLengthOfEncoding();
+                        offset += dimensionEncodings[i].getLengthOfEncoding();
                     }
 
-                    BytesUtil.writeUnsigned(newId, newIdBuf, bufOffset, mergedDict.getSizeOfId());
-                    c.getItem().set(newIdBuf, bufOffset, mergedDict.getSizeOfId());
-                    bufOffset += mergedDict.getSizeOfId();
+                    c.getItem().set(newIdBuf, bufOffset, newKeyLength);
+                    bufOffset += newKeyLength;
                 }
-                return value;
+                return topNCounter;
             }
         };
     }
@@ -171,8 +206,18 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>>
{
 
     @Override
     public List<TblColRef> getColumnsNeedDictionary(FunctionDesc functionDesc) {
-        TblColRef literalCol = functionDesc.getParameter().getColRefs().get(1);
-        return Collections.singletonList(literalCol);
+        List <TblColRef> columnsNeedDict = Lists.newArrayList();
+        List <TblColRef> allCols = functionDesc.getParameter().getColRefs();
+        int start = (functionDesc.getParameter().isColumnType() == true) ? 1 : 0;
+        for (int i = start; i < allCols.size(); i++) {
+            TblColRef tblColRef = allCols.get(i);
+            String encoding = functionDesc.getConfiguration().get(CONFIG_ENCODING_PREFIX
+ tblColRef.getName());
+            if(StringUtils.isEmpty(encoding) || DictionaryDimEnc.ENCODING_NAME.equals(encoding))
{
+                columnsNeedDict.add(tblColRef);
+            }
+        }
+
+        return columnsNeedDict;
     }
 
     @Override
@@ -189,13 +234,13 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>>
{
         if (isTopNCompatibleSum(topN.getFunction(), onlyFunction) == false)
             return null;
 
-        TblColRef literalCol = getTopNLiteralColumn(topN.getFunction());
-        if (unmatchedDimensions.contains(literalCol) == false)
+        List<TblColRef> literalCol = getTopNLiteralColumn(topN.getFunction());
+        if (unmatchedDimensions.containsAll(literalCol) == false)
             return null;
-        if (digest.groupbyColumns.contains(literalCol) == false)
+        if (digest.groupbyColumns.containsAll(literalCol) == false)
             return null;
 
-        unmatchedDimensions.remove(literalCol);
+        unmatchedDimensions.removeAll(literalCol);
         unmatchedAggregations.remove(onlyFunction);
         return new CapabilityInfluence() {
             @Override
@@ -212,11 +257,16 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>>
{
         if (!isTopN(topN) || !sum.isSum())
             return false;
 
-        if (sum.getParameter().getColRefs().isEmpty())
-            return false;
+        TblColRef topnNumCol = getTopNNumericColumn(topN);
+
+        if (sum.getParameter().getColRefs().isEmpty()) {
+            if (topnNumCol == null && sum.getParameter().getValue().equals(topN.getParameter().getValue()))
+                return true;
+            else
+                return false;
+        }
 
         TblColRef sumCol = sum.getParameter().getColRefs().get(0);
-        TblColRef topnNumCol = getTopNNumericColumn(topN);
         return sumCol.equals(topnNumCol);
     }
 
@@ -233,9 +283,9 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>>
{
     @Override
     public void adjustSqlDigest(MeasureDesc measureDesc, SQLDigest sqlDigest) {
         FunctionDesc topnFunc = measureDesc.getFunction();
-        TblColRef topnLiteralCol = getTopNLiteralColumn(topnFunc);
+        List<TblColRef> topnLiteralCol = getTopNLiteralColumn(topnFunc);
 
-        if (sqlDigest.groupbyColumns.contains(topnLiteralCol) == false)
+        if (sqlDigest.groupbyColumns.containsAll(topnLiteralCol) == false)
             return;
 
         if (sqlDigest.aggregations.size() > 1) {
@@ -251,8 +301,8 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>>
{
         }
 
         sqlDigest.aggregations = Lists.newArrayList(topnFunc);
-        sqlDigest.groupbyColumns.remove(topnLiteralCol);
-        sqlDigest.metricColumns.add(topnLiteralCol);
+        sqlDigest.groupbyColumns.removeAll(topnLiteralCol);
+        sqlDigest.metricColumns.addAll(topnLiteralCol);
     }
 
     @Override
@@ -267,12 +317,17 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>>
{
 
     @Override
     public IAdvMeasureFiller getAdvancedTupleFiller(FunctionDesc function, TupleInfo tupleInfo,
Map<TblColRef, Dictionary<String>> dictionaryMap) {
-        final TblColRef literalCol = getTopNLiteralColumn(function);
+        final List<TblColRef> literalCols = getTopNLiteralColumn(function);
         final TblColRef numericCol = getTopNNumericColumn(function);
-        final Dictionary<String> topNColDict = dictionaryMap.get(literalCol);
-        final int literalTupleIdx = tupleInfo.hasColumn(literalCol) ? tupleInfo.getColumnIndex(literalCol)
: -1;
+        final int[] literalTupleIdx = new int[literalCols.size()];
+        final DimensionEncoding[] dimensionEncodings = getDimensionEncodings(function, literalCols,
dictionaryMap);
+        for (int i=0; i< literalCols.size(); i++) {
+            TblColRef colRef = literalCols.get(i);
+            literalTupleIdx[i] = tupleInfo.hasColumn(colRef) ? tupleInfo.getColumnIndex(colRef)
: -1;
+        }
+
         // for TopN, the aggr must be SUM, so the number fill into the column position (without
rewrite)
-        final int numericTupleIdx = tupleInfo.hasColumn(numericCol) ? tupleInfo.getColumnIndex(numericCol)
: -1;
+        final int numericTupleIdx = (numericCol != null && tupleInfo.hasColumn(numericCol))
? tupleInfo.getColumnIndex(numericCol) : -1;
         return new IAdvMeasureFiller() {
             private TopNCounter<ByteArray> topNCounter;
             private Iterator<Counter<ByteArray>> topNCounterIterator;
@@ -297,23 +352,50 @@ public class TopNMeasureType extends MeasureType<TopNCounter<ByteArray>>
{
                     throw new IllegalStateException();
 
                 Counter<ByteArray> counter = topNCounterIterator.next();
-                int key = BytesUtil.readUnsigned(counter.getItem().array(), counter.getItem().offset(),
counter.getItem().length());
-                String colValue = topNColDict.getValueFromId(key);
-                tuple.setDimensionValue(literalTupleIdx, colValue);
+                int offset = counter.getItem().offset();
+                for (int i = 0; i< dimensionEncodings.length; i++) {
+                    String colValue = dimensionEncodings[i].decode(counter.getItem().array(),
offset, dimensionEncodings[i].getLengthOfEncoding());
+                    tuple.setDimensionValue(literalTupleIdx[i], colValue);
+                    offset += dimensionEncodings[i].getLengthOfEncoding();
+                }
                 tuple.setMeasureValue(numericTupleIdx, counter.getCount());
             }
         };
     }
 
+    private static DimensionEncoding[] getDimensionEncodings(FunctionDesc function, List<TblColRef>
literalCols, Map<TblColRef, Dictionary<String>> dictionaryMap) {
+        final DimensionEncoding[] dimensionEncodings = new DimensionEncoding[literalCols.size()];
+        for (int i=0; i< literalCols.size(); i++) {
+            TblColRef colRef = literalCols.get(i);
+            String encoding = function.getConfiguration().get(TopNMeasureType.CONFIG_ENCODING_PREFIX
+ colRef.getName());
+            if(StringUtils.isEmpty(encoding) || DictionaryDimEnc.ENCODING_NAME.equals(encoding))
{
+                dimensionEncodings[i] = new DictionaryDimEnc(dictionaryMap.get(colRef));
+            } else {
+                Object[] encodingConf = DimensionEncoding.parseEncodingConf(encoding);
+                dimensionEncodings[i] = DimensionEncodingFactory.create((String) encodingConf[0],
(String[]) encodingConf[1]);
+            }
+        }
+
+        return dimensionEncodings;
+    }
+
     private TblColRef getTopNNumericColumn(FunctionDesc functionDesc) {
-        return functionDesc.getParameter().getColRefs().get(0);
+        if (functionDesc.getParameter().isColumnType() == true) {
+            return functionDesc.getParameter().getColRefs().get(0);
+        }
+        return null;
     }
 
-    private TblColRef getTopNLiteralColumn(FunctionDesc functionDesc) {
-        return functionDesc.getParameter().getColRefs().get(1);
+    private List<TblColRef> getTopNLiteralColumn(FunctionDesc functionDesc) {
+        List<TblColRef> allColumns = functionDesc.getParameter().getColRefs();
+        if (functionDesc.getParameter().isColumnType() == false) {
+            return allColumns;
+        }
+        return allColumns.subList(1, allColumns.size());
     }
 
     private boolean isTopN(FunctionDesc functionDesc) {
         return FUNC_TOP_N.equalsIgnoreCase(functionDesc.getExpression());
     }
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
index 0906d5f..9450555 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/FunctionDesc.java
@@ -20,6 +20,7 @@ package org.apache.kylin.metadata.model;
 
 import java.util.*;
 
+import com.fasterxml.jackson.annotation.JsonInclude;
 import org.apache.kylin.measure.MeasureType;
 import org.apache.kylin.measure.MeasureTypeFactory;
 import org.apache.kylin.measure.basic.BasicMeasureType;
@@ -61,8 +62,9 @@ public class FunctionDesc {
     @JsonProperty("returntype")
     private String returnType;
 
-    @JsonProperty("configurations")
-    private HashMap<String, String> configurations = new LinkedHashMap<String, String>();
+    @JsonProperty("configuration")
+    @JsonInclude(JsonInclude.Include.NON_NULL)
+    private HashMap<String, String> configuration = new LinkedHashMap<String, String>();
 
     private DataType returnDataType;
     private MeasureType<?> measureType;
@@ -265,12 +267,12 @@ public class FunctionDesc {
         return null;
     }
 
-    public HashMap<String, String> getConfigurations() {
-        return configurations;
+    public HashMap<String, String> getConfiguration() {
+        return configuration;
     }
 
-    public void setConfigurations(HashMap<String, String> configurations) {
-        this.configurations = configurations;
+    public void setConfiguration(HashMap<String, String> configurations) {
+        this.configuration = configurations;
     }
 
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
index c8ac1a9..dde791c 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/MeasureDesc.java
@@ -20,6 +20,7 @@ package org.apache.kylin.metadata.model;
 
 import com.fasterxml.jackson.annotation.JsonAutoDetect;
 import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.annotation.JsonProperty;
 
 /**
@@ -34,6 +35,7 @@ public class MeasureDesc {
     @JsonProperty("function")
     private FunctionDesc function;
     @JsonProperty("dependent_measure_ref")
+    @JsonInclude(JsonInclude.Include.NON_NULL)
     private String dependentMeasureRef;
 
     public String getName() {

http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
index a501bd1..ca1b35c 100644
--- a/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
+++ b/examples/test_case_data/localmeta/cube_desc/test_kylin_cube_without_slr_left_join_desc.json
@@ -156,7 +156,8 @@
           "next_parameter" : null
         }
       },
-      "returntype" : "topn(100)"
+      "returntype" : "topn(100)",
+      "configuration": {"topn.encoding.SELLER_ID" : "int:4"}
     },
     "dependent_measure_ref" : null
   }, {

http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
index a8fe363..f5333b0 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_inner_join_model_desc.json
@@ -38,7 +38,10 @@
         "lstg_format_name",
         "LSTG_SITE_ID",
         "SLR_SEGMENT_CD",
-        "seller_id"
+        "TRANS_ID",
+        "CAL_DT",
+        "LEAF_CATEG_ID",
+        "SELLER_ID"
       ]
     },
     {
@@ -76,20 +79,12 @@
         "cal_dt",
         "week_beg_dt"
       ]
-    },
-    {
-      "table": "default.test_kylin_fact",
-      "columns": [
-        "TRANS_ID",
-        "CAL_DT",
-        "LEAF_CATEG_ID",
-        "SELLER_ID"
-      ]
     }
   ],
   "metrics": [
   "PRICE",
-  "ITEM_COUNT"
+  "ITEM_COUNT",
+  "SELLER_ID"
   ],
   "last_modified" : 1422435345352,
   "fact_table" : "DEFAULT.TEST_KYLIN_FACT",

http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
----------------------------------------------------------------------
diff --git a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
index c85e5d2..8be6dc5 100644
--- a/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
+++ b/examples/test_case_data/localmeta/model_desc/test_kylin_left_join_model_desc.json
@@ -54,10 +54,12 @@
     {
       "table": "default.test_kylin_fact",
       "columns": [
+        "TRANS_ID",
+        "CAL_DT",
         "lstg_format_name",
         "LSTG_SITE_ID",
         "SLR_SEGMENT_CD",
-        "seller_id"
+        "SELLER_ID"
       ]
     },
     {
@@ -95,20 +97,12 @@
         "cal_dt",
         "week_beg_dt"
       ]
-    },
-    {
-      "table": "default.test_kylin_fact",
-      "columns": [
-        "TRANS_ID",
-        "CAL_DT",
-        "LEAF_CATEG_ID",
-        "SELLER_ID"
-      ]
     }
   ],
   "metrics": [
     "PRICE",
-    "ITEM_COUNT"
+    "ITEM_COUNT",
+    "SELLER_ID"
   ],
   "last_modified": 1422435345352,
   "fact_table": "DEFAULT.TEST_KYLIN_FACT",

http://git-wip-us.apache.org/repos/asf/kylin/blob/9113a467/examples/test_case_data/sandbox/kylin_hive_conf.xml
----------------------------------------------------------------------
diff --git a/examples/test_case_data/sandbox/kylin_hive_conf.xml b/examples/test_case_data/sandbox/kylin_hive_conf.xml
index f7ae507..e75f191 100644
--- a/examples/test_case_data/sandbox/kylin_hive_conf.xml
+++ b/examples/test_case_data/sandbox/kylin_hive_conf.xml
@@ -54,18 +54,8 @@
     -->
 
     <property>
-        <name>hive.merge.mapfiles</name>
-        <value>true</value>
-        <description>Enable hive file merge on mapper only job</description>
-    </property>
-    <property>
-        <name>hive.merge.mapredfiles</name>
-        <value>true</value>
-        <description>Enable hive file merge on map-reduce job</description>
-    </property>
-    <property>
         <name>hive.merge.size.per.task</name>
-        <value>64000000</value>
+        <value>32000000</value>
         <description>Size for the merged file</description>
     </property>
 </configuration>
\ No newline at end of file


Mime
View raw message