kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject kylin git commit: KYLIN-1492 Make custom dimension encoding extension point
Date Tue, 22 Mar 2016 02:26:55 GMT
Repository: kylin
Updated Branches:
  refs/heads/master 2850586c8 -> 4bee8d9ef


KYLIN-1492 Make custom dimension encoding extension point


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/4bee8d9e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/4bee8d9e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/4bee8d9e

Branch: refs/heads/master
Commit: 4bee8d9efe489ab874e14a23a78724f3f8e6d596
Parents: 2850586
Author: Li Yang <liyang@apache.org>
Authored: Tue Mar 22 10:26:27 2016 +0800
Committer: Li Yang <liyang@apache.org>
Committed: Tue Mar 22 10:26:40 2016 +0800

----------------------------------------------------------------------
 .../apache/kylin/common/KylinConfigBase.java    |   6 +-
 .../apache/kylin/common/util/StringUtil.java    |   8 +
 .../cube/gridtable/TrimmedCubeCodeSystem.java   |  13 +
 .../org/apache/kylin/cube/kv/CubeDimEncMap.java |   7 +-
 .../apache/kylin/cube/model/RowKeyColDesc.java  | 244 +++++++++----------
 .../org/apache/kylin/cube/model/RowKeyDesc.java |  13 -
 .../kylin/dimension/DictionaryDimEnc.java       |  26 +-
 .../kylin/dimension/DimensionEncoding.java      |   4 +-
 .../apache/kylin/dimension/FixedLenDimEnc.java  |  36 ++-
 9 files changed, 204 insertions(+), 153 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/4bee8d9e/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 82f5434..1603d2a 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -368,13 +368,17 @@ public class KylinConfigBase implements Serializable {
     }
 
     public double getCubeAlgorithmAutoThreshold() {
-        return Double.parseDouble(getOptional("kylin.cube.algorithm.auto.threshold", "8"));
+        return Double.parseDouble(getOptional("kylin.cube.algorithm.autoThreshold", "8"));
     }
 
     public int getCubeAggrGroupMaxSize() {
         return Integer.parseInt(getOptional("kylin.cube.aggrgroup.max.size", "12"));
     }
 
+    public String[] getCubeDimensionCustomEncodingFactories() {
+        return getOptionalStringArray("kylin.cube.dimension.customEncodingFactories", new
String[0]);
+    }
+
     public int getDictionaryMaxCardinality() {
         return Integer.parseInt(getOptional("kylin.dictionary.max.cardinality", "5000000"));
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/4bee8d9e/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
index 842f656..43a469f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/StringUtil.java
@@ -116,4 +116,12 @@ public class StringUtil {
             return s1.compareTo(s2) > 0 ? s1 : s2;
     }
 
+    public static String[] subArray(String[] array, int start, int endExclusive) {
+        if (start < 0 || start > endExclusive || endExclusive > array.length)
+            throw new IllegalArgumentException();
+        String[] result = new String[endExclusive - start];
+        System.arraycopy(array, start, result, 0, endExclusive - start);
+        return result;
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/4bee8d9e/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
index e5169d2..b892520 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/gridtable/TrimmedCubeCodeSystem.java
@@ -20,6 +20,9 @@
 
 package org.apache.kylin.cube.gridtable;
 
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
 import java.nio.ByteBuffer;
 import java.util.Map;
 
@@ -200,6 +203,16 @@ public class TrimmedCubeCodeSystem implements IGTCodeSystem {
         public DataTypeSerializer<Object> asDataTypeSerializer() {
             throw new UnsupportedOperationException();
         }
+
+        @Override
+        public void writeExternal(ObjectOutput out) throws IOException {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException
{
+            throw new UnsupportedOperationException();
+        }
     }
     
     static class TrimmedDimensionSerializer extends DataTypeSerializer<Object> {

http://git-wip-us.apache.org/repos/asf/kylin/blob/4bee8d9e/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java b/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java
index e8c5266..75ea117 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/kv/CubeDimEncMap.java
@@ -26,6 +26,7 @@ import org.apache.kylin.cube.model.RowKeyColDesc;
 import org.apache.kylin.dimension.Dictionary;
 import org.apache.kylin.dimension.DictionaryDimEnc;
 import org.apache.kylin.dimension.DimensionEncoding;
+import org.apache.kylin.dimension.DimensionEncodingFactory;
 import org.apache.kylin.dimension.FixedLenDimEnc;
 import org.apache.kylin.dimension.IDimensionEncodingMap;
 import org.apache.kylin.metadata.model.TblColRef;
@@ -61,7 +62,7 @@ public class CubeDimEncMap implements IDimensionEncodingMap {
         if (result == null) {
             RowKeyColDesc colDesc = cubeDesc.getRowkey().getColDesc(col);
             if (colDesc.isUsingDictionary()) {
-                // dictionary encoding
+                // special dictionary encoding
                 Dictionary<String> dict = getDictionary(col);
                 if (dict == null) {
                     logger.warn("No dictionary found for dict-encoding column " + col + ",
segment " + seg);
@@ -70,8 +71,8 @@ public class CubeDimEncMap implements IDimensionEncodingMap {
                     result = new DictionaryDimEnc(dict);
                 }
             } else {
-                // fixed length encoding
-                result = new FixedLenDimEnc(colDesc.getLength());
+                // normal case
+                DimensionEncodingFactory.create(colDesc.getEncodingName(), colDesc.getEncodingArgs());
             }
             encMap.put(col, result);
         }

http://git-wip-us.apache.org/repos/asf/kylin/blob/4bee8d9e/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
index 7c26e8f..452d689 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyColDesc.java
@@ -1,129 +1,115 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- * 
- *     http://www.apache.org/licenses/LICENSE-2.0
- * 
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.cube.model;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.kylin.cube.kv.RowConstants;
-import org.apache.kylin.metadata.model.TblColRef;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Objects;
-import com.google.common.base.Preconditions;
-
-/**
- * @author yangli9
- * 
- */
-@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility
= Visibility.NONE, setterVisibility = Visibility.NONE)
-public class RowKeyColDesc {
-
-    public enum ColEncodingType {
-        DICT, FIXED_LEN
-    }
-
-    public class ColEncoding {
-        public ColEncodingType type;
-        public Object param;
-
-        public ColEncoding(ColEncodingType type, Object param) {
-            this.type = type;
-            this.param = param;
-        }
-    }
-
-    @JsonProperty("column")
-    private String column;
-    @JsonProperty("encoding")
-    private String encoding;
-
-    // computed
-    private ColEncoding colEncoding;
-    private int bitIndex;
-    private TblColRef colRef;
-
-    public void init() {
-
-        //dict or fix length?
-        Preconditions.checkState(StringUtils.isNotEmpty(this.encoding));
-        if (this.encoding.equalsIgnoreCase("dict")) {
-            this.colEncoding = new ColEncoding(ColEncodingType.DICT, null);
-        } else if (this.encoding.startsWith("fixed_length")) {
-            int length = RowConstants.ROWKEY_COL_DEFAULT_LENGTH;
-            if (this.encoding.indexOf(":") > 0) {
-                length = Integer.parseInt(this.encoding.substring(this.encoding.indexOf(":")
+ 1));
-            }
-            this.colEncoding = new ColEncoding(ColEncodingType.FIXED_LEN, length);
-        } else {
-            throw new IllegalArgumentException("Not supported row key col encoding:" + this.encoding);
-        }
-    }
-
-    public String getEncoding() {
-        return encoding;
-    }
-
-    public void setEncoding(String encoding) {
-        this.encoding = encoding;
-    }
-
-    public String getColumn() {
-        return column;
-    }
-
-    public void setColumn(String column) {
-        this.column = column;
-    }
-
-    public boolean isUsingDictionary() {
-        return this.colEncoding.type == ColEncodingType.DICT;
-
-    }
-
-    public int getLength() {
-        if (this.colEncoding.type == ColEncodingType.FIXED_LEN) {
-            return (Integer) this.colEncoding.param;
-        } else {
-            return 0;
-        }
-    }
-
-    public int getBitIndex() {
-        return bitIndex;
-    }
-
-    void setBitIndex(int index) {
-        this.bitIndex = index;
-    }
-
-    public TblColRef getColRef() {
-        return colRef;
-    }
-
-    void setColRef(TblColRef colRef) {
-        this.colRef = colRef;
-    }
-
-    @Override
-    public String toString() {
-        return Objects.toStringHelper(this).add("column", column).add("encoding", encoding).toString();
-    }
-
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.model;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.kylin.common.util.StringUtil;
+import org.apache.kylin.dimension.DictionaryDimEnc;
+import org.apache.kylin.dimension.DimensionEncodingFactory;
+import org.apache.kylin.metadata.model.TblColRef;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+
+/**
+ * @author yangli9
+ * 
+ */
+@JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility
= Visibility.NONE, setterVisibility = Visibility.NONE)
+public class RowKeyColDesc {
+
+    @JsonProperty("column")
+    private String column;
+    @JsonProperty("encoding")
+    private String encoding;
+
+    // computed
+    private String encodingName;
+    private String[] encodingArgs;
+    private int bitIndex;
+    private TblColRef colRef;
+
+    public void init() {
+        Preconditions.checkState(StringUtils.isNotEmpty(this.encoding));
+
+        String[] parts = this.encoding.split("\\s*[(),:]\\s*");
+        if (parts == null || parts.length == 0 || parts[0].isEmpty())
+            throw new IllegalArgumentException("Not supported row key col encoding: '" +
this.encoding + "'");
+
+        this.encodingName = parts[0];
+        this.encodingArgs = parts[parts.length - 1].isEmpty() //
+                ? StringUtil.subArray(parts, 1, parts.length - 1) : StringUtil.subArray(parts,
1, parts.length);
+
+        if (!DimensionEncodingFactory.isVaildEncoding(this.encodingName))
+            throw new IllegalArgumentException("Not supported row key col encoding: '" +
this.encoding + "'");
+    }
+
+    public String getEncoding() {
+        return encoding;
+    }
+
+    public void setEncoding(String encoding) {
+        this.encoding = encoding;
+    }
+
+    public String getColumn() {
+        return column;
+    }
+
+    public void setColumn(String column) {
+        this.column = column;
+    }
+
+    public String getEncodingName() {
+        return encodingName;
+    }
+
+    public String[] getEncodingArgs() {
+        return encodingArgs;
+    }
+
+    public boolean isUsingDictionary() {
+        return DictionaryDimEnc.ENCODING_NAME.equals(encodingName);
+    }
+
+    public int getBitIndex() {
+        return bitIndex;
+    }
+
+    void setBitIndex(int index) {
+        this.bitIndex = index;
+    }
+
+    public TblColRef getColRef() {
+        return colRef;
+    }
+
+    void setColRef(TblColRef colRef) {
+        this.colRef = colRef;
+    }
+
+    @Override
+    public String toString() {
+        return Objects.toStringHelper(this).add("column", column).add("encoding", encoding).toString();
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/4bee8d9e/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
index 3f33505..6052c08 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/RowKeyDesc.java
@@ -46,19 +46,6 @@ public class RowKeyDesc {
         return rowkeyColumns;
     }
 
-    //    public int getNCuboidBuildLevels() {
-    //        // N aggregation columns requires N levels of cuboid build
-    //        // - N columns requires N-1 levels build
-    //        // - zero tail cuboid needs one more additional level
-    //        Set<String> aggDims = new HashSet<String>();
-    //        for (String[] aggrGroup : aggregationGroups) {
-    //            for (String dim : aggrGroup) {
-    //                aggDims.add(dim);
-    //            }
-    //        }
-    //        return aggDims.size();
-    //    }
-
     public void setCubeDesc(CubeDesc cubeRef) {
         this.cubeDesc = cubeRef;
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/4bee8d9e/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java
b/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java
index bfb0e28..7f80a7f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java
@@ -18,6 +18,9 @@
 
 package org.apache.kylin.dimension;
 
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
 import java.nio.ByteBuffer;
 
 import org.apache.kylin.common.util.Bytes;
@@ -30,6 +33,10 @@ public class DictionaryDimEnc extends DimensionEncoding {
 
     private static final Logger logger = LoggerFactory.getLogger(DictionaryDimEnc.class);
 
+    public static final String ENCODING_NAME = "dict";
+
+    // ============================================================================
+
     // could use a lazy loading trick here, to prevent loading all dictionaries of a segment
at once
     private final Dictionary<String> dict;
     private final int fixedLen;
@@ -52,21 +59,21 @@ public class DictionaryDimEnc extends DimensionEncoding {
     public int getRoundingFlag() {
         return roundingFlag;
     }
-    
+
     public DictionaryDimEnc copy(int roundingFlag) {
         if (this.roundingFlag == roundingFlag)
             return this;
         else
             return new DictionaryDimEnc(dict, roundingFlag, defaultByte);
     }
-    
+
     public DictionaryDimEnc copy(int roundingFlag, byte defaultByte) {
         if (this.roundingFlag == roundingFlag && this.defaultByte == defaultByte)
             return this;
         else
             return new DictionaryDimEnc(dict, roundingFlag, defaultByte);
     }
-    
+
     public Dictionary<String> getDictionary() {
         return dict;
     }
@@ -133,5 +140,16 @@ public class DictionaryDimEnc extends DimensionEncoding {
         public int getStorageBytesEstimate() {
             return dict.getSizeOfId();
         }
-    };
+    }
+
+    @Override
+    public void writeExternal(ObjectOutput out) throws IOException {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+        throw new UnsupportedOperationException();
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/4bee8d9e/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncoding.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncoding.java
b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncoding.java
index 74c0555..5c32114 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncoding.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DimensionEncoding.java
@@ -18,6 +18,8 @@
 
 package org.apache.kylin.dimension;
 
+import java.io.Externalizable;
+
 import org.apache.kylin.metadata.datatype.DataTypeSerializer;
 
 /**
@@ -29,7 +31,7 @@ import org.apache.kylin.metadata.datatype.DataTypeSerializer;
  * hard to enumerate all values of a encoding, thus TupleFilterDictionaryTranslater 
  * cannot work on DimensionEncoding.
  */
-public abstract class DimensionEncoding {
+public abstract class DimensionEncoding implements Externalizable {
 
     // it's convention that all 0xff means NULL
     public static final byte NULL = (byte) 0xff;

http://git-wip-us.apache.org/repos/asf/kylin/blob/4bee8d9e/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java
index 40e9f6d..195bdb9 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java
@@ -18,6 +18,9 @@
 
 package org.apache.kylin.dimension;
 
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 
@@ -33,7 +36,26 @@ public class FixedLenDimEnc extends DimensionEncoding {
     // row key fixed length place holder
     public static final byte ROWKEY_PLACE_HOLDER_BYTE = 9;
 
-    private final int fixedLen;
+    public static final String ENCODING_NAME = "fixed_length";
+    
+    public static DimensionEncodingFactory getFactory() {
+        return new DimensionEncodingFactory() {
+
+            @Override
+            public String getSupportedEncodingName() {
+                return ENCODING_NAME;
+            }
+
+            @Override
+            public DimensionEncoding createDimensionEncoding(String encodingName, String[]
args) {
+                return new FixedLenDimEnc(Integer.parseInt(args[0]));
+            }
+        };
+    };
+
+    // ============================================================================
+
+    private int fixedLen;
 
     transient private int avoidVerbose = 0;
 
@@ -131,6 +153,16 @@ public class FixedLenDimEnc extends DimensionEncoding {
         public Object valueOf(String str) {
             return str;
         }
-    };
+    }
+
+    @Override
+    public void writeExternal(ObjectOutput out) throws IOException {
+        out.writeShort(fixedLen);
+    }
+
+    @Override
+    public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+        fixedLen = in.readShort();
+    }
 
 }


Mime
View raw message