kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From billy...@apache.org
Subject [2/2] kylin git commit: KYLIN-2428 cleanup unnecessary shade dependency in job/coprocessor/spark/jdbc/tool
Date Tue, 14 Feb 2017 05:29:53 GMT
KYLIN-2428 cleanup unnecessary shade dependency in job/coprocessor/spark/jdbc/tool


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f10d9d9b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f10d9d9b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f10d9d9b

Branch: refs/heads/KYLIN-2428
Commit: f10d9d9ba95878a97f70975547763083c000bcfe
Parents: c93ea9c
Author: Billy Liu <billyliu@apache.org>
Authored: Tue Feb 14 13:29:25 2017 +0800
Committer: Billy Liu <billyliu@apache.org>
Committed: Tue Feb 14 13:29:25 2017 +0800

----------------------------------------------------------------------
 assembly/pom.xml                                |  49 +++-
 atopcalcite/pom.xml                             |   1 +
 build/bin/get-properties.sh                     |   3 +-
 build/bin/kylin.sh                              |   8 +-
 build/conf/kylin-tools-log4j.properties         |   9 +-
 build/script/prepare_libs.sh                    |   2 +-
 core-common/pom.xml                             |  22 +-
 .../apache/kylin/common/util/SparkEntry.java    |  42 ---
 core-cube/pom.xml                               |  18 +-
 .../org/apache/kylin/cube/model/CubeDesc.java   |   2 +-
 .../apache/kylin/gridtable/GTInvertedIndex.java | 223 ----------------
 .../gridtable/GTInvertedIndexOfColumn.java      | 133 ----------
 .../gridtable/SimpleInvertedIndexTest.java      | 196 --------------
 core-job/pom.xml                                |   9 +
 core-metadata/pom.xml                           |  23 ++
 .../apache/kylin/dimension/BooleanDimEnc.java   |   5 +-
 .../org/apache/kylin/dimension/DateDimEnc.java  |   4 +-
 .../kylin/dimension/DictionaryDimEnc.java       |   3 +-
 .../apache/kylin/dimension/FixedLenDimEnc.java  |   3 +-
 .../kylin/dimension/FixedLenHexDimEnc.java      |   3 +-
 .../org/apache/kylin/dimension/IntDimEnc.java   |   3 +-
 .../apache/kylin/dimension/IntegerDimEnc.java   |   3 +-
 .../kylin/dimension/OneMoreByteVLongDimEnc.java |   3 +-
 .../org/apache/kylin/dimension/TimeDimEnc.java  |   4 +-
 .../kylin/measure/MeasureTypeFactory.java       |   2 +-
 .../measure/hllc/HLLDistinctCountAggFunc.java   |   2 +-
 .../org/apache/kylin/measure/topn/Counter.java  |   3 +-
 .../kylin/metadata/model/DataModelDesc.java     |   5 +-
 .../apache/kylin/metadata/model/JoinsTree.java  |   2 +-
 .../kylin/metadata/model/PartitionDesc.java     |   2 +-
 .../kylin/metadata/model/SegmentStatusEnum.java |   4 +-
 .../apache/kylin/metadata/model/Segments.java   |   3 +-
 .../realization/RealizationStatusEnum.java      |   4 +-
 core-storage/pom.xml                            |  20 ++
 engine-spark/pom.xml                            |  47 +---
 .../engine/spark/KylinKryoRegistrator.java      | 265 ++++++++++++++-----
 .../spark/SparkBatchCubingJobBuilder2.java      |   1 +
 .../kylin/engine/spark/SparkCubingByLayer.java  |   6 +-
 .../kylin/engine/spark/SparkExecutable.java     |  17 +-
 .../spark/cube/ListBackedCuboidWriter.java      |   4 +-
 .../kylin/engine/spark/util/IteratorUtils.java  |   4 +-
 .../kylin/engine/spark/util/SparkEntry.java     |  48 ++++
 .../engine/spark/util/KyroMappingGenerator.java |  56 ++++
 jdbc/pom.xml                                    |  14 +-
 pom.xml                                         | 216 ++++++++-------
 query/pom.xml                                   |   6 +
 server/pom.xml                                  |   4 +
 source-kafka/pom.xml                            |  16 +-
 storage-hbase/pom.xml                           |  19 +-
 tool-assembly/pom.xml                           | 135 ++++++++++
 tool/pom.xml                                    |  39 ---
 51 files changed, 791 insertions(+), 924 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/assembly/pom.xml
----------------------------------------------------------------------
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 873abff..ba69583 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -19,16 +19,20 @@
 
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>kylin-assembly</artifactId>
+    <name>Apache Kylin - Assembly</name>
+
     <parent>
         <artifactId>kylin</artifactId>
         <groupId>org.apache.kylin</groupId>
         <version>2.0.0-SNAPSHOT</version>
-
     </parent>
-    <modelVersion>4.0.0</modelVersion>
 
-    <artifactId>kylin-assembly</artifactId>
-    <name>Apache Kylin - Assembly</name>
+    <properties>
+        <shadeBase>org.apache.kylin.job.shaded</shadeBase>
+    </properties>
 
     <dependencies>
         <dependency>
@@ -180,8 +184,45 @@
                                     <exclude>org.apache.zookeeper:*</exclude>
                                     <exclude>net.sf.ehcache:*</exclude>
                                     <exclude>org.apache.httpcomponents:*</exclude>
+                                    <exclude>org.apache.commons:commons-email:*</exclude>
+                                    <exclude>com.sun.mail:javax.mail:*</exclude>
+                                    <exclude>javax.activation:activation:*</exclude>
                                 </excludes>
                             </artifactSet>
+                            <relocations>
+                                <relocation>
+                                    <pattern>com.fasterxml.jackson</pattern>
+                                    <shadedPattern>${shadeBase}.com.fasterxml.jackson</shadedPattern>
+                                </relocation>
+                                <relocation>
+                                    <pattern>com.tdunning</pattern>
+                                    <shadedPattern>${shadeBase}.com.tdunning</shadedPattern>
+                                </relocation>
+                                <relocation>
+                                    <pattern>com.fasterxml.jackson</pattern>
+                                    <shadedPattern>${shadeBase}.com.fasterxml.jackson</shadedPattern>
+                                </relocation>
+                                <relocation>
+                                    <pattern>org.apache.commons.codec</pattern>
+                                    <shadedPattern>${shadeBase}.org.apache.commons.codec</shadedPattern>
+                                </relocation>
+                                <relocation>
+                                    <pattern>org.apache.commons.collections</pattern>
+                                    <shadedPattern>${shadeBase}.org.apache.commons.collections</shadedPattern>
+                                </relocation>
+                                <relocation>
+                                    <pattern>org.apache.commons.lang3</pattern>
+                                    <shadedPattern>${shadeBase}.org.apache.commons.lang3</shadedPattern>
+                                </relocation>
+                                <relocation>
+                                    <pattern>org.apache.commons.logging</pattern>
+                                    <shadedPattern>${shadeBase}.org.apache.commons.logging</shadedPattern>
+                                </relocation>
+                                <relocation>
+                                    <pattern>org.roaringbitmap</pattern>
+                                    <shadedPattern>${shadeBase}.org.roaringbitmap</shadedPattern>
+                                </relocation>
+                            </relocations>
                             <filters>
                                 <filter>
                                     <artifact>*:*</artifact>

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/atopcalcite/pom.xml
----------------------------------------------------------------------
diff --git a/atopcalcite/pom.xml b/atopcalcite/pom.xml
index 1b327fe..c65c0d1 100644
--- a/atopcalcite/pom.xml
+++ b/atopcalcite/pom.xml
@@ -43,6 +43,7 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <!-- It should be avatica(the shaded one), not avatica-core, since the inconsistency protobuf dependency with Hadoop -->
         <dependency>
             <groupId>org.apache.calcite.avatica</groupId>
             <artifactId>avatica</artifactId>

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/build/bin/get-properties.sh
----------------------------------------------------------------------
diff --git a/build/bin/get-properties.sh b/build/bin/get-properties.sh
index 170442d..57721d8 100755
--- a/build/bin/get-properties.sh
+++ b/build/bin/get-properties.sh
@@ -25,7 +25,6 @@ then
     exit -1
 fi
 
-job_jar=$(ls $KYLIN_HOME/lib/kylin-job-*.jar)
 tool_jar=$(ls $KYLIN_HOME/tool/kylin-tool-*.jar)
-result=`java -cp $job_jar:$tool_jar org.apache.kylin.tool.KylinConfigCLI $1 2>/dev/null`
+result=`java -cp $tool_jar -Dlog4j.configuration=file:${KYLIN_HOME}/conf/kylin-tools-log4j.properties org.apache.kylin.tool.KylinConfigCLI $1 2>/dev/null`
 echo "$result"

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/build/bin/kylin.sh
----------------------------------------------------------------------
diff --git a/build/bin/kylin.sh b/build/bin/kylin.sh
index cc571e5..15e1aed 100644
--- a/build/bin/kylin.sh
+++ b/build/bin/kylin.sh
@@ -38,7 +38,7 @@ function retrieveDependency() {
         source ${dir}/setenv.sh
     fi
 
-    export HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/tool/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
+    export HBASE_CLASSPATH_PREFIX=${KYLIN_HOME}/conf:${KYLIN_HOME}/lib/*:${KYLIN_HOME}/ext/*:${HBASE_CLASSPATH_PREFIX}
     export HBASE_CLASSPATH=${HBASE_CLASSPATH}:${hive_dependency}
     if [ -n "$KAFKA_HOME" ]
     then
@@ -103,7 +103,7 @@ then
     # KYLIN_EXTRA_START_OPTS is for customized settings, checkout bin/setenv.sh
     hbase ${KYLIN_EXTRA_START_OPTS} \
     -Djava.util.logging.manager=org.apache.juli.ClassLoaderLogManager \
-    -Dlog4j.configuration=kylin-server-log4j.properties \
+    -Dlog4j.configuration=file:${KYLIN_HOME}/conf/kylin-server-log4j.properties \
     -Dorg.apache.tomcat.util.buf.UDecoder.ALLOW_ENCODED_SLASH=true \
     -Dorg.apache.catalina.connector.CoyoteAdapter.ALLOW_BACKSLASH=true \
     -Djava.endorsed.dirs=${tomcat_root}/endorsed  \
@@ -165,8 +165,10 @@ then
         then source ${dir}/setenv-tool.sh
     fi
 
+    hbase_original=${HBASE_CLASSPATH}
+    export HBASE_CLASSPATH=${hbase_original}:${KYLIN_HOME}/tool/*
     exec hbase ${KYLIN_EXTRA_START_OPTS} -Dkylin.hive.dependency=${hive_dependency} -Dkylin.hbase.dependency=${hbase_dependency} -Dlog4j.configuration=file:${KYLIN_HOME}/conf/kylin-tools-log4j.properties "$@"
-
+    export HBASE_CLASSPATH=${hbase_original}
 else
     quit "Usage: 'kylin.sh [-v] start' or 'kylin.sh [-v] stop'"
 fi

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/build/conf/kylin-tools-log4j.properties
----------------------------------------------------------------------
diff --git a/build/conf/kylin-tools-log4j.properties b/build/conf/kylin-tools-log4j.properties
index 0ba7d2e..e975d18 100644
--- a/build/conf/kylin-tools-log4j.properties
+++ b/build/conf/kylin-tools-log4j.properties
@@ -25,11 +25,12 @@
 # It's called kylin-tools-log4j.properties so that it won't distract users from the other more important log4j config file: kylin-server-log4j.properties  
 # enable this by -Dlog4j.configuration=kylin-tools-log4j.properties
 
-log4j.rootLogger=INFO,stdout
+log4j.rootLogger=INFO,stderr
 
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t %c{1}:%L]: %m%n
+log4j.appender.stderr=org.apache.log4j.ConsoleAppender
+log4j.appender.stderr.Target=System.err
+log4j.appender.stderr.layout=org.apache.log4j.PatternLayout
+log4j.appender.stderr.layout.ConversionPattern=%d{ISO8601} %-5p [%t %c{1}:%L]: %m%n
 
 #log4j.logger.org.apache.hadoop=ERROR
 log4j.logger.org.apache.kylin=DEBUG

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/build/script/prepare_libs.sh
----------------------------------------------------------------------
diff --git a/build/script/prepare_libs.sh b/build/script/prepare_libs.sh
index aec9923..3dbdf21 100755
--- a/build/script/prepare_libs.sh
+++ b/build/script/prepare_libs.sh
@@ -33,7 +33,7 @@ mkdir build/lib build/tool
 cp assembly/target/kylin-assembly-${version}-job.jar build/lib/kylin-job-${version}.jar
 cp storage-hbase/target/kylin-storage-hbase-${version}-coprocessor.jar build/lib/kylin-coprocessor-${version}.jar
 cp jdbc/target/kylin-jdbc-${version}.jar build/lib/kylin-jdbc-${version}.jar
-cp tool/target/kylin-tool-${version}-assembly.jar build/tool/kylin-tool-${version}.jar
+cp tool-assembly/target/kylin-tool-assembly-${version}-assembly.jar build/tool/kylin-tool-${version}.jar
 
 # Copied file becomes 000 for some env (e.g. my Cygwin)
 chmod 644 build/lib/kylin-job-${version}.jar

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-common/pom.xml
----------------------------------------------------------------------
diff --git a/core-common/pom.xml b/core-common/pom.xml
index 016d470..95d3c29 100644
--- a/core-common/pom.xml
+++ b/core-common/pom.xml
@@ -35,26 +35,10 @@
     <dependencies>
         <!-- Basic Utilities -->
         <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-databind</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>commons-lang</groupId>
-            <artifactId>commons-lang</artifactId>
-        </dependency>
-        <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>
         </dependency>
         <dependency>
-            <groupId>commons-io</groupId>
-            <artifactId>commons-io</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>commons-cli</groupId>
-            <artifactId>commons-cli</artifactId>
-        </dependency>
-        <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-email</artifactId>
         </dependency>
@@ -63,9 +47,11 @@
             <artifactId>httpclient</artifactId>
         </dependency>
         <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
         </dependency>
+
+        <!-- Provided -->
         <dependency>
             <groupId>com.google.code.findbugs</groupId>
             <artifactId>jsr305</artifactId>

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java b/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java
deleted file mode 100644
index fd324a2..0000000
--- a/core-common/src/main/java/org/apache/kylin/common/util/SparkEntry.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *  
- *     http://www.apache.org/licenses/LICENSE-2.0
- *  
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.kylin.common.util;
-
-import org.apache.commons.lang3.StringUtils;
-
-import com.google.common.base.Preconditions;
-
-/**
- */
-public final class SparkEntry {
-
-    public static void main(String[] args) throws Exception {
-        System.out.println("SparkEntry args:" + StringUtils.join(args, " "));
-        Preconditions.checkArgument(args.length >= 2, "-className is required");
-        Preconditions.checkArgument(args[0].equals("-className"), "-className is required");
-        final String className = args[1];
-        final Object o = Class.<AbstractApplication> forName(className).newInstance();
-        Preconditions.checkArgument(o instanceof AbstractApplication, className + " is not a subClass of AbstractSparkApplication");
-        String[] appArgs = new String[args.length - 2];
-        for (int i = 2; i < args.length; i++) {
-            appArgs[i - 2] = args[i];
-        }
-        AbstractApplication abstractApplication = (AbstractApplication) o;
-        abstractApplication.execute(appArgs);
-    }
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-cube/pom.xml
----------------------------------------------------------------------
diff --git a/core-cube/pom.xml b/core-cube/pom.xml
index 0a30432..fe80783 100644
--- a/core-cube/pom.xml
+++ b/core-cube/pom.xml
@@ -48,9 +48,23 @@
             <groupId>commons-collections</groupId>
             <artifactId>commons-collections</artifactId>
         </dependency>
+
+        <!-- Provided -->
+        <dependency>
+            <groupId>commons-lang</groupId>
+            <artifactId>commons-lang</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
         <dependency>
-            <groupId>com.n3twork.druid</groupId>
-            <artifactId>extendedset</artifactId>
+            <groupId>com.google.code.findbugs</groupId>
+            <artifactId>jsr305</artifactId>
         </dependency>
         <dependency>
             <groupId>com.esotericsoftware</groupId>

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 9acb90c..0c1419d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -97,7 +97,7 @@ public class CubeDesc extends RootPersistentEntity implements IEngineAware {
         }
     }
 
-    public enum DeriveType {
+    public enum DeriveType implements java.io.Serializable{
         LOOKUP, PK_FK, EXTENDED_COLUMN
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java
deleted file mode 100644
index a2c713a..0000000
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndex.java
+++ /dev/null
@@ -1,223 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.gridtable;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.ImmutableBitSet;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-
-import it.uniroma3.mat.extendedset.intset.ConciseSet;
-
-/**
- * A thread-safe inverted index of row blocks in memory.
- * 
- * Note function not() must return all blocks, because index only know what block contains a value,
- * but not sure what block does not contain a value.
- * 
- * @author yangli9
- */
-public class GTInvertedIndex {
-
-    private final GTInfo info;
-    private final ImmutableBitSet colPreferIndex;
-    private final ImmutableBitSet colBlocks;
-    private final GTInvertedIndexOfColumn[] index; // for each column
-
-    private volatile int nIndexedBlocks;
-
-    public GTInvertedIndex(GTInfo info) {
-        this.info = info;
-        this.colPreferIndex = info.colPreferIndex;
-        this.colBlocks = info.selectColumnBlocks(colPreferIndex);
-
-        index = new GTInvertedIndexOfColumn[info.getColumnCount()];
-        for (int i = 0; i < colPreferIndex.trueBitCount(); i++) {
-            int c = colPreferIndex.trueBitAt(i);
-            index[c] = new GTInvertedIndexOfColumn(info.codeSystem.getComparator());
-        }
-    }
-
-    public void add(GTRowBlock block) {
-
-        @SuppressWarnings("unchecked")
-        Set<ByteArray>[] distinctValues = new Set[info.getColumnCount()];
-        for (int i = 0; i < colPreferIndex.trueBitCount(); i++) {
-            int c = colPreferIndex.trueBitAt(i);
-            distinctValues[c] = new HashSet<ByteArray>();
-        }
-
-        GTRowBlock.Reader reader = block.getReader(colBlocks);
-        GTRecord record = new GTRecord(info);
-        while (reader.hasNext()) {
-            reader.fetchNext(record);
-            for (int i = 0; i < colPreferIndex.trueBitCount(); i++) {
-                int c = colPreferIndex.trueBitAt(i);
-                distinctValues[c].add(record.get(c));
-            }
-        }
-
-        for (int i = 0; i < colPreferIndex.trueBitCount(); i++) {
-            int c = colPreferIndex.trueBitAt(i);
-            index[c].add(distinctValues[c], block.getSequenceId());
-        }
-
-        nIndexedBlocks = Math.max(nIndexedBlocks, block.seqId + 1);
-    }
-
-    public ConciseSet filter(TupleFilter filter) {
-        return filter(filter, nIndexedBlocks);
-    }
-
-    public ConciseSet filter(TupleFilter filter, int totalBlocks) {
-        // number of indexed blocks may increase as we do evaluation
-        int indexedBlocks = nIndexedBlocks;
-
-        Evaluator evaluator = new Evaluator(indexedBlocks);
-        ConciseSet r = evaluator.evaluate(filter);
-
-        // add blocks that have not been indexed
-        for (int i = indexedBlocks; i < totalBlocks; i++) {
-            r.add(i);
-        }
-
-        return r;
-    }
-
-    private class Evaluator {
-        private int indexedBlocks;
-
-        Evaluator(int indexedBlocks) {
-            this.indexedBlocks = indexedBlocks;
-        }
-
-        public ConciseSet evaluate(TupleFilter filter) {
-            if (filter == null) {
-                return all();
-            }
-
-            if (filter instanceof LogicalTupleFilter)
-                return evalLogical((LogicalTupleFilter) filter);
-
-            if (filter instanceof CompareTupleFilter)
-                return evalCompare((CompareTupleFilter) filter);
-
-            // unable to evaluate
-            return all();
-        }
-
-        @SuppressWarnings("unchecked")
-        private ConciseSet evalCompare(CompareTupleFilter filter) {
-            int col = col(filter);
-            if (index[col] == null)
-                return all();
-
-            switch (filter.getOperator()) {
-            case ISNULL:
-                return index[col].getNull();
-            case ISNOTNULL:
-                return all();
-            case EQ:
-                return index[col].getEquals((ByteArray) filter.getFirstValue());
-            case NEQ:
-                return all();
-            case IN:
-                return index[col].getIn((Iterable<ByteArray>) filter.getValues());
-            case NOTIN:
-                return all();
-            case LT:
-                return index[col].getRange(null, false, (ByteArray) filter.getFirstValue(), false);
-            case LTE:
-                return index[col].getRange(null, false, (ByteArray) filter.getFirstValue(), true);
-            case GT:
-                return index[col].getRange((ByteArray) filter.getFirstValue(), false, null, false);
-            case GTE:
-                return index[col].getRange((ByteArray) filter.getFirstValue(), true, null, false);
-            default:
-                throw new IllegalStateException("Unsupported operator " + filter.getOperator());
-            }
-        }
-
-        private ConciseSet evalLogical(LogicalTupleFilter filter) {
-            List<? extends TupleFilter> children = filter.getChildren();
-
-            switch (filter.getOperator()) {
-            case AND:
-                return evalLogicalAnd(children);
-            case OR:
-                return evalLogicalOr(children);
-            case NOT:
-                return evalLogicalNot(children);
-            default:
-                throw new IllegalStateException("Unsupported operator " + filter.getOperator());
-            }
-        }
-
-        private ConciseSet evalLogicalAnd(List<? extends TupleFilter> children) {
-            ConciseSet set = all();
-
-            for (TupleFilter c : children) {
-                ConciseSet t = evaluate(c);
-                if (t == null)
-                    continue; // because it's AND
-
-                set.retainAll(t);
-            }
-            return set;
-        }
-
-        private ConciseSet evalLogicalOr(List<? extends TupleFilter> children) {
-            ConciseSet set = new ConciseSet();
-
-            for (TupleFilter c : children) {
-                ConciseSet t = evaluate(c);
-                if (t == null)
-                    return null; // because it's OR
-
-                set.addAll(t);
-            }
-            return set;
-        }
-
-        private ConciseSet evalLogicalNot(List<? extends TupleFilter> children) {
-            return all();
-        }
-
-        private ConciseSet all() {
-            return not(new ConciseSet());
-        }
-
-        private ConciseSet not(ConciseSet set) {
-            set.add(indexedBlocks);
-            set.complement();
-            return set;
-        }
-
-        private int col(CompareTupleFilter filter) {
-            return filter.getColumn().getColumnDesc().getZeroBasedIndex();
-        }
-
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java b/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java
deleted file mode 100644
index bfacc0f..0000000
--- a/core-cube/src/main/java/org/apache/kylin/gridtable/GTInvertedIndexOfColumn.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.gridtable;
-
-import java.util.NavigableMap;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import org.apache.kylin.common.util.ByteArray;
-
-import com.google.common.collect.Maps;
-
-import it.uniroma3.mat.extendedset.intset.ConciseSet;
-
-public class GTInvertedIndexOfColumn {
-
-    final private IGTComparator comparator;
-    final private ReentrantReadWriteLock rwLock;
-
-    private int nBlocks;
-    private NavigableMap<ByteArray, ConciseSet> rangeIndex;
-    private ConciseSet nullIndex;
-
-    public GTInvertedIndexOfColumn(IGTComparator comparator) {
-        this.comparator = comparator;
-        this.rwLock = new ReentrantReadWriteLock();
-        this.rangeIndex = Maps.newTreeMap(comparator);
-        this.nullIndex = new ConciseSet();
-    }
-
-    public void add(Iterable<ByteArray> codes, int blockId) {
-        rwLock.writeLock().lock();
-        try {
-            for (ByteArray code : codes) {
-                if (comparator.isNull(code)) {
-                    nullIndex.add(blockId);
-                    continue;
-                }
-                ConciseSet set = rangeIndex.get(code);
-                if (set == null) {
-                    set = new ConciseSet();
-                    rangeIndex.put(code.copy(), set);
-                }
-                set.add(blockId);
-            }
-
-            if (blockId >= nBlocks) {
-                nBlocks = blockId + 1;
-            }
-
-        } finally {
-            rwLock.writeLock().unlock();
-        }
-    }
-
-    public ConciseSet getNull() {
-        rwLock.readLock().lock();
-        try {
-            return nullIndex.clone();
-        } finally {
-            rwLock.readLock().unlock();
-        }
-    }
-
-    public ConciseSet getEquals(ByteArray code) {
-        rwLock.readLock().lock();
-        try {
-            ConciseSet set = rangeIndex.get(code);
-            if (set == null)
-                return new ConciseSet();
-            else
-                return set.clone();
-        } finally {
-            rwLock.readLock().unlock();
-        }
-    }
-
-    public ConciseSet getIn(Iterable<ByteArray> codes) {
-        rwLock.readLock().lock();
-        try {
-            ConciseSet r = new ConciseSet();
-            for (ByteArray code : codes) {
-                ConciseSet set = rangeIndex.get(code);
-                if (set != null)
-                    r.addAll(set);
-            }
-            return r;
-        } finally {
-            rwLock.readLock().unlock();
-        }
-    }
-
-    public ConciseSet getRange(ByteArray from, boolean fromInclusive, ByteArray to, boolean toInclusive) {
-        rwLock.readLock().lock();
-        try {
-            ConciseSet r = new ConciseSet();
-            if (from == null && to == null) {
-                r.add(nBlocks);
-                r.complement();
-                return r;
-            }
-            NavigableMap<ByteArray, ConciseSet> subMap;
-            if (from == null) {
-                subMap = rangeIndex.headMap(to, toInclusive);
-            } else if (to == null) {
-                subMap = rangeIndex.tailMap(from, fromInclusive);
-            } else {
-                subMap = rangeIndex.subMap(from, fromInclusive, to, toInclusive);
-            }
-            for (ConciseSet set : subMap.values()) {
-                r.addAll(set);
-            }
-            return r;
-        } finally {
-            rwLock.readLock().unlock();
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
----------------------------------------------------------------------
diff --git a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java b/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
deleted file mode 100644
index 5e3e771..0000000
--- a/core-cube/src/test/java/org/apache/kylin/gridtable/SimpleInvertedIndexTest.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.gridtable;
-
-import static org.junit.Assert.assertEquals;
-
-import java.math.BigDecimal;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-
-import org.apache.kylin.common.util.ByteArray;
-import org.apache.kylin.common.util.LocalFileMetadataTestCase;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.apache.kylin.metadata.datatype.StringSerializer;
-import org.apache.kylin.metadata.filter.ColumnTupleFilter;
-import org.apache.kylin.metadata.filter.CompareTupleFilter;
-import org.apache.kylin.metadata.filter.ConstantTupleFilter;
-import org.apache.kylin.metadata.filter.LogicalTupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter;
-import org.apache.kylin.metadata.filter.TupleFilter.FilterOperatorEnum;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import com.google.common.collect.Lists;
-
-import it.uniroma3.mat.extendedset.intset.ConciseSet;
-
-public class SimpleInvertedIndexTest extends LocalFileMetadataTestCase {
-
-    GTInfo info;
-    GTInvertedIndex index;
-    ArrayList<CompareTupleFilter> basicFilters = Lists.newArrayList();
-    ArrayList<ConciseSet> basicResults = Lists.newArrayList();
-
-    @BeforeClass
-    public static void setUp() throws Exception {
-        staticCreateTestMetadata();
-    }
-
-    @AfterClass
-    public static void after() throws Exception {
-        cleanAfterClass();
-    }
-
-    public SimpleInvertedIndexTest() {
-
-        info = UnitTestSupport.advancedInfo();
-        TblColRef colA = info.colRef(0);
-
-        // block i contains value "i", the last is NULL
-        index = new GTInvertedIndex(info);
-        GTRowBlock mockBlock = GTRowBlock.allocate(info);
-        GTRowBlock.Writer writer = mockBlock.getWriter();
-        GTRecord record = new GTRecord(info);
-        for (int i = 0; i < 10; i++) {
-            record.setValues(i < 9 ? "" + i : null, "", "", new Long(0), new BigDecimal(0));
-            for (int j = 0; j < info.getRowBlockSize(); j++) {
-                writer.append(record);
-            }
-            writer.readyForFlush();
-            index.add(mockBlock);
-
-            writer.clearForNext();
-        }
-
-        basicFilters.add(compare(colA, FilterOperatorEnum.ISNULL));
-        basicResults.add(set(9));
-
-        basicFilters.add(compare(colA, FilterOperatorEnum.ISNOTNULL));
-        basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
-
-        basicFilters.add(compare(colA, FilterOperatorEnum.EQ, 0));
-        basicResults.add(set(0));
-
-        basicFilters.add(compare(colA, FilterOperatorEnum.NEQ, 0));
-        basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
-
-        basicFilters.add(compare(colA, FilterOperatorEnum.IN, 0, 5));
-        basicResults.add(set(0, 5));
-
-        basicFilters.add(compare(colA, FilterOperatorEnum.NOTIN, 0, 5));
-        basicResults.add(set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9));
-
-        basicFilters.add(compare(colA, FilterOperatorEnum.LT, 3));
-        basicResults.add(set(0, 1, 2));
-
-        basicFilters.add(compare(colA, FilterOperatorEnum.LTE, 3));
-        basicResults.add(set(0, 1, 2, 3));
-
-        basicFilters.add(compare(colA, FilterOperatorEnum.GT, 3));
-        basicResults.add(set(4, 5, 6, 7, 8));
-
-        basicFilters.add(compare(colA, FilterOperatorEnum.GTE, 3));
-        basicResults.add(set(3, 4, 5, 6, 7, 8));
-    }
-
-    @Test
-    public void testBasics() {
-        for (int i = 0; i < basicFilters.size(); i++) {
-            assertEquals(basicResults.get(i), index.filter(basicFilters.get(i)));
-        }
-    }
-
-    @Test
-    public void testLogicalAnd() {
-        for (int i = 0; i < basicFilters.size(); i++) {
-            for (int j = 0; j < basicFilters.size(); j++) {
-                LogicalTupleFilter f = logical(FilterOperatorEnum.AND, basicFilters.get(i), basicFilters.get(j));
-                ConciseSet r = basicResults.get(i).clone();
-                r.retainAll(basicResults.get(j));
-                assertEquals(r, index.filter(f));
-            }
-        }
-    }
-
-    @Test
-    public void testLogicalOr() {
-        for (int i = 0; i < basicFilters.size(); i++) {
-            for (int j = 0; j < basicFilters.size(); j++) {
-                LogicalTupleFilter f = logical(FilterOperatorEnum.OR, basicFilters.get(i), basicFilters.get(j));
-                ConciseSet r = basicResults.get(i).clone();
-                r.addAll(basicResults.get(j));
-                assertEquals(r, index.filter(f));
-            }
-        }
-    }
-
-    @Test
-    public void testNotEvaluable() {
-        ConciseSet all = set(0, 1, 2, 3, 4, 5, 6, 7, 8, 9);
-
-        CompareTupleFilter notEvaluable = compare(info.colRef(1), FilterOperatorEnum.EQ, 0);
-        assertEquals(all, index.filter(notEvaluable));
-
-        LogicalTupleFilter or = logical(FilterOperatorEnum.OR, basicFilters.get(0), notEvaluable);
-        assertEquals(all, index.filter(or));
-
-        LogicalTupleFilter and = logical(FilterOperatorEnum.AND, basicFilters.get(0), notEvaluable);
-        assertEquals(basicResults.get(0), index.filter(and));
-    }
-
-    public static CompareTupleFilter compare(TblColRef col, TupleFilter.FilterOperatorEnum op, int... ids) {
-        CompareTupleFilter filter = new CompareTupleFilter(op);
-        filter.addChild(columnFilter(col));
-        for (int i : ids) {
-            filter.addChild(constFilter(i));
-        }
-        return filter;
-    }
-
-    public static LogicalTupleFilter logical(TupleFilter.FilterOperatorEnum op, TupleFilter... filters) {
-        LogicalTupleFilter filter = new LogicalTupleFilter(op);
-        for (TupleFilter f : filters)
-            filter.addChild(f);
-        return filter;
-    }
-
-    public static ColumnTupleFilter columnFilter(TblColRef col) {
-        return new ColumnTupleFilter(col);
-    }
-
-    public static ConstantTupleFilter constFilter(int id) {
-        byte[] space = new byte[10];
-        ByteBuffer buf = ByteBuffer.wrap(space);
-        StringSerializer stringSerializer = new StringSerializer(DataType.getType("string"));
-        stringSerializer.serialize("" + id, buf);
-        ByteArray data = new ByteArray(buf.array(), buf.arrayOffset(), buf.position());
-        return new ConstantTupleFilter(data);
-    }
-
-    public static ConciseSet set(int... ints) {
-        ConciseSet set = new ConciseSet();
-        for (int i : ints)
-            set.add(i);
-        return set;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-job/pom.xml
----------------------------------------------------------------------
diff --git a/core-job/pom.xml b/core-job/pom.xml
index 36d34c8..9478555 100644
--- a/core-job/pom.xml
+++ b/core-job/pom.xml
@@ -39,6 +39,15 @@
             <artifactId>kylin-core-cube</artifactId>
         </dependency>
 
+        <!-- Provided -->
+        <dependency>
+            <groupId>commons-lang</groupId>
+            <artifactId>commons-lang</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.apache.curator</groupId>
             <artifactId>curator-recipes</artifactId>

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/pom.xml
----------------------------------------------------------------------
diff --git a/core-metadata/pom.xml b/core-metadata/pom.xml
index 87c4438..0832e80 100644
--- a/core-metadata/pom.xml
+++ b/core-metadata/pom.xml
@@ -39,6 +39,29 @@
             <artifactId>kylin-core-common</artifactId>
         </dependency>
 
+        <!-- Provided -->
+        <dependency>
+            <groupId>commons-cli</groupId>
+            <artifactId>commons-cli</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>commons-lang</groupId>
+            <artifactId>commons-lang</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.google.code.findbugs</groupId>
+            <artifactId>jsr305</artifactId>
+        </dependency>
+
+        <!-- Compiled -->
         <dependency>
             <groupId>net.sf.ehcache</groupId>
             <artifactId>ehcache</artifactId>

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java
index 12162b2..fbdb0bb 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/BooleanDimEnc.java
@@ -21,6 +21,7 @@ package org.apache.kylin.dimension;
 import java.io.IOException;
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
+import java.io.Serializable;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 import java.util.Map;
@@ -33,7 +34,7 @@ import com.google.common.collect.Maps;
 /**
  * Encoding Boolean values to bytes
  */
-public class BooleanDimEnc extends DimensionEncoding {
+public class BooleanDimEnc extends DimensionEncoding implements Serializable{
     private static final long serialVersionUID = 1L;
 
     public static final String ENCODING_NAME = "boolean";
@@ -108,7 +109,7 @@ public class BooleanDimEnc extends DimensionEncoding {
         return new BooleanSerializer();
     }
 
-    private class BooleanSerializer extends DataTypeSerializer<Object> {
+    public class BooleanSerializer extends DataTypeSerializer<Object> {
 
         private byte[] currentBuf() {
             byte[] buf = (byte[]) current.get();

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java
index 79e00ce..3b390d9 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DateDimEnc.java
@@ -18,10 +18,12 @@
 
 package org.apache.kylin.dimension;
 
+import java.io.Serializable;
+
 /**
  * This encoding is meant to be IDENTICAL to DateStrDictionary for 100% backward compatibility.
  */
-public class DateDimEnc extends AbstractDateDimEnc {
+public class DateDimEnc extends AbstractDateDimEnc implements Serializable{
     private static final long serialVersionUID = 1L;
 
     public static final int ID_9999_12_31 = 3652426;

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java
index 48238dc..cfa10b9 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/DictionaryDimEnc.java
@@ -21,6 +21,7 @@ package org.apache.kylin.dimension;
 import java.io.IOException;
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
+import java.io.Serializable;
 import java.nio.ByteBuffer;
 
 import org.apache.kylin.common.util.BytesUtil;
@@ -29,7 +30,7 @@ import org.apache.kylin.metadata.datatype.DataTypeSerializer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class DictionaryDimEnc extends DimensionEncoding {
+public class DictionaryDimEnc extends DimensionEncoding implements Serializable{
     private static final long serialVersionUID = 1L;
 
     private static final Logger logger = LoggerFactory.getLogger(DictionaryDimEnc.class);

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java
index f7f02a0..9ce1577 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenDimEnc.java
@@ -21,6 +21,7 @@ package org.apache.kylin.dimension;
 import java.io.IOException;
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
+import java.io.Serializable;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 
@@ -29,7 +30,7 @@ import org.apache.kylin.metadata.datatype.DataTypeSerializer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class FixedLenDimEnc extends DimensionEncoding {
+public class FixedLenDimEnc extends DimensionEncoding implements Serializable{
     private static final long serialVersionUID = 1L;
 
     private static Logger logger = LoggerFactory.getLogger(FixedLenDimEnc.class);

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java
index f90a40e..a931450 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/FixedLenHexDimEnc.java
@@ -21,6 +21,7 @@ package org.apache.kylin.dimension;
 import java.io.IOException;
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
+import java.io.Serializable;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 
@@ -43,7 +44,7 @@ import com.google.common.base.Preconditions;
  * <p>
  * Due to these limitations hex representation of hash values(with no padding, better with even characters) is more suitable
  */
-public class FixedLenHexDimEnc extends DimensionEncoding {
+public class FixedLenHexDimEnc extends DimensionEncoding implements Serializable{
     private static final long serialVersionUID = 1L;
 
     private static Logger logger = LoggerFactory.getLogger(FixedLenHexDimEnc.class);

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java
index aa954da..3650200 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntDimEnc.java
@@ -21,6 +21,7 @@ package org.apache.kylin.dimension;
 import java.io.IOException;
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
+import java.io.Serializable;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 
@@ -33,7 +34,7 @@ import org.slf4j.LoggerFactory;
  * deprecated use IntegerDimEnc instead
  * @deprecated
  */
-public class IntDimEnc extends DimensionEncoding {
+public class IntDimEnc extends DimensionEncoding implements Serializable{
     private static final long serialVersionUID = 1L;
 
     private static Logger logger = LoggerFactory.getLogger(IntDimEnc.class);

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
index 0875a7f..e024696 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/IntegerDimEnc.java
@@ -21,6 +21,7 @@ package org.apache.kylin.dimension;
 import java.io.IOException;
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
+import java.io.Serializable;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 
@@ -36,7 +37,7 @@ import org.slf4j.LoggerFactory;
  * -2^(8*N-1) is not supported because the slot is reserved for null values.
  * -2^(8*N-1) will be encoded with warn, and its output will be null
  */
-public class IntegerDimEnc extends DimensionEncoding {
+public class IntegerDimEnc extends DimensionEncoding implements Serializable{
     private static final long serialVersionUID = 1L;
 
     private static Logger logger = LoggerFactory.getLogger(IntegerDimEnc.class);

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java
index c0c52d1..d998f44 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/OneMoreByteVLongDimEnc.java
@@ -26,13 +26,14 @@ import org.slf4j.LoggerFactory;
 import java.io.IOException;
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
+import java.io.Serializable;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
 
 /**
  * not being used yet, prepared for future
  */
-public class OneMoreByteVLongDimEnc extends DimensionEncoding {
+public class OneMoreByteVLongDimEnc extends DimensionEncoding implements Serializable{
     private static final long serialVersionUID = 1L;
 
     private static Logger logger = LoggerFactory.getLogger(OneMoreByteVLongDimEnc.class);

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java b/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java
index 8c66d5d..4bef2b8 100644
--- a/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/dimension/TimeDimEnc.java
@@ -18,10 +18,12 @@
 
 package org.apache.kylin.dimension;
 
+import java.io.Serializable;
+
 /**
  * This encoding is meant to be IDENTICAL to TimeStrDictionary for 100% backward compatibility.
  */
-public class TimeDimEnc extends AbstractDateDimEnc {
+public class TimeDimEnc extends AbstractDateDimEnc implements Serializable{
     private static final long serialVersionUID = 1L;
 
     public static final String ENCODING_NAME = "time";

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
index 5d0e007..7f3a5f1 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/MeasureTypeFactory.java
@@ -218,7 +218,7 @@ abstract public class MeasureTypeFactory<T> {
     }
 
     @SuppressWarnings("rawtypes")
-    private static class NeedRewriteOnlyMeasureType extends MeasureType {
+    public static class NeedRewriteOnlyMeasureType extends MeasureType {
 
         private Boolean needRewrite;
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLDistinctCountAggFunc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLDistinctCountAggFunc.java b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLDistinctCountAggFunc.java
index 438a33f..c635cd6 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLDistinctCountAggFunc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/hllc/HLLDistinctCountAggFunc.java
@@ -77,7 +77,7 @@ public class HLLDistinctCountAggFunc {
     }
 
     @SuppressWarnings("serial")
-    private static class FixedValueHLLCMockup extends HLLCounter {
+    public static class FixedValueHLLCMockup extends HLLCounter {
 
         private Long value = null;
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/measure/topn/Counter.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/measure/topn/Counter.java b/core-metadata/src/main/java/org/apache/kylin/measure/topn/Counter.java
index 2f625af..d8fdc6e 100644
--- a/core-metadata/src/main/java/org/apache/kylin/measure/topn/Counter.java
+++ b/core-metadata/src/main/java/org/apache/kylin/measure/topn/Counter.java
@@ -22,13 +22,14 @@ import java.io.Externalizable;
 import java.io.IOException;
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
+import java.io.Serializable;
 
 /**
  * Modified from Counter.java in https://github.com/addthis/stream-lib
  * 
  * @param <T>
  */
-public class Counter<T> implements Externalizable {
+public class Counter<T> implements Externalizable, Serializable{
 
     protected T item;
     protected double count;

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 86eea55..c04de0d 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -18,6 +18,7 @@
 
 package org.apache.kylin.metadata.model;
 
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashSet;
@@ -48,11 +49,11 @@ import com.google.common.collect.Sets;
 public class DataModelDesc extends RootPersistentEntity {
     private static final Logger logger = LoggerFactory.getLogger(DataModelDesc.class);
 
-    public static enum TableKind {
+    public static enum TableKind implements Serializable{
         FACT, LOOKUP
     }
 
-    public static enum RealizationCapacity {
+    public static enum RealizationCapacity implements Serializable{
         SMALL, MEDIUM, LARGE
     }
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
index 3ab9a46..c7666cb 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
@@ -113,7 +113,7 @@ public class JoinsTree  implements Serializable {
             return chain.join;
     }
 
-    static class Chain implements Serializable {
+    public static class Chain implements Serializable {
         private static final long serialVersionUID = 1L;
         
         TableRef table; // pk side

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
index c6e6425..38c2de5 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/PartitionDesc.java
@@ -34,7 +34,7 @@ import java.io.Serializable;
 @JsonAutoDetect(fieldVisibility = Visibility.NONE, getterVisibility = Visibility.NONE, isGetterVisibility = Visibility.NONE, setterVisibility = Visibility.NONE)
 public class PartitionDesc implements Serializable {
 
-    public static enum PartitionType {
+    public static enum PartitionType implements Serializable{
         APPEND, //
         UPDATE_INSERT // not used since 0.7.1
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/metadata/model/SegmentStatusEnum.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/SegmentStatusEnum.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/SegmentStatusEnum.java
index 65c44df..689a314 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/SegmentStatusEnum.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/SegmentStatusEnum.java
@@ -18,10 +18,12 @@
 
 package org.apache.kylin.metadata.model;
 
+import java.io.Serializable;
+
 /**
  * @author xduo
  * 
  */
-public enum SegmentStatusEnum {
+public enum SegmentStatusEnum implements Serializable{
     NEW, READY, READY_PENDING
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
index 9371f76..9d22dc6 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/Segments.java
@@ -19,6 +19,7 @@
 package org.apache.kylin.metadata.model;
 
 import java.io.IOException;
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -27,7 +28,7 @@ import org.apache.kylin.common.util.Pair;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-public class Segments<T extends ISegment> extends ArrayList<T> {
+public class Segments<T extends ISegment> extends ArrayList<T> implements Serializable{
 
     private static final long serialVersionUID = 1L;
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java
index 27e2d57..872d2e4 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/realization/RealizationStatusEnum.java
@@ -18,7 +18,9 @@
 
 package org.apache.kylin.metadata.realization;
 
-public enum RealizationStatusEnum {
+import java.io.Serializable;
+
+public enum RealizationStatusEnum implements Serializable{
 
     DISABLED, READY, DESCBROKEN
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/core-storage/pom.xml
----------------------------------------------------------------------
diff --git a/core-storage/pom.xml b/core-storage/pom.xml
index 9bd3f04..8c3da5f 100644
--- a/core-storage/pom.xml
+++ b/core-storage/pom.xml
@@ -39,8 +39,28 @@
             <artifactId>kylin-core-cube</artifactId>
         </dependency>
 
+        <!-- Provided -->
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.google.code.findbugs</groupId>
+            <artifactId>jsr305</artifactId>
+        </dependency>
+
         <!-- Env & Test -->
         <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>commons-lang</groupId>
+            <artifactId>commons-lang</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
             <groupId>org.apache.kylin</groupId>
             <artifactId>kylin-core-common</artifactId>
             <type>test-jar</type>

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/engine-spark/pom.xml
----------------------------------------------------------------------
diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml
index be2cdf6..7a478f3 100644
--- a/engine-spark/pom.xml
+++ b/engine-spark/pom.xml
@@ -67,11 +67,6 @@
             <scope>provided</scope>
         </dependency>
 
-        <dependency>
-            <groupId>org.reflections</groupId>
-            <artifactId>reflections</artifactId>
-        </dependency>
-
         <!-- Hadoop dependency -->
         <dependency>
             <groupId>org.apache.hadoop</groupId>
@@ -116,45 +111,13 @@
             <artifactId>maven-model</artifactId>
             <scope>test</scope>
         </dependency>
+
         <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
+            <groupId>org.reflections</groupId>
+            <artifactId>reflections</artifactId>
+            <version>0.9.10</version>
+            <scope>test</scope>
         </dependency>
-
-
     </dependencies>
 
-    <build>
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-shade-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>shade</goal>
-                        </goals>
-                        <configuration>
-                            <minimizeJar>false</minimizeJar>
-                            <shadedArtifactAttached>true</shadedArtifactAttached>
-                            <shadedClassifierName>job</shadedClassifierName>
-                            <filters>
-                                <filter>
-                                    <artifact>*:*</artifact>
-                                    <excludes>
-                                        <exclude>META-INF/*.SF</exclude>
-                                        <exclude>META-INF/*.DSA</exclude>
-                                        <exclude>META-INF/*.RSA</exclude>
-                                    </excludes>
-                                </filter>
-                            </filters>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-
-
 </project>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/f10d9d9b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
index 3d33aa8..20b39ee 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
@@ -19,15 +19,11 @@
 package org.apache.kylin.engine.spark;
 
 import com.esotericsoftware.kryo.Kryo;
-import com.google.common.collect.Sets;
-import com.google.common.hash.Hashing;
-import org.apache.kylin.measure.MeasureIngester;
 import org.apache.spark.serializer.KryoRegistrator;
-import org.reflections.Reflections;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.Serializable;
+import java.util.LinkedHashSet;
 import java.util.Set;
 
 /**
@@ -39,7 +35,7 @@ public class KylinKryoRegistrator implements KryoRegistrator {
     @Override
     public void registerClasses(Kryo kryo) {
 
-        Set<Class> kyroClasses = Sets.newLinkedHashSet();
+        Set<Class> kyroClasses = new LinkedHashSet();
         kyroClasses.add(byte[].class);
         kyroClasses.add(int[].class);
         kyroClasses.add(byte[][].class);
@@ -55,15 +51,11 @@ public class KylinKryoRegistrator implements KryoRegistrator {
         kyroClasses.add(java.util.HashMap.class);
         kyroClasses.add(java.util.TreeMap.class);
         kyroClasses.add(java.util.Properties.class);
-        kyroClasses.addAll(new Reflections("org.apache.kylin").getSubTypesOf(Serializable.class));
-        kyroClasses.addAll(new Reflections("org.apache.kylin.dimension").getSubTypesOf(Serializable.class));
-        kyroClasses.addAll(new Reflections("org.apache.kylin.cube").getSubTypesOf(Serializable.class));
-        kyroClasses.addAll(new Reflections("org.apache.kylin.cube.model").getSubTypesOf(Object.class));
-        kyroClasses.addAll(new Reflections("org.apache.kylin.metadata").getSubTypesOf(Object.class));
-        kyroClasses.addAll(new Reflections("org.apache.kylin.metadata.model").getSubTypesOf(Object.class));
-        kyroClasses.addAll(new Reflections("org.apache.kylin.metadata.measure").getSubTypesOf(Object.class));
-        kyroClasses.addAll(new Reflections("org.apache.kylin.metadata.datatype").getSubTypesOf(org.apache.kylin.common.util.BytesSerializer.class));
-        kyroClasses.addAll(new Reflections("org.apache.kylin.measure").getSubTypesOf(MeasureIngester.class));
+        kyroClasses.add(java.math.MathContext.class);
+        kyroClasses.add(java.math.RoundingMode.class);
+        kyroClasses.add(java.util.concurrent.ConcurrentHashMap.class);
+        kyroClasses.add(java.util.Random.class);
+        kyroClasses.add(java.util.concurrent.atomic.AtomicLong.class);
 
         kyroClasses.add(org.apache.spark.sql.Row[].class);
         kyroClasses.add(org.apache.spark.sql.Row.class);
@@ -74,65 +66,29 @@ public class KylinKryoRegistrator implements KryoRegistrator {
         kyroClasses.add(org.apache.spark.sql.types.DateType$.class);
         kyroClasses.add(org.apache.spark.sql.types.Metadata.class);
         kyroClasses.add(org.apache.spark.sql.types.StringType$.class);
-        kyroClasses.add(Hashing.murmur3_128().getClass());
         kyroClasses.add(org.apache.spark.sql.execution.columnar.CachedBatch.class);
         kyroClasses.add(org.apache.spark.sql.types.Decimal.class);
         kyroClasses.add(scala.math.BigDecimal.class);
-        kyroClasses.add(java.math.MathContext.class);
-        kyroClasses.add(java.math.RoundingMode.class);
-        kyroClasses.add(java.util.concurrent.ConcurrentHashMap.class);
-        kyroClasses.add(java.util.Random.class);
-        kyroClasses.add(java.util.concurrent.atomic.AtomicLong.class);
 
+        kyroClasses.add(org.apache.kylin.common.util.SplittedBytes[].class);
         kyroClasses.add(org.apache.kylin.metadata.model.ColumnDesc[].class);
         kyroClasses.add(org.apache.kylin.metadata.model.JoinTableDesc[].class);
         kyroClasses.add(org.apache.kylin.metadata.model.TblColRef[].class);
-        kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.RealizationCapacity.class);
-        kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.TableKind.class);
-        kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.DefaultPartitionConditionBuilder.class);
-        kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.PartitionType.class);
-        kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.DeriveInfo.class);
-        kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.DeriveType.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.MeasureDesc[].class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.DataTypeSerializer[].class);
+        kyroClasses.add(org.apache.kylin.measure.MeasureAggregator[].class);
         kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnFamilyDesc[].class);
         kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnDesc[].class);
-        kyroClasses.add(org.apache.kylin.metadata.model.MeasureDesc[].class);
         kyroClasses.add(org.apache.kylin.cube.model.RowKeyColDesc[].class);
-        kyroClasses.add(org.apache.kylin.common.util.Array.class);
-        kyroClasses.add(org.apache.kylin.metadata.model.Segments.class);
-        kyroClasses.add(org.apache.kylin.metadata.realization.RealizationStatusEnum.class);
-        kyroClasses.add(org.apache.kylin.metadata.model.SegmentStatusEnum.class);
-        kyroClasses.add(org.apache.kylin.measure.BufferedMeasureCodec.class);
-        kyroClasses.add(org.apache.kylin.cube.kv.RowKeyColumnIO.class);
-        kyroClasses.add(org.apache.kylin.measure.MeasureCodec.class);
-        kyroClasses.add(org.apache.kylin.measure.MeasureAggregator[].class);
-        kyroClasses.add(org.apache.kylin.metadata.datatype.DataTypeSerializer[].class);
-        kyroClasses.add(org.apache.kylin.cube.kv.CubeDimEncMap.class);
-        kyroClasses.add(org.apache.kylin.measure.basic.BasicMeasureType.class);
-        kyroClasses.add(org.apache.kylin.common.util.SplittedBytes[].class);
-        kyroClasses.add(org.apache.kylin.common.util.SplittedBytes.class);
-        kyroClasses.add(org.apache.kylin.cube.kv.RowKeyEncoderProvider.class);
-        kyroClasses.add(org.apache.kylin.cube.kv.RowKeyEncoder.class);
-        kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalIngester.class);
-        kyroClasses.add(org.apache.kylin.dimension.DictionaryDimEnc.class);
-        kyroClasses.add(org.apache.kylin.dimension.IntDimEnc.class);
-        kyroClasses.add(org.apache.kylin.dimension.BooleanDimEnc.class);
-        kyroClasses.add(org.apache.kylin.dimension.DateDimEnc.class);
-        kyroClasses.add(org.apache.kylin.dimension.FixedLenDimEnc.class);
-        kyroClasses.add(org.apache.kylin.dimension.FixedLenHexDimEnc.class);
-        kyroClasses.add(org.apache.kylin.dimension.IntegerDimEnc.class);
-        kyroClasses.add(org.apache.kylin.dimension.OneMoreByteVLongDimEnc.class);
-        kyroClasses.add(org.apache.kylin.dimension.TimeDimEnc.class);
-        kyroClasses.add(org.apache.kylin.cube.model.AggregationGroup.HierarchyMask.class);
-        kyroClasses.add(org.apache.kylin.measure.topn.DoubleDeltaSerializer.class);
-        kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounter.class);
+
+        kylinClassByReflection1(kyroClasses);
+        kylinClassByReflection2(kyroClasses);
+
+        kyroClasses.add(com.google.common.hash.Hashing.murmur3_128().getClass());
         kyroClasses.add(org.roaringbitmap.buffer.MutableRoaringArray.class);
         kyroClasses.add(org.roaringbitmap.buffer.MappeableContainer[].class);
         kyroClasses.add(org.roaringbitmap.buffer.MutableRoaringBitmap.class);
         kyroClasses.add(org.roaringbitmap.buffer.MappeableArrayContainer.class);
-        kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounterFactory.class);
-        kyroClasses.add(org.apache.kylin.measure.topn.Counter.class);
-        kyroClasses.add(org.apache.kylin.measure.topn.TopNCounter.class);
-        kyroClasses.add(org.apache.kylin.measure.percentile.PercentileSerializer.class);
         kyroClasses.add(com.tdunning.math.stats.AVLTreeDigest.class);
         kyroClasses.add(com.tdunning.math.stats.Centroid.class);
 
@@ -151,6 +107,195 @@ public class KylinKryoRegistrator implements KryoRegistrator {
         //        kryo.register(PercentileCounter.class, new JavaSerializer());
     }
 
+    /**
+     * Generated by KyroMappingGenerator, method exceeds 150 lines, splits into two methods
+     * @param kyroClasses
+     */
+    private void kylinClassByReflection1(Set<Class> kyroClasses) {
+        kyroClasses.add(org.apache.kylin.common.KylinConfig.class);
+        kyroClasses.add(org.apache.kylin.common.KylinConfigBase.class);
+        kyroClasses.add(org.apache.kylin.common.KylinConfigExt.class);
+        kyroClasses.add(org.apache.kylin.common.persistence.RootPersistentEntity.class);
+        kyroClasses.add(org.apache.kylin.common.util.Array.class);
+        kyroClasses.add(org.apache.kylin.common.util.ByteArray.class);
+        kyroClasses.add(org.apache.kylin.common.util.ByteArray.ImmutableByteArray.class);
+        kyroClasses.add(org.apache.kylin.common.util.Dictionary.class);
+        kyroClasses.add(org.apache.kylin.common.util.OrderedProperties.class);
+        kyroClasses.add(org.apache.kylin.common.util.Pair.class);
+        kyroClasses.add(org.apache.kylin.common.util.SplittedBytes.class);
+        kyroClasses.add(org.apache.kylin.cube.CubeInstance.class);
+        kyroClasses.add(org.apache.kylin.cube.CubeSegment.class);
+        kyroClasses.add(org.apache.kylin.cube.common.RowKeySplitter.class);
+        kyroClasses.add(org.apache.kylin.cube.cuboid.Cuboid.class);
+        kyroClasses.add(org.apache.kylin.cube.cuboid.CuboidScheduler.class);
+        kyroClasses.add(org.apache.kylin.cube.gridtable.TrimmedDimensionSerializer.class);
+        kyroClasses.add(org.apache.kylin.cube.kv.AbstractRowKeyEncoder.class);
+        kyroClasses.add(org.apache.kylin.cube.kv.CubeDimEncMap.class);
+        kyroClasses.add(org.apache.kylin.cube.kv.FuzzyKeyEncoder.class);
+        kyroClasses.add(org.apache.kylin.cube.kv.FuzzyMaskEncoder.class);
+        kyroClasses.add(org.apache.kylin.cube.kv.LazyRowKeyEncoder.class);
+        kyroClasses.add(org.apache.kylin.cube.kv.RowKeyColumnIO.class);
+        kyroClasses.add(org.apache.kylin.cube.kv.RowKeyEncoder.class);
+        kyroClasses.add(org.apache.kylin.cube.kv.RowKeyEncoderProvider.class);
+        kyroClasses.add(org.apache.kylin.cube.model.AggregationGroup.class);
+        kyroClasses.add(org.apache.kylin.cube.model.AggregationGroup.HierarchyMask.class);
+        kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.class);
+        kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.DeriveInfo.class);
+        kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.DeriveType.class);
+        kyroClasses.add(org.apache.kylin.cube.model.CubeJoinedFlatTableDesc.class);
+        kyroClasses.add(org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich.class);
+        kyroClasses.add(org.apache.kylin.cube.model.DictionaryDesc.class);
+        kyroClasses.add(org.apache.kylin.cube.model.DimensionDesc.class);
+        kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnDesc.class);
+        kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnFamilyDesc.class);
+        kyroClasses.add(org.apache.kylin.cube.model.HBaseMappingDesc.class);
+        kyroClasses.add(org.apache.kylin.cube.model.HierarchyDesc.class);
+        kyroClasses.add(org.apache.kylin.cube.model.RowKeyColDesc.class);
+        kyroClasses.add(org.apache.kylin.cube.model.RowKeyDesc.class);
+        kyroClasses.add(org.apache.kylin.cube.model.SelectRule.class);
+        kyroClasses.add(org.apache.kylin.cube.model.v1_4_0.CubeDesc.class);
+        kyroClasses.add(org.apache.kylin.dict.AppendTrieDictionary.class);
+        kyroClasses.add(org.apache.kylin.dict.AppendTrieDictionary.DictNode.class);
+        kyroClasses.add(org.apache.kylin.dict.AppendTrieDictionary.DictSlice.class);
+        kyroClasses.add(org.apache.kylin.dict.AppendTrieDictionary.DictSliceKey.class);
+        kyroClasses.add(org.apache.kylin.dict.CacheDictionary.class);
+        kyroClasses.add(org.apache.kylin.dict.DateStrDictionary.class);
+        kyroClasses.add(org.apache.kylin.dict.DictionaryInfo.class);
+        kyroClasses.add(org.apache.kylin.dict.NumberDictionary.class);
+        kyroClasses.add(org.apache.kylin.dict.NumberDictionary2.class);
+        kyroClasses.add(org.apache.kylin.dict.NumberDictionaryForestBuilder.Number2BytesConverter.class);
+        kyroClasses.add(org.apache.kylin.dict.StringBytesConverter.class);
+        kyroClasses.add(org.apache.kylin.dict.TimeStrDictionary.class);
+        kyroClasses.add(org.apache.kylin.dict.TrieDictionary.class);
+        kyroClasses.add(org.apache.kylin.dict.TrieDictionaryForest.class);
+        kyroClasses.add(org.apache.kylin.dict.lookup.SnapshotTable.class);
+        kyroClasses.add(org.apache.kylin.dimension.BooleanDimEnc.class);
+        kyroClasses.add(org.apache.kylin.dimension.BooleanDimEnc.BooleanSerializer.class);
+        kyroClasses.add(org.apache.kylin.dimension.DateDimEnc.class);
+        kyroClasses.add(org.apache.kylin.dimension.DictionaryDimEnc.class);
+        kyroClasses.add(org.apache.kylin.dimension.DictionaryDimEnc.DictionarySerializer.class);
+        kyroClasses.add(org.apache.kylin.dimension.FixedLenDimEnc.class);
+        kyroClasses.add(org.apache.kylin.dimension.FixedLenDimEnc.FixedLenSerializer.class);
+        kyroClasses.add(org.apache.kylin.dimension.FixedLenHexDimEnc.class);
+        kyroClasses.add(org.apache.kylin.dimension.FixedLenHexDimEnc.FixedLenSerializer.class);
+        kyroClasses.add(org.apache.kylin.dimension.IntDimEnc.class);
+        kyroClasses.add(org.apache.kylin.dimension.IntDimEnc.IntegerSerializer.class);
+        kyroClasses.add(org.apache.kylin.dimension.IntegerDimEnc.class);
+        kyroClasses.add(org.apache.kylin.dimension.IntegerDimEnc.IntegerSerializer.class);
+        kyroClasses.add(org.apache.kylin.dimension.OneMoreByteVLongDimEnc.class);
+        kyroClasses.add(org.apache.kylin.dimension.OneMoreByteVLongDimEnc.VLongSerializer.class);
+        kyroClasses.add(org.apache.kylin.dimension.TimeDimEnc.class);
+        kyroClasses.add(org.apache.kylin.engine.mr.common.BaseCuboidBuilder.class);
+        kyroClasses.add(org.apache.kylin.engine.mr.common.NDCuboidBuilder.class);
+        kyroClasses.add(org.apache.kylin.engine.spark.SparkCubingByLayer.class);
+        kyroClasses.add(org.apache.kylin.job.JobInstance.class);
+        kyroClasses.add(org.apache.kylin.job.dao.ExecutableOutputPO.class);
+        kyroClasses.add(org.apache.kylin.job.dao.ExecutablePO.class);
+    }
+
+    /**
+     * Generated by KyroMappingGenerator
+     * @param kyroClasses
+     */
+    private void kylinClassByReflection2(Set<Class> kyroClasses) {
+        kyroClasses.add(org.apache.kylin.measure.BufferedMeasureCodec.class);
+        kyroClasses.add(org.apache.kylin.measure.MeasureAggregator.class);
+        kyroClasses.add(org.apache.kylin.measure.MeasureAggregators.class);
+        kyroClasses.add(org.apache.kylin.measure.MeasureCodec.class);
+        kyroClasses.add(org.apache.kylin.measure.MeasureIngester.class);
+        kyroClasses.add(org.apache.kylin.measure.MeasureType.class);
+        kyroClasses.add(org.apache.kylin.measure.MeasureTypeFactory.NeedRewriteOnlyMeasureType.class);
+        kyroClasses.add(org.apache.kylin.measure.basic.BasicMeasureType.class);
+        kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalIngester.class);
+        kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalMaxAggregator.class);
+        kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalMinAggregator.class);
+        kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalSumAggregator.class);
+        kyroClasses.add(org.apache.kylin.measure.basic.DoubleIngester.class);
+        kyroClasses.add(org.apache.kylin.measure.basic.DoubleMaxAggregator.class);
+        kyroClasses.add(org.apache.kylin.measure.basic.DoubleMinAggregator.class);
+        kyroClasses.add(org.apache.kylin.measure.basic.DoubleSumAggregator.class);
+        kyroClasses.add(org.apache.kylin.measure.basic.LongIngester.class);
+        kyroClasses.add(org.apache.kylin.measure.basic.LongMaxAggregator.class);
+        kyroClasses.add(org.apache.kylin.measure.basic.LongMinAggregator.class);
+        kyroClasses.add(org.apache.kylin.measure.basic.LongSumAggregator.class);
+        kyroClasses.add(org.apache.kylin.measure.bitmap.BitmapAggregator.class);
+        kyroClasses.add(org.apache.kylin.measure.bitmap.BitmapMeasureType.class);
+        kyroClasses.add(org.apache.kylin.measure.bitmap.BitmapSerializer.class);
+        kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounter.class);
+        kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounterFactory.class);
+        kyroClasses.add(org.apache.kylin.measure.dim.DimCountDistinctMeasureType.class);
+        kyroClasses.add(org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType.class);
+        kyroClasses.add(org.apache.kylin.measure.extendedcolumn.ExtendedColumnSerializer.class);
+        kyroClasses.add(org.apache.kylin.measure.hllc.DenseRegister.class);
+        kyroClasses.add(org.apache.kylin.measure.hllc.HLLCAggregator.class);
+        kyroClasses.add(org.apache.kylin.measure.hllc.HLLCMeasureType.class);
+        kyroClasses.add(org.apache.kylin.measure.hllc.HLLCSerializer.class);
+        kyroClasses.add(org.apache.kylin.measure.hllc.HLLCounter.class);
+        kyroClasses.add(org.apache.kylin.measure.hllc.HLLCounterOld.class);
+        kyroClasses.add(org.apache.kylin.measure.hllc.HLLDistinctCountAggFunc.FixedValueHLLCMockup.class);
+        kyroClasses.add(org.apache.kylin.measure.hllc.HyperLogLogPlusTable.class);
+        kyroClasses.add(org.apache.kylin.measure.hllc.SingleValueRegister.class);
+        kyroClasses.add(org.apache.kylin.measure.hllc.SparseRegister.class);
+        kyroClasses.add(org.apache.kylin.measure.percentile.PercentileAggregator.class);
+        kyroClasses.add(org.apache.kylin.measure.percentile.PercentileCounter.class);
+        kyroClasses.add(org.apache.kylin.measure.percentile.PercentileMeasureType.class);
+        kyroClasses.add(org.apache.kylin.measure.percentile.PercentileSerializer.class);
+        kyroClasses.add(org.apache.kylin.measure.raw.RawAggregator.class);
+        kyroClasses.add(org.apache.kylin.measure.raw.RawMeasureType.class);
+        kyroClasses.add(org.apache.kylin.measure.raw.RawSerializer.class);
+        kyroClasses.add(org.apache.kylin.measure.topn.Counter.class);
+        kyroClasses.add(org.apache.kylin.measure.topn.DoubleDeltaSerializer.class);
+        kyroClasses.add(org.apache.kylin.measure.topn.TopNAggregator.class);
+        kyroClasses.add(org.apache.kylin.measure.topn.TopNCounter.class);
+        kyroClasses.add(org.apache.kylin.measure.topn.TopNCounterSerializer.class);
+        kyroClasses.add(org.apache.kylin.measure.topn.TopNMeasureType.class);
+        kyroClasses.add(org.apache.kylin.metadata.badquery.BadQueryEntry.class);
+        kyroClasses.add(org.apache.kylin.metadata.badquery.BadQueryHistory.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.BigDecimalSerializer.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.BooleanSerializer.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.DataType.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.DataTypeSerializer.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.DateTimeSerializer.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.DoubleMutable.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.DoubleSerializer.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.Int4Serializer.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.IntMutable.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.Long8Serializer.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.LongMutable.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.LongSerializer.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.StringSerializer.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.ColumnDesc.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.RealizationCapacity.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.TableKind.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.DatabaseDesc.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.ExternalFilterDesc.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.FunctionDesc.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.JoinDesc.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.JoinTableDesc.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.JoinsTree.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.JoinsTree.Chain.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.MeasureDesc.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.ModelDimensionDesc.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.ParameterDesc.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.PartitionType.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.DefaultPartitionConditionBuilder.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.Segments.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.SegmentStatusEnum.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.TableDesc.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.TableExtDesc.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.TableExtDesc.ColumnStats.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.TableRef.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.TblColRef.class);
+        kyroClasses.add(org.apache.kylin.metadata.project.ProjectInstance.class);
+        kyroClasses.add(org.apache.kylin.metadata.project.RealizationEntry.class);
+        kyroClasses.add(org.apache.kylin.metadata.realization.RealizationStatusEnum.class);
+        kyroClasses.add(org.apache.kylin.metadata.streaming.StreamingConfig.class);
+        kyroClasses.add(org.apache.kylin.source.ReadableTable.TableSignature.class);
+        kyroClasses.add(org.apache.kylin.storage.hybrid.HybridInstance.class);
+    }
+
     private static void addClassQuitely(Set<Class> kyroClasses, String className) {
         try {
             kyroClasses.add(Class.forName(className));


Mime
View raw message