kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From billy...@apache.org
Subject kylin git commit: KYLIN-2428 Generate kyro mapping by Reflection
Date Mon, 13 Feb 2017 23:33:27 GMT
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2428 900436e36 -> 62463b741


KYLIN-2428 Generate kyro mapping by Reflection


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/62463b74
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/62463b74
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/62463b74

Branch: refs/heads/KYLIN-2428
Commit: 62463b7413f34718ee29005c414b0b86bb751d8e
Parents: 900436e
Author: Billy Liu <billyliu@apache.org>
Authored: Tue Feb 14 07:33:13 2017 +0800
Committer: Billy Liu <billyliu@apache.org>
Committed: Tue Feb 14 07:33:13 2017 +0800

----------------------------------------------------------------------
 .../apache/kylin/metadata/model/JoinsTree.java  |   2 +-
 engine-spark/pom.xml                            |   7 +
 .../engine/spark/KylinKryoRegistrator.java      | 187 ++++++++++++++++++-
 .../engine/spark/util/KyroMappingGenerator.java |  51 +++++
 4 files changed, 245 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/62463b74/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
index 3ab9a46..c7666cb 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/JoinsTree.java
@@ -113,7 +113,7 @@ public class JoinsTree  implements Serializable {
             return chain.join;
     }
 
-    static class Chain implements Serializable {
+    public static class Chain implements Serializable {
         private static final long serialVersionUID = 1L;
         
         TableRef table; // pk side

http://git-wip-us.apache.org/repos/asf/kylin/blob/62463b74/engine-spark/pom.xml
----------------------------------------------------------------------
diff --git a/engine-spark/pom.xml b/engine-spark/pom.xml
index 258232d..7a478f3 100644
--- a/engine-spark/pom.xml
+++ b/engine-spark/pom.xml
@@ -111,6 +111,13 @@
             <artifactId>maven-model</artifactId>
             <scope>test</scope>
         </dependency>
+
+        <dependency>
+            <groupId>org.reflections</groupId>
+            <artifactId>reflections</artifactId>
+            <version>0.9.10</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
 </project>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/kylin/blob/62463b74/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
index c99ee6e..d9bfe99 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/KylinKryoRegistrator.java
@@ -61,7 +61,7 @@ public class KylinKryoRegistrator implements KryoRegistrator {
         kyroClasses.add(org.apache.spark.sql.types.DateType$.class);
         kyroClasses.add(org.apache.spark.sql.types.Metadata.class);
         kyroClasses.add(org.apache.spark.sql.types.StringType$.class);
-//        kyroClasses.add(Hashing.murmur3_128().getClass());
+        //        kyroClasses.add(Hashing.murmur3_128().getClass());
         kyroClasses.add(org.apache.spark.sql.execution.columnar.CachedBatch.class);
         kyroClasses.add(org.apache.spark.sql.types.Decimal.class);
         kyroClasses.add(scala.math.BigDecimal.class);
@@ -146,6 +146,9 @@ public class KylinKryoRegistrator implements KryoRegistrator {
         kyroClasses.add(org.apache.kylin.dimension.IntegerDimEnc.class);
         kyroClasses.add(org.apache.kylin.dimension.OneMoreByteVLongDimEnc.class);
         kyroClasses.add(org.apache.kylin.dimension.TimeDimEnc.class);
+        kylinClassByReflection1(kyroClasses);
+        kylinClassByReflection2(kyroClasses);
+
         kyroClasses.add(org.roaringbitmap.buffer.MutableRoaringArray.class);
         kyroClasses.add(org.roaringbitmap.buffer.MappeableContainer[].class);
         kyroClasses.add(org.roaringbitmap.buffer.MutableRoaringBitmap.class);
@@ -168,6 +171,188 @@ public class KylinKryoRegistrator implements KryoRegistrator {
         //        kryo.register(PercentileCounter.class, new JavaSerializer());
     }
 
+    /**
+     * Generated by KyroMappingGenerator, method exceeds 150 lines, splits into two methods
+     * @param kyroClasses
+     */
+    private void kylinClassByReflection1(Set<Class> kyroClasses) {
+        kyroClasses.add(org.apache.kylin.metadata.model.ColumnDesc[].class);
+        kyroClasses.add(org.apache.kylin.cube.model.RowKeyDesc.class);
+        kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalMaxAggregator.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.DataModelDesc.class);
+        kyroClasses.add(org.apache.kylin.source.ReadableTable.TableSignature.class);
+        kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.DeriveInfo.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.ParameterDesc.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.ColumnDesc.class);
+        kyroClasses.add(org.apache.kylin.measure.raw.RawSerializer.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.LongMutable.class);
+        kyroClasses.add(org.apache.kylin.dimension.FixedLenDimEnc.FixedLenSerializer.class);
+        kyroClasses.add(org.apache.kylin.cube.model.AggregationGroup.class);
+        kyroClasses.add(org.apache.kylin.dict.TrieDictionary.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.Int4Serializer.class);
+        kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalIngester.class);
+        kyroClasses.add(org.apache.kylin.measure.hllc.DenseRegister.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.DefaultPartitionConditionBuilder.class);
+        kyroClasses.add(org.apache.kylin.measure.extendedcolumn.ExtendedColumnSerializer.class);
+        kyroClasses.add(org.apache.kylin.dict.lookup.SnapshotTable.class);
+        kyroClasses.add(org.apache.kylin.measure.percentile.PercentileMeasureType.class);
+        kyroClasses.add(org.apache.kylin.measure.MeasureType.class);
+        kyroClasses.add(org.apache.kylin.cube.gridtable.TrimmedDimensionSerializer.class);
+        kyroClasses.add(org.apache.kylin.metadata.project.RealizationEntry.class);
+        kyroClasses.add(org.apache.kylin.cube.kv.AbstractRowKeyEncoder.class);
+        kyroClasses.add(org.apache.kylin.common.KylinConfigBase.class);
+        kyroClasses.add(org.apache.kylin.cube.model.v1_4_0.CubeDesc.class);
+        kyroClasses.add(org.apache.kylin.cube.kv.FuzzyKeyEncoder.class);
+        kyroClasses.add(org.apache.kylin.cube.kv.RowKeyEncoderProvider.class);
+        kyroClasses.add(org.apache.kylin.cube.model.DimensionDesc.class);
+        kyroClasses.add(org.apache.kylin.measure.raw.RawAggregator.class);
+        kyroClasses.add(org.apache.kylin.measure.basic.DoubleSumAggregator.class);
+        kyroClasses.add(org.apache.kylin.dimension.IntegerDimEnc.IntegerSerializer.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.ModelDimensionDesc.class);
+        kyroClasses.add(org.apache.kylin.cube.model.SelectRule.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.DatabaseDesc.class);
+        kyroClasses.add(org.apache.kylin.common.util.Array.class);
+        kyroClasses.add(org.apache.kylin.engine.mr.common.NDCuboidBuilder.class);
+        kyroClasses.add(org.apache.kylin.metadata.badquery.BadQueryHistory.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.JoinTableDesc.class);
+        kyroClasses.add(org.apache.kylin.measure.hllc.HLLCounterOld.class);
+        kyroClasses.add(org.apache.kylin.measure.basic.LongMinAggregator.class);
+        kyroClasses.add(org.apache.kylin.measure.percentile.PercentileCounter.class);
+        kyroClasses.add(org.apache.kylin.metadata.project.ProjectInstance.class);
+        kyroClasses.add(org.apache.kylin.measure.basic.BasicMeasureType.class);
+        kyroClasses.add(org.apache.kylin.measure.percentile.PercentileAggregator.class);
+        kyroClasses.add(org.apache.kylin.common.util.ByteArray.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.FunctionDesc.class);
+        kyroClasses.add(org.apache.kylin.measure.topn.DoubleDeltaSerializer.class);
+        kyroClasses.add(org.apache.kylin.measure.hllc.HLLCounter.class);
+        kyroClasses.add(org.apache.kylin.cube.kv.LazyRowKeyEncoder.class);
+        kyroClasses.add(org.apache.kylin.measure.hllc.HLLCAggregator.class);
+        kyroClasses.add(org.apache.kylin.measure.bitmap.BitmapMeasureType.class);
+        kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnDesc.class);
+        kyroClasses.add(org.apache.kylin.measure.hllc.HLLCSerializer.class);
+        kyroClasses.add(org.apache.kylin.cube.cuboid.CuboidScheduler.class);
+        kyroClasses.add(org.apache.kylin.measure.bitmap.BitmapAggregator.class);
+        kyroClasses.add(org.apache.kylin.job.dao.ExecutablePO.class);
+        kyroClasses.add(org.apache.kylin.measure.raw.RawMeasureType.class);
+        kyroClasses.add(org.apache.kylin.common.util.OrderedProperties.class);
+        kyroClasses.add(org.apache.kylin.measure.bitmap.BitmapSerializer.class);
+        kyroClasses.add(org.apache.kylin.cube.kv.RowKeyColumnIO.class);
+        kyroClasses.add(org.apache.kylin.cube.cuboid.Cuboid.class);
+        kyroClasses.add(org.apache.kylin.measure.MeasureAggregator.class);
+        kyroClasses.add(org.apache.kylin.cube.model.DictionaryDesc.class);
+        kyroClasses.add(org.apache.kylin.measure.dim.DimCountDistinctMeasureType.class);
+        kyroClasses.add(org.apache.kylin.cube.model.RowKeyColDesc.class);
+        kyroClasses.add(org.apache.kylin.measure.topn.TopNCounter.class);
+        kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalMinAggregator.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.TableExtDesc.ColumnStats.class);
+        kyroClasses.add(org.apache.kylin.metadata.streaming.StreamingConfig.class);
+        kyroClasses.add(org.apache.kylin.dict.AppendTrieDictionary.class);
+        kyroClasses.add(org.apache.kylin.dimension.IntDimEnc.IntegerSerializer.class);
+        kyroClasses.add(org.apache.kylin.common.KylinConfig.class);
+        kyroClasses.add(org.apache.kylin.cube.model.AggregationGroup.HierarchyMask.class);
+        kyroClasses.add(org.apache.kylin.measure.BufferedMeasureCodec.class);
+        kyroClasses.add(org.apache.kylin.cube.model.CubeDesc.class);
+        kyroClasses.add(org.apache.kylin.cube.CubeInstance.class);
+    }
+
+    /**
+     * Generated by KyroMappingGenerator
+     * @param kyroClasses
+     */
+    private void kylinClassByReflection2(Set<Class> kyroClasses) {
+        kyroClasses.add(org.apache.kylin.measure.basic.DoubleMaxAggregator.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.DateTimeSerializer.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.BigDecimalSerializer.class);
+        kyroClasses.add(org.apache.kylin.dict.NumberDictionary2.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.IntMutable.class);
+        kyroClasses.add(org.apache.kylin.storage.hybrid.HybridInstance.class);
+        kyroClasses.add(org.apache.kylin.cube.model.HBaseColumnFamilyDesc.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.PartitionDesc.class);
+        kyroClasses.add(org.apache.kylin.cube.CubeSegment.class);
+        kyroClasses.add(org.apache.kylin.common.util.ByteArray.ImmutableByteArray.class);
+        kyroClasses.add(org.apache.kylin.cube.model.CubeJoinedFlatTableDesc.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.MeasureDesc.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.DoubleMutable.class);
+        kyroClasses.add(org.apache.kylin.common.util.Pair.class);
+        kyroClasses.add(org.apache.kylin.measure.topn.TopNAggregator.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.DataType.class);
+        kyroClasses.add(org.apache.kylin.measure.basic.LongSumAggregator.class);
+        kyroClasses.add(org.apache.kylin.measure.hllc.SparseRegister.class);
+        kyroClasses.add(org.apache.kylin.dict.AppendTrieDictionary.DictSlice.class);
+        kyroClasses.add(org.apache.kylin.dict.TrieDictionaryForest.class);
+        kyroClasses.add(org.apache.kylin.measure.basic.DoubleMinAggregator.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.TableExtDesc.class);
+        kyroClasses.add(org.apache.kylin.measure.MeasureAggregators.class);
+        kyroClasses.add(org.apache.kylin.dict.CacheDictionary.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.Long8Serializer.class);
+        kyroClasses.add(org.apache.kylin.dict.NumberDictionaryForestBuilder.Number2BytesConverter.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.TableRef.class);
+        kyroClasses.add(org.apache.kylin.metadata.badquery.BadQueryEntry.class);
+        kyroClasses.add(org.apache.kylin.cube.kv.RowKeyEncoder.class);
+        kyroClasses.add(org.apache.kylin.measure.extendedcolumn.ExtendedColumnMeasureType.class);
+        kyroClasses.add(org.apache.kylin.common.KylinConfigExt.class);
+        kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounter.class);
+        kyroClasses.add(org.apache.kylin.common.util.Dictionary.class);
+        kyroClasses.add(org.apache.kylin.measure.topn.TopNCounterSerializer.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.JoinsTree.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.JoinsTree.Chain.class);
+        kyroClasses.add(org.apache.kylin.cube.model.HBaseMappingDesc.class);
+        kyroClasses.add(org.apache.kylin.job.dao.ExecutableOutputPO.class);
+        kyroClasses.add(org.apache.kylin.job.JobInstance.class);
+        kyroClasses.add(org.apache.kylin.dimension.FixedLenHexDimEnc.FixedLenSerializer.class);
+        kyroClasses.add(org.apache.kylin.measure.topn.TopNMeasureType.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.JoinDesc.class);
+        kyroClasses.add(org.apache.kylin.dict.NumberDictionary.class);
+        kyroClasses.add(org.apache.kylin.dimension.OneMoreByteVLongDimEnc.VLongSerializer.class);
+        kyroClasses.add(org.apache.kylin.dict.TimeStrDictionary.class);
+        kyroClasses.add(org.apache.kylin.measure.hllc.SingleValueRegister.class);
+        kyroClasses.add(org.apache.kylin.dict.StringBytesConverter.class);
+        kyroClasses.add(org.apache.kylin.cube.model.CubeJoinedFlatTableEnrich.class);
+        kyroClasses.add(org.apache.kylin.measure.hllc.HLLCMeasureType.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.DoubleSerializer.class);
+        kyroClasses.add(org.apache.kylin.engine.spark.SparkCubingByLayer.class);
+        kyroClasses.add(org.apache.kylin.cube.common.RowKeySplitter.class);
+        kyroClasses.add(org.apache.kylin.measure.basic.DoubleIngester.class);
+        kyroClasses.add(org.apache.kylin.cube.kv.FuzzyMaskEncoder.class);
+        kyroClasses.add(org.apache.kylin.dimension.DictionaryDimEnc.DictionarySerializer.class);
+        kyroClasses.add(org.apache.kylin.common.util.SplittedBytes.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.DataTypeSerializer.class);
+        kyroClasses.add(org.apache.kylin.dict.DictionaryInfo.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.TableDesc.class);
+        kyroClasses.add(org.apache.kylin.engine.mr.common.BaseCuboidBuilder.class);
+        kyroClasses.add(org.apache.kylin.measure.percentile.PercentileSerializer.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.LongSerializer.class);
+        kyroClasses.add(org.apache.kylin.measure.MeasureCodec.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.ExternalFilterDesc.class);
+        kyroClasses.add(org.apache.kylin.dict.DateStrDictionary.class);
+        kyroClasses.add(org.apache.kylin.measure.MeasureIngester.class);
+        kyroClasses.add(org.apache.kylin.dict.AppendTrieDictionary.DictNode.class);
+        kyroClasses.add(org.apache.kylin.cube.kv.CubeDimEncMap.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.BooleanSerializer.class);
+        kyroClasses.add(org.apache.kylin.dict.AppendTrieDictionary.DictSliceKey.class);
+        kyroClasses.add(org.apache.kylin.measure.basic.LongMaxAggregator.class);
+        kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalSumAggregator.class);
+        kyroClasses.add(org.apache.kylin.measure.basic.LongIngester.class);
+        kyroClasses.add(org.apache.kylin.common.persistence.RootPersistentEntity.class);
+        kyroClasses.add(org.apache.kylin.metadata.model.TblColRef.class);
+        kyroClasses.add(org.apache.kylin.cube.model.HierarchyDesc.class);
+        kyroClasses.add(org.apache.kylin.measure.bitmap.RoaringBitmapCounterFactory.class);
+        kyroClasses.add(org.apache.kylin.measure.hllc.HyperLogLogPlusTable.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.StringSerializer.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.LongSerializer.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.DoubleSerializer.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.Long8Serializer.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.BooleanSerializer.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.DateTimeSerializer.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.BigDecimalSerializer.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.Int4Serializer.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.DataTypeSerializer.class);
+        kyroClasses.add(org.apache.kylin.metadata.datatype.StringSerializer.class);
+        kyroClasses.add(org.apache.kylin.measure.basic.BigDecimalIngester.class);
+        kyroClasses.add(org.apache.kylin.measure.basic.DoubleIngester.class);
+        kyroClasses.add(org.apache.kylin.measure.basic.LongIngester.class);
+    }
+
     private static void addClassQuitely(Set<Class> kyroClasses, String className) {
         try {
             kyroClasses.add(Class.forName(className));

http://git-wip-us.apache.org/repos/asf/kylin/blob/62463b74/engine-spark/src/test/java/org/apache/kylin/engine/spark/util/KyroMappingGenerator.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/test/java/org/apache/kylin/engine/spark/util/KyroMappingGenerator.java
b/engine-spark/src/test/java/org/apache/kylin/engine/spark/util/KyroMappingGenerator.java
new file mode 100644
index 0000000..b4960e9
--- /dev/null
+++ b/engine-spark/src/test/java/org/apache/kylin/engine/spark/util/KyroMappingGenerator.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.engine.spark.util;
+
+import org.apache.kylin.common.util.BytesSerializer;
+import org.apache.kylin.measure.MeasureIngester;
+import org.reflections.Reflections;
+
+import java.io.Serializable;
+import java.util.Set;
+
+/**
+ * Generate Kyro Registrator class
+ */
+public class KyroMappingGenerator {
+    public static void main(String[] args) {
+        Set<Class<? extends Serializable>> subTypesOfSerializable = new Reflections("org.apache.kylin").getSubTypesOf(Serializable.class);
+        String begin = "kyroClasses.add(";
+        String end = ".class);";
+        for (Class clazz : subTypesOfSerializable) {
+            if (clazz.getCanonicalName() != null)
+                System.out.println(begin + clazz.getCanonicalName() + end);
+        }
+        Set<Class<? extends BytesSerializer>> subTypesOfBytes = new Reflections("org.apache.kylin.metadata.datatype").getSubTypesOf(BytesSerializer.class);
+        for (Class clazz : subTypesOfBytes) {
+            if (clazz.getCanonicalName() != null)
+                System.out.println(begin + clazz.getCanonicalName() + end);
+        }
+        Set<Class<? extends MeasureIngester>> subTypesOfMeasure = new Reflections("org.apache.kylin.measure").getSubTypesOf(MeasureIngester.class);
+        for (Class clazz : subTypesOfMeasure) {
+            if (clazz.getCanonicalName() != null)
+                System.out.println(begin + clazz.getCanonicalName() + end);
+        }
+    }
+}


Mime
View raw message