kylin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From liy...@apache.org
Subject kylin git commit: KYLIN-2497 Optimize the performance of SelfDefineSortableKey
Date Fri, 10 Mar 2017 10:41:10 GMT
Repository: kylin
Updated Branches:
  refs/heads/KYLIN-2497 [created] 6e89e7796


KYLIN-2497 Optimize the performance of SelfDefineSortableKey

Signed-off-by: Hongbin Ma <mahongbin@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6e89e779
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6e89e779
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6e89e779

Branch: refs/heads/KYLIN-2497
Commit: 6e89e7796455e85065730353216ae9b6891b5e0e
Parents: 8edf234
Author: xiefan46 <958034172@qq.com>
Authored: Fri Mar 10 17:20:53 2017 +0800
Committer: Hongbin Ma <mahongbin@apache.org>
Committed: Fri Mar 10 18:38:05 2017 +0800

----------------------------------------------------------------------
 .../mr/steps/FactDistinctColumnsMapper.java     |  18 ++-
 .../mr/steps/FactDistinctColumnsMapperBase.java |   2 +-
 .../engine/mr/steps/SelfDefineSortableKey.java  | 102 ++++++-------
 .../mr/steps/SelfDefineSortableKeyOld.java      | 143 +++++++++++++++++++
 .../mr/steps/NumberDictionaryForestTest.java    |  18 +--
 .../mr/steps/SelfDefineSortableKeyTest.java     |  82 +++++++++--
 6 files changed, 278 insertions(+), 87 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kylin/blob/6e89e779/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
index 07e788c..9f65163 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapper.java
@@ -31,6 +31,7 @@ import org.apache.kylin.cube.cuboid.CuboidScheduler;
 import org.apache.kylin.engine.mr.common.BatchConstants;
 import org.apache.kylin.measure.BufferedMeasureCodec;
 import org.apache.kylin.measure.hllc.HLLCounter;
+import org.apache.kylin.metadata.datatype.DataType;
 import org.apache.kylin.metadata.model.TblColRef;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -40,6 +41,7 @@ import com.google.common.hash.HashFunction;
 import com.google.common.hash.Hasher;
 import com.google.common.hash.Hashing;
 
+
 /**
  */
 public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperBase<KEYIN,
Object> {
@@ -50,6 +52,7 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
         BYTES
     }
 
+
     protected boolean collectStatistics = false;
     protected CuboidScheduler cuboidScheduler = null;
     protected int nRowKey;
@@ -68,6 +71,8 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
     private int partitionColumnIndex = -1;
     private boolean needFetchPartitionCol = true;
 
+    private SelfDefineSortableKey sortableKey = new SelfDefineSortableKey();
+
     @Override
     protected void setup(Context context) throws IOException {
         super.setup(context);
@@ -137,7 +142,6 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
         String[] row = flatTableInputFormat.parseMapperInput(record);
 
         context.getCounter(RawDataCounter.BYTES).increment(countSizeInBytes(row));
-
         for (int i = 0; i < factDictCols.size(); i++) {
             String fieldValue = row[dictionaryColumnIndex[i]];
             if (fieldValue == null)
@@ -161,9 +165,9 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
             tmpbuf.put(Bytes.toBytes(reducerIndex)[3]);
             tmpbuf.put(valueBytes);
             outputKey.set(tmpbuf.array(), 0, tmpbuf.position());
-            sortableKey.setText(outputKey);
+            DataType type = factDictCols.get(i).getType();
+            sortableKey.init(outputKey, type);
             //judge type
-            sortableKey.setTypeIdByDatatype(factDictCols.get(i).getType());
             context.write(sortableKey, EMPTY_TEXT);
 
             // log a few rows for troubleshooting
@@ -189,8 +193,7 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
                     tmpbuf.put(MARK_FOR_PARTITION_COL);
                     tmpbuf.put(valueBytes);
                     outputKey.set(tmpbuf.array(), 0, tmpbuf.position());
-                    sortableKey.setText(outputKey);
-                    sortableKey.setTypeId((byte) 0);
+                    sortableKey.init(outputKey, (byte) 0);
                     context.write(sortableKey, EMPTY_TEXT);
                 }
             }
@@ -247,13 +250,13 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
                 hllBuf.clear();
                 hll.writeRegisters(hllBuf);
                 outputValue.set(hllBuf.array(), 0, hllBuf.position());
-                sortableKey.setText(outputKey);
-                sortableKey.setTypeId((byte) 0);
+                sortableKey.init(outputKey, (byte) 0);
                 context.write(sortableKey, outputValue);
             }
         }
     }
 
+
     private int countNewSize(int oldSize, int dataSize) {
         int newSize = oldSize * 2;
         while (newSize < dataSize) {
@@ -261,4 +264,5 @@ public class FactDistinctColumnsMapper<KEYIN> extends FactDistinctColumnsMapperB
         }
         return newSize;
     }
+
 }

http://git-wip-us.apache.org/repos/asf/kylin/blob/6e89e779/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
index 34624e3..458af69 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsMapperBase.java
@@ -54,7 +54,7 @@ abstract public class FactDistinctColumnsMapperBase<KEYIN, VALUEIN>
extends Kyli
     protected IMRTableInputFormat flatTableInputFormat;
 
     protected Text outputKey = new Text();
-    protected SelfDefineSortableKey sortableKey = new SelfDefineSortableKey();
+    //protected SelfDefineSortableKey sortableKey = new SelfDefineSortableKey();
     protected Text outputValue = new Text();
     protected int errorRecordCounter = 0;
 

http://git-wip-us.apache.org/repos/asf/kylin/blob/6e89e779/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
index 74525d3..48c7596 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKey.java
@@ -17,20 +17,19 @@
 */
 package org.apache.kylin.engine.mr.steps;
 
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.WritableComparable;
-import org.apache.kylin.metadata.datatype.DataType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import java.io.DataInput;
 import java.io.DataOutput;
 import java.io.IOException;
 
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.kylin.metadata.datatype.DataType;
+
 /**
- * Created by xiefan on 16-11-1.
+ * Created by xiefan on 17-3-10.
  */
 public class SelfDefineSortableKey implements WritableComparable<SelfDefineSortableKey>
{
+
     public enum TypeFlag {
         NONE_NUMERIC_TYPE,
         INTEGER_FAMILY_TYPE,
@@ -39,51 +38,44 @@ public class SelfDefineSortableKey implements WritableComparable<SelfDefineSorta
 
     private byte typeId; //non-numeric(0000 0000) int(0000 0001) other numberic(0000 0010)
 
-    private Text text;
+    private Object keyInObj;
 
-    private static final Logger logger = LoggerFactory.getLogger(SelfDefineSortableKey.class);
 
     public SelfDefineSortableKey() {
     }
 
-    public SelfDefineSortableKey(byte typeId, Text text) {
+    public SelfDefineSortableKey(Text key, DataType type) {
+        init(key, getTypeIdByDatatype(type));
+    }
+
+    public void init(Text key, byte typeId) {
         this.typeId = typeId;
-        this.text = text;
+        if (isNumberFamily()) {
+            if (isIntegerFamily()) {
+                this.keyInObj = Integer.parseInt(new String(key.getBytes()));
+            } else {
+                this.keyInObj = Double.parseDouble(new String(key.getBytes()));
+            }
+        } else {
+            this.keyInObj = key;
+        }
+    }
+
+    public void init(Text key, DataType type) {
+        init(key, getTypeIdByDatatype(type));
     }
 
     @Override
     public int compareTo(SelfDefineSortableKey o) {
-        if (!o.isNumberFamily()) {
-            return this.text.compareTo(o.text);
+        if (this.typeId != o.typeId)
+            throw new IllegalStateException("Error. Incompatible types");
+        if (!isNumberFamily()) {
+            return ((Text) this.keyInObj).compareTo(((Text) o.keyInObj));
         } else {
-            byte[] data1 = this.text.getBytes();
-            byte[] data2 = o.text.getBytes();
-            String str1 = new String(data1, 1, data1.length - 1);
-            String str2 = new String(data2, 1, data2.length - 1);
-            if (str1.equals("") || str2.equals("")) {
-                //should not achieve here
-                logger.error("none numeric value!");
-                return 0;
-            }
-            if (o.isIntegerFamily()) { //integer type
-                try {
-                    Long num1 = Long.parseLong(str1);
-                    Long num2 = Long.parseLong(str2);
-                    return num1.compareTo(num2);
-                } catch (NumberFormatException e) {
-                    logger.error("NumberFormatException when parse integer family number.str1:"
+ str1 + " str2:" + str2);
-                    e.printStackTrace();
-                    return 0;
-                }
-            } else { //other numeric type
-                try {
-                    Double num1 = Double.parseDouble(str1);
-                    Double num2 = Double.parseDouble(str2);
-                    return num1.compareTo(num2);
-                } catch (NumberFormatException e) {
-                    logger.error("NumberFormatException when parse doul family number.str1:"
+ str1 + " str2:" + str2);
-                    return 0;
-                }
+            if (isIntegerFamily()) {
+                return Integer.compare((Integer) this.keyInObj, (Integer) o.keyInObj);
+            } else {
+                return Double.compare((Double) this.keyInObj, (Double) o.keyInObj);
             }
         }
     }
@@ -91,14 +83,15 @@ public class SelfDefineSortableKey implements WritableComparable<SelfDefineSorta
     @Override
     public void write(DataOutput dataOutput) throws IOException {
         dataOutput.writeByte(typeId);
-        text.write(dataOutput);
+        Text outputKey = new Text(keyInObj.toString());
+        outputKey.write(dataOutput);
     }
 
     @Override
     public void readFields(DataInput dataInput) throws IOException {
         this.typeId = dataInput.readByte();
-        this.text = new Text();
-        text.readFields(dataInput);
+        Text inputKey = new Text();
+        init(inputKey, typeId);
     }
 
     public short getTypeId() {
@@ -106,7 +99,8 @@ public class SelfDefineSortableKey implements WritableComparable<SelfDefineSorta
     }
 
     public Text getText() {
-        return text;
+        Text key = new Text(keyInObj.toString());
+        return key;
     }
 
     public boolean isNumberFamily() {
@@ -119,25 +113,21 @@ public class SelfDefineSortableKey implements WritableComparable<SelfDefineSorta
         return (typeId == TypeFlag.INTEGER_FAMILY_TYPE.ordinal());
     }
 
-    public boolean isOtherNumericFamily() {
-        return (typeId == TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
-    }
 
-    public void setTypeIdByDatatype(DataType type) {
+    public byte getTypeIdByDatatype(DataType type) {
         if (!type.isNumberFamily()) {
-            this.typeId = (byte) TypeFlag.NONE_NUMERIC_TYPE.ordinal();
+            return (byte) TypeFlag.NONE_NUMERIC_TYPE.ordinal();
         } else if (type.isIntegerFamily()) {
-            this.typeId = (byte) TypeFlag.INTEGER_FAMILY_TYPE.ordinal();
+            return (byte) TypeFlag.INTEGER_FAMILY_TYPE.ordinal();
         } else {
-            this.typeId = (byte) TypeFlag.DOUBLE_FAMILY_TYPE.ordinal();
+            return (byte) TypeFlag.DOUBLE_FAMILY_TYPE.ordinal();
         }
     }
-    
+
     public void setTypeId(byte typeId) {
         this.typeId = typeId;
     }
 
-    public void setText(Text text) {
-        this.text = text;
-    }
 }
+
+

http://git-wip-us.apache.org/repos/asf/kylin/blob/6e89e779/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyOld.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyOld.java
b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyOld.java
new file mode 100644
index 0000000..e360b49
--- /dev/null
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyOld.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+package org.apache.kylin.engine.mr.steps;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.kylin.metadata.datatype.DataType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+/**
+ * Created by xiefan on 16-11-1.
+ */
+public class SelfDefineSortableKeyOld implements WritableComparable<SelfDefineSortableKeyOld>
{
+    public enum TypeFlag {
+        NONE_NUMERIC_TYPE,
+        INTEGER_FAMILY_TYPE,
+        DOUBLE_FAMILY_TYPE
+    }
+
+    private byte typeId; //non-numeric(0000 0000) int(0000 0001) other numberic(0000 0010)
+
+    private Text text;
+
+    private static final Logger logger = LoggerFactory.getLogger(SelfDefineSortableKeyOld.class);
+
+    public SelfDefineSortableKeyOld() {
+    }
+
+    public SelfDefineSortableKeyOld(byte typeId, Text text) {
+        this.typeId = typeId;
+        this.text = text;
+    }
+
+    @Override
+    public int compareTo(SelfDefineSortableKeyOld o) {
+        if (!o.isNumberFamily()) {
+            return this.text.compareTo(o.text);
+        } else {
+            byte[] data1 = this.text.getBytes();
+            byte[] data2 = o.text.getBytes();
+            String str1 = new String(data1, 1, data1.length - 1);
+            String str2 = new String(data2, 1, data2.length - 1);
+            if (str1.equals("") || str2.equals("")) {
+                //should not achieve here
+                logger.error("none numeric value!");
+                return 0;
+            }
+            if (o.isIntegerFamily()) { //integer type
+                try {
+                    Long num1 = Long.parseLong(str1);
+                    Long num2 = Long.parseLong(str2);
+                    return num1.compareTo(num2);
+                } catch (NumberFormatException e) {
+                    logger.error("NumberFormatException when parse integer family number.str1:"
+ str1 + " str2:" + str2);
+                    e.printStackTrace();
+                    return 0;
+                }
+            } else { //other numeric type
+                try {
+                    Double num1 = Double.parseDouble(str1);
+                    Double num2 = Double.parseDouble(str2);
+                    return num1.compareTo(num2);
+                } catch (NumberFormatException e) {
+                    logger.error("NumberFormatException when parse doul family number.str1:"
+ str1 + " str2:" + str2);
+                    return 0;
+                }
+            }
+        }
+    }
+
+    @Override
+    public void write(DataOutput dataOutput) throws IOException {
+        dataOutput.writeByte(typeId);
+        text.write(dataOutput);
+    }
+
+    @Override
+    public void readFields(DataInput dataInput) throws IOException {
+        this.typeId = dataInput.readByte();
+        this.text = new Text();
+        text.readFields(dataInput);
+    }
+
+    public short getTypeId() {
+        return typeId;
+    }
+
+    public Text getText() {
+        return text;
+    }
+
+    public boolean isNumberFamily() {
+        if (typeId == TypeFlag.NONE_NUMERIC_TYPE.ordinal())
+            return false;
+        return true;
+    }
+
+    public boolean isIntegerFamily() {
+        return (typeId == TypeFlag.INTEGER_FAMILY_TYPE.ordinal());
+    }
+
+    public boolean isOtherNumericFamily() {
+        return (typeId == TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
+    }
+
+    public void setTypeIdByDatatype(DataType type) {
+        if (!type.isNumberFamily()) {
+            this.typeId = (byte) TypeFlag.NONE_NUMERIC_TYPE.ordinal();
+        } else if (type.isIntegerFamily()) {
+            this.typeId = (byte) TypeFlag.INTEGER_FAMILY_TYPE.ordinal();
+        } else {
+            this.typeId = (byte) TypeFlag.DOUBLE_FAMILY_TYPE.ordinal();
+        }
+    }
+    
+    public void setTypeId(byte typeId) {
+        this.typeId = typeId;
+    }
+
+    public void setText(Text text) {
+        this.text = text;
+    }
+}

http://git-wip-us.apache.org/repos/asf/kylin/blob/6e89e779/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
index 1908992..f3209cc 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/NumberDictionaryForestTest.java
@@ -39,7 +39,7 @@ import org.apache.kylin.dict.NumberDictionary;
 import org.apache.kylin.dict.NumberDictionaryBuilder;
 import org.apache.kylin.dict.NumberDictionaryForestBuilder;
 import org.apache.kylin.dict.TrieDictionaryForest;
-import org.apache.kylin.engine.mr.steps.SelfDefineSortableKey.TypeFlag;
+import org.apache.kylin.engine.mr.steps.SelfDefineSortableKeyOld.TypeFlag;
 import org.junit.Ignore;
 import org.junit.Test;
 
@@ -63,19 +63,19 @@ public class NumberDictionaryForestTest {
 
     private void testData(List<String> list, TypeFlag flag) {
         //stimulate map-reduce job
-        ArrayList<SelfDefineSortableKey> keyList = createKeyList(list, (byte) flag.ordinal());
+        ArrayList<SelfDefineSortableKeyOld> keyList = createKeyList(list, (byte) flag.ordinal());
         Collections.sort(keyList);
         //build tree
         NumberDictionaryForestBuilder b = new NumberDictionaryForestBuilder(0, 0);
 
-        for (SelfDefineSortableKey key : keyList) {
+        for (SelfDefineSortableKeyOld key : keyList) {
             String fieldValue = printKey(key);
             b.addValue(fieldValue);
         }
         TrieDictionaryForest<String> dict = b.build();
         dict.dump(System.out);
         ArrayList<Integer> resultIds = new ArrayList<>();
-        for (SelfDefineSortableKey key : keyList) {
+        for (SelfDefineSortableKeyOld key : keyList) {
             String fieldValue = getFieldValue(key);
             resultIds.add(dict.getIdFromValue(fieldValue));
             assertEquals(fieldValue, dict.getValueFromId(dict.getIdFromValue(fieldValue)));
@@ -221,9 +221,9 @@ public class NumberDictionaryForestTest {
         return list;
     }
 
-    private ArrayList<SelfDefineSortableKey> createKeyList(List<String> strNumList,
byte typeFlag) {
+    private ArrayList<SelfDefineSortableKeyOld> createKeyList(List<String> strNumList,
byte typeFlag) {
         int partationId = 0;
-        ArrayList<SelfDefineSortableKey> keyList = new ArrayList<>();
+        ArrayList<SelfDefineSortableKeyOld> keyList = new ArrayList<>();
         for (String str : strNumList) {
             ByteBuffer keyBuffer = ByteBuffer.allocate(4096);
             int offset = keyBuffer.position();
@@ -235,20 +235,20 @@ public class NumberDictionaryForestTest {
             //System.out.println("arrays toString:"+Arrays.toString(valueField));
             Text outputKey = new Text();
             outputKey.set(keyBuffer.array(), offset, keyBuffer.position() - offset);
-            SelfDefineSortableKey sortableKey = new SelfDefineSortableKey(typeFlag, outputKey);
+            SelfDefineSortableKeyOld sortableKey = new SelfDefineSortableKeyOld(typeFlag,
outputKey);
             keyList.add(sortableKey);
         }
         return keyList;
     }
 
-    private String printKey(SelfDefineSortableKey key) {
+    private String printKey(SelfDefineSortableKeyOld key) {
         Text data = key.getText();
         String fieldValue = Bytes.toString(data.getBytes(), 1, data.getLength() - 1);
         System.out.println("type flag:" + key.getTypeId() + " fieldValue:" + fieldValue);
         return fieldValue;
     }
 
-    private String getFieldValue(SelfDefineSortableKey key) {
+    private String getFieldValue(SelfDefineSortableKeyOld key) {
         Text data = key.getText();
         return Bytes.toString(data.getBytes(), 1, data.getLength() - 1);
     }

http://git-wip-us.apache.org/repos/asf/kylin/blob/6e89e779/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java
b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java
index df6287f..dc30029 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/SelfDefineSortableKeyTest.java
@@ -29,8 +29,9 @@ import java.util.Random;
 import java.util.UUID;
 
 import org.apache.hadoop.io.Text;
+import org.apache.kylin.common.util.ByteArray;
 import org.apache.kylin.common.util.Bytes;
-import org.apache.kylin.engine.mr.steps.SelfDefineSortableKey.TypeFlag;
+import org.apache.kylin.engine.mr.steps.SelfDefineSortableKeyOld.TypeFlag;
 import org.junit.Test;
 
 /**
@@ -55,11 +56,11 @@ public class SelfDefineSortableKeyTest {
         System.out.println("test numbers:" + longList);
         ArrayList<String> strNumList = listToStringList(longList);
         //System.out.println("test num strs list:"+strNumList);
-        ArrayList<SelfDefineSortableKey> keyList = createKeyList(strNumList, (byte)
TypeFlag.INTEGER_FAMILY_TYPE.ordinal());
+        ArrayList<SelfDefineSortableKeyOld> keyList = createKeyList(strNumList, (byte)
TypeFlag.INTEGER_FAMILY_TYPE.ordinal());
         System.out.println(keyList.get(0).isIntegerFamily());
         Collections.sort(keyList);
         ArrayList<String> strListAftereSort = new ArrayList<>();
-        for (SelfDefineSortableKey key : keyList) {
+        for (SelfDefineSortableKeyOld key : keyList) {
             String str = printKey(key);
             strListAftereSort.add(str);
         }
@@ -91,11 +92,11 @@ public class SelfDefineSortableKeyTest {
         System.out.println("test numbers:" + doubleList);
         ArrayList<String> strNumList = listToStringList(doubleList);
         //System.out.println("test num strs list:"+strNumList);
-        ArrayList<SelfDefineSortableKey> keyList = createKeyList(strNumList, (byte)
TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
+        ArrayList<SelfDefineSortableKeyOld> keyList = createKeyList(strNumList, (byte)
TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
         System.out.println(keyList.get(0).isOtherNumericFamily());
         Collections.sort(keyList);
         ArrayList<String> strListAftereSort = new ArrayList<>();
-        for (SelfDefineSortableKey key : keyList) {
+        for (SelfDefineSortableKeyOld key : keyList) {
             String str = printKey(key);
             strListAftereSort.add(str);
         }
@@ -121,11 +122,11 @@ public class SelfDefineSortableKeyTest {
         strList.add("hello"); //duplicate
         strList.add("123");
         strList.add("");
-        ArrayList<SelfDefineSortableKey> keyList = createKeyList(strList, (byte) TypeFlag.NONE_NUMERIC_TYPE.ordinal());
+        ArrayList<SelfDefineSortableKeyOld> keyList = createKeyList(strList, (byte)
TypeFlag.NONE_NUMERIC_TYPE.ordinal());
         System.out.println(keyList.get(0).isOtherNumericFamily());
         Collections.sort(keyList);
         ArrayList<String> strListAftereSort = new ArrayList<>();
-        for (SelfDefineSortableKey key : keyList) {
+        for (SelfDefineSortableKeyOld key : keyList) {
             String str = printKey(key);
             strListAftereSort.add(str);
         }
@@ -138,6 +139,60 @@ public class SelfDefineSortableKeyTest {
     }
 
     @Test
+    public void testPerformance() {
+        Random rand = new Random(System.currentTimeMillis());
+        ArrayList<Double> doubleList = new ArrayList<>();
+        int count = 10 * 10000;
+        for (int i = 0; i < count; i++) {
+            doubleList.add(rand.nextDouble());
+        }
+        doubleList.add(0.0);
+        doubleList.add(0.0); //test duplicate
+        doubleList.add(-1.0); //test negative number
+        doubleList.add(Double.MAX_VALUE);
+        doubleList.add(-Double.MAX_VALUE);
+        //System.out.println(Double.MIN_VALUE);
+
+
+        ArrayList<String> strNumList = listToStringList(doubleList);
+        //System.out.println("test num strs list:"+strNumList);
+        ArrayList<SelfDefineSortableKeyOld> keyList = createKeyList(strNumList, (byte)
TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
+
+        System.out.println("start to test str sort");
+        long start = System.currentTimeMillis();
+        Collections.sort(strNumList);
+        System.out.println("sort time : " + (System.currentTimeMillis() - start));
+
+
+        System.out.println("start to test double sort");
+        start = System.currentTimeMillis();
+        Collections.sort(keyList);
+        System.out.println("sort time : " + (System.currentTimeMillis() - start));
+
+        List<ByteArray> byteList = new ArrayList<>();
+        for (String str : strNumList) {
+            byteList.add(new ByteArray(Bytes.toBytes(str)));
+        }
+        System.out.println("start to test byte array sort");
+        start = System.currentTimeMillis();
+        Collections.sort(byteList);
+        System.out.println("sort time : " + (System.currentTimeMillis() - start));
+
+        //test new key
+        List<SelfDefineSortableKey> newKeyList = new ArrayList<>();
+        for (String str : strNumList) {
+            SelfDefineSortableKey key = new SelfDefineSortableKey();
+            key.init(new Text(str), (byte) TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
+            newKeyList.add(key);
+        }
+        System.out.println("start to test new sortable key");
+        start = System.currentTimeMillis();
+        Collections.sort(newKeyList);
+        System.out.println("sort time : " + (System.currentTimeMillis() - start));
+    }
+
+
+    @Test
     public void testIllegalNumber() {
         Random rand = new Random(System.currentTimeMillis());
         ArrayList<Double> doubleList = new ArrayList<>();
@@ -156,10 +211,10 @@ public class SelfDefineSortableKeyTest {
         ArrayList<String> strNumList = listToStringList(doubleList);
         strNumList.add("fjaeif"); //illegal type
         //System.out.println("test num strs list:"+strNumList);
-        ArrayList<SelfDefineSortableKey> keyList = createKeyList(strNumList, (byte)
TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
+        ArrayList<SelfDefineSortableKeyOld> keyList = createKeyList(strNumList, (byte)
TypeFlag.DOUBLE_FAMILY_TYPE.ordinal());
         System.out.println(keyList.get(0).isOtherNumericFamily());
         Collections.sort(keyList);
-        for (SelfDefineSortableKey key : keyList) {
+        for (SelfDefineSortableKeyOld key : keyList) {
             printKey(key);
         }
 
@@ -176,15 +231,14 @@ public class SelfDefineSortableKeyTest {
     private <T> ArrayList<String> listToStringList(ArrayList<T> list) {
         ArrayList<String> strList = new ArrayList<>();
         for (T t : list) {
-            System.out.println(t.toString());
             strList.add(t.toString());
         }
         return strList;
     }
 
-    private ArrayList<SelfDefineSortableKey> createKeyList(List<String> strNumList,
byte typeFlag) {
+    private ArrayList<SelfDefineSortableKeyOld> createKeyList(List<String> strNumList,
byte typeFlag) {
         int partationId = 0;
-        ArrayList<SelfDefineSortableKey> keyList = new ArrayList<>();
+        ArrayList<SelfDefineSortableKeyOld> keyList = new ArrayList<>();
         for (String str : strNumList) {
             ByteBuffer keyBuffer = ByteBuffer.allocate(4096);
             int offset = keyBuffer.position();
@@ -193,13 +247,13 @@ public class SelfDefineSortableKeyTest {
             Bytes.copy(keyBuffer.array(), 1, keyBuffer.position() - offset - 1);
             Text outputKey = new Text();
             outputKey.set(keyBuffer.array(), offset, keyBuffer.position() - offset);
-            SelfDefineSortableKey sortableKey = new SelfDefineSortableKey(typeFlag, outputKey);
+            SelfDefineSortableKeyOld sortableKey = new SelfDefineSortableKeyOld(typeFlag,
outputKey);
             keyList.add(sortableKey);
         }
         return keyList;
     }
 
-    private String printKey(SelfDefineSortableKey key) {
+    private String printKey(SelfDefineSortableKeyOld key) {
         Text data = key.getText();
         String fieldValue = Bytes.toString(data.getBytes(), 1, data.getLength() - 1);
         System.out.println("type flag:" + key.getTypeId() + " fieldValue:" + fieldValue);


Mime
View raw message