carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jack...@apache.org
Subject [3/5] carbondata git commit: refactor write step based on ColumnPage
Date Wed, 10 May 2017 01:17:05 GMT
http://git-wip-us.apache.org/repos/asf/carbondata/blob/a79a3a16/core/src/main/java/org/apache/carbondata/core/datastore/page/ComplexColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/ComplexColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/ComplexColumnPage.java
new file mode 100644
index 0000000..024c341
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/ComplexColumnPage.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.page;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.carbondata.common.CarbonIterator;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+
+// Represent a complex column page, e.g. Array, Struct type column
+public class ComplexColumnPage extends ColumnPage {
+
+  // Holds data for all rows in this page in columnar layout.
+  // After the complex data expand, it is of type byte[][], the first level array in the byte[][]
+  // representing a sub-column in the complex type, which can be retrieved by giving the depth
+  // of the complex type.
+  // TODO: further optimize it to make it more memory efficient
+  private List<ArrayList<byte[]>> complexColumnData;
+
+  // depth is the number of column after complex type is expanded. It is from 1 to N
+  private final int depth;
+
+  public ComplexColumnPage(int pageSize, int depth) {
+    super(DataType.BYTE_ARRAY, pageSize);
+    this.depth = depth;
+    complexColumnData = new ArrayList<>(depth);
+    for (int i = 0; i < depth; i++) {
+      complexColumnData.add(new ArrayList<byte[]>(pageSize));
+    }
+  }
+
+  public void putComplexData(int rowId, int depth, List<byte[]> value) {
+    assert (depth <= this.depth);
+    ArrayList<byte[]> subColumnPage = complexColumnData.get(depth);
+    subColumnPage.addAll(value);
+  }
+
+  // iterate on the sub-column after complex type is expanded, return columnar page of
+  // each sub-column
+  public Iterator<byte[][]> iterator() {
+
+    return new CarbonIterator<byte[][]>() {
+      private int index = 0;
+      @Override public boolean hasNext() {
+        return index < depth;
+      }
+
+      @Override public byte[][] next() {
+        // convert the subColumnPage from ArrayList<byte[]> to byte[][]
+        ArrayList<byte[]> subColumnPage = complexColumnData.get(index);
+        index++;
+        return subColumnPage.toArray(new byte[subColumnPage.size()][]);
+      }
+    };
+  }
+
+  public int getDepth() {
+    return depth;
+  }
+}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a79a3a16/core/src/main/java/org/apache/carbondata/core/datastore/page/FixLengthColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/FixLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/FixLengthColumnPage.java
new file mode 100644
index 0000000..a56563e
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/FixLengthColumnPage.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.page;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import java.util.BitSet;
+
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.util.DataTypeUtil;
+
+// Represent a columnar data in one page for one column.
+public class FixLengthColumnPage extends ColumnPage {
+
+  // Only one of following fields will be used
+  private byte[] byteData;
+  private short[] shortData;
+  private int[] intData;
+  private long[] longData;
+  private double[] doubleData;
+
+  private byte[][] byteArrayData;
+
+  // The index of the rowId whose value is null, will be set to 1
+  private BitSet nullBitSet;
+
+  public FixLengthColumnPage(DataType dataType, int pageSize) {
+    super(dataType, pageSize);
+    nullBitSet = new BitSet(pageSize);
+    switch (dataType) {
+      case SHORT:
+      case INT:
+      case LONG:
+        longData = new long[pageSize];
+        break;
+      case DOUBLE:
+        doubleData = new double[pageSize];
+        break;
+      case DECIMAL:
+        byteArrayData = new byte[pageSize][];
+        break;
+      default:
+        throw new RuntimeException("Unsupported data dataType: " + dataType);
+    }
+  }
+
+  public DataType getDataType() {
+    return dataType;
+  }
+
+  private void putByte(int rowId, byte value) {
+    byteData[rowId] = value;
+  }
+
+  private void putShort(int rowId, short value) {
+    shortData[rowId] = value;
+  }
+
+  private void putInt(int rowId, int value) {
+    intData[rowId] = value;
+  }
+
+  private void putLong(int rowId, long value) {
+    longData[rowId] = value;
+  }
+
+  private void putDouble(int rowId, double value) {
+    doubleData[rowId] = value;
+  }
+
+  // This method will do LV (length value) coded of input bytes
+  private void putDecimalBytes(int rowId, byte[] decimalInBytes) {
+    ByteBuffer byteBuffer = ByteBuffer.allocate(decimalInBytes.length +
+        CarbonCommonConstants.INT_SIZE_IN_BYTE);
+    byteBuffer.putInt(decimalInBytes.length);
+    byteBuffer.put(decimalInBytes);
+    byteBuffer.flip();
+    byteArrayData[rowId] = byteBuffer.array();
+  }
+
+  public void putData(int rowId, Object value) {
+    if (value == null) {
+      putNull(rowId);
+      return;
+    }
+    switch (dataType) {
+      case BYTE:
+      case SHORT:
+      case INT:
+      case LONG:
+        putLong(rowId, (long) value);
+        break;
+      case DOUBLE:
+        putDouble(rowId, (double) value);
+        break;
+      case DECIMAL:
+        putDecimalBytes(rowId, (byte[]) value);
+        break;
+      default:
+        throw new RuntimeException("unsupported data type: " + dataType);
+    }
+    updateStatistics(value);
+  }
+
+  private void putNull(int rowId) {
+    nullBitSet.set(rowId);
+    switch (dataType) {
+      case BYTE:
+      case SHORT:
+      case INT:
+      case LONG:
+        putLong(rowId, 0L);
+        break;
+      case DOUBLE:
+        putDouble(rowId, 0.0);
+        break;
+      case DECIMAL:
+        byte[] decimalInBytes = DataTypeUtil.bigDecimalToByte(BigDecimal.ZERO);
+        putDecimalBytes(rowId, decimalInBytes);
+        break;
+    }
+  }
+
+  public long[] getLongPage() {
+    return longData;
+  }
+
+  public double[] getDoublePage() {
+    return doubleData;
+  }
+
+  public byte[][] getDecimalPage() {
+    return byteArrayData;
+  }
+
+  public BitSet getNullBitSet() {
+    return nullBitSet;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a79a3a16/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPage.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPage.java
new file mode 100644
index 0000000..d5e9ce3
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/VarLengthColumnPage.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.page;
+
+import org.apache.carbondata.core.metadata.datatype.DataType;
+
+// Represent a variable length columnar data in one page, e.g. for dictionary columns.
+public class VarLengthColumnPage extends ColumnPage {
+
+  // TODO: further optimizite it, to store length and data separately
+  private byte[][] byteArrayData;
+
+  public VarLengthColumnPage(int pageSize) {
+    super(DataType.BYTE_ARRAY, pageSize);
+    byteArrayData = new byte[pageSize][];
+  }
+
+  public void putByteArray(int rowId, byte[] value) {
+    byteArrayData[rowId] = value;
+    updateStatistics(value);
+  }
+
+  public byte[][] getByteArrayPage() {
+    return byteArrayData;
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a79a3a16/core/src/main/java/org/apache/carbondata/core/datastore/page/compression/Compression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/compression/Compression.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/compression/Compression.java
new file mode 100644
index 0000000..c954a33
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/compression/Compression.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.page.compression;
+
+public interface Compression {
+  byte[] compress(byte[] input);
+  byte[] decompress(byte[] input);
+}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a79a3a16/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnCodec.java
new file mode 100644
index 0000000..e870ad6
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/ColumnCodec.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.page.encoding;
+
+import org.apache.carbondata.core.datastore.page.ColumnPage;
+
+/**
+ *  Codec for a column page data, implementation should not keep state across pages,
+ *  caller will use the same object to encode multiple pages.
+ */
+public interface ColumnCodec {
+
+  /** Codec name will be stored in BlockletHeader (DataChunk3) */
+  String getName();
+
+  byte[] encode(ColumnPage columnPage);
+
+  ColumnPage decode(byte[] encoded);
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a79a3a16/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DummyCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DummyCodec.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DummyCodec.java
new file mode 100644
index 0000000..0dd23c7
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/DummyCodec.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.page.encoding;
+
+import org.apache.carbondata.core.datastore.page.ColumnPage;
+
+public class DummyCodec implements ColumnCodec {
+  @Override
+  public String getName() {
+    return "DummyCodec";
+  }
+
+  @Override
+  public byte[] encode(ColumnPage columnPage) {
+    return null;
+  }
+
+  @Override
+  public ColumnPage decode(byte[] encoded) {
+    return null;
+  }
+}

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a79a3a16/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PageStatistics.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PageStatistics.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PageStatistics.java
new file mode 100644
index 0000000..3ecf1da
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/PageStatistics.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.page.statistics;
+
+import java.math.BigDecimal;
+
+import org.apache.carbondata.core.metadata.datatype.DataType;
+import org.apache.carbondata.core.util.DataTypeUtil;
+
+/** statics for one column page */
+public class PageStatistics {
+  private DataType dataType;
+
+  /** min and max value of the measures */
+  private Object min, max;
+
+  /**
+   * the unique value is the non-exist value in the row,
+   * and will be used as storage key for null values of measures
+   */
+  private Object uniqueValue;
+
+  /** decimal count of the measures */
+  private int decimal;
+
+  public PageStatistics(DataType dataType) {
+    this.dataType = dataType;
+    switch (dataType) {
+      case SHORT:
+      case INT:
+      case LONG:
+        max = Long.MIN_VALUE;
+        min = Long.MAX_VALUE;
+        uniqueValue = Long.MIN_VALUE;
+        break;
+      case DOUBLE:
+        max = Double.MIN_VALUE;
+        min = Double.MAX_VALUE;
+        uniqueValue = Double.MIN_VALUE;
+        break;
+      case DECIMAL:
+        max = new BigDecimal(Double.MIN_VALUE);
+        min = new BigDecimal(Double.MAX_VALUE);
+        uniqueValue = new BigDecimal(Double.MIN_VALUE);
+        break;
+    }
+    decimal = 0;
+  }
+
+  /**
+   * update the statistics for the input row
+   */
+  public void update(Object value) {
+    switch (dataType) {
+      case SHORT:
+      case INT:
+      case LONG:
+        max = ((long) max > (long) value) ? max : value;
+        min = ((long) min < (long) value) ? min : value;
+        uniqueValue = (long) min - 1;
+        break;
+      case DOUBLE:
+        max = ((double) max > (double) value) ? max : value;
+        min = ((double) min < (double) value) ? min : value;
+        int num = getDecimalCount((double) value);
+        decimal = decimal > num ? decimal : num;
+        uniqueValue = (double) min - 1;
+        break;
+      case DECIMAL:
+        BigDecimal decimalValue = DataTypeUtil.byteToBigDecimal((byte[]) value);
+        decimal = decimalValue.scale();
+        BigDecimal val = (BigDecimal) min;
+        uniqueValue = (val.subtract(new BigDecimal(1.0)));
+        break;
+      case ARRAY:
+      case STRUCT:
+        // for complex type column, writer is not going to use stats, so, do nothing
+    }
+  }
+
+  /**
+   * return no of digit after decimal
+   */
+  private int getDecimalCount(double value) {
+    String strValue = BigDecimal.valueOf(Math.abs(value)).toPlainString();
+    int integerPlaces = strValue.indexOf('.');
+    int decimalPlaces = 0;
+    if (-1 != integerPlaces) {
+      decimalPlaces = strValue.length() - integerPlaces - 1;
+    }
+    return decimalPlaces;
+  }
+
+  public Object getMin() {
+    return min;
+  }
+
+  public Object getMax() {
+    return max;
+  }
+
+  public Object getUniqueValue() {
+    return uniqueValue;
+  }
+
+  public int getDecimal() {
+    return decimal;
+  }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a79a3a16/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/StatisticsCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/StatisticsCollector.java b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/StatisticsCollector.java
new file mode 100644
index 0000000..f8b336c
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/datastore/page/statistics/StatisticsCollector.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.datastore.page.statistics;
+
+import org.apache.carbondata.core.datastore.page.ColumnPage;
+
+/**
+ * Calculate the statistics for a column page and blocklet
+ */
+public interface StatisticsCollector {
+
+  /**
+   * name will be stored in Header
+   */
+  String getName();
+
+  void startPage(int pageID);
+
+  void endPage(int pageID);
+
+  void startBlocklet(int blockletID);
+
+  void endBlocklet(int blockletID);
+
+  void startBlock(int blocklID);
+
+  void endBlock(int blockID);
+
+  /**
+   * Update the stats for the input batch
+   */
+  void update(ColumnPage batch);
+
+  /**
+   * Ouput will be written to DataChunk2 (page header)
+   */
+  byte[] getPageStatistisc();
+
+  /**
+   * Output will be written to DataChunk3 (blocklet header)
+   */
+  byte[] getBlockletStatistics();
+
+  /**
+   * Output will be written to Footer
+   */
+  byte[] getBlockStatistics();
+}
+
+
+

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a79a3a16/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java b/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java
index 4a9007c..741b999 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/ValueEncoderMeta.java
@@ -19,6 +19,9 @@ package org.apache.carbondata.core.metadata;
 
 import java.io.Serializable;
 
+import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.metadata.datatype.DataType;
+
 /**
  * DO NOT MODIFY THIS CLASS AND PACKAGE NAME, BECAUSE
  * IT IS SERIALIZE TO STORE
@@ -78,7 +81,20 @@ public class ValueEncoderMeta implements Serializable {
     this.decimal = decimal;
   }
 
-  public char getType() {
+  public DataType getType() {
+    switch (type) {
+      case CarbonCommonConstants.BIG_INT_MEASURE:
+        return DataType.LONG;
+      case CarbonCommonConstants.DOUBLE_MEASURE:
+        return DataType.DOUBLE;
+      case CarbonCommonConstants.BIG_DECIMAL_MEASURE:
+        return DataType.DECIMAL;
+      default:
+        throw new RuntimeException("Unexpected type: " + type);
+    }
+  }
+
+  public char getTypeInChar() {
     return type;
   }
 

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a79a3a16/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataType.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataType.java b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataType.java
index d77406c..da13d5c 100644
--- a/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataType.java
+++ b/core/src/main/java/org/apache/carbondata/core/metadata/datatype/DataType.java
@@ -32,7 +32,11 @@ public enum DataType {
   DECIMAL(8, "DECIMAL"),
   ARRAY(9, "ARRAY"),
   STRUCT(10, "STRUCT"),
-  MAP(11, "MAP");
+  MAP(11, "MAP"),
+  BYTE(12, "BYTE"),
+
+  // internal use only
+  BYTE_ARRAY(13, "BYTE ARRAY");
 
   private int precedenceOrder;
   private String name ;

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a79a3a16/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
index f4ab982..caba75f 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ByteUtil.java
@@ -39,6 +39,8 @@ public final class ByteUtil {
 
   public static final String UTF8_CSN = StandardCharsets.UTF_8.name();
 
+  public static final byte[] ZERO_IN_BYTES = toBytes(0);
+
   private ByteUtil() {
 
   }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a79a3a16/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
index 6398f30..6fe38e2 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonMetadataUtil.java
@@ -554,7 +554,7 @@ public class CarbonMetadataUtil {
     Object[] minValue = new Object[encoderMetas.length];
     int[] decimalLength = new int[encoderMetas.length];
     Object[] uniqueValue = new Object[encoderMetas.length];
-    char[] aggType = new char[encoderMetas.length];
+    DataType[] aggType = new DataType[encoderMetas.length];
     byte[] dataTypeSelected = new byte[encoderMetas.length];
     for (int i = 0; i < encoderMetas.length; i++) {
       maxValue[i] = encoderMetas[i].getMaxValue();
@@ -827,25 +827,29 @@ public class CarbonMetadataUtil {
 
   public static byte[] serializeEncodeMetaUsingByteBuffer(ValueEncoderMeta valueEncoderMeta) {
     ByteBuffer buffer = null;
-    if (valueEncoderMeta.getType() == CarbonCommonConstants.DOUBLE_MEASURE) {
-      buffer = ByteBuffer.allocate(
-          (CarbonCommonConstants.DOUBLE_SIZE_IN_BYTE * 3) + CarbonCommonConstants.INT_SIZE_IN_BYTE
-              + 3);
-      buffer.putChar(valueEncoderMeta.getType());
-      buffer.putDouble((Double) valueEncoderMeta.getMaxValue());
-      buffer.putDouble((Double) valueEncoderMeta.getMinValue());
-      buffer.putDouble((Double) valueEncoderMeta.getUniqueValue());
-    } else if (valueEncoderMeta.getType() == CarbonCommonConstants.BIG_INT_MEASURE) {
-      buffer = ByteBuffer.allocate(
-          (CarbonCommonConstants.LONG_SIZE_IN_BYTE * 3) + CarbonCommonConstants.INT_SIZE_IN_BYTE
-              + 3);
-      buffer.putChar(valueEncoderMeta.getType());
-      buffer.putLong((Long) valueEncoderMeta.getMaxValue());
-      buffer.putLong((Long) valueEncoderMeta.getMinValue());
-      buffer.putLong((Long) valueEncoderMeta.getUniqueValue());
-    } else {
-      buffer = ByteBuffer.allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE + 3);
-      buffer.putChar(valueEncoderMeta.getType());
+    switch (valueEncoderMeta.getType()) {
+      case LONG:
+        buffer = ByteBuffer.allocate(
+            (CarbonCommonConstants.LONG_SIZE_IN_BYTE * 3) + CarbonCommonConstants.INT_SIZE_IN_BYTE
+                + 3);
+        buffer.putChar(valueEncoderMeta.getTypeInChar());
+        buffer.putLong((Long) valueEncoderMeta.getMaxValue());
+        buffer.putLong((Long) valueEncoderMeta.getMinValue());
+        buffer.putLong((Long) valueEncoderMeta.getUniqueValue());
+        break;
+      case DOUBLE:
+        buffer = ByteBuffer.allocate(
+            (CarbonCommonConstants.DOUBLE_SIZE_IN_BYTE * 3) + CarbonCommonConstants.INT_SIZE_IN_BYTE
+                + 3);
+        buffer.putChar(valueEncoderMeta.getTypeInChar());
+        buffer.putDouble((Double) valueEncoderMeta.getMaxValue());
+        buffer.putDouble((Double) valueEncoderMeta.getMinValue());
+        buffer.putDouble((Double) valueEncoderMeta.getUniqueValue());
+        break;
+      case DECIMAL:
+        buffer = ByteBuffer.allocate(CarbonCommonConstants.INT_SIZE_IN_BYTE + 3);
+        buffer.putChar(valueEncoderMeta.getTypeInChar());
+        break;
     }
     buffer.putInt(valueEncoderMeta.getDecimal());
     buffer.put(valueEncoderMeta.getDataTypeSelected());

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a79a3a16/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index 92c85a1..496adff 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -749,5 +749,4 @@ public final class CarbonProperties {
     }
     return numberOfDeltaFilesThreshold;
   }
-
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a79a3a16/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
index 99463de..8e4df1a 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java
@@ -833,7 +833,7 @@ public final class CarbonUtil {
     Object[] minValue = new Object[encodeMetaList.size()];
     Object[] uniqueValue = new Object[encodeMetaList.size()];
     int[] decimal = new int[encodeMetaList.size()];
-    char[] type = new char[encodeMetaList.size()];
+    DataType[] type = new DataType[encodeMetaList.size()];
     byte[] dataTypeSelected = new byte[encodeMetaList.size()];
 
     /*

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a79a3a16/core/src/main/java/org/apache/carbondata/core/util/CompressionFinder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CompressionFinder.java b/core/src/main/java/org/apache/carbondata/core/util/CompressionFinder.java
index d931af6..732d053 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CompressionFinder.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CompressionFinder.java
@@ -16,8 +16,8 @@
  */
 package org.apache.carbondata.core.util;
 
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.util.ValueCompressionUtil.COMPRESSION_TYPE;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
 
 
 /**
@@ -37,7 +37,7 @@ public class CompressionFinder implements Comparable<CompressionFinder> {
 
   private PRIORITY priority;
 
-  private char measureStoreType;
+  private DataType measureStoreType;
 
   /**
    * CompressionFinder constructor.
@@ -47,7 +47,7 @@ public class CompressionFinder implements Comparable<CompressionFinder> {
    * @param convertedDataType
    */
   CompressionFinder(COMPRESSION_TYPE compType, DataType actualDataType,
-      DataType convertedDataType, char measureStoreType) {
+      DataType convertedDataType, DataType measureStoreType) {
     super();
     this.compType = compType;
     this.actualDataType = actualDataType;
@@ -65,7 +65,7 @@ public class CompressionFinder implements Comparable<CompressionFinder> {
    */
 
   CompressionFinder(COMPRESSION_TYPE compType, DataType actualDataType, DataType convertedDataType,
-      PRIORITY priority, char measureStoreType) {
+      PRIORITY priority, DataType measureStoreType) {
     super();
     this.actualDataType = actualDataType;
     this.convertedDataType = convertedDataType;
@@ -155,7 +155,7 @@ public class CompressionFinder implements Comparable<CompressionFinder> {
     return priority;
   }
 
-  public char getMeasureStoreType() {
+  public DataType getMeasureStoreType() {
     return measureStoreType;
   }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a79a3a16/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
index 80c9e72..e33d198 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/DataTypeUtil.java
@@ -156,9 +156,6 @@ public final class DataTypeUtil {
     }
   }
 
-  // bytes of 0 in BigDecimal
-  public static final byte[] zeroBigDecimalBytes = bigDecimalToByte(BigDecimal.valueOf(0));
-
   /**
    * This method will convert a big decimal value to bytes
    *

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a79a3a16/core/src/main/java/org/apache/carbondata/core/util/NodeHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/NodeHolder.java b/core/src/main/java/org/apache/carbondata/core/util/NodeHolder.java
index 69ed9f8..a37a9a7 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/NodeHolder.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/NodeHolder.java
@@ -57,16 +57,6 @@ public class NodeHolder {
   private int[] keyLengths;
 
   /**
-   * dataAfterCompression
-   */
-  private short[][] dataAfterCompression;
-
-  /**
-   * indexMap
-   */
-  private short[][] indexMap;
-
-  /**
    * keyIndexBlockLenght
    */
   private int[] keyBlockIndexLength;
@@ -86,11 +76,6 @@ public class NodeHolder {
   private int[] dataIndexMapLength;
 
   /**
-   * dataIndexMap
-   */
-  private int[] dataIndexMapOffsets;
-
-  /**
    * compressedDataIndex
    */
   private byte[][] compressedDataIndex;
@@ -120,19 +105,9 @@ public class NodeHolder {
   private boolean[] aggBlocks;
 
   /**
-   * all columns max value
-   */
-  private byte[][] allMaxValue;
-
-  /**
-   * all column max value
-   */
-  private byte[][] allMinValue;
-
-  /**
    * true if given index is colgroup block
    */
-  private boolean[] colGrpBlock;
+  private boolean[] colGrpBlocks;
 
   /**
    * bit set which will holds the measure
@@ -383,14 +358,14 @@ public class NodeHolder {
    * @return
    */
   public boolean[] getColGrpBlocks() {
-    return this.colGrpBlock;
+    return this.colGrpBlocks;
   }
 
   /**
    * @param colGrpBlock true if block is column group
    */
   public void setColGrpBlocks(boolean[] colGrpBlock) {
-    this.colGrpBlock = colGrpBlock;
+    this.colGrpBlocks = colGrpBlock;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a79a3a16/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
index c8a9397..5020acb 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/ValueCompressionUtil.java
@@ -28,10 +28,29 @@ import org.apache.carbondata.core.datastore.compression.MeasureMetaDataModel;
 import org.apache.carbondata.core.datastore.compression.ReaderCompressModel;
 import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
 import org.apache.carbondata.core.datastore.compression.WriterCompressModel;
-import org.apache.carbondata.core.datastore.compression.decimal.*;
-import org.apache.carbondata.core.datastore.compression.nondecimal.*;
-import org.apache.carbondata.core.datastore.compression.none.*;
+import org.apache.carbondata.core.datastore.compression.decimal.CompressByteArray;
+import org.apache.carbondata.core.datastore.compression.decimal.CompressionMaxMinByte;
+import org.apache.carbondata.core.datastore.compression.decimal.CompressionMaxMinDefault;
+import org.apache.carbondata.core.datastore.compression.decimal.CompressionMaxMinInt;
+import org.apache.carbondata.core.datastore.compression.decimal.CompressionMaxMinLong;
+import org.apache.carbondata.core.datastore.compression.decimal.CompressionMaxMinShort;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalByte;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalDefault;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalInt;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalLong;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalMaxMinByte;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalMaxMinDefault;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalMaxMinInt;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalMaxMinLong;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalMaxMinShort;
+import org.apache.carbondata.core.datastore.compression.nondecimal.CompressionNonDecimalShort;
+import org.apache.carbondata.core.datastore.compression.none.CompressionNoneByte;
+import org.apache.carbondata.core.datastore.compression.none.CompressionNoneDefault;
+import org.apache.carbondata.core.datastore.compression.none.CompressionNoneInt;
+import org.apache.carbondata.core.datastore.compression.none.CompressionNoneLong;
+import org.apache.carbondata.core.datastore.compression.none.CompressionNoneShort;
 import org.apache.carbondata.core.metadata.ValueEncoderMeta;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 
 public final class ValueCompressionUtil {
 
@@ -47,29 +66,28 @@ public final class ValueCompressionUtil {
    * @see
    */
   private static DataType getDataType(double value, int mantissa, byte dataTypeSelected) {
-    DataType dataType = DataType.DATA_DOUBLE;
+    DataType dataType = DataType.DOUBLE;
     if (mantissa == 0) {
       if (value <= Byte.MAX_VALUE && value >= Byte.MIN_VALUE) {
-        dataType = DataType.DATA_BYTE;
+        dataType = DataType.BYTE;
       } else if (value <= Short.MAX_VALUE && value >= Short.MIN_VALUE) {
-        dataType = DataType.DATA_SHORT;
+        dataType = DataType.SHORT;
       } else if (value <= Integer.MAX_VALUE && value >= Integer.MIN_VALUE) {
-        dataType = DataType.DATA_INT;
+        dataType = DataType.INT;
       } else if (value <= Long.MAX_VALUE && value >= Long.MIN_VALUE) {
-        dataType = DataType.DATA_LONG;
+        dataType = DataType.LONG;
       }
     } else {
       if (dataTypeSelected == 1) {
         if (value <= Float.MAX_VALUE && value >= Float.MIN_VALUE) {
           float floatValue = (float) value;
           if (floatValue - value != 0) {
-            dataType = DataType.DATA_DOUBLE;
-
+            dataType = DataType.DOUBLE;
           } else {
-            dataType = DataType.DATA_FLOAT;
+            dataType = DataType.FLOAT;
           }
         } else if (value <= Double.MAX_VALUE && value >= Double.MIN_VALUE) {
-          dataType = DataType.DATA_DOUBLE;
+          dataType = DataType.DOUBLE;
         }
       }
     }
@@ -84,14 +102,14 @@ public final class ValueCompressionUtil {
    * @see
    */
   public static int getSize(DataType dataType) {
-
     switch (dataType) {
-      case DATA_BYTE:
+      case BOOLEAN:
+      case BYTE:
         return 1;
-      case DATA_SHORT:
+      case SHORT:
         return 2;
-      case DATA_INT:
-      case DATA_FLOAT:
+      case INT:
+      case FLOAT:
         return 4;
       default:
         return 8;
@@ -110,19 +128,17 @@ public final class ValueCompressionUtil {
    * @see
    */
   public static CompressionFinder getCompressionFinder(Object maxValue, Object minValue,
-      int mantissa, char measureStoreType, byte dataTypeSelected) {
-    // ''l' for long, 'n' for double
+      int mantissa, DataType measureStoreType, byte dataTypeSelected) {
     switch (measureStoreType) {
-      case 'b':
-        return new CompressionFinder(COMPRESSION_TYPE.BIGDECIMAL, DataType.DATA_BYTE,
-            DataType.DATA_BYTE, measureStoreType);
-      case 'd':
+      case DECIMAL:
+        return new CompressionFinder(COMPRESSION_TYPE.BIGDECIMAL, DataType.BYTE,
+            DataType.BYTE, measureStoreType);
+      case SHORT:
+      case INT:
+      case LONG:
         return getLongCompressorFinder(maxValue, minValue, mantissa, dataTypeSelected,
             measureStoreType);
-      case 'l':
-        return new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE,
-            DataType.DATA_BIGINT, DataType.DATA_BIGINT, measureStoreType);
-      case 'n':
+      case DOUBLE:
         return getDoubleCompressorFinder(maxValue, minValue, mantissa, dataTypeSelected,
             measureStoreType);
       default:
@@ -131,7 +147,7 @@ public final class ValueCompressionUtil {
   }
 
   private static CompressionFinder getDoubleCompressorFinder(Object maxValue, Object minValue,
-      int mantissa, byte dataTypeSelected, char measureStoreType) {
+      int mantissa, byte dataTypeSelected, DataType measureStoreType) {
     //Here we should use the Max abs as max to getDatatype, let's say -1 and -10000000, -1 is max,
     //but we can't use -1 to getDatatype, we should use -10000000.
     double absMaxValue = Math.abs((double) maxValue) >= Math.abs((double) minValue) ?
@@ -145,13 +161,13 @@ public final class ValueCompressionUtil {
       int adaptiveSize = getSize(adaptiveDataType);
       int deltaSize = getSize(deltaDataType);
       if (adaptiveSize > deltaSize) {
-        return new CompressionFinder(COMPRESSION_TYPE.DELTA_DOUBLE, DataType.DATA_DOUBLE,
+        return new CompressionFinder(COMPRESSION_TYPE.DELTA_DOUBLE, DataType.DOUBLE,
             deltaDataType, measureStoreType);
       } else if (adaptiveSize < deltaSize) {
-        return new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, DataType.DATA_DOUBLE,
+        return new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, DataType.DOUBLE,
             deltaDataType, measureStoreType);
       } else {
-        return new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, DataType.DATA_DOUBLE,
+        return new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, DataType.DOUBLE,
             adaptiveDataType, measureStoreType);
       }
     } else {
@@ -178,7 +194,7 @@ public final class ValueCompressionUtil {
   }
 
   private static CompressionFinder getLongCompressorFinder(Object maxValue, Object minValue,
-      int mantissa, byte dataTypeSelected, char measureStoreType) {
+      int mantissa, byte dataTypeSelected, DataType measureStoreType) {
     DataType adaptiveDataType = getDataType((long) maxValue, mantissa, dataTypeSelected);
     int adaptiveSize = getSize(adaptiveDataType);
     DataType deltaDataType = null;
@@ -186,20 +202,20 @@ public final class ValueCompressionUtil {
     // consider the scenario when max and min value are equal to is long max and min value OR
     // when the max and min value are resulting in a value greater than long max value, then
     // it is not possible to determine the compression type.
-    if (adaptiveDataType == DataType.DATA_LONG) {
-      deltaDataType = DataType.DATA_BIGINT;
+    if (adaptiveDataType == DataType.LONG) {
+      deltaDataType = DataType.LONG;
     } else {
       deltaDataType = getDataType((long) maxValue - (long) minValue, mantissa, dataTypeSelected);
     }
     int deltaSize = getSize(deltaDataType);
     if (adaptiveSize > deltaSize) {
-      return new CompressionFinder(COMPRESSION_TYPE.DELTA_DOUBLE, DataType.DATA_BIGINT,
+      return new CompressionFinder(COMPRESSION_TYPE.DELTA_DOUBLE, DataType.LONG,
           deltaDataType, measureStoreType);
     } else if (adaptiveSize < deltaSize) {
-      return new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, DataType.DATA_BIGINT,
+      return new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, DataType.LONG,
           deltaDataType, measureStoreType);
     } else {
-      return new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, DataType.DATA_BIGINT,
+      return new CompressionFinder(COMPRESSION_TYPE.ADAPTIVE, DataType.LONG,
           adaptiveDataType, measureStoreType);
     }
   }
@@ -234,7 +250,9 @@ public final class ValueCompressionUtil {
    */
   public static ValueCompressor getValueCompressor(CompressionFinder compressorFinder) {
     switch (compressorFinder.getMeasureStoreType()) {
-      case 'd':
+      case SHORT:
+      case INT:
+      case LONG:
         return new BigIntCompressor();
       default:
         return new DoubleCompressor();
@@ -295,7 +313,7 @@ public final class ValueCompressionUtil {
   private static Object compressNone(DataType changedDataType, double[] value) {
     int i = 0;
     switch (changedDataType) {
-      case DATA_BYTE:
+      case BYTE:
         byte[] result = new byte[value.length];
 
         for (double a : value) {
@@ -304,7 +322,7 @@ public final class ValueCompressionUtil {
         }
         return result;
 
-      case DATA_SHORT:
+      case SHORT:
         short[] shortResult = new short[value.length];
 
         for (double a : value) {
@@ -313,7 +331,7 @@ public final class ValueCompressionUtil {
         }
         return shortResult;
 
-      case DATA_INT:
+      case INT:
         int[] intResult = new int[value.length];
 
         for (double a : value) {
@@ -322,8 +340,7 @@ public final class ValueCompressionUtil {
         }
         return intResult;
 
-      case DATA_LONG:
-      case DATA_BIGINT:
+      case LONG:
         long[] longResult = new long[value.length];
 
         for (double a : value) {
@@ -332,7 +349,7 @@ public final class ValueCompressionUtil {
         }
         return longResult;
 
-      case DATA_FLOAT:
+      case FLOAT:
         float[] floatResult = new float[value.length];
 
         for (double a : value) {
@@ -353,7 +370,7 @@ public final class ValueCompressionUtil {
   private static Object compressMaxMin(DataType changedDataType, double[] value, double maxValue) {
     int i = 0;
     switch (changedDataType) {
-      case DATA_BYTE:
+      case BYTE:
 
         byte[] result = new byte[value.length];
         for (double a : value) {
@@ -362,7 +379,7 @@ public final class ValueCompressionUtil {
         }
         return result;
 
-      case DATA_SHORT:
+      case SHORT:
 
         short[] shortResult = new short[value.length];
 
@@ -372,7 +389,7 @@ public final class ValueCompressionUtil {
         }
         return shortResult;
 
-      case DATA_INT:
+      case INT:
 
         int[] intResult = new int[value.length];
 
@@ -382,7 +399,7 @@ public final class ValueCompressionUtil {
         }
         return intResult;
 
-      case DATA_LONG:
+      case LONG:
 
         long[] longResult = new long[value.length];
 
@@ -392,7 +409,7 @@ public final class ValueCompressionUtil {
         }
         return longResult;
 
-      case DATA_FLOAT:
+      case FLOAT:
 
         float[] floatResult = new float[value.length];
 
@@ -422,7 +439,7 @@ public final class ValueCompressionUtil {
   private static Object compressNonDecimal(DataType changedDataType, double[] value, int mantissa) {
     int i = 0;
     switch (changedDataType) {
-      case DATA_BYTE:
+      case BYTE:
         byte[] result = new byte[value.length];
 
         for (double a : value) {
@@ -430,7 +447,7 @@ public final class ValueCompressionUtil {
           i++;
         }
         return result;
-      case DATA_SHORT:
+      case SHORT:
         short[] shortResult = new short[value.length];
 
         for (double a : value) {
@@ -438,7 +455,7 @@ public final class ValueCompressionUtil {
           i++;
         }
         return shortResult;
-      case DATA_INT:
+      case INT:
 
         int[] intResult = new int[value.length];
 
@@ -448,7 +465,7 @@ public final class ValueCompressionUtil {
         }
         return intResult;
 
-      case DATA_LONG:
+      case LONG:
 
         long[] longResult = new long[value.length];
 
@@ -458,7 +475,7 @@ public final class ValueCompressionUtil {
         }
         return longResult;
 
-      case DATA_FLOAT:
+      case FLOAT:
 
         float[] floatResult = new float[value.length];
 
@@ -489,7 +506,7 @@ public final class ValueCompressionUtil {
     int i = 0;
     BigDecimal max = BigDecimal.valueOf(maxValue);
     switch (changedDataType) {
-      case DATA_BYTE:
+      case BYTE:
 
         byte[] result = new byte[value.length];
 
@@ -501,7 +518,7 @@ public final class ValueCompressionUtil {
         }
         return result;
 
-      case DATA_SHORT:
+      case SHORT:
 
         short[] shortResult = new short[value.length];
 
@@ -513,7 +530,7 @@ public final class ValueCompressionUtil {
         }
         return shortResult;
 
-      case DATA_INT:
+      case INT:
 
         int[] intResult = new int[value.length];
 
@@ -525,7 +542,7 @@ public final class ValueCompressionUtil {
         }
         return intResult;
 
-      case DATA_LONG:
+      case LONG:
 
         long[] longResult = new long[value.length];
 
@@ -537,7 +554,7 @@ public final class ValueCompressionUtil {
         }
         return longResult;
 
-      case DATA_FLOAT:
+      case FLOAT:
 
         float[] floatResult = new float[value.length];
 
@@ -570,14 +587,13 @@ public final class ValueCompressionUtil {
   public static ValueCompressionHolder getCompressionNone(DataType compDataType,
       DataType actualDataType) {
     switch (compDataType) {
-      case DATA_BYTE:
+      case BYTE:
         return new CompressionNoneByte(actualDataType);
-      case DATA_SHORT:
+      case SHORT:
         return new CompressionNoneShort(actualDataType);
-      case DATA_INT:
+      case INT:
         return new CompressionNoneInt(actualDataType);
-      case DATA_LONG:
-      case DATA_BIGINT:
+      case LONG:
         return new CompressionNoneLong(actualDataType);
       default:
         return new CompressionNoneDefault(actualDataType);
@@ -590,13 +606,13 @@ public final class ValueCompressionUtil {
   public static ValueCompressionHolder getCompressionDecimalMaxMin(
       DataType compDataType, DataType actualDataType) {
     switch (compDataType) {
-      case DATA_BYTE:
+      case BYTE:
         return new CompressionMaxMinByte(actualDataType);
-      case DATA_SHORT:
+      case SHORT:
         return new CompressionMaxMinShort(actualDataType);
-      case DATA_INT:
+      case INT:
         return new CompressionMaxMinInt(actualDataType);
-      case DATA_LONG:
+      case LONG:
         return new CompressionMaxMinLong(actualDataType);
       default:
         return new CompressionMaxMinDefault(actualDataType);
@@ -609,13 +625,13 @@ public final class ValueCompressionUtil {
   public static ValueCompressionHolder getCompressionNonDecimal(
       DataType compDataType) {
     switch (compDataType) {
-      case DATA_BYTE:
+      case BYTE:
         return new CompressionNonDecimalByte();
-      case DATA_SHORT:
+      case SHORT:
         return new CompressionNonDecimalShort();
-      case DATA_INT:
+      case INT:
         return new CompressionNonDecimalInt();
-      case DATA_LONG:
+      case LONG:
         return new CompressionNonDecimalLong();
       default:
         return new CompressionNonDecimalDefault();
@@ -628,13 +644,13 @@ public final class ValueCompressionUtil {
   public static ValueCompressionHolder getCompressionNonDecimalMaxMin(
       DataType compDataType) {
     switch (compDataType) {
-      case DATA_BYTE:
+      case BYTE:
         return new CompressionNonDecimalMaxMinByte();
-      case DATA_SHORT:
+      case SHORT:
         return new CompressionNonDecimalMaxMinShort();
-      case DATA_INT:
+      case INT:
         return new CompressionNonDecimalMaxMinInt();
-      case DATA_LONG:
+      case LONG:
         return new CompressionNonDecimalMaxMinLong();
       default:
         return new CompressionNonDecimalMaxMinDefault();
@@ -645,10 +661,10 @@ public final class ValueCompressionUtil {
    * Create Value compression model for write path
    */
   public static WriterCompressModel getWriterCompressModel(Object[] maxValue, Object[] minValue,
-      int[] mantissa, Object[] uniqueValue, char[] aggType, byte[] dataTypeSelected) {
+      int[] mantissa, Object[] uniqueValue, DataType[] dataType, byte[] dataTypeSelected) {
     MeasureMetaDataModel metaDataModel =
         new MeasureMetaDataModel(minValue, maxValue, mantissa, maxValue.length, uniqueValue,
-            aggType, dataTypeSelected);
+            dataType, dataTypeSelected);
     return getWriterCompressModel(metaDataModel);
   }
 
@@ -661,7 +677,7 @@ public final class ValueCompressionUtil {
     Object[] maxValue = measureMDMdl.getMaxValue();
     Object[] uniqueValue = measureMDMdl.getUniqueValue();
     int[] mantissa = measureMDMdl.getMantissa();
-    char[] type = measureMDMdl.getType();
+    DataType[] type = measureMDMdl.getType();
     byte[] dataTypeSelected = measureMDMdl.getDataTypeSelected();
     WriterCompressModel compressionModel = new WriterCompressModel();
     DataType[] actualType = new DataType[measureCount];
@@ -772,20 +788,4 @@ public final class ValueCompressionUtil {
      */
     BIGDECIMAL
   }
-
-  /**
-   * use to identify the type of data.
-   */
-  public enum DataType {
-    DATA_BYTE(),
-    DATA_SHORT(),
-    DATA_INT(),
-    DATA_FLOAT(),
-    DATA_LONG(),
-    DATA_BIGINT(),
-    DATA_DOUBLE(),
-    DATA_BIGDECIMAL();
-    DataType() {
-    }
-  }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a79a3a16/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
index 2c6c890..ddcc8a4 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/CarbonMetadataUtilTest.java
@@ -28,6 +28,8 @@ import org.apache.carbondata.core.metadata.ValueEncoderMeta;
 import org.apache.carbondata.format.*;
 import org.apache.carbondata.format.BlockletMinMaxIndex;
 import org.apache.carbondata.format.ColumnSchema;
+import org.apache.carbondata.format.DataType;
+
 import org.junit.BeforeClass;
 import org.junit.Test;
 
@@ -169,7 +171,12 @@ public class CarbonMetadataUtilTest {
     long[] longArr = { 1, 2, 3, 4, 5 };
     byte[][] maxByteArr = { { 1, 2 }, { 3, 4 }, { 5, 6 }, { 2, 4 }, { 1, 2 } };
     int[] cardinality = { 1, 2, 3, 4, 5 };
-    char[] charArr = { 'a', 's', 'd', 'g', 'h' };
+    org.apache.carbondata.core.metadata.datatype.DataType[] dataType = {
+        org.apache.carbondata.core.metadata.datatype.DataType.INT,
+        org.apache.carbondata.core.metadata.datatype.DataType.INT,
+        org.apache.carbondata.core.metadata.datatype.DataType.INT,
+        org.apache.carbondata.core.metadata.datatype.DataType.INT,
+        org.apache.carbondata.core.metadata.datatype.DataType.INT };
 
     org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema colSchema =
         new org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema();
@@ -191,7 +198,7 @@ public class CarbonMetadataUtilTest {
     writerCompressModel.setMinValue(objMinArr);
     writerCompressModel.setDataTypeSelected(byteArr);
     writerCompressModel.setMantissa(intArr);
-    writerCompressModel.setType(charArr);
+    writerCompressModel.setType(dataType);
     writerCompressModel.setUniqueValue(objMinArr);
 
     BlockletInfoColumnar blockletInfoColumnar = new BlockletInfoColumnar();

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a79a3a16/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java b/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java
index 6252ca1..3032085 100644
--- a/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/util/ValueCompressionUtilTest.java
@@ -27,7 +27,7 @@ import org.apache.carbondata.core.datastore.compression.ValueCompressionHolder;
 import org.apache.carbondata.core.datastore.compression.decimal.*;
 import org.apache.carbondata.core.datastore.compression.nondecimal.*;
 import org.apache.carbondata.core.datastore.compression.none.*;
-import org.apache.carbondata.core.util.ValueCompressionUtil.DataType;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 
 import org.junit.Test;
 
@@ -36,8 +36,8 @@ public class ValueCompressionUtilTest {
 
   @Test public void testGetSize() {
     DataType[] dataTypes =
-        { DataType.DATA_BIGINT, DataType.DATA_INT, DataType.DATA_BYTE, DataType.DATA_SHORT,
-            DataType.DATA_FLOAT };
+        { DataType.LONG, DataType.INT, DataType.BOOLEAN, DataType.SHORT,
+            DataType.FLOAT };
     int[] expectedSizes = { 8, 4, 1, 2, 4 };
     for (int i = 0; i < dataTypes.length; i++) {
       assertEquals(expectedSizes[i], ValueCompressionUtil.getSize(dataTypes[i]));
@@ -48,7 +48,7 @@ public class ValueCompressionUtilTest {
     double[] values = { 25, 12, 22 };
     int[] result = (int[]) ValueCompressionUtil
         .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, values,
-            DataType.DATA_INT, 22, 0);
+            DataType.INT, 22, 0);
     int[] expectedResult = { -3, 10, 0 };
     for (int i = 0; i < result.length; i++) {
       assertEquals(result[i], expectedResult[i]);
@@ -59,7 +59,7 @@ public class ValueCompressionUtilTest {
     double[] values = { 20, 21, 22 };
     byte[] result = (byte[]) ValueCompressionUtil
         .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, values,
-            DataType.DATA_BYTE, 22, 0);
+            DataType.BYTE, 22, 0);
     byte[] expectedResult = { 2, 1, 0 };
     for (int i = 0; i < result.length; i++) {
       assertEquals(result[i], expectedResult[i]);
@@ -70,7 +70,7 @@ public class ValueCompressionUtilTest {
     double[] values = { 200, 21, 22 };
     short[] result = (short[]) ValueCompressionUtil
         .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, values,
-            DataType.DATA_SHORT, 22, 0);
+            DataType.SHORT, 22, 0);
     short[] expectedResult = { -178, 1, 0 };
     for (int i = 0; i < result.length; i++) {
       assertEquals(result[i], expectedResult[i]);
@@ -81,7 +81,7 @@ public class ValueCompressionUtilTest {
     double[] values = { 2000, 2100, 2002 };
     long[] result = (long[]) ValueCompressionUtil
         .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, values,
-            DataType.DATA_LONG, 2125, 0);
+            DataType.LONG, 2125, 0);
     long[] expectedResult = { 125, 25, 123 };
     for (int i = 0; i < result.length; i++) {
       assertEquals(result[i], expectedResult[i]);
@@ -92,7 +92,7 @@ public class ValueCompressionUtilTest {
     double[] values = { 20.121, 21.223, 22.345 };
     float[] result = (float[]) ValueCompressionUtil
         .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, values,
-            DataType.DATA_FLOAT, 22.345, 3);
+            DataType.FLOAT, 22.345, 3);
     float[] expectedResult = { 2.224f, 1.122f, 0f };
     for (int i = 0; i < result.length; i++) {
     	assertTrue(result[i]-expectedResult[i]==0);
@@ -103,7 +103,7 @@ public class ValueCompressionUtilTest {
     double[] values = { 20.121, 21.223, 22.345 };
     double[] result = (double[]) ValueCompressionUtil
         .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, values,
-            DataType.DATA_DOUBLE, 102.345, 3);
+            DataType.DOUBLE, 102.345, 3);
     double[] expectedResult = { 82.224, 81.122, 80.0 };
     for (int i = 0; i < result.length; i++) {
       assertTrue(result[i]-expectedResult[i]==0);
@@ -114,7 +114,7 @@ public class ValueCompressionUtilTest {
     double[] values = { 20.121, 21.223, 22.345 };
     long[] result = (long[]) ValueCompressionUtil
         .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values,
-            DataType.DATA_BIGINT, 22, 0);
+            DataType.LONG, 22, 0);
     long[] expectedResult = { 20, 21, 22 };
     for (int i = 0; i < result.length; i++) {
       assertEquals(result[i], expectedResult[i]);
@@ -124,7 +124,7 @@ public class ValueCompressionUtilTest {
   @Test public void testToGetCompressedValuesWithCompressionTypeNoneForDataByte() {
     double[] values = { 20, 21, 22 };
     byte[] result = (byte[]) ValueCompressionUtil
-        .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values, DataType.DATA_BYTE,
+        .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values, DataType.BYTE,
             22, 0);
     byte[] expectedResult = { 20, 21, 22 };
     for (int i = 0; i < result.length; i++) {
@@ -136,7 +136,7 @@ public class ValueCompressionUtilTest {
     double[] values = { 200000, 21, 22 };
     short[] result = (short[]) ValueCompressionUtil
         .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values,
-            DataType.DATA_SHORT, 22, 0);
+            DataType.SHORT, 22, 0);
     short[] expectedResult = { 3392, 21, 22 };
     for (int i = 0; i < result.length; i++) {
       assertEquals(result[i], expectedResult[i]);
@@ -146,7 +146,7 @@ public class ValueCompressionUtilTest {
   @Test public void testToGetCompressedValuesWithCompressionTypeNoneForDataInt() {
     double[] values = { 20, 21, 22 };
     int[] result = (int[]) ValueCompressionUtil
-        .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values, DataType.DATA_INT,
+        .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values, DataType.INT,
             22, 0);
     int[] expectedResult = { 20, 21, 22 };
     for (int i = 0; i < result.length; i++) {
@@ -157,7 +157,7 @@ public class ValueCompressionUtilTest {
   @Test public void testToGetCompressedValuesWithCompressionTypeNoneForDataLong() {
     double[] values = { 20, 21, 22 };
     long[] result = (long[]) ValueCompressionUtil
-        .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values, DataType.DATA_LONG,
+        .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values, DataType.LONG,
             22, 0);
     long[] expectedResult = { 20, 21, 22 };
     for (int i = 0; i < result.length; i++) {
@@ -169,7 +169,7 @@ public class ValueCompressionUtilTest {
     double[] values = { 20.121, 21.223, 22.345 };
     float[] result = (float[]) ValueCompressionUtil
         .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values,
-            DataType.DATA_FLOAT, 22, 3);
+            DataType.FLOAT, 22, 3);
     float[] expectedResult = { 20.121f, 21.223f, 22.345f };
     for (int i = 0; i < result.length; i++) {
       assertEquals(result[i], expectedResult[i],3);
@@ -180,7 +180,7 @@ public class ValueCompressionUtilTest {
     double[] values = { 20.121, 21.223, 22.345 };
     double[] result = (double[]) ValueCompressionUtil
         .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, values,
-            DataType.DATA_DOUBLE, 22, 3);
+            DataType.DOUBLE, 22, 3);
     double[] expectedResult = { 20.121, 21.223, 22.345 };
     for (int i = 0; i < result.length; i++) {
       assertEquals(result[i], expectedResult[i],3);
@@ -191,7 +191,7 @@ public class ValueCompressionUtilTest {
     double[] values = { 20.1, 21.2, 22.3 };
     float[] result = (float[]) ValueCompressionUtil
         .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.BIGINT, values,
-            DataType.DATA_FLOAT, 22, 1);
+            DataType.FLOAT, 22, 1);
     float[] expectedResult = { 201f, 212f, 223f };
     for (int i = 0; i < result.length; i++) {
       assertEquals(result[i], expectedResult[i],0);
@@ -202,7 +202,7 @@ public class ValueCompressionUtilTest {
     double[] values = { 20.1, 21.2, 22.3 };
     byte[] result = (byte[]) ValueCompressionUtil
         .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.BIGINT, values,
-            DataType.DATA_BYTE, 22, 1);
+            DataType.BYTE, 22, 1);
     byte[] expectedResult = { -55, -44, -33 };
     for (int i = 0; i < result.length; i++) {
       assertEquals(result[i], expectedResult[i]);
@@ -213,7 +213,7 @@ public class ValueCompressionUtilTest {
     double[] values = { 20.1, 21.2, 22.3 };
     short[] result = (short[]) ValueCompressionUtil
         .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.BIGINT, values,
-            DataType.DATA_SHORT, 22, 1);
+            DataType.SHORT, 22, 1);
     short[] expectedResult = { 201, 212, 223 };
     for (int i = 0; i < result.length; i++) {
       assertEquals(result[i], expectedResult[i]);
@@ -224,7 +224,7 @@ public class ValueCompressionUtilTest {
     double[] values = { 20.1, 21.2, 22.3 };
     int[] result = (int[]) ValueCompressionUtil
         .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.BIGINT, values,
-            DataType.DATA_INT, 22, 1);
+            DataType.INT, 22, 1);
     int[] expectedResult = { 201, 212, 223 };
     for (int i = 0; i < result.length; i++) {
       assertEquals(result[i], expectedResult[i]);
@@ -235,7 +235,7 @@ public class ValueCompressionUtilTest {
     double[] values = { 20.1, 21.2, 22.3 };
     long[] result = (long[]) ValueCompressionUtil
         .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.BIGINT, values,
-            DataType.DATA_LONG, 22, 1);
+            DataType.LONG, 22, 1);
     long[] expectedResult = { 201, 212, 223 };
     for (int i = 0; i < result.length; i++) {
       assertEquals(result[i], expectedResult[i]);
@@ -246,7 +246,7 @@ public class ValueCompressionUtilTest {
     double[] values = { 20.1, 21.2, 22.3 };
     double[] result = (double[]) ValueCompressionUtil
         .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.BIGINT, values,
-            DataType.DATA_DOUBLE, 22, 1);
+            DataType.DOUBLE, 22, 1);
     double[] expectedResult = { 201, 212, 223 };
     for (int i = 0; i < result.length; i++) {
       assertEquals(result[i], expectedResult[i],0);
@@ -257,7 +257,7 @@ public class ValueCompressionUtilTest {
     double[] values = { 20, 21, 22 };
     byte[] result = (byte[]) ValueCompressionUtil
         .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_NON_DECIMAL, values,
-            DataType.DATA_BYTE, 22, 1);
+            DataType.BYTE, 22, 1);
     byte[] expectedResult = { 20, 10, 0 };
     for (int i = 0; i < result.length; i++) {
       assertEquals(result[i], expectedResult[i]);
@@ -268,7 +268,7 @@ public class ValueCompressionUtilTest {
     double[] values = { 20, 21, 22 };
     int[] result = (int[]) ValueCompressionUtil
         .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_NON_DECIMAL, values,
-            DataType.DATA_INT, 22, 1);
+            DataType.INT, 22, 1);
     int[] expectedResult = { 20, 10, 0 };
     for (int i = 0; i < result.length; i++) {
       assertEquals(result[i], expectedResult[i]);
@@ -279,7 +279,7 @@ public class ValueCompressionUtilTest {
     double[] values = { 20, 21, 22 };
     double[] result = (double[]) ValueCompressionUtil
         .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_NON_DECIMAL, values,
-            DataType.DATA_DOUBLE, 22, 1);
+            DataType.DOUBLE, 22, 1);
     double[] expectedResult = { 20, 10, 0 };
     for (int i = 0; i < result.length; i++) {
       assertEquals(result[i], expectedResult[i],0);
@@ -290,7 +290,7 @@ public class ValueCompressionUtilTest {
     double[] values = { 20000, 21, 22 };
     short[] result = (short[]) ValueCompressionUtil
         .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_NON_DECIMAL, values,
-            DataType.DATA_SHORT, 22, 1);
+            DataType.SHORT, 22, 1);
     short[] expectedResult = { -3172, 10, 0 };
     for (int i = 0; i < result.length; i++) {
       assertEquals(result[i], expectedResult[i]);
@@ -301,7 +301,7 @@ public class ValueCompressionUtilTest {
     double[] values = { 20, 21, 22 };
     long[] result = (long[]) ValueCompressionUtil
         .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_NON_DECIMAL, values,
-            DataType.DATA_LONG, 22, 1);
+            DataType.LONG, 22, 1);
     long[] expectedResult = { 20, 10, 0 };
     for (int i = 0; i < result.length; i++) {
       assertEquals(result[i], expectedResult[i]);
@@ -312,7 +312,7 @@ public class ValueCompressionUtilTest {
     double[] values = { 20, 21, 22 };
     float[] result = (float[]) ValueCompressionUtil
         .getCompressedValues(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_NON_DECIMAL, values,
-            DataType.DATA_FLOAT, 22, 1);
+            DataType.FLOAT, 22, 1);
     float[] expectedResult = { 20f, 10f, 0f };
     for (int i = 0; i < result.length; i++) {
       assertEquals(result[i], expectedResult[i],0);
@@ -321,127 +321,127 @@ public class ValueCompressionUtilTest {
 
   @Test public void testToUnCompressNone() {
     ValueCompressionHolder result =
-        ValueCompressionUtil.getCompressionNone(DataType.DATA_BIGINT, DataType.DATA_BIGINT);
+        ValueCompressionUtil.getCompressionNone(DataType.LONG, DataType.LONG);
     assertEquals(result.getClass(), CompressionNoneLong.class);
   }
 
   @Test public void testToUnCompressNoneForByte() {
     ValueCompressionHolder result =
-        ValueCompressionUtil.getCompressionNone(DataType.DATA_BYTE, DataType.DATA_FLOAT);
+        ValueCompressionUtil.getCompressionNone(DataType.BYTE, DataType.FLOAT);
     assertEquals(result.getClass(), CompressionNoneByte.class);
   }
 
   @Test public void testToUnCompressNoneForLong() {
     ValueCompressionHolder result =
-        ValueCompressionUtil.getCompressionNone(DataType.DATA_LONG, DataType.DATA_FLOAT);
+        ValueCompressionUtil.getCompressionNone(DataType.LONG, DataType.FLOAT);
     assertEquals(result.getClass(), CompressionNoneLong.class);
   }
 
   @Test public void testToUnCompressNoneForShort() {
     ValueCompressionHolder result =
-        ValueCompressionUtil.getCompressionNone(DataType.DATA_SHORT, DataType.DATA_FLOAT);
+        ValueCompressionUtil.getCompressionNone(DataType.SHORT, DataType.FLOAT);
     assertEquals(result.getClass(), CompressionNoneShort.class);
   }
 
   @Test public void testToUnCompressNoneForInt() {
     ValueCompressionHolder result =
-        ValueCompressionUtil.getCompressionNone(DataType.DATA_INT, DataType.DATA_FLOAT);
+        ValueCompressionUtil.getCompressionNone(DataType.INT, DataType.FLOAT);
     assertEquals(result.getClass(), CompressionNoneInt.class);
   }
 
   @Test public void testToUnCompressNoneForDouble() {
     ValueCompressionHolder result =
-        ValueCompressionUtil.getCompressionNone(DataType.DATA_DOUBLE, DataType.DATA_FLOAT);
+        ValueCompressionUtil.getCompressionNone(DataType.DOUBLE, DataType.FLOAT);
     assertEquals(result.getClass(), CompressionNoneDefault.class);
   }
 
   @Test public void testToUnCompressMaxMinForDouble() {
     ValueCompressionHolder result =
-        ValueCompressionUtil.getCompressionDecimalMaxMin(DataType.DATA_DOUBLE, null);
+        ValueCompressionUtil.getCompressionDecimalMaxMin(DataType.DOUBLE, null);
     assertEquals(result.getClass(), CompressionMaxMinDefault.class);
   }
 
   @Test public void testToUnCompressMaxMinForInt() {
     ValueCompressionHolder result =
-        ValueCompressionUtil.getCompressionDecimalMaxMin(DataType.DATA_INT, null);
+        ValueCompressionUtil.getCompressionDecimalMaxMin(DataType.INT, null);
     assertEquals(result.getClass(), CompressionMaxMinInt.class);
   }
 
   @Test public void testToUnCompressMaxMinForLong() {
     ValueCompressionHolder result =
-        ValueCompressionUtil.getCompressionDecimalMaxMin(DataType.DATA_LONG, null);
+        ValueCompressionUtil.getCompressionDecimalMaxMin(DataType.LONG, null);
     assertEquals(result.getClass(), CompressionMaxMinLong.class);
   }
 
   @Test public void testToUnCompressMaxMinForByte() {
     ValueCompressionHolder result =
-        ValueCompressionUtil.getCompressionDecimalMaxMin(DataType.DATA_BYTE, null);
+        ValueCompressionUtil.getCompressionDecimalMaxMin(DataType.BYTE, null);
     assertEquals(result.getClass(), CompressionMaxMinByte.class);
   }
 
   @Test public void testToUnCompressMaxMinForShort() {
     ValueCompressionHolder result =
-        ValueCompressionUtil.getCompressionDecimalMaxMin(DataType.DATA_SHORT, null);
+        ValueCompressionUtil.getCompressionDecimalMaxMin(DataType.SHORT, null);
     assertEquals(result.getClass(), CompressionMaxMinShort.class);
   }
 
   @Test public void testToUnCompressNonDecimalForDouble() {
     ValueCompressionHolder result =
-        ValueCompressionUtil.getCompressionNonDecimal(DataType.DATA_DOUBLE);
+        ValueCompressionUtil.getCompressionNonDecimal(DataType.DOUBLE);
     assertEquals(result.getClass(), CompressionNonDecimalDefault.class);
   }
 
   @Test public void testToUnCompressNonDecimalForInt() {
     ValueCompressionHolder result =
-        ValueCompressionUtil.getCompressionNonDecimal(DataType.DATA_INT);
+        ValueCompressionUtil.getCompressionNonDecimal(DataType.INT);
     assertEquals(result.getClass(), CompressionNonDecimalInt.class);
   }
 
   @Test public void testToUnCompressNonDecimalForLong() {
     ValueCompressionHolder result =
-        ValueCompressionUtil.getCompressionNonDecimal(DataType.DATA_LONG);
+        ValueCompressionUtil.getCompressionNonDecimal(DataType.LONG);
     assertEquals(result.getClass(), CompressionNonDecimalLong.class);
   }
 
   @Test public void testToUnCompressNonDecimalForByte() {
     ValueCompressionHolder result =
-        ValueCompressionUtil.getCompressionNonDecimal(DataType.DATA_BYTE);
+        ValueCompressionUtil.getCompressionNonDecimal(DataType.BYTE);
     assertEquals(result.getClass(), CompressionNonDecimalByte.class);
   }
 
   @Test public void testToUnCompressNonDecimalForShort() {
     ValueCompressionHolder result =
-        ValueCompressionUtil.getCompressionNonDecimal(DataType.DATA_SHORT);
+        ValueCompressionUtil.getCompressionNonDecimal(DataType.SHORT);
     assertEquals(result.getClass(), CompressionNonDecimalShort.class);
   }
 
   @Test public void testToUnCompressNonDecimalMaxMinForDouble() {
     ValueCompressionHolder result =
-        ValueCompressionUtil.getCompressionNonDecimalMaxMin(DataType.DATA_DOUBLE);
+        ValueCompressionUtil.getCompressionNonDecimalMaxMin(DataType.DOUBLE);
     assertEquals(result.getClass(), CompressionNonDecimalMaxMinDefault.class);
   }
 
   @Test public void testToUnCompressNonDecimalMaxMinForInt() {
     ValueCompressionHolder result =
-        ValueCompressionUtil.getCompressionNonDecimalMaxMin(DataType.DATA_INT);
+        ValueCompressionUtil.getCompressionNonDecimalMaxMin(DataType.INT);
     assertEquals(result.getClass(), CompressionNonDecimalMaxMinInt.class);
   }
 
   @Test public void testToUnCompressNonDecimalMaxMinForLong() {
     ValueCompressionHolder result =
-        ValueCompressionUtil.getCompressionNonDecimalMaxMin(DataType.DATA_LONG);
+        ValueCompressionUtil.getCompressionNonDecimalMaxMin(DataType.LONG);
     assertEquals(result.getClass(), CompressionNonDecimalMaxMinLong.class);
   }
 
   @Test public void testToUnCompressNonDecimalMaxMinForByte() {
     ValueCompressionHolder result =
-        ValueCompressionUtil.getCompressionNonDecimalMaxMin(DataType.DATA_BYTE);
+        ValueCompressionUtil.getCompressionNonDecimalMaxMin(DataType.BYTE);
     assertEquals(result.getClass(), CompressionNonDecimalMaxMinByte.class);
   }
 
   @Test public void testToUnCompressNonDecimalMaxMinForShort() {
     ValueCompressionHolder result =
-        ValueCompressionUtil.getCompressionNonDecimalMaxMin(DataType.DATA_SHORT);
+        ValueCompressionUtil.getCompressionNonDecimalMaxMin(DataType.SHORT);
     assertEquals(result.getClass(), CompressionNonDecimalMaxMinShort.class);
   }
 
@@ -490,7 +490,7 @@ public class ValueCompressionUtilTest {
     Object[] minValues = { 1L, 2L, 3L };
     int[] decimalLength = { 0, 0, 0 };
     Object[] uniqueValues = { 5, new Long[]{2L,4L}, 2L};
-    char[] types = { 'l', 'l', 'l' };
+    DataType[] types = { DataType.LONG, DataType.LONG, DataType.LONG };
     byte[] dataTypeSelected = { 1, 2, 4 };
     MeasureMetaDataModel measureMetaDataModel =
         new MeasureMetaDataModel(maxValues, minValues, decimalLength, 3, uniqueValues, types,
@@ -510,7 +510,7 @@ public class ValueCompressionUtilTest {
     Object[] minValues = { 1.0 };
     int[] decimalLength = { 0 };
     Object[] uniqueValues = { 5 };
-    char[] types = { 'n' };
+    DataType[] types = { DataType.DOUBLE };
     byte[] dataTypeSelected = { 1 };
     MeasureMetaDataModel measureMetaDataModel =
         new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types,
@@ -526,7 +526,7 @@ public class ValueCompressionUtilTest {
     Object[] minValues = { 32500.00 };
     int[] decimalLength = { 0 };
     Object[] uniqueValues = { 5 };
-    char[] types = { 'n' };
+    DataType[] types = { DataType.DOUBLE };
     byte[] dataTypeSelected = { 1 };
     MeasureMetaDataModel measureMetaDataModel =
         new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types,
@@ -542,7 +542,7 @@ public class ValueCompressionUtilTest {
     Object[] minValues = { 1111078433.0 };
     int[] decimalLength = { 0 };
     Object[] uniqueValues = { 5 };
-    char[] types = { 'n' };
+    DataType[] types = { DataType.DOUBLE };
     byte[] dataTypeSelected = { 1 };
     MeasureMetaDataModel measureMetaDataModel =
         new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types,
@@ -558,7 +558,7 @@ public class ValueCompressionUtilTest {
     Object[] minValues = { 32744.0 };
     int[] decimalLength = { 0 };
     Object[] uniqueValues = { 5 };
-    char[] types = { 'n' };
+    DataType[] types = { DataType.DOUBLE};
     byte[] dataTypeSelected = { 1 };
     MeasureMetaDataModel measureMetaDataModel =
         new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types,
@@ -574,7 +574,7 @@ public class ValueCompressionUtilTest {
     Object[] minValues = { 32744.0 };
     int[] decimalLength = { 1 };
     Object[] uniqueValues = { 5 };
-    char[] types = { 'n' };
+    DataType[] types = { DataType.DOUBLE };
     byte[] dataTypeSelected = { 1 };
     MeasureMetaDataModel measureMetaDataModel =
         new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types,
@@ -590,7 +590,7 @@ public class ValueCompressionUtilTest {
     Object[] minValues = { 32744.0 };
     int[] decimalLength = { 1 };
     Object[] uniqueValues = { 5 };
-    char[] types = { 'n' };
+    DataType[] types = { DataType.DOUBLE };
     byte[] dataTypeSelected = { 0 };
     MeasureMetaDataModel measureMetaDataModel =
         new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types,
@@ -606,7 +606,7 @@ public class ValueCompressionUtilTest {
     Object[] minValues = { 32744.0 };
     int[] decimalLength = { 1 };
     Object[] uniqueValues = { 5 };
-    char[] types = { 'n' };
+    DataType[] types = { DataType.DOUBLE };
     byte[] dataTypeSelected = { 1 };
     MeasureMetaDataModel measureMetaDataModel =
         new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types,

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a79a3a16/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java b/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java
index d02e25f..480ed04 100644
--- a/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/writer/CarbonFooterWriterTest.java
@@ -184,7 +184,7 @@ public class CarbonFooterWriterTest extends TestCase{
     compressionModel.setMaxValue(new Object[] { 44d, 55d });
     compressionModel.setMinValue(new Object[] { 0d, 0d });
     compressionModel.setMantissa(new int[] { 0, 0 });
-    compressionModel.setType(new char[] { 'n', 'n' });
+    compressionModel.setType(new DataType[] { DataType.DOUBLE, DataType.DOUBLE });
     compressionModel.setUniqueValue(new Object[] { 0d, 0d });
     compressionModel.setDataTypeSelected(new byte[2]);
     infoColumnar.setCompressionModel(compressionModel);

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a79a3a16/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
----------------------------------------------------------------------
diff --git a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
index 371b9bb..9ae01b8 100644
--- a/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
+++ b/examples/spark2/src/main/scala/org/apache/carbondata/examples/CarbonSessionExample.scala
@@ -43,6 +43,7 @@ object CarbonSessionExample {
       .master("local")
       .appName("CarbonSessionExample")
       .config("spark.sql.warehouse.dir", warehouse)
+      .config("spark.driver.host", "localhost")
       .getOrCreateCarbonSession(storeLocation, metastoredb)
 
     spark.sparkContext.setLogLevel("WARN")

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a79a3a16/format/src/main/thrift/carbondata.thrift
----------------------------------------------------------------------
diff --git a/format/src/main/thrift/carbondata.thrift b/format/src/main/thrift/carbondata.thrift
index 937108c..b4cbc4e 100644
--- a/format/src/main/thrift/carbondata.thrift
+++ b/format/src/main/thrift/carbondata.thrift
@@ -114,8 +114,12 @@ struct DataChunk{
 }
 
 /**
- * Represents a chunk of data. The chunk can be a single column stored in Column Major format or a group of columns stored in Row Major Format.
- * For V2 format.
+ * Represents the metadata of a data chunk.
+ * The chunk can be a single column stored in Column Major format or a group of columns stored
+ * in Row Major format.
+ *
+ * For V3, one data chunk is one page data of 32K rows.
+ * For V2 & V1, one data chunk is one blocklet data.
  */
 struct DataChunk2{
     1: required ChunkCompressionMeta chunk_meta; // The metadata of a chunk

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a79a3a16/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/emptyrow/TestEmptyRows.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/emptyrow/TestEmptyRows.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/emptyrow/TestEmptyRows.scala
index f31d434..5d6c07a 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/emptyrow/TestEmptyRows.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/integration/spark/testsuite/emptyrow/TestEmptyRows.scala
@@ -28,7 +28,7 @@ class TestEmptyRows extends QueryTest with BeforeAndAfterAll {
   override def beforeAll {
     sql("drop table if exists emptyRowCarbonTable")
     sql("drop table if exists emptyRowHiveTable")
-    //eid,ename,sal,presal,comm,deptno,Desc
+
     sql(
       "create table if not exists emptyRowCarbonTable (eid int,ename String,sal decimal,presal " +
         "decimal,comm decimal" +

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a79a3a16/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala
index 3d85814..fa7b970 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestLoadDataWithNoMeasure.scala
@@ -28,7 +28,7 @@ import org.scalatest.BeforeAndAfterAll
 class TestLoadDataWithNoMeasure extends QueryTest with BeforeAndAfterAll {
 
   override def beforeAll {
-    sql("DROP TABLE IF EXISTS nomeasureTest_sd")
+    sql("DROP TABLE IF EXISTS nomeasureTest")
     sql(
       "CREATE TABLE nomeasureTest (empno String, doj String) STORED BY 'org.apache.carbondata" +
         ".format'"
@@ -106,8 +106,8 @@ class TestLoadDataWithNoMeasure extends QueryTest with BeforeAndAfterAll {
   }
 
   override def afterAll {
-    sql("drop table nomeasureTest")
-    sql("drop table nomeasureTest_sd")
-    sql("drop table nomeasureTest_scd")
+    sql("drop table if exists nomeasureTest")
+    sql("drop table if exists nomeasureTest_sd")
+    sql("drop table if exists nomeasureTest_scd")
   }
 }

http://git-wip-us.apache.org/repos/asf/carbondata/blob/a79a3a16/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala
index f1c1d69..ab003c0 100644
--- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala
+++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/detailquery/ColumnGroupDataTypesTestCase.scala
@@ -27,6 +27,12 @@ import org.scalatest.BeforeAndAfterAll
 class ColumnGroupDataTypesTestCase extends QueryTest with BeforeAndAfterAll {
 
   override def beforeAll {
+    sql("drop table if exists colgrp")
+    sql("drop table if exists normal")
+    sql("drop table if exists colgrp_dictexclude_before")
+    sql("drop table if exists colgrp_dictexclude_after")
+    sql("drop table if exists colgrp_disorder")
+
     sql("create table colgrp (column1 string,column2 string,column3 string,column4 string,column5 string,column6 string,column7 string,column8 string,column9 string,column10 string,measure1 int,measure2 int,measure3 int,measure4 int) STORED BY 'org.apache.carbondata.format' TBLPROPERTIES (\"COLUMN_GROUPS\"=\"(column2,column3,column4),(column7,column8,column9)\")")
     sql(s"LOAD DATA LOCAL INPATH '$resourcesPath/10dim_4msr.csv' INTO table colgrp options('FILEHEADER'='column1,column2,column3,column4,column5,column6,column7,column8,column9,column10,measure1,measure2,measure3,measure4')");
     sql("create table normal (column1 string,column2 string,column3 string,column4 string,column5 string,column6 string,column7 string,column8 string,column9 string,column10 string,measure1 int,measure2 int,measure3 int,measure4 int) STORED BY 'org.apache.carbondata.format'")


Mime
View raw message