carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenliang...@apache.org
Subject [10/52] [partial] incubator-carbondata git commit: Renamed packages to org.apache.carbondata and fixed errors
Date Mon, 15 Aug 2016 07:08:55 GMT
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/util/LoadStatistics.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/util/LoadStatistics.java b/core/src/main/java/org/carbondata/core/util/LoadStatistics.java
deleted file mode 100644
index e5f24e6..0000000
--- a/core/src/main/java/org/carbondata/core/util/LoadStatistics.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.core.util;
-
-public interface LoadStatistics {
-  //Init PartitonInfo
-  void  initPartitonInfo(String PartitionId);
-
-  //Record the time
-  void recordDicShuffleAndWriteTime();
-
-  void recordLoadCsvfilesToDfTime();
-
-  void recordDictionaryValuesTotalTime(String partitionID,
-      Long dictionaryValuesTotalTimeTimePoint);
-
-  void recordCsvInputStepTime(String partitionID,
-      Long csvInputStepTimePoint);
-
-  void recordLruCacheLoadTime(double lruCacheLoadTime);
-
-  void recordGeneratingDictionaryValuesTime(String partitionID,
-      Long generatingDictionaryValuesTimePoint);
-
-  void recordSortRowsStepTotalTime(String partitionID,
-      Long sortRowsStepTotalTimePoint);
-
-  void recordMdkGenerateTotalTime(String partitionID,
-      Long mdkGenerateTotalTimePoint);
-
-  void recordDictionaryValue2MdkAdd2FileTime(String partitionID,
-      Long dictionaryValue2MdkAdd2FileTimePoint);
-
-  //Record the node blocks information map
-  void recordHostBlockMap(String host, Integer numBlocks);
-
-  //Record the partition blocks information map
-  void recordPartitionBlockMap(String partitionID, Integer numBlocks);
-
-  //Record total num of records processed
-  void recordTotalRecords(long totalRecords);
-
-  //Print the statistics information
-  void printStatisticsInfo(String partitionID);
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/util/ValueCompressionUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/util/ValueCompressionUtil.java b/core/src/main/java/org/carbondata/core/util/ValueCompressionUtil.java
deleted file mode 100644
index ac2281d..0000000
--- a/core/src/main/java/org/carbondata/core/util/ValueCompressionUtil.java
+++ /dev/null
@@ -1,1027 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.core.util;
-
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-
-import org.carbondata.common.logging.LogService;
-import org.carbondata.common.logging.LogServiceFactory;
-import org.carbondata.core.datastorage.store.compression.MeasureMetaDataModel;
-import org.carbondata.core.datastorage.store.compression.ValueCompressionModel;
-import org.carbondata.core.datastorage.store.compression.ValueCompressonHolder;
-import org.carbondata.core.datastorage.store.compression.type.UnCompressByteArray;
-import org.carbondata.core.datastorage.store.compression.type.UnCompressDefaultLong;
-import org.carbondata.core.datastorage.store.compression.type.UnCompressMaxMinByte;
-import org.carbondata.core.datastorage.store.compression.type.UnCompressMaxMinDefault;
-import org.carbondata.core.datastorage.store.compression.type.UnCompressMaxMinFloat;
-import org.carbondata.core.datastorage.store.compression.type.UnCompressMaxMinInt;
-import org.carbondata.core.datastorage.store.compression.type.UnCompressMaxMinLong;
-import org.carbondata.core.datastorage.store.compression.type.UnCompressMaxMinShort;
-import org.carbondata.core.datastorage.store.compression.type.UnCompressNonDecimalByte;
-import org.carbondata.core.datastorage.store.compression.type.UnCompressNonDecimalDefault;
-import org.carbondata.core.datastorage.store.compression.type.UnCompressNonDecimalFloat;
-import org.carbondata.core.datastorage.store.compression.type.UnCompressNonDecimalInt;
-import org.carbondata.core.datastorage.store.compression.type.UnCompressNonDecimalLong;
-import org.carbondata.core.datastorage.store.compression.type.UnCompressNonDecimalMaxMinByte;
-import org.carbondata.core.datastorage.store.compression.type.UnCompressNonDecimalMaxMinDefault;
-import org.carbondata.core.datastorage.store.compression.type.UnCompressNonDecimalMaxMinFloat;
-import org.carbondata.core.datastorage.store.compression.type.UnCompressNonDecimalMaxMinInt;
-import org.carbondata.core.datastorage.store.compression.type.UnCompressNonDecimalMaxMinLong;
-import org.carbondata.core.datastorage.store.compression.type.UnCompressNonDecimalMaxMinShort;
-import org.carbondata.core.datastorage.store.compression.type.UnCompressNonDecimalShort;
-import org.carbondata.core.datastorage.store.compression.type.UnCompressNoneByte;
-import org.carbondata.core.datastorage.store.compression.type.UnCompressNoneDefault;
-import org.carbondata.core.datastorage.store.compression.type.UnCompressNoneFloat;
-import org.carbondata.core.datastorage.store.compression.type.UnCompressNoneInt;
-import org.carbondata.core.datastorage.store.compression.type.UnCompressNoneLong;
-import org.carbondata.core.datastorage.store.compression.type.UnCompressNoneShort;
-
-public final class ValueCompressionUtil {
-
-  /**
-   * Attribute for Carbon LOGGER
-   */
-  private static final LogService LOGGER =
-      LogServiceFactory.getLogService(ValueCompressionUtil.class.getName());
-
-  private ValueCompressionUtil() {
-
-  }
-
-  /**
-   * decide actual type of value
-   *
-   * @param value   :the measure value
-   * @param decimal :
-   * @return: actual type of value
-   * @see
-   */
-  private static DataType getDataType(double value, int decimal, byte dataTypeSelected) {
-    DataType dataType = DataType.DATA_DOUBLE;
-    if (decimal == 0) {
-      if (value < Byte.MAX_VALUE) {
-        dataType = DataType.DATA_BYTE;
-      } else if (value < Short.MAX_VALUE) {
-        dataType = DataType.DATA_SHORT;
-      } else if (value < Integer.MAX_VALUE) {
-        dataType = DataType.DATA_INT;
-      } else if (value < Long.MAX_VALUE) {
-        dataType = DataType.DATA_LONG;
-      }
-    } else {
-      if (dataTypeSelected == 1) {
-        if (value < Float.MAX_VALUE) {
-          float floatValue = (float) value;
-          if (floatValue - value != 0) {
-            dataType = DataType.DATA_DOUBLE;
-
-          } else {
-            dataType = DataType.DATA_FLOAT;
-          }
-        } else if (value < Double.MAX_VALUE) {
-          dataType = DataType.DATA_DOUBLE;
-        }
-      }
-    }
-    return dataType;
-  }
-
-  /**
-   * Gives the size of datatype
-   *
-   * @param dataType : measure value type
-   * @return: the size of DataType
-   * @see
-   */
-  public static int getSize(DataType dataType) {
-
-    switch (dataType) {
-      case DATA_BYTE:
-        return 1;
-      case DATA_SHORT:
-        return 2;
-      case DATA_INT:
-      case DATA_FLOAT:
-        return 4;
-      default:
-        return 8;
-    }
-  }
-
-  /**
-   * get the best compression type. priority list,from high to low:
-   * COMPRESSION_TYPE.NONE COMPRESSION_TYPE.MAX_MIN
-   * COMPRESSION_TYPE.NON_DECIMAL_CONVERT COMPRESSION_TYPE.MAX_MIN_NDC
-   *
-   * @param maxValue : max value of one measure
-   * @param minValue : min value of one measure
-   * @param decimal  : decimal num of one measure
-   * @return : the best compression type
-   * @see
-   */
-  private static CompressionFinder getCompressionType(Object maxValue, Object minValue, int decimal,
-      char aggregatorType, byte dataTypeSelected) {
-    // 'c' for aggregate table,'b' fo rBigdecimal, 'l' for long,'n' for double
-    switch (aggregatorType) {
-      case 'c':
-        return new CompressionFinder(COMPRESSION_TYPE.CUSTOM, DataType.DATA_BYTE,
-            DataType.DATA_BYTE);
-      case 'b':
-        return new CompressionFinder(COMPRESSION_TYPE.CUSTOM_BIGDECIMAL, DataType.DATA_BYTE,
-            DataType.DATA_BYTE);
-      case 'l':
-        return new CompressionFinder(COMPRESSION_TYPE.NONE,
-                DataType.DATA_BIGINT, DataType.DATA_BIGINT);
-      default:
-        break;
-    }
-    // None Decimal
-    if (decimal == 0) {
-      if (getSize(getDataType((double) maxValue, decimal, dataTypeSelected)) > getSize(
-          getDataType((double) maxValue - (double) minValue, decimal, dataTypeSelected))) {
-        return new CompressionFinder(COMPRESSION_TYPE.MAX_MIN, DataType.DATA_DOUBLE,
-            getDataType((double) maxValue - (double) minValue, decimal, dataTypeSelected));
-      } else if (getSize(getDataType((double) maxValue, decimal, dataTypeSelected)) < getSize(
-              getDataType((double) maxValue - (double) minValue, decimal, dataTypeSelected))) {
-        return new CompressionFinder(COMPRESSION_TYPE.NONE, DataType.DATA_DOUBLE,
-                getDataType((double) maxValue - (double) minValue, decimal, dataTypeSelected));
-      } else {
-        return new CompressionFinder(COMPRESSION_TYPE.NONE, DataType.DATA_DOUBLE,
-            getDataType((double) maxValue, decimal, dataTypeSelected));
-      }
-    }
-    // decimal
-    else {
-      DataType actualDataType = getDataType((double) maxValue, decimal, dataTypeSelected);
-      DataType diffDataType =
-          getDataType((double) maxValue - (double) minValue, decimal, dataTypeSelected);
-      DataType maxNonDecDataType =
-          getDataType(Math.pow(10, decimal) * (double) maxValue, 0, dataTypeSelected);
-      DataType diffNonDecDataType =
-          getDataType(Math.pow(10, decimal) * ((double) maxValue - (double) minValue), 0,
-              dataTypeSelected);
-
-      CompressionFinder[] finders = new CompressionFinder[] {
-          new CompressionFinder(actualDataType, actualDataType, CompressionFinder.PRIORITY.ACTUAL,
-              COMPRESSION_TYPE.NONE),
-          new CompressionFinder(actualDataType, diffDataType, CompressionFinder.PRIORITY.DIFFSIZE,
-              COMPRESSION_TYPE.MAX_MIN), new CompressionFinder(actualDataType, maxNonDecDataType,
-          CompressionFinder.PRIORITY.MAXNONDECIMAL, COMPRESSION_TYPE.NON_DECIMAL_CONVERT),
-          new CompressionFinder(actualDataType, diffNonDecDataType,
-              CompressionFinder.PRIORITY.DIFFNONDECIMAL, COMPRESSION_TYPE.MAX_MIN_NDC) };
-      // sort the compressionFinder.The top have the highest priority
-      Arrays.sort(finders);
-      CompressionFinder compression = finders[0];
-      return compression;
-    }
-  }
-
-  /**
-   * @param compType        : compression type
-   * @param values          : the data of one measure
-   * @param changedDataType : changed data type
-   * @param maxValue        : the max value of one measure
-   * @param decimal         : the decimal length of one measure
-   * @return: the compress data array
-   * @see
-   */
-  public static Object getCompressedValues(COMPRESSION_TYPE compType, double[] values,
-      DataType changedDataType, double maxValue, int decimal) {
-    Object o;
-    switch (compType) {
-      case NONE:
-
-        o = compressNone(changedDataType, values);
-        return o;
-
-      case MAX_MIN:
-
-        o = compressMaxMin(changedDataType, values, maxValue);
-        return o;
-
-      case NON_DECIMAL_CONVERT:
-
-        o = compressNonDecimal(changedDataType, values, decimal);
-        return o;
-
-      default:
-        o = compressNonDecimalMaxMin(changedDataType, values, decimal, maxValue);
-        return o;
-    }
-  }
-
-  public static Object getCompressedValues(COMPRESSION_TYPE compType, long[] values,
-      DataType changedDataType, long maxValue, int decimal) {
-    Object o;
-    switch (compType) {
-      case NONE:
-      default:
-        return values;
-    }
-  }
-
-  private static ValueCompressonHolder.UnCompressValue[] getUncompressedValues(
-      COMPRESSION_TYPE[] compType, DataType[] actualDataType, DataType[] changedDataType) {
-
-    ValueCompressonHolder.UnCompressValue[] compressValue =
-        new ValueCompressonHolder.UnCompressValue[changedDataType.length];
-    for (int i = 0; i < changedDataType.length; i++) {
-      switch (compType[i]) {
-        case NONE:
-
-          compressValue[i] = unCompressNone(changedDataType[i], actualDataType[i]);
-          break;
-
-        case MAX_MIN:
-
-          compressValue[i] = unCompressMaxMin(changedDataType[i], actualDataType[i]);
-          break;
-
-        case NON_DECIMAL_CONVERT:
-
-          compressValue[i] = unCompressNonDecimal(changedDataType[i], DataType.DATA_DOUBLE);
-          break;
-
-        case CUSTOM:
-          compressValue[i] = new UnCompressByteArray(UnCompressByteArray.ByteArrayType.BYTE_ARRAY);
-          break;
-
-        case CUSTOM_BIGDECIMAL:
-          compressValue[i] = new UnCompressByteArray(UnCompressByteArray.ByteArrayType.BIG_DECIMAL);
-          break;
-
-        default:
-          compressValue[i] = unCompressNonDecimalMaxMin(changedDataType[i], null);
-      }
-    }
-    return compressValue;
-
-  }
-
-  /**
-   * compress data to other type for example: double -> int
-   */
-  private static Object compressNone(DataType changedDataType, double[] value) {
-    int i = 0;
-    switch (changedDataType) {
-
-      case DATA_BYTE:
-
-        byte[] result = new byte[value.length];
-
-        for (double a : value) {
-          result[i] = (byte) a;
-          i++;
-        }
-        return result;
-
-      case DATA_SHORT:
-
-        short[] shortResult = new short[value.length];
-
-        for (double a : value) {
-          shortResult[i] = (short) a;
-          i++;
-        }
-        return shortResult;
-
-      case DATA_INT:
-
-        int[] intResult = new int[value.length];
-
-        for (double a : value) {
-          intResult[i] = (int) a;
-          i++;
-        }
-        return intResult;
-
-      case DATA_LONG:
-      case DATA_BIGINT:
-
-        long[] longResult = new long[value.length];
-
-        for (double a : value) {
-          longResult[i] = (long) a;
-          i++;
-        }
-        return longResult;
-
-      case DATA_FLOAT:
-
-        float[] floatResult = new float[value.length];
-
-        for (double a : value) {
-          floatResult[i] = (float) a;
-          i++;
-        }
-        return floatResult;
-
-      default:
-
-        return value;
-
-    }
-  }
-
-  /**
-   * compress data to other type through sub value for example: 1. subValue =
-   * maxValue - value 2. subValue: double->int
-   */
-  private static Object compressMaxMin(DataType changedDataType, double[] value, double maxValue) {
-    int i = 0;
-    switch (changedDataType) {
-      case DATA_BYTE:
-
-        byte[] result = new byte[value.length];
-        for (double a : value) {
-          result[i] = (byte) (maxValue - a);
-          i++;
-        }
-        return result;
-
-      case DATA_SHORT:
-
-        short[] shortResult = new short[value.length];
-
-        for (double a : value) {
-          shortResult[i] = (short) (maxValue - a);
-          i++;
-        }
-        return shortResult;
-
-      case DATA_INT:
-
-        int[] intResult = new int[value.length];
-
-        for (double a : value) {
-          intResult[i] = (int) (maxValue - a);
-          i++;
-        }
-        return intResult;
-
-      case DATA_LONG:
-
-        long[] longResult = new long[value.length];
-
-        for (double a : value) {
-          longResult[i] = (long) (maxValue - a);
-          i++;
-        }
-        return longResult;
-
-      case DATA_FLOAT:
-
-        float[] floatResult = new float[value.length];
-
-        for (double a : value) {
-          floatResult[i] = (float) (maxValue - a);
-          i++;
-        }
-        return floatResult;
-
-      default:
-
-        double[] defaultResult = new double[value.length];
-
-        for (double a : value) {
-          defaultResult[i] = (double) (maxValue - a);
-          i++;
-        }
-        return defaultResult;
-
-    }
-  }
-
-  /**
-   * compress data to other type through sub value for example: 1. subValue =
-   * value * Math.pow(10, decimal) 2. subValue: double->int
-   */
-  private static Object compressNonDecimal(DataType changedDataType, double[] value, int decimal) {
-    int i = 0;
-    switch (changedDataType) {
-      case DATA_BYTE:
-        byte[] result = new byte[value.length];
-
-        for (double a : value) {
-          result[i] = (byte) (Math.round(Math.pow(10, decimal) * a));
-          i++;
-        }
-        return result;
-      case DATA_SHORT:
-        short[] shortResult = new short[value.length];
-
-        for (double a : value) {
-          shortResult[i] = (short) (Math.round(Math.pow(10, decimal) * a));
-          i++;
-        }
-        return shortResult;
-      case DATA_INT:
-
-        int[] intResult = new int[value.length];
-
-        for (double a : value) {
-          intResult[i] = (int) (Math.round(Math.pow(10, decimal) * a));
-          i++;
-        }
-        return intResult;
-
-      case DATA_LONG:
-
-        long[] longResult = new long[value.length];
-
-        for (double a : value) {
-          longResult[i] = (long) (Math.round(Math.pow(10, decimal) * a));
-          i++;
-        }
-        return longResult;
-
-      case DATA_FLOAT:
-
-        float[] floatResult = new float[value.length];
-
-        for (double a : value) {
-          floatResult[i] = (float) (Math.round(Math.pow(10, decimal) * a));
-          i++;
-        }
-        return floatResult;
-
-      default:
-        double[] defaultResult = new double[value.length];
-
-        for (double a : value) {
-          defaultResult[i] = (double) (Math.round(Math.pow(10, decimal) * a));
-          i++;
-        }
-        return defaultResult;
-    }
-  }
-
-  /**
-   * compress data to other type through sub value for example: 1. subValue =
-   * maxValue - value 2. subValue = subValue * Math.pow(10, decimal) 3.
-   * subValue: double->int
-   */
-  private static Object compressNonDecimalMaxMin(DataType changedDataType, double[] value,
-      int decimal, double maxValue) {
-    int i = 0;
-    switch (changedDataType) {
-      case DATA_BYTE:
-
-        byte[] result = new byte[value.length];
-
-        for (double a : value) {
-          result[i] = (byte) (Math.round((maxValue - a) * Math.pow(10, decimal)));
-          i++;
-        }
-        return result;
-
-      case DATA_SHORT:
-
-        short[] shortResult = new short[value.length];
-
-        for (double a : value) {
-          shortResult[i] = (byte) (Math.round((maxValue - a) * Math.pow(10, decimal)));
-          i++;
-        }
-        return shortResult;
-
-      case DATA_INT:
-
-        int[] intResult = new int[value.length];
-
-        for (double a : value) {
-          intResult[i] = (byte) (Math.round((maxValue - a) * Math.pow(10, decimal)));
-          i++;
-        }
-        return intResult;
-
-      case DATA_LONG:
-
-        long[] longResult = new long[value.length];
-
-        for (double a : value) {
-          longResult[i] = (byte) (Math.round((maxValue - a) * Math.pow(10, decimal)));
-          i++;
-        }
-        return longResult;
-
-      case DATA_FLOAT:
-
-        float[] floatResult = new float[value.length];
-
-        for (double a : value) {
-          floatResult[i] = (byte) (Math.round((maxValue - a) * Math.pow(10, decimal)));
-          i++;
-        }
-        return floatResult;
-
-      default:
-
-        double[] defaultResult = new double[value.length];
-
-        for (double a : value) {
-          defaultResult[i] = (byte) (Math.round((maxValue - a) * Math.pow(10, decimal)));
-          i++;
-        }
-        return defaultResult;
-
-    }
-  }
-
-  /**
-   * uncompress data for example: int -> double
-   */
-  public static ValueCompressonHolder.UnCompressValue unCompressNone(DataType compDataType,
-      DataType actualDataType) {
-    if (actualDataType == DataType.DATA_BIGINT) {
-      return new UnCompressDefaultLong();
-    } else {
-      switch (compDataType) {
-        case DATA_BYTE:
-
-          return new UnCompressNoneByte();
-
-        case DATA_SHORT:
-
-          return new UnCompressNoneShort();
-
-        case DATA_INT:
-
-          return new UnCompressNoneInt();
-
-        case DATA_LONG:
-
-          return new UnCompressNoneLong();
-
-        case DATA_FLOAT:
-
-          return new UnCompressNoneFloat();
-
-        default:
-
-          return new UnCompressNoneDefault();
-      }
-    }
-  }
-
-  /**
-   * uncompress data 1. value = maxValue - subValue 2. value: int->double
-   */
-  public static ValueCompressonHolder.UnCompressValue unCompressMaxMin(DataType compDataType,
-      DataType actualDataType) {
-    switch (compDataType) {
-      case DATA_BYTE:
-
-        return new UnCompressMaxMinByte();
-
-      case DATA_SHORT:
-
-        return new UnCompressMaxMinShort();
-
-      case DATA_INT:
-
-        return new UnCompressMaxMinInt();
-
-      case DATA_LONG:
-
-        return new UnCompressMaxMinLong();
-
-      case DATA_FLOAT:
-
-        return new UnCompressMaxMinFloat();
-
-      default:
-
-        return new UnCompressMaxMinDefault();
-
-    }
-  }
-
-  /**
-   * uncompress data value = value/Math.pow(10, decimal)
-   */
-  public static ValueCompressonHolder.UnCompressValue unCompressNonDecimal(DataType compDataType,
-      DataType actualDataType) {
-    switch (compDataType) {
-      case DATA_BYTE:
-
-        return new UnCompressNonDecimalByte();
-
-      case DATA_SHORT:
-
-        return new UnCompressNonDecimalShort();
-
-      case DATA_INT:
-
-        return new UnCompressNonDecimalInt();
-
-      case DATA_LONG:
-
-        return new UnCompressNonDecimalLong();
-
-      case DATA_FLOAT:
-
-        return new UnCompressNonDecimalFloat();
-
-      default:
-
-        return new UnCompressNonDecimalDefault();
-
-    }
-  }
-
-  /**
-   * uncompress data value = (maxValue - subValue)/Math.pow(10, decimal)
-   */
-  public static ValueCompressonHolder.UnCompressValue unCompressNonDecimalMaxMin(
-      DataType compDataType, DataType actualDataType) {
-    switch (compDataType) {
-      case DATA_BYTE:
-
-        return new UnCompressNonDecimalMaxMinByte();
-
-      case DATA_SHORT:
-
-        return new UnCompressNonDecimalMaxMinShort();
-
-      case DATA_INT:
-
-        return new UnCompressNonDecimalMaxMinInt();
-
-      case DATA_LONG:
-
-        return new UnCompressNonDecimalMaxMinLong();
-
-      case DATA_FLOAT:
-
-        return new UnCompressNonDecimalMaxMinFloat();
-
-      default:
-
-        return new UnCompressNonDecimalMaxMinDefault();
-
-    }
-  }
-
-  /**
-   * Create Value compression model
-   *
-   * @param maxValue
-   * @param minValue
-   * @param decimalLength
-   * @param uniqueValue
-   * @param aggType
-   * @param dataTypeSelected
-   * @return
-   */
-  public static ValueCompressionModel getValueCompressionModel(Object[] maxValue, Object[] minValue,
-      int[] decimalLength, Object[] uniqueValue, char[] aggType, byte[] dataTypeSelected) {
-
-    MeasureMetaDataModel metaDataModel =
-        new MeasureMetaDataModel(minValue, maxValue, decimalLength, maxValue.length, uniqueValue,
-            aggType, dataTypeSelected);
-    return getValueCompressionModel(metaDataModel);
-  }
-
-  public static ValueCompressionModel getValueCompressionModel(MeasureMetaDataModel measureMDMdl) {
-    int measureCount = measureMDMdl.getMeasureCount();
-    Object[] minValue = measureMDMdl.getMinValue();
-    Object[] maxValue = measureMDMdl.getMaxValue();
-    Object[] uniqueValue = measureMDMdl.getUniqueValue();
-    int[] decimal = measureMDMdl.getDecimal();
-    char[] type = measureMDMdl.getType();
-    byte[] dataTypeSelected = measureMDMdl.getDataTypeSelected();
-    ValueCompressionModel compressionModel = new ValueCompressionModel();
-    DataType[] actualType = new DataType[measureCount];
-    DataType[] changedType = new DataType[measureCount];
-    COMPRESSION_TYPE[] compType = new COMPRESSION_TYPE[measureCount];
-    for (int i = 0; i < measureCount; i++) {
-      CompressionFinder compresssionFinder = ValueCompressionUtil
-          .getCompressionType(maxValue[i], minValue[i], decimal[i], type[i], dataTypeSelected[i]);
-      actualType[i] = compresssionFinder.actualDataType;
-      changedType[i] = compresssionFinder.changedDataType;
-      compType[i] = compresssionFinder.compType;
-    }
-    compressionModel.setMaxValue(maxValue);
-    compressionModel.setDecimal(decimal);
-    compressionModel.setChangedDataType(changedType);
-    compressionModel.setCompType(compType);
-    compressionModel.setActualDataType(actualType);
-    compressionModel.setMinValue(minValue);
-    compressionModel.setUniqueValue(uniqueValue);
-    compressionModel.setType(type);
-    compressionModel.setMinValueFactForAgg(measureMDMdl.getMinValueFactForAgg());
-    compressionModel.setDataTypeSelected(dataTypeSelected);
-    ValueCompressonHolder.UnCompressValue[] values = ValueCompressionUtil
-        .getUncompressedValues(compressionModel.getCompType(), compressionModel.getActualDataType(),
-            compressionModel.getChangedDataType());
-    compressionModel.setUnCompressValues(values);
-    return compressionModel;
-  }
-
-  public static byte[] convertToBytes(short[] values) {
-    ByteBuffer buffer = ByteBuffer.allocate(values.length * 2);
-    for (short val : values) {
-      buffer.putShort(val);
-    }
-    return buffer.array();
-  }
-
-  public static byte[] convertToBytes(int[] values) {
-    ByteBuffer buffer = ByteBuffer.allocate(values.length * 4);
-    for (int val : values) {
-      buffer.putInt(val);
-    }
-    return buffer.array();
-  }
-
-  public static byte[] convertToBytes(float[] values) {
-    ByteBuffer buffer = ByteBuffer.allocate(values.length * 4);
-    for (float val : values) {
-      buffer.putFloat(val);
-    }
-    return buffer.array();
-  }
-
-  public static byte[] convertToBytes(long[] values) {
-    ByteBuffer buffer = ByteBuffer.allocate(values.length * 8);
-    for (long val : values) {
-      buffer.putLong(val);
-    }
-    return buffer.array();
-  }
-
-  public static byte[] convertToBytes(double[] values) {
-    ByteBuffer buffer = ByteBuffer.allocate(values.length * 8);
-    for (double val : values) {
-      buffer.putDouble(val);
-    }
-    return buffer.array();
-  }
-
-  public static short[] convertToShortArray(ByteBuffer buffer, int length) {
-    buffer.rewind();
-    short[] values = new short[length / 2];
-
-    for (int i = 0; i < values.length; i++) {
-      values[i] = buffer.getShort();
-    }
-    return values;
-  }
-
-  public static int[] convertToIntArray(ByteBuffer buffer, int length) {
-    buffer.rewind();
-    int[] values = new int[length / 4];
-
-    for (int i = 0; i < values.length; i++) {
-      values[i] = buffer.getInt();
-    }
-    return values;
-  }
-
-  public static float[] convertToFloatArray(ByteBuffer buffer, int length) {
-    buffer.rewind();
-    float[] values = new float[length / 4];
-
-    for (int i = 0; i < values.length; i++) {
-      values[i] = buffer.getFloat();
-    }
-    return values;
-  }
-
-  public static long[] convertToLongArray(ByteBuffer buffer, int length) {
-    buffer.rewind();
-    long[] values = new long[length / 8];
-    for (int i = 0; i < values.length; i++) {
-      values[i] = buffer.getLong();
-    }
-    return values;
-  }
-
-  public static double[] convertToDoubleArray(ByteBuffer buffer, int length) {
-    buffer.rewind();
-    double[] values = new double[length / 8];
-    for (int i = 0; i < values.length; i++) {
-      values[i] = buffer.getDouble();
-    }
-    return values;
-  }
-
-  /**
-   * use to identify compression type.
-   */
-  public static enum COMPRESSION_TYPE {
-    /**
-     *
-     */
-    NONE, /**
-     *
-     */
-    MAX_MIN, /**
-     *
-     */
-    NON_DECIMAL_CONVERT, /**
-     *
-     */
-    MAX_MIN_NDC,
-
-    /**
-     * custome
-     */
-    CUSTOM,
-
-    CUSTOM_BIGDECIMAL
-  }
-
-  /**
-   * use to identify the type of data.
-   */
-  public static enum DataType {
-    /**
-     *
-     */
-    DATA_BYTE(), /**
-     *
-     */
-    DATA_SHORT(), /**
-     *
-     */
-    DATA_INT(), /**
-     *
-     */
-    DATA_FLOAT(), /**
-     *
-     */
-    DATA_LONG(), /**
-     *
-     */
-    DATA_BIGINT(), /**
-     *
-     */
-    DATA_DOUBLE();
-
-    /**
-     * DataType.
-     */
-    private DataType() {
-      //this.size = size;
-    }
-
-  }
-
-  /**
-   * through the size of data type,priority and compression type, select the
-   * best compression type
-   */
-  private static class CompressionFinder implements Comparable<CompressionFinder> {
-    /**
-     * compType.
-     */
-    private COMPRESSION_TYPE compType;
-    /**
-     * actualDataType.
-     */
-    private DataType actualDataType;
-    /**
-     * changedDataType.
-     */
-    private DataType changedDataType;
-    /**
-     * the size of changed data
-     */
-    private int size;
-    /**
-     * priority.
-     */
-    private PRIORITY priority;
-
-    /**
-     * CompressionFinder constructor.
-     *
-     * @param compType
-     * @param actualDataType
-     * @param changedDataType
-     */
-    CompressionFinder(COMPRESSION_TYPE compType, DataType actualDataType,
-        DataType changedDataType) {
-      super();
-      this.compType = compType;
-      this.actualDataType = actualDataType;
-      this.changedDataType = changedDataType;
-    }
-
-    /**
-     * CompressionFinder overloaded constructor.
-     *
-     * @param actualDataType
-     * @param changedDataType
-     * @param priority
-     * @param compType
-     */
-
-    CompressionFinder(DataType actualDataType, DataType changedDataType, PRIORITY priority,
-        COMPRESSION_TYPE compType) {
-      super();
-      this.actualDataType = actualDataType;
-      this.changedDataType = changedDataType;
-      this.size = getSize(changedDataType);
-      this.priority = priority;
-      this.compType = compType;
-    }
-
-    @Override public boolean equals(Object obj) {
-      boolean equals = false;
-      if (obj instanceof CompressionFinder) {
-        CompressionFinder cf = (CompressionFinder) obj;
-
-        if (this.size == cf.size && this.priority == cf.priority) {
-          equals = true;
-        }
-
-      }
-      return equals;
-    }
-
-    @Override public int hashCode() {
-      final int code = 31;
-      int result = 1;
-
-      result = code * result + this.size;
-      result = code * result + ((priority == null) ? 0 : priority.hashCode());
-      return result;
-    }
-
-    @Override public int compareTo(CompressionFinder o) {
-      int returnVal = 0;
-      // the big size have high priority
-      if (this.equals(o)) {
-        returnVal = 0;
-      } else if (this.size == o.size) {
-        // the compression type priority
-        if (priority.priority > o.priority.priority) {
-          returnVal = 1;
-        } else if (priority.priority < o.priority.priority) {
-          returnVal = -1;
-        }
-
-      } else if (this.size > o.size) {
-        returnVal = 1;
-      } else {
-        returnVal = -1;
-      }
-      return returnVal;
-    }
-
-    /**
-     * Compression type priority.
-     * ACTUAL is the highest priority and DIFFNONDECIMAL is the lowest
-     * priority
-     */
-    static enum PRIORITY {
-      /**
-       *
-       */
-      ACTUAL(0), /**
-       *
-       */
-      DIFFSIZE(1), /**
-       *
-       */
-      MAXNONDECIMAL(2), /**
-       *
-       */
-      DIFFNONDECIMAL(3);
-
-      /**
-       * priority.
-       */
-      private int priority;
-
-      private PRIORITY(int priority) {
-        this.priority = priority;
-      }
-    }
-  }
-
-}
-
-
-

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/writer/ByteArrayHolder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/writer/ByteArrayHolder.java b/core/src/main/java/org/carbondata/core/writer/ByteArrayHolder.java
deleted file mode 100644
index 599fe09..0000000
--- a/core/src/main/java/org/carbondata/core/writer/ByteArrayHolder.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.core.writer;
-
-import java.util.Arrays;
-
-import org.carbondata.core.util.ByteUtil;
-
-public class ByteArrayHolder implements Comparable<ByteArrayHolder> {
-
-  /**
-   * mdkey
-   */
-  private byte[] mdKey;
-
-  /**
-   * primary key
-   */
-  private int primaryKey;
-
-  /**
-   * @param mdKey
-   * @param primaryKey
-   */
-  public ByteArrayHolder(byte[] mdKey, int primaryKey) {
-    this.mdKey = mdKey;
-    this.primaryKey = primaryKey;
-  }
-
-  @Override public int compareTo(ByteArrayHolder o) {
-    return ByteUtil.compare(mdKey, o.mdKey);
-  }
-
-  @Override public boolean equals(Object obj) {
-    // TODO Auto-generated method stub
-    if (obj instanceof ByteArrayHolder) {
-      if (0 == ByteUtil.compare(mdKey, ((ByteArrayHolder) obj).mdKey)) {
-        return true;
-      }
-    }
-    return false;
-  }
-
-  @Override public int hashCode() {
-    int prime = 31;
-    int result = prime * Arrays.hashCode(mdKey);
-    result = result + prime * primaryKey;
-    return result;
-  }
-
-  public byte[] getMdKey() {
-    return mdKey;
-  }
-
-  public int getPrimaryKey() {
-    return primaryKey;
-  }
-
-}
-

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/writer/CarbonDictionaryWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/writer/CarbonDictionaryWriter.java b/core/src/main/java/org/carbondata/core/writer/CarbonDictionaryWriter.java
deleted file mode 100644
index e13c32f..0000000
--- a/core/src/main/java/org/carbondata/core/writer/CarbonDictionaryWriter.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.carbondata.core.writer;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-
-/**
- * dictionary writer interface
- */
-public interface CarbonDictionaryWriter extends Closeable {
-  /**
-   * write method that accepts one value at a time
-   * This method can be used when data is huge and memory is les. In that
-   * case data can be stored to a file and an iterator can iterate over it and
-   * pass one value at a time
-   *
-   * @param value unique dictionary value
-   * @throws IOException if an I/O error occurs
-   */
-  void write(String value) throws IOException;
-
-  /**
-   * write method that accepts one value at a time
-   * This method can be used when data is huge and memory is les. In that
-   * case data can be stored to a file and an iterator can iterate over it and
-   * pass one value at a time
-   *
-   * @param value unique dictionary value
-   * @throws IOException if an I/O error occurs
-   */
-  void write(byte[] value) throws IOException;
-
-  /**
-   * write method that accepts list of byte arrays as value
-   * This can be used when data is less, then string can be converted
-   * to byte array for each value and added to a list
-   *
-   * @param valueList list of byte array. Each byte array is unique dictionary value
-   * @throws IOException if an I/O error occurs
-   */
-  void write(List<byte[]> valueList) throws IOException;
-
-
-  void commit() throws IOException;
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/writer/CarbonDictionaryWriterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/writer/CarbonDictionaryWriterImpl.java b/core/src/main/java/org/carbondata/core/writer/CarbonDictionaryWriterImpl.java
deleted file mode 100644
index 2508c86..0000000
--- a/core/src/main/java/org/carbondata/core/writer/CarbonDictionaryWriterImpl.java
+++ /dev/null
@@ -1,422 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.core.writer;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.carbondata.common.factory.CarbonCommonFactory;
-import org.carbondata.common.logging.LogService;
-import org.carbondata.common.logging.LogServiceFactory;
-import org.carbondata.core.carbon.CarbonTableIdentifier;
-import org.carbondata.core.carbon.ColumnIdentifier;
-import org.carbondata.core.carbon.path.CarbonTablePath;
-import org.carbondata.core.constants.CarbonCommonConstants;
-import org.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.carbondata.core.datastorage.store.impl.FileFactory;
-import org.carbondata.core.reader.CarbonDictionaryColumnMetaChunk;
-import org.carbondata.core.reader.CarbonDictionaryMetadataReader;
-import org.carbondata.core.reader.CarbonDictionaryMetadataReaderImpl;
-import org.carbondata.core.service.PathService;
-import org.carbondata.core.util.CarbonProperties;
-import org.carbondata.core.util.CarbonUtil;
-import org.carbondata.format.ColumnDictionaryChunk;
-import org.carbondata.format.ColumnDictionaryChunkMeta;
-
-import org.apache.thrift.TBase;
-
-/**
- * This class is responsible for writing the dictionary file and its metadata
- */
-public class CarbonDictionaryWriterImpl implements CarbonDictionaryWriter {
-
-  /**
-   * LOGGER
-   */
-  private static final LogService LOGGER =
-      LogServiceFactory.getLogService(CarbonDictionaryWriterImpl.class.getName());
-
-  /**
-   * carbon type identifier
-   */
-  protected CarbonTableIdentifier carbonTableIdentifier;
-
-  /**
-   * list which will hold values upto maximum of one dictionary chunk size
-   */
-  private List<ByteBuffer> oneDictionaryChunkList;
-
-  /**
-   * Meta object which will hold last segment entry details
-   */
-  private CarbonDictionaryColumnMetaChunk chunkMetaObjectForLastSegmentEntry;
-
-  /**
-   * dictionary file and meta thrift writer
-   */
-  private ThriftWriter dictionaryThriftWriter;
-
-  /**
-   * column identifier
-   */
-  protected ColumnIdentifier columnIdentifier;
-
-  /**
-   * HDFS store path
-   */
-  protected String hdfsStorePath;
-
-  /**
-   * dictionary file path
-   */
-  protected String dictionaryFilePath;
-
-  /**
-   * dictionary metadata file path
-   */
-  protected String dictionaryMetaFilePath;
-
-  /**
-   * start offset of dictionary chunk  for a segment
-   */
-  private long chunk_start_offset;
-
-  /**
-   * end offset of a dictionary chunk for a segment
-   */
-  private long chunk_end_offset;
-
-  /**
-   * total dictionary value record count for one segment
-   */
-  private int totalRecordCount;
-
-  /**
-   * total thrift object chunk count written for one segment
-   */
-  private int chunk_count;
-
-  /**
-   * chunk size for a dictionary file after which data will be written to disk
-   */
-  private int dictionary_one_chunk_size;
-
-  /**
-   * flag to check whether write method is called for first time
-   */
-  private boolean isFirstTime;
-
-  private static final Charset defaultCharset = Charset.forName(
-      CarbonCommonConstants.DEFAULT_CHARSET);
-
-  /**
-   * Constructor
-   *
-   * @param hdfsStorePath         HDFS store path
-   * @param carbonTableIdentifier table identifier which will give table name and database name
-   * @param columnIdentifier      column unique identifier
-   */
-  public CarbonDictionaryWriterImpl(String hdfsStorePath,
-      CarbonTableIdentifier carbonTableIdentifier, ColumnIdentifier columnIdentifier) {
-    this.carbonTableIdentifier = carbonTableIdentifier;
-    this.columnIdentifier = columnIdentifier;
-    this.hdfsStorePath = hdfsStorePath;
-    this.isFirstTime = true;
-  }
-
-  /**
-   * This method will write the data in thrift format to disk. This method will be guided by
-   * parameter dictionary_one_chunk_size and data will be divided into chunks
-   * based on this parameter
-   *
-   * @param value unique dictionary value
-   * @throws IOException if an I/O error occurs
-   */
-  @Override public void write(String value) throws IOException {
-    write(value.getBytes(defaultCharset));
-  }
-
-  /**
-   * This method will write the data in thrift format to disk. This method will be guided by
-   * parameter dictionary_one_chunk_size and data will be divided into chunks
-   * based on this parameter
-   *
-   * @param value unique dictionary value
-   * @throws IOException if an I/O error occurs
-   */
-  @Override public void write(byte[] value) throws IOException {
-    if (isFirstTime) {
-      init();
-      isFirstTime = false;
-    }
-    // if one chunk size is equal to list size then write the data to file
-    checkAndWriteDictionaryChunkToFile();
-    oneDictionaryChunkList.add(ByteBuffer.wrap(value));
-    totalRecordCount++;
-  }
-
-  /**
-   * This method will write the data in thrift format to disk. This method will not be guided by
-   * parameter dictionary_one_chunk_size and complete data will be written as one chunk
-   *
-   * @param valueList list of byte array. Each byte array is unique dictionary value
-   * @throws IOException if an I/O error occurs
-   */
-  @Override public void write(List<byte[]> valueList) throws IOException {
-    if (isFirstTime) {
-      init();
-      isFirstTime = false;
-    }
-    for (byte[] value : valueList) {
-      oneDictionaryChunkList.add(ByteBuffer.wrap(value));
-      totalRecordCount++;
-    }
-  }
-
-  /**
-   * write dictionary metadata file and close thrift object
-   *
-   * @throws IOException if an I/O error occurs
-   */
-  @Override public void close() throws IOException {
-    if (null != dictionaryThriftWriter) {
-      writeDictionaryFile();
-      // close the thrift writer for dictionary file
-      closeThriftWriter();
-    }
-  }
-
-  /**
-   * check if the threshold has been reached for the number of
-   * values that can kept in memory and then flush the data to file
-   *
-   * @throws IOException if an I/O error occurs
-   */
-  private void checkAndWriteDictionaryChunkToFile() throws IOException {
-    if (oneDictionaryChunkList.size() >= dictionary_one_chunk_size) {
-      writeDictionaryFile();
-      createChunkList();
-    }
-  }
-
-  /**
-   * This method will serialize the object of dictionary file
-   *
-   * @throws IOException if an I/O error occurs
-   */
-  private void writeDictionaryFile() throws IOException {
-    ColumnDictionaryChunk columnDictionaryChunk = new ColumnDictionaryChunk();
-    columnDictionaryChunk.setValues(oneDictionaryChunkList);
-    writeThriftObject(columnDictionaryChunk);
-  }
-
-  /**
-   * This method will check and created the directory path where dictionary file has to be created
-   *
-   * @throws IOException if an I/O error occurs
-   */
-  private void init() throws IOException {
-    initDictionaryChunkSize();
-    initPaths();
-    boolean dictFileExists = CarbonUtil.isFileExists(this.dictionaryFilePath);
-    if (dictFileExists && CarbonUtil.isFileExists(this.dictionaryMetaFilePath)) {
-      this.chunk_start_offset = CarbonUtil.getFileSize(this.dictionaryFilePath);
-      validateDictionaryFileOffsetWithLastSegmentEntryOffset();
-    } else if (dictFileExists) {
-      FileFactory.getCarbonFile(dictionaryFilePath, FileFactory.getFileType(dictionaryFilePath))
-          .delete();
-    }
-    openThriftWriter(this.dictionaryFilePath);
-    createChunkList();
-  }
-
-  protected void initPaths() {
-    PathService pathService = CarbonCommonFactory.getPathService();
-    CarbonTablePath carbonTablePath = pathService.getCarbonTablePath(columnIdentifier,
-            this.hdfsStorePath, carbonTableIdentifier);
-    this.dictionaryFilePath = carbonTablePath.getDictionaryFilePath(columnIdentifier.getColumnId());
-    this.dictionaryMetaFilePath =
-        carbonTablePath.getDictionaryMetaFilePath(columnIdentifier.getColumnId());
-  }
-
-  /**
-   * initialize the value of dictionary chunk that can be kept in memory at a time
-   */
-  private void initDictionaryChunkSize() {
-    try {
-      dictionary_one_chunk_size = Integer.parseInt(CarbonProperties.getInstance()
-          .getProperty(CarbonCommonConstants.DICTIONARY_ONE_CHUNK_SIZE,
-              CarbonCommonConstants.DICTIONARY_ONE_CHUNK_SIZE_DEFAULT));
-    } catch (NumberFormatException e) {
-      dictionary_one_chunk_size =
-          Integer.parseInt(CarbonCommonConstants.DICTIONARY_ONE_CHUNK_SIZE_DEFAULT);
-      LOGGER.error("Dictionary chunk size not configured properly. Taking default size "
-              + dictionary_one_chunk_size);
-    }
-  }
-
-  /**
-   * initialise one dictionary size chunk list and increment chunk count
-   */
-  private void createChunkList() {
-    this.oneDictionaryChunkList = new ArrayList<ByteBuffer>(dictionary_one_chunk_size);
-    chunk_count++;
-  }
-
-  /**
-   * if file already exists then read metadata file and
-   * validate the last entry end offset with file size. If
-   * they are not equal that means some invalid data is present which needs
-   * to be truncated
-   *
-   * @throws IOException if an I/O error occurs
-   */
-  private void validateDictionaryFileOffsetWithLastSegmentEntryOffset() throws IOException {
-    // read last dictionary chunk meta entry from dictionary metadata file
-    chunkMetaObjectForLastSegmentEntry = getChunkMetaObjectForLastSegmentEntry();
-    int bytesToTruncate =
-        (int) (chunk_start_offset - chunkMetaObjectForLastSegmentEntry.getEnd_offset());
-    if (bytesToTruncate > 0) {
-      LOGGER.info("some inconsistency in dictionary file for column " + this.columnIdentifier);
-      // truncate the dictionary data till chunk meta end offset
-      FileFactory.FileType fileType = FileFactory.getFileType(this.dictionaryFilePath);
-      CarbonFile carbonFile = FileFactory.getCarbonFile(this.dictionaryFilePath, fileType);
-      boolean truncateSuccess = carbonFile
-          .truncate(this.dictionaryFilePath, chunkMetaObjectForLastSegmentEntry.getEnd_offset());
-      if (!truncateSuccess) {
-        LOGGER.info("Diction file not truncated successfully for column " + this.columnIdentifier);
-      }
-    }
-  }
-
-  /**
-   * This method will write the dictionary metadata file for a given column
-   *
-   * @throws IOException if an I/O error occurs
-   */
-  private void writeDictionaryMetadataFile() throws IOException {
-    // Format of dictionary metadata file
-    // min, max, start offset, end offset and chunk count
-    int min_surrogate_key = 0;
-    int max_surrogate_key = 0;
-    // case 1: first time dictionary writing
-    // previousMax = 0, totalRecordCount = 5, min = 1, max= 5
-    // case2: file already exists
-    // previousMax = 5, totalRecordCount = 10, min = 6, max = 15
-    // case 3: no unique values, total records 0
-    // previousMax = 15, totalRecordCount = 0, min = 15, max = 15
-    // both min and max equal to previous max
-    if (null != chunkMetaObjectForLastSegmentEntry) {
-      if (0 == totalRecordCount) {
-        min_surrogate_key = chunkMetaObjectForLastSegmentEntry.getMax_surrogate_key();
-      } else {
-        min_surrogate_key = chunkMetaObjectForLastSegmentEntry.getMax_surrogate_key() + 1;
-      }
-      max_surrogate_key =
-          chunkMetaObjectForLastSegmentEntry.getMax_surrogate_key() + totalRecordCount;
-    } else {
-      if (totalRecordCount > 0) {
-        min_surrogate_key = 1;
-      }
-      max_surrogate_key = totalRecordCount;
-    }
-    ColumnDictionaryChunkMeta dictionaryChunkMeta =
-        new ColumnDictionaryChunkMeta(min_surrogate_key, max_surrogate_key, chunk_start_offset,
-            chunk_end_offset, chunk_count);
-    openThriftWriter(this.dictionaryMetaFilePath);
-    // write dictionary metadata file
-    writeThriftObject(dictionaryChunkMeta);
-    closeThriftWriter();
-    LOGGER.info("Dictionary metadata file written successfully for column " + this.columnIdentifier
-            + " at path " + this.dictionaryMetaFilePath);
-  }
-
-  /**
-   * open thrift writer for writing dictionary chunk/meta object
-   *
-   * @param dictionaryFile can be dictionary file name or dictionary metadata file name
-   * @throws IOException if an I/O error occurs
-   */
-  private void openThriftWriter(String dictionaryFile) throws IOException {
-    // create thrift writer instance
-    dictionaryThriftWriter = new ThriftWriter(dictionaryFile, true);
-    // open the file stream
-    dictionaryThriftWriter.open();
-  }
-
-  /**
-   * This method will write the thrift object to a file
-   *
-   * @param dictionaryThriftObject can be dictionary thrift object or dictionary metadata
-   *                               thrift object
-   * @throws IOException if an I/O error occurs
-   */
-  private void writeThriftObject(TBase dictionaryThriftObject) throws IOException {
-    dictionaryThriftWriter.write(dictionaryThriftObject);
-  }
-
-  /**
-   * close dictionary thrift writer
-   */
-  private void closeThriftWriter() {
-    if (null != dictionaryThriftWriter) {
-      dictionaryThriftWriter.close();
-    }
-  }
-
-  /**
-   * This method will read the dictionary chunk metadata thrift object for last entry
-   *
-   * @return last entry of dictionary meta chunk
-   * @throws IOException if an I/O error occurs
-   */
-  private CarbonDictionaryColumnMetaChunk getChunkMetaObjectForLastSegmentEntry()
-      throws IOException {
-    CarbonDictionaryColumnMetaChunk carbonDictionaryColumnMetaChunk = null;
-    CarbonDictionaryMetadataReader columnMetadataReaderImpl = getDictionaryMetadataReader();
-    try {
-      // read the last segment entry for dictionary metadata
-      carbonDictionaryColumnMetaChunk =
-          columnMetadataReaderImpl.readLastEntryOfDictionaryMetaChunk();
-    } finally {
-      // Close metadata reader
-      columnMetadataReaderImpl.close();
-    }
-    return carbonDictionaryColumnMetaChunk;
-  }
-
-  /**
-   * @return
-   */
-  protected CarbonDictionaryMetadataReader getDictionaryMetadataReader() {
-    return new CarbonDictionaryMetadataReaderImpl(hdfsStorePath, carbonTableIdentifier,
-        columnIdentifier);
-  }
-
-  @Override public void commit() throws IOException {
-    if (null != dictionaryThriftWriter) {
-      this.chunk_end_offset = CarbonUtil.getFileSize(this.dictionaryFilePath);
-      writeDictionaryMetadataFile();
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/writer/CarbonFooterWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/writer/CarbonFooterWriter.java b/core/src/main/java/org/carbondata/core/writer/CarbonFooterWriter.java
deleted file mode 100644
index 17b5686..0000000
--- a/core/src/main/java/org/carbondata/core/writer/CarbonFooterWriter.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.carbondata.core.writer;
-
-import java.io.IOException;
-
-import org.carbondata.format.FileFooter;
-
-/**
- * Writes metadata block to the fact table file in thrift format org.carbondata.format.FileFooter
- */
-public class CarbonFooterWriter {
-
-  // It is version number of this format class.
-  private static int VERSION_NUMBER = 1;
-
-  // Fact file path
-  private String filePath;
-
-  public CarbonFooterWriter(String filePath) {
-    this.filePath = filePath;
-  }
-
-  /**
-   * It writes FileFooter thrift format object to file.
-   *
-   * @param footer
-   * @param currentPosition At where this metadata is going to be written.
-   * @throws IOException
-   */
-  public void writeFooter(FileFooter footer, long currentPosition) throws IOException {
-
-    ThriftWriter thriftWriter = openThriftWriter(filePath);
-    footer.setVersion(VERSION_NUMBER);
-    try {
-      thriftWriter.write(footer);
-      thriftWriter.writeOffset(currentPosition);
-    } catch (Exception e) {
-      throw e;
-    } finally {
-      thriftWriter.close();
-    }
-  }
-
-  /**
-   * open thrift writer for writing dictionary chunk/meta object
-   */
-  private ThriftWriter openThriftWriter(String filePath) throws IOException {
-    // create thrift writer instance
-    ThriftWriter thriftWriter = new ThriftWriter(filePath, true);
-    // open the file stream
-    thriftWriter.open();
-    return thriftWriter;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/writer/CarbonIndexFileWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/writer/CarbonIndexFileWriter.java b/core/src/main/java/org/carbondata/core/writer/CarbonIndexFileWriter.java
deleted file mode 100644
index 5ae7b33..0000000
--- a/core/src/main/java/org/carbondata/core/writer/CarbonIndexFileWriter.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.carbondata.core.writer;
-
-import java.io.IOException;
-
-import org.apache.thrift.TBase;
-
-/**
- * Reader class which will be used to read the index file
- */
-public class CarbonIndexFileWriter {
-
-  /**
-   * thrift writer object
-   */
-  private ThriftWriter thriftWriter;
-
-  /**
-   * It writes thrift object to file
-   *
-   * @param footer
-   * @throws IOException
-   */
-  public void writeThrift(TBase indexObject) throws IOException {
-    thriftWriter.write(indexObject);
-  }
-
-  /**
-   * Below method will be used to open the thrift writer
-   *
-   * @param filePath file path where data need to be written
-   * @throws IOException throws io exception in case of any failure
-   */
-  public void openThriftWriter(String filePath) throws IOException {
-    // create thrift writer instance
-    thriftWriter = new ThriftWriter(filePath, true);
-    // open the file stream
-    thriftWriter.open();
-  }
-
-  /**
-   * Below method will be used to close the thrift object
-   */
-  public void close() {
-    thriftWriter.close();
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/writer/HierarchyValueWriterForCSV.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/writer/HierarchyValueWriterForCSV.java b/core/src/main/java/org/carbondata/core/writer/HierarchyValueWriterForCSV.java
deleted file mode 100644
index d75ac6f..0000000
--- a/core/src/main/java/org/carbondata/core/writer/HierarchyValueWriterForCSV.java
+++ /dev/null
@@ -1,320 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.core.writer;
-
-import java.io.Closeable;
-import java.io.File;
-import java.io.FileFilter;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-
-import org.carbondata.common.logging.LogService;
-import org.carbondata.common.logging.LogServiceFactory;
-import org.carbondata.core.constants.CarbonCommonConstants;
-import org.carbondata.core.util.CarbonProperties;
-import org.carbondata.core.util.CarbonUtil;
-
-import org.pentaho.di.core.exception.KettleException;
-
-public class HierarchyValueWriterForCSV {
-
-  /**
-   * Comment for <code>LOGGER</code>
-   */
-  private static final LogService LOGGER =
-      LogServiceFactory.getLogService(HierarchyValueWriterForCSV.class.getName());
-  /**
-   * hierarchyName
-   */
-  private String hierarchyName;
-
-  /**
-   * bufferedOutStream
-   */
-  private FileChannel outPutFileChannel;
-
-  /**
-   * storeFolderLocation
-   */
-  private String storeFolderLocation;
-
-  /**
-   * intialized
-   */
-  private boolean intialized;
-
-  /**
-   * counter the number of files.
-   */
-  private int counter;
-
-  /**
-   * byteArrayList
-   */
-  private List<ByteArrayHolder> byteArrayholder =
-      new ArrayList<ByteArrayHolder>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
-
-  /**
-   * toflush
-   */
-  private int toflush;
-
-  public HierarchyValueWriterForCSV(String hierarchy, String storeFolderLocation) {
-    this.hierarchyName = hierarchy;
-    this.storeFolderLocation = storeFolderLocation;
-
-    CarbonProperties instance = CarbonProperties.getInstance();
-
-    this.toflush = Integer.parseInt(instance
-        .getProperty(CarbonCommonConstants.SORT_SIZE, CarbonCommonConstants.SORT_SIZE_DEFAULT_VAL));
-
-    int rowSetSize = Integer.parseInt(instance.getProperty(CarbonCommonConstants.GRAPH_ROWSET_SIZE,
-        CarbonCommonConstants.GRAPH_ROWSET_SIZE_DEFAULT));
-
-    if (this.toflush > rowSetSize) {
-      this.toflush = rowSetSize;
-    }
-
-    updateCounter(hierarchy, storeFolderLocation);
-  }
-
-  /**
-   * @return Returns the byteArrayList.
-   */
-  public List<ByteArrayHolder> getByteArrayList() throws KettleException {
-    return byteArrayholder;
-  }
-
-  public FileChannel getBufferedOutStream() {
-    return outPutFileChannel;
-  }
-
-  private void updateCounter(final String meString, String storeFolderLocation) {
-    File storeFolder = new File(storeFolderLocation);
-
-    File[] listFiles = storeFolder.listFiles(new FileFilter() {
-
-      @Override public boolean accept(File file) {
-        if (file.getName().indexOf(meString) > -1)
-
-        {
-          return true;
-        }
-        return false;
-      }
-    });
-
-    if (null == listFiles || listFiles.length == 0) {
-      counter = 0;
-      return;
-    }
-
-    for (File hierFile : listFiles) {
-      String hierFileName = hierFile.getName();
-
-      if (hierFileName.endsWith(CarbonCommonConstants.FILE_INPROGRESS_STATUS)) {
-        hierFileName = hierFileName.substring(0, hierFileName.lastIndexOf('.'));
-        try {
-          counter = Integer.parseInt(hierFileName.substring(hierFileName.length() - 1));
-        } catch (NumberFormatException nfe) {
-
-          if (new File(hierFileName + '0' + CarbonCommonConstants.LEVEL_FILE_EXTENSION).exists()) {
-            // Need to skip because the case can come in which server went down while files were
-            // merging and the other hierarchy files were not deleted, and the current file
-            // status is inrogress. so again we will merge the files and rename to normal file
-            LOGGER.info("Need to skip as this can be case in which hierarchy file already renamed");
-            if (hierFile.delete()) {
-              LOGGER.info("Deleted the Inprogress hierarchy Files.");
-            }
-          } else {
-            // levelfileName0.level file not exist that means files is merged and other
-            // files got deleted. while renaming this file from inprogress to normal file,
-            // server got restarted/killed. so we need to rename the file to normal.
-
-            File inprogressFile = new File(storeFolder + File.separator + hierFile.getName());
-            File changetoName = new File(storeFolder + File.separator + hierFileName);
-
-            if (inprogressFile.renameTo(changetoName)) {
-              LOGGER.info(
-                  "Renaming the level Files while creating the new instance on server startup.");
-            }
-
-          }
-
-        }
-      }
-
-      String val = hierFileName.substring(hierFileName.length() - 1);
-
-      int parsedVal = getIntValue(val);
-
-      if (counter < parsedVal) {
-        counter = parsedVal;
-      }
-    }
-    counter++;
-  }
-
-  private int getIntValue(String val) {
-    int parsedVal = 0;
-    try {
-      parsedVal = Integer.parseInt(val);
-    } catch (NumberFormatException nfe) {
-      LOGGER.info("Hierarchy File is already renamed so there will not be"
-              + "any need to keep the counter");
-    }
-    return parsedVal;
-  }
-
-  private void intialize() throws KettleException {
-    intialized = true;
-
-    File f = new File(storeFolderLocation + File.separator + hierarchyName + counter
-        + CarbonCommonConstants.FILE_INPROGRESS_STATUS);
-
-    counter++;
-
-    FileOutputStream fos = null;
-
-    boolean isFileCreated = false;
-    if (!f.exists()) {
-      try {
-        isFileCreated = f.createNewFile();
-
-      } catch (IOException e) {
-        //not required: findbugs fix
-        throw new KettleException("unable to create member mapping file", e);
-      }
-      if (!isFileCreated) {
-        throw new KettleException("unable to create file" + f.getAbsolutePath());
-      }
-    }
-
-    try {
-      fos = new FileOutputStream(f);
-
-      outPutFileChannel = fos.getChannel();
-    } catch (FileNotFoundException e) {
-      closeStreamAndDeleteFile(f, outPutFileChannel, fos);
-      throw new KettleException("member Mapping File not found to write mapping info", e);
-    }
-  }
-
-  public void writeIntoHierarchyFile(byte[] bytes, int primaryKey) throws KettleException {
-    if (!intialized) {
-      intialize();
-    }
-
-    ByteBuffer byteBuffer = storeValueInCache(bytes, primaryKey);
-
-    try {
-      byteBuffer.flip();
-      outPutFileChannel.write(byteBuffer);
-    } catch (IOException e) {
-      throw new KettleException("Error while writting in the hierarchy mapping file", e);
-    }
-  }
-
-  private ByteBuffer storeValueInCache(byte[] bytes, int primaryKey) {
-
-    // adding 4 to store the total length of the row at the beginning
-    ByteBuffer buffer = ByteBuffer.allocate(bytes.length + 4);
-
-    buffer.put(bytes);
-    buffer.putInt(primaryKey);
-
-    return buffer;
-  }
-
-  public void performRequiredOperation() throws KettleException {
-    if (byteArrayholder.size() == 0) {
-      return;
-    }
-    //write to the file and close the stream.
-    Collections.sort(byteArrayholder);
-
-    for (ByteArrayHolder byteArray : byteArrayholder) {
-      writeIntoHierarchyFile(byteArray.getMdKey(), byteArray.getPrimaryKey());
-    }
-
-    CarbonUtil.closeStreams(outPutFileChannel);
-
-    //rename the inprogress file to normal .level file
-    String filePath = this.storeFolderLocation + File.separator + hierarchyName + (counter - 1)
-        + CarbonCommonConstants.FILE_INPROGRESS_STATUS;
-    File inProgressFile = new File(filePath);
-    String inprogressFileName = inProgressFile.getName();
-
-    String changedFileName = inprogressFileName.substring(0, inprogressFileName.lastIndexOf('.'));
-
-    File orgFinalName = new File(this.storeFolderLocation + File.separator + changedFileName);
-
-    if (!inProgressFile.renameTo(orgFinalName)) {
-      LOGGER.error("Not able to rename file : " + inprogressFileName);
-    }
-
-    //create the new outputStream
-    try {
-      intialize();
-    } catch (KettleException e) {
-      LOGGER.error("Not able to create output stream for file:" + hierarchyName + (counter - 1));
-    }
-
-    //clear the byte array holder also.
-    byteArrayholder.clear();
-  }
-
-  private void closeStreamAndDeleteFile(File f, Closeable... streams) throws KettleException {
-    boolean isDeleted = false;
-    for (Closeable stream : streams) {
-      if (null != stream) {
-        try {
-          stream.close();
-        } catch (IOException e) {
-          LOGGER.error(e, "unable to close the stream ");
-        }
-
-      }
-    }
-
-    // delete the file
-    isDeleted = f.delete();
-    if (!isDeleted) {
-      LOGGER.error("Unable to delete the file " + f.getAbsolutePath());
-    }
-
-  }
-
-  public String getHierarchyName() {
-    return hierarchyName;
-  }
-
-  public int getCounter() {
-    return counter;
-  }
-
-}
-

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/writer/ThriftWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/writer/ThriftWriter.java b/core/src/main/java/org/carbondata/core/writer/ThriftWriter.java
deleted file mode 100644
index 2c5ee1d..0000000
--- a/core/src/main/java/org/carbondata/core/writer/ThriftWriter.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.core.writer;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import org.carbondata.core.datastorage.store.impl.FileFactory;
-import org.carbondata.core.util.CarbonUtil;
-
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.thrift.TBase;
-import org.apache.thrift.TException;
-import org.apache.thrift.protocol.TCompactProtocol;
-import org.apache.thrift.protocol.TProtocol;
-import org.apache.thrift.transport.TIOStreamTransport;
-
-/**
- * Simple class that makes it easy to write Thrift objects to disk.
- */
-public class ThriftWriter {
-
-  /**
-   * buffer size
-   */
-  private static final int bufferSize = 2048;
-
-  /**
-   * File to write to.
-   */
-  private String fileName;
-
-  /**
-   * For writing to the file.
-   */
-  private DataOutputStream dataOutputStream;
-
-  /**
-   * For binary serialization of objects.
-   */
-  private TProtocol binaryOut;
-
-  /**
-   * flag to append to existing file
-   */
-  private boolean append;
-
-  /**
-   * Constructor.
-   */
-  public ThriftWriter(String fileName, boolean append) {
-    this.fileName = fileName;
-    this.append = append;
-  }
-
-  /**
-   * Open the file for writing.
-   */
-  public void open() throws IOException {
-    FileFactory.FileType fileType = FileFactory.getFileType(fileName);
-    dataOutputStream = FileFactory.getDataOutputStream(fileName, fileType, bufferSize, append);
-    binaryOut = new TCompactProtocol(new TIOStreamTransport(dataOutputStream));
-  }
-
-  /**
-   * Write the object to disk.
-   */
-  public void write(TBase t) throws IOException {
-    try {
-      t.write(binaryOut);
-      dataOutputStream.flush();
-    } catch (TException e) {
-      throw new IOException(e);
-    }
-  }
-
-  /**
-   * Write the offset to the file
-   *
-   * @param offset
-   * @throws IOException
-   */
-  public void writeOffset(long offset) throws IOException {
-    dataOutputStream.writeLong(offset);
-  }
-
-  /**
-   * Close the file stream.
-   */
-  public void close() {
-    CarbonUtil.closeStreams(dataOutputStream);
-  }
-
-  /**
-   * Flush data to HDFS file
-   */
-  public void sync() throws IOException {
-    if (dataOutputStream instanceof FSDataOutputStream) {
-      ((FSDataOutputStream) dataOutputStream).hsync();
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/writer/exception/CarbonDataWriterException.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/writer/exception/CarbonDataWriterException.java b/core/src/main/java/org/carbondata/core/writer/exception/CarbonDataWriterException.java
deleted file mode 100644
index 1e9ee18..0000000
--- a/core/src/main/java/org/carbondata/core/writer/exception/CarbonDataWriterException.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.carbondata.core.writer.exception;
-
-import java.util.Locale;
-
-public class CarbonDataWriterException extends Exception {
-
-  /**
-   * default serial version ID.
-   */
-  private static final long serialVersionUID = 1L;
-
-  /**
-   * The Error message.
-   */
-  private String msg = "";
-
-  /**
-   * Constructor
-   *
-   * @param msg The error message for this exception.
-   */
-  public CarbonDataWriterException(String msg) {
-    super(msg);
-    this.msg = msg;
-  }
-
-  /**
-   * Constructor
-   *
-   * @param msg The error message for this exception.
-   */
-  public CarbonDataWriterException(String msg, Throwable t) {
-    super(msg, t);
-    this.msg = msg;
-  }
-
-  /**
-   * getLocalizedMessage
-   */
-  @Override public String getLocalizedMessage() {
-    return super.getLocalizedMessage();
-  }
-
-  /**
-   * getMessage
-   */
-  public String getMessage() {
-    return this.msg;
-  }
-
-  /**
-   * This method is used to get the localized message.
-   *
-   * @param locale - A Locale object represents a specific geographical,
-   *               political, or cultural region.
-   * @return - Localized error message.
-   */
-  public String getLocalizedMessage(Locale locale) {
-    return "";
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriter.java b/core/src/main/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriter.java
deleted file mode 100644
index 385efbe..0000000
--- a/core/src/main/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriter.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.carbondata.core.writer.sortindex;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Interface for writing the dictionary sort index and sort index revers data.
- */
-public interface CarbonDictionarySortIndexWriter extends Closeable {
-
-  /**
-   * The method is used write the dictionary sortIndex data to columns
-   * sortedIndex file in thrif format.
-   *
-   * @param sortIndexList list of sortIndex
-   * @throws IOException In Case of any I/O errors occurs.
-   */
-  public void writeSortIndex(List<Integer> sortIndexList) throws IOException;
-
-  /**
-   * The method is used write the dictionary sortIndexInverted data to columns
-   * sortedIndex file in thrif format.
-   *
-   * @param invertedSortIndexList list of  sortIndexInverted
-   * @throws IOException In Case of any I/O errors occurs.
-   */
-  public void writeInvertedSortIndex(List<Integer> invertedSortIndexList) throws IOException;
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java b/core/src/main/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
deleted file mode 100644
index 9c398cb..0000000
--- a/core/src/main/java/org/carbondata/core/writer/sortindex/CarbonDictionarySortIndexWriterImpl.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.carbondata.core.writer.sortindex;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.List;
-
-import org.carbondata.common.factory.CarbonCommonFactory;
-import org.carbondata.common.logging.LogService;
-import org.carbondata.common.logging.LogServiceFactory;
-import org.carbondata.core.carbon.CarbonTableIdentifier;
-import org.carbondata.core.carbon.ColumnIdentifier;
-import org.carbondata.core.carbon.path.CarbonTablePath;
-import org.carbondata.core.constants.CarbonCommonConstants;
-import org.carbondata.core.datastorage.store.filesystem.CarbonFile;
-import org.carbondata.core.datastorage.store.impl.FileFactory;
-import org.carbondata.core.service.PathService;
-import org.carbondata.core.util.CarbonProperties;
-import org.carbondata.core.util.CarbonUtil;
-import org.carbondata.core.writer.ThriftWriter;
-import org.carbondata.format.ColumnSortInfo;
-
-/**
- * The class responsible for writing the dictionary/column sort index and sort index inverted data
- * in the thrift format
- */
-public class CarbonDictionarySortIndexWriterImpl implements CarbonDictionarySortIndexWriter {
-
-  /**
-   * carbonTable Identifier holding the info of databaseName and tableName
-   */
-  protected CarbonTableIdentifier carbonTableIdentifier;
-
-  /**
-   * column name
-   */
-  protected ColumnIdentifier columnIdentifier;
-
-  /**
-   * carbon store location
-   */
-  protected String carbonStorePath;
-  /**
-   * Path of dictionary sort index file for which the sortIndex to be written
-   */
-  protected String sortIndexFilePath;
-  /**
-   * Instance of thrift writer to write the data
-   */
-  private ThriftWriter sortIndexThriftWriter;
-
-  /**
-   * Column sort info thrift instance.
-   */
-  private ColumnSortInfo columnSortInfo = new ColumnSortInfo();
-
-  /**
-   * Comment for <code>LOGGER</code>
-   */
-  private static final LogService LOGGER =
-      LogServiceFactory.getLogService(CarbonDictionarySortIndexWriterImpl.class.getName());
-
-  /**
-   * @param carbonStorePath       Carbon store path
-   * @param carbonTableIdentifier table identifier which will give table name and database name
-   * @param columnIdentifier      column unique identifier
-   */
-  public CarbonDictionarySortIndexWriterImpl(final CarbonTableIdentifier carbonTableIdentifier,
-      final ColumnIdentifier columnIdentifier, final String carbonStorePath) {
-    this.carbonTableIdentifier = carbonTableIdentifier;
-    this.columnIdentifier = columnIdentifier;
-    this.carbonStorePath = carbonStorePath;
-  }
-
-  /**
-   * The method is used populate the dictionary sortIndex data to columnSortInfo
-   * in thrif format.
-   *
-   * @param sortIndexList list of sortIndex
-   * @throws IOException In Case of any I/O errors occurs.
-   */
-  @Override public void writeSortIndex(List<Integer> sortIndexList) throws IOException {
-    columnSortInfo.setSort_index(sortIndexList);
-  }
-
-  /**
-   * The method is used populate the dictionary Inverted sortIndex data to columnSortInfo
-   * in thrif format.
-   *
-   * @param invertedSortIndexList list of  sortIndexInverted
-   * @throws IOException In Case of any I/O errors occurs.
-   */
-  @Override public void writeInvertedSortIndex(List<Integer> invertedSortIndexList)
-      throws IOException {
-    columnSortInfo.setSort_index_inverted(invertedSortIndexList);
-  }
-
-  /**
-   * Initialize the sortIndexFilePath and open writing stream
-   * for dictionary sortIndex file thrif writer
-   * write the column sort info to the store when both sort index  and sort index
-   * inverted are populated.
-   * existing sort index file has to be overwritten with new sort index data
-   * columnSortInfo having null sortIndex and invertedSortIndex will not be written
-   */
-  private void writeColumnSortInfo() throws IOException {
-    boolean isNotNull =
-        null != columnSortInfo.getSort_index() && null != columnSortInfo.sort_index_inverted;
-    if (isNotNull) {
-      initPath();
-      String folderContainingFile = CarbonTablePath.getFolderContainingFile(this.sortIndexFilePath);
-      boolean created = CarbonUtil.checkAndCreateFolder(folderContainingFile);
-      if (!created) {
-        LOGGER.error("Database metadata folder creation status :: " + created);
-        throw new IOException("Failed to created database metadata folder");
-      }
-      try {
-
-        this.sortIndexThriftWriter = new ThriftWriter(this.sortIndexFilePath, false);
-        this.sortIndexThriftWriter.open();
-        sortIndexThriftWriter.write(columnSortInfo);
-      } catch (IOException ie) {
-        LOGGER.error(ie,
-            "problem while writing the dictionary sort index file.");
-        throw new IOException("problem while writing the dictionary sort index file.", ie);
-      } finally {
-        if (null != sortIndexThriftWriter) {
-          this.sortIndexThriftWriter.close();
-        }
-        this.sortIndexFilePath = null;
-      }
-    }
-  }
-
-  protected void initPath() {
-    PathService pathService = CarbonCommonFactory.getPathService();
-    CarbonTablePath carbonTablePath = pathService
-        .getCarbonTablePath(columnIdentifier, carbonStorePath, carbonTableIdentifier);
-    String dictionaryPath = carbonTablePath.getDictionaryFilePath(columnIdentifier.getColumnId());
-    long dictOffset = CarbonUtil.getFileSize(dictionaryPath);
-    this.sortIndexFilePath =
-        carbonTablePath.getSortIndexFilePath(columnIdentifier.getColumnId(), dictOffset);
-    cleanUpOldSortIndex(carbonTablePath, dictionaryPath);
-  }
-
-  /**
-   * It cleans up old unused sortindex file
-   *
-   * @param carbonTablePath
-   */
-  protected void cleanUpOldSortIndex(CarbonTablePath carbonTablePath, String dictPath) {
-    CarbonFile dictFile =
-        FileFactory.getCarbonFile(dictPath, FileFactory.getFileType(dictPath));
-    CarbonFile[] files =
-        carbonTablePath.getSortIndexFiles(dictFile.getParentFile(),
-            columnIdentifier.getColumnId());
-    int maxTime;
-    try {
-      maxTime = Integer.parseInt(CarbonProperties.getInstance()
-          .getProperty(CarbonCommonConstants.MAX_QUERY_EXECUTION_TIME));
-    } catch (NumberFormatException e) {
-      maxTime = CarbonCommonConstants.DEFAULT_MAX_QUERY_EXECUTION_TIME;
-    }
-    if (null != files) {
-      Arrays.sort(files, new Comparator<CarbonFile>() {
-        @Override public int compare(CarbonFile o1, CarbonFile o2) {
-          return o1.getName().compareTo(o2.getName());
-        }
-      });
-      for (int i = 0; i < files.length - 1; i++) {
-        long difference = System.currentTimeMillis() - files[i].getLastModifiedTime();
-        long minutesElapsed = (difference / (1000 * 60));
-        if (minutesElapsed > maxTime) {
-          if (!files[i].delete()) {
-            LOGGER.warn("Failed to delete sortindex file." + files[i].getAbsolutePath());
-          } else {
-            LOGGER.info("Sort index file is deleted." + files[i].getAbsolutePath());
-          }
-        }
-      }
-    }
-  }
-
-  /**
-   * Closes this stream and releases any system resources associated
-   * with it. If the stream is already closed then invoking this
-   * method has no effect.
-   *
-   * @throws IOException if an I/O error occurs
-   */
-  @Override public void close() throws IOException {
-    writeColumnSortInfo();
-    if (null != sortIndexThriftWriter) {
-      sortIndexThriftWriter.close();
-    }
-  }
-}


Mime
View raw message