carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chenliang...@apache.org
Subject [1/3] incubator-carbondata git commit: wip
Date Thu, 29 Dec 2016 08:13:25 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master 93f23ccb8 -> 241f45f8a


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/AbstractTempSortFileReader.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/AbstractTempSortFileReader.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/AbstractTempSortFileReader.java
deleted file mode 100644
index 9d7a7ba..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/AbstractTempSortFileReader.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.processing.sortandgroupby.sortdata;
-
-import java.io.File;
-import java.nio.ByteBuffer;
-
-import org.apache.carbondata.core.datastorage.store.FileHolder;
-import org.apache.carbondata.core.datastorage.store.impl.MemoryMappedFileHolderImpl;
-
-public abstract class AbstractTempSortFileReader implements TempSortFileReader {
-  /**
-   * measure count
-   */
-  protected int measureCount;
-
-  /**
-   * Measure count
-   */
-  protected int dimensionCount;
-
-  /**
-   * complexDimension count
-   */
-  protected int complexDimensionCount;
-
-  /**
-   * entryCount
-   */
-  protected int entryCount;
-
-  /**
-   * fileHolder
-   */
-  protected FileHolder fileHolder;
-
-  /**
-   * Temp file path
-   */
-  protected String filePath;
-
-  /**
-   * eachRecordSize
-   */
-  protected int eachRecordSize;
-
-  /**
-   * AbstractTempSortFileReader
-   *
-   * @param measureCount
-   * @param dimensionCount
-   * @param tempFile
-   */
-  public AbstractTempSortFileReader(int dimensionCount, int complexDimensionCount, int measureCount,
-      File tempFile, int noDictionaryCount) {
-    this.measureCount = measureCount;
-    this.dimensionCount = dimensionCount;
-    this.complexDimensionCount = complexDimensionCount;
-    this.fileHolder = new MemoryMappedFileHolderImpl(1);
-    this.filePath = tempFile.getAbsolutePath();
-    entryCount = fileHolder.readInt(filePath);
-    eachRecordSize = dimensionCount + complexDimensionCount + measureCount;
-  }
-
-  /**
-   * below method will be used to close the file holder
-   */
-  public void finish() {
-    this.fileHolder.finish();
-  }
-
-  /**
-   * Below method will be used to get the total row count in temp file
-   *
-   * @return
-   */
-  public int getEntryCount() {
-    return entryCount;
-  }
-
-  /**
-   * Below method will be used to get the row
-   */
-  public abstract Object[][] getRow();
-
-  protected Object[][] prepareRecordFromByteBuffer(int recordLength, byte[] byteArrayFromFile) {
-    Object[][] records = new Object[recordLength][];
-    Object[] record = null;
-    ByteBuffer buffer = ByteBuffer.allocate(byteArrayFromFile.length);
-
-    buffer.put(byteArrayFromFile);
-    buffer.rewind();
-
-    int index = 0;
-    byte b = 0;
-
-    for (int i = 0; i < recordLength; i++) {
-      record = new Object[eachRecordSize];
-      index = 0;
-
-      for (int j = 0; j < dimensionCount; j++) {
-        record[index++] = buffer.getInt();
-      }
-
-      for (int j = 0; j < complexDimensionCount; j++) {
-        byte[] complexByteArray = new byte[buffer.getInt()];
-        buffer.get(complexByteArray);
-        record[index++] = complexByteArray;
-      }
-
-      for (int j = 0; j < measureCount; j++) {
-        b = buffer.get();
-        if (b == 1) {
-          record[index++] = buffer.getDouble();
-        } else {
-          record[index++] = null;
-        }
-      }
-
-      records[i] = record;
-    }
-    return records;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/CompressedTempSortFileReader.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/CompressedTempSortFileReader.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/CompressedTempSortFileReader.java
deleted file mode 100644
index 46e7df0..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/CompressedTempSortFileReader.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.processing.sortandgroupby.sortdata;
-
-import java.io.File;
-
-import org.apache.carbondata.core.datastorage.store.compression.CompressorFactory;
-
-public class CompressedTempSortFileReader extends AbstractTempSortFileReader {
-
-  /**
-   * CompressedTempSortFileReader
-   *
-   * @param measureCount
-   * @param dimensionCount
-   * @param tempFile
-   */
-  public CompressedTempSortFileReader(int dimensionCount, int complexDimensionCount,
-      int measureCount, File tempFile, int noDictionaryCount) {
-    super(dimensionCount, complexDimensionCount, measureCount, tempFile, noDictionaryCount);
-  }
-
-  /**
-   * below method will be used to get chunk of rows
-   *
-   * @return row
-   */
-  @Override public Object[][] getRow() {
-    int recordLength = fileHolder.readInt(filePath);
-    int byteArrayLength = fileHolder.readInt(filePath);
-    byte[] byteArrayFromFile = CompressorFactory.getInstance()
-        .unCompressByte(fileHolder.readByteArray(filePath, byteArrayLength));
-    return prepareRecordFromByteBuffer(recordLength, byteArrayFromFile);
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
index c24472b..f3c45b0 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/SortParameters.java
@@ -438,8 +438,8 @@ public class SortParameters {
   }
 
   public static SortParameters createSortParameters(String databaseName, String tableName,
-      int dimColCount, int complexDimColCount, int measureColCount, SortObserver observer,
-      int noDictionaryCount, String partitionID, String segmentId, String taskNo,
+      int dimColCount, int complexDimColCount, int measureColCount, int noDictionaryCount,
+      String partitionID, String segmentId, String taskNo,
       boolean[] noDictionaryColMaping) {
     SortParameters parameters = new SortParameters();
     CarbonProperties carbonProperties = CarbonProperties.getInstance();

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/TempSortFileReaderFactory.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/TempSortFileReaderFactory.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/TempSortFileReaderFactory.java
deleted file mode 100644
index 63ed3a5..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/TempSortFileReaderFactory.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.processing.sortandgroupby.sortdata;
-
-import java.io.File;
-
-public final class TempSortFileReaderFactory {
-  private static final TempSortFileReaderFactory READERFACTORY = new TempSortFileReaderFactory();
-
-  private TempSortFileReaderFactory() {
-
-  }
-
-  public static TempSortFileReaderFactory getInstance() {
-    return READERFACTORY;
-  }
-
-  public TempSortFileReader getTempSortFileReader(boolean isCompressionEnabled, int dimensionCount,
-      int complexDimensionCount, int measureCount, File tempFile, int noDictionaryCount) {
-    if (isCompressionEnabled) {
-      return new CompressedTempSortFileReader(dimensionCount, complexDimensionCount, measureCount,
-          tempFile, noDictionaryCount);
-    } else {
-      return new UnCompressedTempSortFileReader(dimensionCount, complexDimensionCount, measureCount,
-          tempFile, noDictionaryCount);
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/UnCompressedTempSortFileReader.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/UnCompressedTempSortFileReader.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/UnCompressedTempSortFileReader.java
deleted file mode 100644
index d70e46c..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdata/UnCompressedTempSortFileReader.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.processing.sortandgroupby.sortdata;
-
-import java.io.File;
-
-public class UnCompressedTempSortFileReader extends AbstractTempSortFileReader {
-
-  /**
-   * UnCompressedTempSortFileReader
-   *
-   * @param measureCount
-   * @param dimensionCount
-   * @param tempFile
-   */
-  public UnCompressedTempSortFileReader(int dimensionCount, int complexDimensionCount,
-      int measureCount, File tempFile, int noDictionaryCount) {
-    super(dimensionCount, complexDimensionCount, measureCount, tempFile, noDictionaryCount);
-  }
-
-  /**
-   * below method will be used to get chunk of rows
-   *
-   * @return row
-   */
-  @Override public Object[][] getRow() {
-    int recordLength = fileHolder.readInt(filePath);
-    int byteArrayLength = fileHolder.readInt(filePath);
-    byte[] byteArrayFromFile = fileHolder.readByteArray(filePath, byteArrayLength);
-    return prepareRecordFromByteBuffer(recordLength, byteArrayFromFile);
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStep.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStep.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStep.java
index 0ff09e1..abbb641 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStep.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStep.java
@@ -180,7 +180,7 @@ public class SortKeyStep extends BaseStep {
       SortParameters parameters =
           SortParameters.createSortParameters(meta.getDatabaseName(), meta.getTabelName(),
               meta.getDimensionCount(), meta.getComplexDimensionCount(), meta.getMeasureCount(),
-              this.observer, meta.getNoDictionaryCount(), meta.getPartitionID(),
+              meta.getNoDictionaryCount(), meta.getPartitionID(),
               meta.getSegmentId() + "", meta.getTaskNo(), this.noDictionaryColMaping);
       intermediateFileMerger = new SortIntermediateFileMerger(parameters);
       this.sortDataRows = new SortDataRows(parameters, intermediateFileMerger);

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStepMeta.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStepMeta.java b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStepMeta.java
index 2519f73..c2f0611 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStepMeta.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/sortandgroupby/sortdatastep/SortKeyStepMeta.java
@@ -347,13 +347,6 @@ public class SortKeyStepMeta extends BaseStepMeta implements StepMetaInterface {
   }
 
   /**
-   * @return the tableName
-   */
-  public String getTableName() {
-    return tableName;
-  }
-
-  /**
    * @param tableName the tableName to set
    */
   public void setTableName(String tableName) {
@@ -411,37 +404,12 @@ public class SortKeyStepMeta extends BaseStepMeta implements StepMetaInterface {
   }
 
   /**
-   * @return the factDimLensString
-   */
-  public String getFactDimLensString() {
-    return factDimLensString;
-  }
-
-  /**
-   * @param factDimLensString the factDimLensString to set
-   */
-  public void setFactDimLensString(String factDimLensString) {
-    this.factDimLensString = factDimLensString;
-  }
-
-  /**
-   * @return the isUpdateMemberRequest
-   */
-  public boolean isUpdateMemberRequest() {
-    return Boolean.parseBoolean(updateMemberRequest);
-  }
-
-  /**
    * @param isUpdateMemberRequest the isUpdateMemberRequest to set
    */
   public void setIsUpdateMemberRequest(String isUpdateMemberRequest) {
     this.updateMemberRequest = isUpdateMemberRequest;
   }
 
-  public String getMeasureDataType() {
-    return measureDataType;
-  }
-
   public void setMeasureDataType(String measureDataType) {
     this.measureDataType = measureDataType;
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
index f6a729d..96f8a04 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerColumnar.java
@@ -161,30 +161,13 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
   private boolean[] aggKeyBlock;
   private boolean[] isNoDictionary;
   private boolean isAggKeyBlock;
-  private boolean enableInvertedIndex;
   private long processedDataCount;
   /**
    * thread pool size to be used for block sort
    */
   private int thread_pool_size;
-  /**
-   * factLevels
-   */
-  private int[] surrogateIndex;
-  /**
-   * factKeyGenerator
-   */
-  private KeyGenerator factKeyGenerator;
-  /**
-   * aggKeyGenerator
-   */
-  private KeyGenerator keyGenerator;
   private KeyGenerator[] complexKeyGenerator;
   /**
-   * maskedByteRanges
-   */
-  private int[] maskedByte;
-  /**
    * isDataWritingRequest
    */
   //    private boolean isDataWritingRequest;
@@ -1142,22 +1125,6 @@ public class CarbonFactDataHandlerColumnar implements CarbonFactHandler {
     }
   }
 
-  private byte[] getAggregateTableMdkey(byte[] maksedKey) throws CarbonDataWriterException {
-    long[] keyArray = this.factKeyGenerator.getKeyArray(maksedKey, maskedByte);
-
-    int[] aggSurrogateKey = new int[surrogateIndex.length];
-
-    for (int j = 0; j < aggSurrogateKey.length; j++) {
-      aggSurrogateKey[j] = (int) keyArray[surrogateIndex[j]];
-    }
-
-    try {
-      return keyGenerator.generateKey(aggSurrogateKey);
-    } catch (KeyGenException e) {
-      throw new CarbonDataWriterException("Problem while generating the mdkeyfor aggregate ", e);
-    }
-  }
-
   private int getColsCount(int columnSplit) {
     int count = 0;
     for (int i = 0; i < columnSplit; i++) {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
index 51663fc..c9548c0 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/CarbonFactDataHandlerModel.java
@@ -68,10 +68,6 @@ public class CarbonFactDataHandlerModel {
    */
   private int blockSize;
   /**
-   * flag to check whether to group the similar data
-   */
-  private boolean isGroupByEnabled;
-  /**
    * total count of measures in table
    */
   private int measureCount;
@@ -104,10 +100,6 @@ public class CarbonFactDataHandlerModel {
    */
   private boolean isMergingRequestForCustomAgg;
   /**
-   * flag to check whether the request is for updating a member
-   */
-  private boolean isUpdateMemberRequest;
-  /**
    * flag to check whether use inverted index
    */
   private boolean[] isUseInvertedIndex;
@@ -349,10 +341,6 @@ public class CarbonFactDataHandlerModel {
     this.tableName = tableName;
   }
 
-  public boolean isGroupByEnabled() {
-    return isGroupByEnabled;
-  }
-
   public int getMeasureCount() {
     return measureCount;
   }
@@ -377,22 +365,6 @@ public class CarbonFactDataHandlerModel {
     this.mdKeyIndex = mdKeyIndex;
   }
 
-  public String[] getAggregators() {
-    return aggregators;
-  }
-
-  public void setAggregators(String[] aggregators) {
-    this.aggregators = aggregators;
-  }
-
-  public String[] getAggregatorClass() {
-    return aggregatorClass;
-  }
-
-  public void setAggregatorClass(String[] aggregatorClass) {
-    this.aggregatorClass = aggregatorClass;
-  }
-
   public String getStoreLocation() {
     return storeLocation;
   }
@@ -401,26 +373,10 @@ public class CarbonFactDataHandlerModel {
     this.storeLocation = storeLocation;
   }
 
-  public int[] getFactDimLens() {
-    return factDimLens;
-  }
-
   public void setFactDimLens(int[] factDimLens) {
     this.factDimLens = factDimLens;
   }
 
-  public boolean isMergingRequestForCustomAgg() {
-    return isMergingRequestForCustomAgg;
-  }
-
-  public void setMergingRequestForCustomAgg(boolean mergingRequestForCustomAgg) {
-    isMergingRequestForCustomAgg = mergingRequestForCustomAgg;
-  }
-
-  public boolean isUpdateMemberRequest() {
-    return isUpdateMemberRequest;
-  }
-
   public int[] getDimLens() {
     return dimLens;
   }
@@ -429,26 +385,6 @@ public class CarbonFactDataHandlerModel {
     this.dimLens = dimLens;
   }
 
-  public String[] getFactLevels() {
-    return factLevels;
-  }
-
-  public void setFactLevels(String[] factLevels) {
-    this.factLevels = factLevels;
-  }
-
-  public String[] getAggLevels() {
-    return aggLevels;
-  }
-
-  public void setAggLevels(String[] aggLevels) {
-    this.aggLevels = aggLevels;
-  }
-
-  public boolean isDataWritingRequest() {
-    return isDataWritingRequest;
-  }
-
   public void setDataWritingRequest(boolean dataWritingRequest) {
     isDataWritingRequest = dataWritingRequest;
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
index 0195392..9e1a67a 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/AbstractFactDataWriter.java
@@ -390,13 +390,6 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
   }
 
   /**
-   * This method will be used to get the blocklet metadata
-   *
-   * @return BlockletInfo - blocklet metadata
-   */
-  protected abstract BlockletInfoColumnar getBlockletInfo(NodeHolder nodeHolder, long offset);
-
-  /**
    * Method will be used to close the open file channel
    *
    * @throws CarbonDataWriterException
@@ -559,14 +552,6 @@ public abstract class AbstractFactDataWriter<T> implements CarbonFactDataWriter<
    */
   public abstract void writeBlockletData(NodeHolder nodeHolder) throws CarbonDataWriterException;
 
-  @Override public int getLeafMetadataSize() {
-    return blockletInfoList.size();
-  }
-
-  @Override public String getTempStoreLocation() {
-    return this.fileName;
-  }
-
   protected byte[][] fillAndCompressedKeyBlockData(IndexStorage<int[]>[] keyStorageArray,
       int entryCount) {
     byte[][] keyBlockData = new byte[keyStorageArray.length][];

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonDataWriterVo.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonDataWriterVo.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonDataWriterVo.java
index e46430b..1fd5fe3 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonDataWriterVo.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonDataWriterVo.java
@@ -97,13 +97,6 @@ public class CarbonDataWriterVo {
   }
 
   /**
-   * @return the mdKeyLength
-   */
-  public int getMdKeyLength() {
-    return mdKeyLength;
-  }
-
-  /**
    * @param mdKeyLength the mdKeyLength to set
    */
   public void setMdKeyLength(int mdKeyLength) {
@@ -237,20 +230,6 @@ public class CarbonDataWriterVo {
   }
 
   /**
-   * @return the numberOfNoDictionaryColumn
-   */
-  public int getNumberOfNoDictionaryColumn() {
-    return numberOfNoDictionaryColumn;
-  }
-
-  /**
-   * @param numberOfNoDictionaryColumn the numberOfNoDictionaryColumn to set
-   */
-  public void setNumberOfNoDictionaryColumn(int numberOfNoDictionaryColumn) {
-    this.numberOfNoDictionaryColumn = numberOfNoDictionaryColumn;
-  }
-
-  /**
    * @return the isDictionaryColumn
    */
   public boolean[] getIsDictionaryColumn() {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java
index 5745e9f..6fa1522 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/CarbonFactDataWriter.java
@@ -70,18 +70,6 @@ public interface CarbonFactDataWriter<T> {
   void closeWriter() throws CarbonDataWriterException;
 
   /**
-   * Below method will be used to get the leaf meta data size
-   */
-  int getLeafMetadataSize();
-
-  /**
-   * For getting TempLocation
-   *
-   * @return
-   */
-  String getTempStoreLocation();
-
-  /**
    * @param isNoDictionary
    */
   void setIsNoDictionary(boolean[] isNoDictionary);

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/store/writer/NodeHolder.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/NodeHolder.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/NodeHolder.java
index 0864458..30c9d7c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/NodeHolder.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/NodeHolder.java
@@ -244,34 +244,6 @@ public class NodeHolder {
   }
 
   /**
-   * @return the dataAfterCompression
-   */
-  public short[][] getDataAfterCompression() {
-    return dataAfterCompression;
-  }
-
-  /**
-   * @param dataAfterCompression the dataAfterCompression to set
-   */
-  public void setDataAfterCompression(short[][] dataAfterCompression) {
-    this.dataAfterCompression = dataAfterCompression;
-  }
-
-  /**
-   * @return the indexMap
-   */
-  public short[][] getIndexMap() {
-    return indexMap;
-  }
-
-  /**
-   * @param indexMap the indexMap to set
-   */
-  public void setIndexMap(short[][] indexMap) {
-    this.indexMap = indexMap;
-  }
-
-  /**
    * @return the keyBlockIndexLength
    */
   public int[] getKeyBlockIndexLength() {
@@ -339,20 +311,6 @@ public class NodeHolder {
   }
 
   /**
-   * @return the dataIndexMapOffsets
-   */
-  public int[] getDataIndexMapOffsets() {
-    return dataIndexMapOffsets;
-  }
-
-  /**
-   * @param dataIndexMapOffsets the dataIndexMapOffsets to set
-   */
-  public void setDataIndexMapOffsets(int[] dataIndexMapOffsets) {
-    this.dataIndexMapOffsets = dataIndexMapOffsets;
-  }
-
-  /**
    * @return the dataIndexMapLength
    */
   public int[] getDataIndexMapLength() {
@@ -409,34 +367,6 @@ public class NodeHolder {
   }
 
   /**
-   * @return the allMaxValue
-   */
-  public byte[][] getAllMaxValue() {
-    return allMaxValue;
-  }
-
-  /**
-   * @param allMaxValue the allMaxValue to set
-   */
-  public void setAllMaxValue(byte[][] allMaxValue) {
-    this.allMaxValue = allMaxValue;
-  }
-
-  /**
-   * @return the allMinValue
-   */
-  public byte[][] getAllMinValue() {
-    return allMinValue;
-  }
-
-  /**
-   * @param allMinValue the allMinValue to set
-   */
-  public void setAllMinValue(byte[][] allMinValue) {
-    this.allMinValue = allMinValue;
-  }
-
-  /**
    * @return
    */
   public boolean[] getColGrpBlocks() {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java
index 754ac48..fda840c 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedDimSurrogateKeyGen.java
@@ -34,7 +34,6 @@ import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.cache.dictionary.Dictionary;
 import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.keygenerator.KeyGenException;
 import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
 import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
 import org.apache.carbondata.core.writer.HierarchyValueWriterForCSV;
@@ -70,7 +69,6 @@ public abstract class CarbonCSVBasedDimSurrogateKeyGen {
    * columnsInfo
    */
   protected ColumnsInfo columnsInfo;
-  protected IFileManagerComposite measureFilemanager;
   /**
    * primary key max surrogate key map
    */
@@ -228,20 +226,6 @@ public abstract class CarbonCSVBasedDimSurrogateKeyGen {
     }
   }
 
-  public abstract void writeDataToFileAndCloseStreams() throws KettleException, KeyGenException;
-
-  /**
-   * Search entry and insert if not found in store.
-   *
-   * @param val
-   * @param hier
-   * @return
-   * @throws KeyGenException
-   * @throws KettleException
-   */
-  protected abstract byte[] getHierFromStore(int[] val, String hier, int primaryKey)
-      throws KettleException;
-
   /**
    * Search entry and insert if not found in store.
    *

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenData.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenData.java b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenData.java
index 7fe1665..2895fa8 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenData.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenData.java
@@ -88,27 +88,6 @@ public class CarbonCSVBasedSeqGenData extends BaseStepData implements StepDataIn
   }
 
   /**
-   * @return Returns the defaultObjects.
-   */
-  public Object[] getDefaultObjects() {
-    return defaultObjects;
-  }
-
-  /**
-   * @param defaultObjects The defaultObjects to set.
-   */
-  public void setDefaultObjects(Object[] defaultObjects) {
-    this.defaultObjects = defaultObjects;
-  }
-
-  /**
-   * @return Returns the inputSize.
-   */
-  public int getInputSize() {
-    return inputSize;
-  }
-
-  /**
    * @param inputSize The inputSize to set.
    */
   public void setInputSize(int inputSize) {
@@ -116,20 +95,6 @@ public class CarbonCSVBasedSeqGenData extends BaseStepData implements StepDataIn
   }
 
   /**
-   * @return Returns the columnIndex.
-   */
-  public Map<String, int[]> getColumnIndex() {
-    return columnIndex;
-  }
-
-  /**
-   * @return Returns the generator.
-   */
-  public KeyGenerator getGenerator() {
-    return generator;
-  }
-
-  /**
    * @param generator The generator to set.
    */
   public void setGenerator(KeyGenerator generator) {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java
index 248a6e0..61fb54d 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenMeta.java
@@ -22,7 +22,6 @@ package org.apache.carbondata.processing.surrogatekeysgenerator.csvbased;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
@@ -96,10 +95,6 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn
    */
   protected int[] msrs;
   /**
-   * timehierName
-   */
-  protected String timehierName;
-  /**
    * hirches
    */
   protected Map<String, int[]> hirches;
@@ -116,10 +111,6 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn
    */
   protected int timeDimeIndex = -1;
   /**
-   * timeLevels
-   */
-  protected String[] timeLevels = new String[0];
-  /**
    * timeOrdinalCols
    */
   protected String[] timeOrdinalCols = new String[0];
@@ -144,10 +135,6 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn
    */
   protected String[] measureColumn;
   /**
-   * column data types
-   */
-  protected String[] dimColsDataType;
-  /**
    * array of carbon measures
    */
   protected CarbonMeasure[] carbonMeasures;
@@ -156,15 +143,10 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn
    */
   protected int[] msrMapping;
   /**
-   * dimMapping
-   */
-  protected int[] dimMapping;
-  /**
    * dims
    */
   protected boolean[] dimPresent;
   protected int normLength;
-  protected List<String> normHierList;
   /**
    * actualDimArrays
    */
@@ -240,10 +222,6 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn
    */
   private String metaHeirSQLQuery;
   /**
-   * carbonMetaHier
-   */
-  private String carbonMetaHier;
-  /**
    * Foreign key and respective hierarchy String
    */
   private String foreignKeyHierarchyString;
@@ -394,10 +372,6 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn
     return complexTypes;
   }
 
-  public void setComplexTypes(Map<String, GenericDataType> complexTypes) {
-    this.complexTypes = complexTypes;
-  }
-
   public String getComplexDelimiterLevel1() {
     return complexDelimiterLevel1;
   }
@@ -414,10 +388,6 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn
     this.complexDelimiterLevel2 = complexDelimiterLevel2;
   }
 
-  public String getComplexTypeString() {
-    return complexTypeString;
-  }
-
   public void setComplexTypeString(String complexTypeString) {
     this.complexTypeString = complexTypeString;
   }
@@ -430,22 +400,6 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn
     return complexTypeColumns;
   }
 
-  public void setComplexTypeColumns(String[] complexTypeColumns) {
-    this.complexTypeColumns = complexTypeColumns;
-  }
-
-  public String getCarbonMetaHier() {
-    return carbonMetaHier;
-  }
-
-  public void setCarbonMetaHier(String carbonMetaHier) {
-    this.carbonMetaHier = carbonMetaHier;
-  }
-
-  public String getMetaHeirSQLQueries() {
-    return metaHeirSQLQuery;
-  }
-
   public void setMetaMetaHeirSQLQueries(String metaHeirSQLQuery) {
     this.metaHeirSQLQuery = metaHeirSQLQuery;
   }
@@ -470,89 +424,39 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn
     return storeType;
   }
 
-  public void setStoreType(String storeType) {
-    this.storeType = storeType;
-  }
-
-  public String getCarbonSchema() {
-    return carbonSchema;
-  }
-
-  public void setCarbonSchema(String carbonSchema) {
-    this.carbonSchema = carbonSchema;
-  }
-
   public List<HierarchiesInfo> getMetahierVoList() {
     return metahierVoList;
   }
 
-  public void setMetahierVoList(List<HierarchiesInfo> metahierVoList) {
-    this.metahierVoList = metahierVoList;
-  }
-
   public String getCarbonTime() {
     return carbonTime;
   }
 
-  public void setCarbonTime(String carbonTime) {
-    this.carbonTime = carbonTime;
-  }
-
   // getters and setters for the step settings
 
-  public String getCarbonProps() {
-    return carbonProps;
-  }
-
   public void setCarbonProps(String carbonProps) {
     this.carbonProps = carbonProps;
   }
 
-  public String getCarbonmsr() {
-    return carbonmsr;
-  }
-
   public void setCarbonmsr(String carbonmsr) {
     this.carbonmsr = carbonmsr;
   }
 
-  public String getCarbondim() {
-    return carbondim;
-  }
-
   public void setCarbondim(String carbondim) {
     this.carbondim = carbondim;
   }
 
-  public String getCarbonHier() {
-    return carbonhier;
-  }
-
   public void setCarbonhier(String carbonhier) {
     this.carbonhier = carbonhier;
   }
 
   /**
-   * @return the connectionURL
-   */
-  public String getConnectionURL() {
-    return connectionURL;
-  }
-
-  /**
    * @param connectionURL the connectionURL to set
    */
   public void setConnectionURL(String connectionURL) {
     this.connectionURL = connectionURL;
   }
 
-  /**
-   * @return the driverClass
-   */
-  public String getDriverClass() {
-    return driverClass;
-  }
-
   //TODO SIMIAN
 
   /**
@@ -563,13 +467,6 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn
   }
 
   /**
-   * @return the userName
-   */
-  public String getUserName() {
-    return userName;
-  }
-
-  /**
    * @param userName the userName to set
    */
   public void setUserName(String userName) {
@@ -577,35 +474,12 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn
   }
 
   /**
-   * @return the password
-   */
-  public String getPassword() {
-    return password;
-  }
-
-  /**
    * @param password the password to set
    */
   public void setPassword(String password) {
     this.password = password;
   }
 
-  /**
-   * @return Returns the generateDimFiles.
-   */
-  public boolean isGenerateDimFiles() {
-    return generateDimFiles;
-  }
-
-  /**
-   * @param generateDimFiles The generateDimFiles to set.
-   */
-  public void setGenerateDimFiles(boolean generateDimFiles) {
-    this.generateDimFiles = generateDimFiles;
-  }
-
-  public String getDateFormat() { return dateFormat; }
-
   public void setDateFormat(String dateFormat) { this.dateFormat = dateFormat; }
 
   /**
@@ -627,7 +501,6 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn
     storeType = "";
     isAggregate = false;
     metaHeirSQLQuery = "";
-    carbonMetaHier = "";
     dimesionTableNames = "";
     dimensionColumnIds = "";
     noDictionaryDims = "";
@@ -678,7 +551,6 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn
     retval.append("    ").append(XMLHandler.addTagValue("isAggregate", isAggregate));
     retval.append("    ").append(XMLHandler.addTagValue("storeType", storeType));
     retval.append("    ").append(XMLHandler.addTagValue("metadataFilePath", metaHeirSQLQuery));
-    retval.append("    ").append(XMLHandler.addTagValue("carbonMetaHier", carbonMetaHier));
     retval.append("    ")
         .append(XMLHandler.addTagValue("foreignKeyHierarchyString", foreignKeyHierarchyString));
     retval.append("    ").append(XMLHandler.addTagValue("complexTypeString", complexTypeString));
@@ -738,7 +610,6 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn
       connectionURL = XMLHandler.getTagValue(stepnode, "connectionURL");
       userName = XMLHandler.getTagValue(stepnode, "userName");
       password = XMLHandler.getTagValue(stepnode, "password");
-      carbonMetaHier = XMLHandler.getTagValue(stepnode, "carbonMetaHier");
       carbonhierColumn = XMLHandler.getTagValue(stepnode, "carbonhierColumn");
       foreignKeyHierarchyString = XMLHandler.getTagValue(stepnode, "foreignKeyHierarchyString");
       complexTypeString = XMLHandler.getTagValue(stepnode, "complexTypeString");
@@ -823,16 +694,9 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn
 
     actualDimArray = getActualDimensionArray(actualDimNames);
 
-    normHierList = getNormHierList(normHiers);
-
     //update non time dimension properties
     updateDimProperties();
 
-    //update the meta Hierarichies list
-    getMetaHierarichies(carbonMetaHier);
-
-    updateMetaHierarichiesWithQueries(metaHeirSQLQuery);
-
     updateMeasureAggregator(msrAggregatorString);
 
     measureSurrogateRequired = getMeasureDatatypeMap(measureDataType);
@@ -1085,79 +949,6 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn
     return columns;
   }
 
-  private void getMetaHierarichies(String carbonMetaHier) {
-    if (null == carbonMetaHier || "".equals(carbonMetaHier)) {
-      return;
-    }
-    String[] metaHier = carbonMetaHier.split(CarbonCommonConstants.AMPERSAND_SPC_CHARACTER);
-    metahierVoList = new ArrayList<HierarchiesInfo>(metaHier.length);
-    Map<String, String[]> columnPropsMap =
-        new HashMap<String, String[]>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-    for (int i = 0; i < metaHier.length; i++) {
-      HierarchiesInfo hierarichiesVo = new HierarchiesInfo();
-      String[] isTimeDim = metaHier[i].split(CarbonCommonConstants.HASH_SPC_CHARACTER);
-      String[] split = isTimeDim[0].split(CarbonCommonConstants.COLON_SPC_CHARACTER);
-      String[] columnNames = new String[split.length - 1];
-      int[] columnIndex = new int[split.length - 1];
-      hierarichiesVo.setHierarichieName(split[0]);
-      if (null != hirches.get(split[0])) {
-        hierarichiesVo.setLoadToHierarichiTable(true);
-      }
-      int index = 0;
-      for (int j = 1; j < split.length; j++) {
-        String[] columnAndPropertyNames = split[j].split(
-            CarbonCommonConstants.COMA_SPC_CHARACTER);//CHECKSTYLE:OFF    Approval No:Approval-323
-        columnNames[index] = columnAndPropertyNames[0];//CHECKSTYLE:ON
-        columnIndex[index] = getColumnIndex(columnNames[index]);
-        String[] properties = new String[columnAndPropertyNames.length - 1];
-        System
-            .arraycopy(columnAndPropertyNames, 1, properties, 0, columnAndPropertyNames.length - 1);
-        if (null == columnPropsMap.get(columnNames[index])) {
-          columnPropsMap.put(columnNames[index], properties);
-        }
-        index++;
-      }
-      hierarichiesVo.setColumnIndex(columnIndex);
-      hierarichiesVo.setColumnNames(columnNames);
-      hierarichiesVo.setColumnPropMap(columnPropsMap);
-      metahierVoList.add(hierarichiesVo);
-    }
-  }
-
-  private void updateMetaHierarichiesWithQueries(String carbonLocation) {
-    //
-    if (null == carbonLocation) {
-      return;
-    }
-    String[] hierWithQueries = carbonLocation.split(CarbonCommonConstants.HASH_SPC_CHARACTER);
-    //
-    for (String hierarchyWithQuery : hierWithQueries) {
-      String[] hierQueryStrings =
-          hierarchyWithQuery.split(CarbonCommonConstants.COLON_SPC_CHARACTER);
-
-      Iterator<HierarchiesInfo> iterator = metahierVoList.iterator();
-      while (iterator.hasNext()) {
-        //
-        HierarchiesInfo next = iterator.next();
-        if (hierQueryStrings[0].equalsIgnoreCase(next.getHierarichieName())) {
-          next.setQuery(hierQueryStrings[1]);
-          break;
-        }
-
-      }
-    }
-
-  }
-
-  private int getColumnIndex(String colNames) {
-    for (int j = 0; j < dimColNames.length; j++) {
-      if (dimColNames[j].equalsIgnoreCase(colNames)) {
-        return j;
-      }
-    }
-    return -1;
-  }
-
   /**
    * Parse the properties string.
    * Level Entries separated by '&'
@@ -1309,7 +1100,6 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn
       password = rep.getStepAttributeString(idStep, "password");
       isAggregate = rep.getStepAttributeBoolean(idStep, "isAggregate");
       metaHeirSQLQuery = rep.getStepAttributeString(idStep, "metadataFilePath");
-      carbonMetaHier = rep.getStepAttributeString(idStep, "carbonMetaHier");
       carbonhierColumn = rep.getStepAttributeString(idStep, "carbonhierColumn");
       foreignKeyHierarchyString = rep.getStepAttributeString(idStep, "foreignKeyHierarchyString");
       primaryKeysString = rep.getStepAttributeString(idStep, "primaryKeysString");
@@ -1365,7 +1155,6 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn
       rep.saveStepAttribute(idTransformation, idStep, "password", password);
       rep.saveStepAttribute(idTransformation, idStep, "isInitialLoad", isAggregate);
       rep.saveStepAttribute(idTransformation, idStep, "metadataFilePath", metaHeirSQLQuery);
-      rep.saveStepAttribute(idTransformation, idStep, "carbonMetaHier", carbonMetaHier);
       rep.saveStepAttribute(idTransformation, idStep, "batchSize", batchSize);
       rep.saveStepAttribute(idTransformation, idStep, "dimHierReleation", dimesionTableNames);
       rep.saveStepAttribute(idTransformation, idStep, "dimensionColumnIds", dimensionColumnIds);
@@ -1427,10 +1216,6 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn
     return propTypes;
   }
 
-  public String getTableNames() {
-    return dimesionTableNames;
-  }
-
   public void setTableNames(String dimHierReleation) {
     this.dimesionTableNames = dimHierReleation;
   }
@@ -1459,10 +1244,6 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn
     this.tableName = tableName;
   }
 
-  public String[] getModifiedDimension() {
-    return modifiedDimension;
-  }
-
   public void setModifiedDimension(String[] modifiedDimension) {
     this.modifiedDimension = modifiedDimension;
   }
@@ -1483,66 +1264,34 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn
     return primaryKeyMap;
   }
 
-  public String getCarbonMeasureNames() {
-    return carbonMeasureNames;
-  }
-
   public void setCarbonMeasureNames(String carbonMeasureNames) {
     this.carbonMeasureNames = carbonMeasureNames;
   }
 
-  public String getActualDimNames() {
-    return actualDimNames;
-  }
-
   public void setActualDimNames(String actualDimNames) {
     this.actualDimNames = actualDimNames;
   }
 
-  public String getNormHiers() {
-    return normHiers;
-  }
-
   public void setNormHiers(String normHiers) {
     this.normHiers = normHiers;
   }
 
-  public String getMsrAggregatorString() {
-    return msrAggregatorString;
-  }
-
   public void setMsrAggregatorString(String msrAggregatorString) {
     this.msrAggregatorString = msrAggregatorString;
   }
 
-  public String getHeirKeySize() {
-    return heirKeySize;
-  }
-
   public void setHeirKeySize(String heirKeySize) {
     this.heirKeySize = heirKeySize;
   }
 
-  public String getForgienKeyPrimayKeyString() {
-    return forgienKeyPrimayKeyString;
-  }
-
   public void setForgienKeyPrimayKeyString(String forgienKeyPrimayKeyString) {
     this.forgienKeyPrimayKeyString = forgienKeyPrimayKeyString;
   }
 
-  public String getHeirNadDimsLensString() {
-    return heirNadDimsLensString;
-  }
-
   public void setHeirNadDimsLensString(String heirNadDimsLensString) {
     this.heirNadDimsLensString = heirNadDimsLensString;
   }
 
-  public String getMeasureDataType() {
-    return measureDataType;
-  }
-
   public void setMeasureDataType(String measureDataType) {
     this.measureDataType = measureDataType;
   }
@@ -1551,10 +1300,6 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn
     return measureSurrogateRequired;
   }
 
-  public void setMeasureSurrogateRequired(Map<String, Boolean> measureSurrogateRequired) {
-    this.measureSurrogateRequired = measureSurrogateRequired;
-  }
-
   public Map<String, String> getHierDimTableMap() {
     return hierDimTableMap;
   }
@@ -1563,10 +1308,6 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn
     return dimTableArray;
   }
 
-  public String getColumnAndTableNameColumnMapForAggString() {
-    return columnAndTableNameColumnMapForAggString;
-  }
-
   public void setColumnAndTableNameColumnMapForAggString(
       String columnAndTableNameColumnMapForAggString) {
     this.columnAndTableNameColumnMapForAggString = columnAndTableNameColumnMapForAggString;
@@ -1580,10 +1321,6 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn
     this.databaseName = databaseName;
   }
 
-  public String getDenormColumNames() {
-    return denormColumNames;
-  }
-
   public void setDenormColumNames(String denormColumNames) {
     this.denormColumNames = denormColumNames;
   }
@@ -1597,13 +1334,6 @@ public class CarbonCSVBasedSeqGenMeta extends BaseStepMeta implements StepMetaIn
   }
 
   /**
-   * @return columngroups
-   */
-  public String getDimensionColumnsDataType() {
-    return columnsDataTypeString;
-  }
-
-  /**
    * @param columnsDataTypeString
    */
   public void setDimensionColumnsDataType(String columnsDataTypeString) {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
index b1ef7fc..db2c367 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/CarbonCSVBasedSeqGenStep.java
@@ -166,7 +166,6 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
    * measureCol
    */
   private List<String> measureCol;
-  private boolean isTerminated;
   /**
    * dimPresentCsvOrder - Dim present In CSV order
    */
@@ -180,7 +179,6 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
    */
   private Future[] resultArray;
 
-  private int outSize;
   /**
    * denormHierarchies
    */
@@ -400,7 +398,6 @@ public class CarbonCSVBasedSeqGenStep extends BaseStep {
 
             ValueMetaInterface[] out = null;
             out = new ValueMetaInterface[meta.normLength + meta.msrMapping.length];
-            this.outSize = out.length;
             int outCounter = 0;
             for (int i = 0; i < meta.actualDimArray.length; i++) {
               if (meta.dimPresent[i]) {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
index 113121b..9b3bec7 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
@@ -298,18 +298,6 @@ public class FileStoreSurrogateKeyGenForCSV extends CarbonCSVBasedDimSurrogateKe
     }
   }
 
-  @Override protected byte[] getHierFromStore(int[] val, String hier, int primaryKey)
-      throws KettleException {
-    byte[] bytes;
-    try {
-      bytes = columnsInfo.getKeyGenerators().get(hier).generateKey(val);
-      hierValueWriter.get(hier).getByteArrayList().add(new ByteArrayHolder(bytes, primaryKey));
-    } catch (KeyGenException e) {
-      throw new KettleException(e);
-    }
-    return bytes;
-  }
-
   @Override protected int getSurrogateFromStore(String value, int index, Object[] properties)
       throws KettleException {
     max[index]++;
@@ -375,28 +363,4 @@ public class FileStoreSurrogateKeyGenForCSV extends CarbonCSVBasedDimSurrogateKe
     return measureSurrogate;
   }
 
-  @Override public void writeDataToFileAndCloseStreams() throws KettleException, KeyGenException {
-
-    // For closing stream inside hierarchy writer
-
-    for (Entry<String, String> entry : hierInsertFileNames.entrySet()) {
-
-      String hierFileName = hierValueWriter.get(entry.getKey()).getHierarchyName();
-
-      int size = fileManager.size();
-      for (int j = 0; j < size; j++) {
-        FileData fileData = (FileData) fileManager.get(j);
-        String fileName = fileData.getFileName();
-        if (hierFileName.equals(fileName)) {
-          HierarchyValueWriterForCSV hierarchyValueWriter = fileData.getHierarchyValueWriter();
-          hierarchyValueWriter.performRequiredOperation();
-
-          break;
-        }
-
-      }
-    }
-
-  }
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/dbbased/CarbonDimSurrogateKeyGen.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/dbbased/CarbonDimSurrogateKeyGen.java b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/dbbased/CarbonDimSurrogateKeyGen.java
deleted file mode 100644
index 4759853..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/dbbased/CarbonDimSurrogateKeyGen.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.processing.surrogatekeysgenerator.dbbased;
-
-import java.sql.Connection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-import org.apache.carbondata.core.cache.dictionary.Dictionary;
-import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.processing.schema.metadata.ColumnsInfo;
-
-import org.pentaho.di.core.exception.KettleException;
-
-public abstract class CarbonDimSurrogateKeyGen {
-  /**
-   * HIERARCHY_FILE_EXTENSION
-   */
-  protected static final String HIERARCHY_FILE_EXTENSION = ".hierarchy";
-  /**
-   * Comment for <code>LOGGER</code>
-   */
-  private static final LogService LOGGER =
-      LogServiceFactory.getLogService(CarbonDimSurrogateKeyGen.class.getName());
-  /**
-   * Cache should be map only. because, multiple levels can map to same
-   * database column. This case duplicate storage should be avoided.
-   */
-  protected Map<String, Dictionary> dictionaryCaches;
-  /**
-   * dimsFiles
-   */
-  protected String[] dimsFiles;
-  /**
-   * max
-   */
-  protected int[] max;
-  /**
-   * connection
-   */
-  protected Connection connection;
-  /**
-   * hierInsertFileNames
-   */
-  protected Map<String, String> hierInsertFileNames;
-  /**
-   * dimInsertFileNames
-   */
-  protected String[] dimInsertFileNames;
-  /**
-   * hierCache
-   */
-  protected Map<String, Map<IntArrayWrapper, Boolean>> hierCache =
-      new HashMap<String, Map<IntArrayWrapper, Boolean>>(
-          CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-  /**
-   * columnsInfo
-   */
-  protected ColumnsInfo columnsInfo;
-  /**
-   * rwLock
-   */
-  private ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock();
-  /**
-   * wLock
-   */
-  private Lock wLock = rwLock.writeLock();
-  /**
-   * rwLock2
-   */
-  private ReentrantReadWriteLock rwLock2 = new ReentrantReadWriteLock();
-  /**
-   * wLock2
-   */
-  protected Lock wLock2 = rwLock2.writeLock();
-
-  /**
-   * @param columnsInfo ColumnsInfo With all the required details for surrogate key generation and
-   *                    hierarchy entries.
-   */
-  public CarbonDimSurrogateKeyGen(ColumnsInfo columnsInfo) {
-    this.columnsInfo = columnsInfo;
-
-    setDimensionTables(columnsInfo.getDimColNames());
-    setHierFileNames(columnsInfo.getHierTables());
-  }
-
-  public Object[] generateSurrogateKeys(Object[] tuple, Object[] out,
-      List<Integer> timeOrdinalColValues) throws KettleException {
-    boolean[] dimsPresent = columnsInfo.getDimsPresent();
-    int[] dims = columnsInfo.getDims();
-
-    String[] dimColNames = columnsInfo.getDimColNames();
-    int k = 0;
-    for (int i = 0; i < dims.length; i++) {
-      Integer key = null;
-      Object value = null;
-      if (columnsInfo.isAggregateLoad()) {
-        value = tuple[i];
-      } else {
-        if (dimsPresent[i]) {
-          value = tuple[k];
-
-        } else {
-          continue;
-        }
-      }
-
-      if (value == null) {
-        value = "null";
-      }
-      String dimS = value.toString().trim();
-      // getting values from local cache
-      Dictionary dicCache = dictionaryCaches.get(dimColNames[i]);
-
-      key = dicCache.getSurrogateKey(dimS);
-
-      // Update the generated key in output.
-      out[k] = key;
-      k++;
-    }
-    return out;
-  }
-
-  private Object[] getProperties(Object[] tuple, List<Integer> timeOrdinalColValues, int i) {
-    Object[] props = new Object[0];
-    if (columnsInfo.getTimDimIndex() != -1 && i >= columnsInfo.getTimDimIndex() && i < columnsInfo
-        .getTimDimIndexEnd()) {
-      //For time dimensions only ordinal columns is considered.
-      int ordinalIndx = columnsInfo.getTimeOrdinalIndices()[i - columnsInfo.getTimDimIndexEnd()];
-      if (ordinalIndx != -1) {
-        props = new Object[1];
-        props[0] = timeOrdinalColValues.get(ordinalIndx);
-      }
-    } else {
-      if (columnsInfo.getPropIndx() != null) {
-        int[] pIndices = columnsInfo.getPropIndx()[i];
-        props = new Object[pIndices.length];
-        for (int j = 0; j < props.length; j++) {
-          props[j] = tuple[pIndices[j]];
-        }
-      }
-    }
-    return props;
-  }
-
-  public void checkHierExists(int[] val, String hier) throws KettleException {
-    IntArrayWrapper wrapper = new IntArrayWrapper(val, 0);
-    Map<IntArrayWrapper, Boolean> hCache = hierCache.get(hier);
-    Boolean b = hCache.get(wrapper);
-    if (b != null) {
-      return;
-    }
-
-    wLock2.lock();
-    try {
-      if (null == hCache.get(wrapper)) {
-        getHierFromStore(val, hier);
-        // Store in cache
-        hCache.put(wrapper, true);
-      }
-    } finally {
-      wLock2.unlock();
-    }
-  }
-
-  public void close() throws Exception {
-    if (null != connection) {
-      connection.close();
-    }
-    hierCache.clear();
-  }
-
-  public abstract void writeHeirDataToFileAndCloseStreams() throws KettleException;
-
-  /**
-   * Search entry and insert if not found in store.
-   *
-   * @param val
-   * @param hier
-   * @return
-   * @throws KeyGenException
-   * @throws KettleException
-   */
-  protected abstract byte[] getHierFromStore(int[] val, String hier) throws KettleException;
-
-  /**
-   * Search entry and insert if not found in store.
-   *
-   * @param val
-   * @param hier
-   * @return
-   * @throws KeyGenException
-   * @throws KettleException
-   */
-  protected abstract byte[] getNormalizedHierFromStore(int[] val, String hier,
-      HierarchyValueWriter hierWriter) throws KettleException;
-
-  /**
-   * Search entry and insert if not found in store.
-   *
-   * @param value
-   * @param index
-   * @param properties - Ordinal column, name column and all other properties
-   * @return
-   * @throws KettleException
-   */
-  protected abstract int getSurrogateFromStore(String value, int index, Object[] properties)
-      throws KettleException;
-
-  private Map<IntArrayWrapper, Boolean> getHCache(String hName) {
-    Map<IntArrayWrapper, Boolean> hCache = hierCache.get(hName);
-    if (hCache == null) {
-      hCache = new HashMap<IntArrayWrapper, Boolean>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-      hierCache.put(hName, hCache);
-    }
-
-    return hCache;
-  }
-
-  private void setHierFileNames(Set<String> set) {
-    hierInsertFileNames =
-        new HashMap<String, String>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-    for (String s : set) {
-      hierInsertFileNames.put(s, s + HIERARCHY_FILE_EXTENSION);
-
-      // fix hierStream is null issue
-      getHCache(s);
-    }
-  }
-
-  private void setDimensionTables(String[] dimeFileNames) {
-    this.dimsFiles = dimeFileNames;
-    max = new int[dimeFileNames.length];
-    createRespectiveDimFilesForDimTables();
-  }
-
-  private void createRespectiveDimFilesForDimTables() {
-    int dimCount = this.dimsFiles.length;
-    dimInsertFileNames = new String[dimCount];
-    System.arraycopy(dimsFiles, 0, dimInsertFileNames, 0, dimCount);
-  }
-
-  public void checkNormalizedHierExists(int[] val, String hier, HierarchyValueWriter hierWriter)
-      throws KettleException {
-    IntArrayWrapper wrapper = new IntArrayWrapper(val, 0);
-    Map<IntArrayWrapper, Boolean> hCache = hierCache.get(hier);
-
-    Boolean b = hCache.get(wrapper);
-
-    if (b != null) {
-      return;
-    } else {
-      wLock2.lock();
-      try {
-        getNormalizedHierFromStore(val, hier, hierWriter);
-      } finally {
-        wLock2.unlock();
-      }
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/dbbased/HierarchyValueWriter.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/dbbased/HierarchyValueWriter.java b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/dbbased/HierarchyValueWriter.java
deleted file mode 100644
index 0912310..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/dbbased/HierarchyValueWriter.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.processing.surrogatekeysgenerator.dbbased;
-
-import java.io.BufferedOutputStream;
-import java.io.Closeable;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
-
-import org.pentaho.di.core.exception.KettleException;
-
-public class HierarchyValueWriter {
-
-  /**
-   * Comment for <code>LOGGER</code>
-   */
-  private static final LogService LOGGER =
-      LogServiceFactory.getLogService(HierarchyValueWriter.class.getName());
-  /**
-   * BUFFFER_SIZE
-   */
-  private static final int BUFFFER_SIZE = 32768;
-  /**
-   * hierarchyName
-   */
-  private String hierarchyName;
-  /**
-   * bufferedOutStream
-   */
-  private BufferedOutputStream bufferedOutStream;
-  /**
-   * storeFolderLocation
-   */
-  private String storeFolderLocation;
-
-  /**
-   * byteArrayList
-   */
-  private List<byte[]> byteArrayList = new ArrayList<byte[]>();
-
-  public HierarchyValueWriter(String hierarchy, String storeFolderLocation) {
-    this.hierarchyName = hierarchy;
-    this.storeFolderLocation = storeFolderLocation;
-  }
-
-  /**
-   * @return Returns the byteArrayList.
-   */
-  public List<byte[]> getByteArrayList() {
-    return byteArrayList;
-  }
-
-  /**
-   * @param byteArrayList The byteArrayList to set.
-   */
-  public void setByteArrayList(List<byte[]> byteArrayList) {
-    this.byteArrayList = byteArrayList;
-  }
-
-  /**
-   * @return Returns the bufferedOutStream.
-   */
-  public BufferedOutputStream getBufferedOutStream() {
-    return bufferedOutStream;
-  }
-
-  public void writeIntoHierarchyFile(byte[] bytes) throws KettleException {
-    File f = new File(storeFolderLocation + File.separator + hierarchyName);
-
-    FileOutputStream fos = null;
-
-    boolean isFileCreated = false;
-    if (!f.exists()) {
-      try {
-        isFileCreated = f.createNewFile();
-
-      } catch (IOException e) {
-        throw new KettleException("unable to create file", e);
-      }
-      if (!isFileCreated) {
-        throw new KettleException("unable to create file" + f.getAbsolutePath());
-      }
-    }
-    try {
-      if (null == bufferedOutStream) {
-        fos = new FileOutputStream(f);
-        bufferedOutStream = new BufferedOutputStream(fos, BUFFFER_SIZE);
-
-      }
-      bufferedOutStream.write(bytes);
-
-    } catch (FileNotFoundException e) {
-      closeStreamAndDeleteFile(f, bufferedOutStream, fos);
-      throw new KettleException("hierarchy mapping file not found", e);
-    } catch (IOException e) {
-      closeStreamAndDeleteFile(f, bufferedOutStream, fos);
-      throw new KettleException("Error while writting in the hierarchy mapping file", e);
-    }
-  }
-
-  private void closeStreamAndDeleteFile(File f, Closeable... streams) throws KettleException {
-    boolean isDeleted = false;
-    for (Closeable stream : streams) {
-      if (null != stream) {
-        try {
-          stream.close();
-        } catch (IOException e) {
-          LOGGER.error(e,
-              "unable to close the stream ");
-        }
-
-      }
-    }
-
-    // delete the file
-    isDeleted = f.delete();
-    if (!isDeleted) {
-      LOGGER.error("Unable to delete the file " + f.getAbsolutePath());
-    }
-
-  }
-
-  /**
-   * @return Returns the hierarchyName.
-   */
-  public String getHierarchyName() {
-    return hierarchyName;
-  }
-
-}
-

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/dbbased/IntArrayWrapper.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/dbbased/IntArrayWrapper.java b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/dbbased/IntArrayWrapper.java
deleted file mode 100644
index 35efbbc..0000000
--- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/dbbased/IntArrayWrapper.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.processing.surrogatekeysgenerator.dbbased;
-
-import java.io.Serializable;
-import java.util.Arrays;
-
-public class IntArrayWrapper implements Serializable, Comparable<IntArrayWrapper> {
-
-  /**
-   * serialVersionUID
-   */
-  private static final long serialVersionUID = 1L;
-
-  private int[] data;
-
-  private int primaryKey;
-
-  public IntArrayWrapper(int[] data, int primaryKey) {
-    initialize(data);
-    this.primaryKey = primaryKey;
-  }
-
-  /**
-   * This method is used to initialize data array
-   *
-   * @param data
-   */
-  public void initialize(int[] data) {
-    if (data == null) {
-      throw new IllegalArgumentException(" Data Array is NUll");
-    }
-    this.data = data;
-  }
-
-  /**
-   * This method will be used check to ArrayWrapper object is equal or not
-   *
-   * @return boolean
-   * equal or not
-   */
-  @Override public boolean equals(Object other) {
-    if (other instanceof IntArrayWrapper) {
-      return Arrays.equals(data, ((IntArrayWrapper) other).data);
-    }
-    return false;
-  }
-
-  /**
-   * This method will be used to get the hascode, this will be used to the
-   * index for inserting ArrayWrapper object as a key in Map
-   *
-   * @return hascode
-   */
-  @Override public int hashCode() {
-    return Arrays.hashCode(data);
-  }
-
-  /**
-   * This method will be used to get the long array surrogate keys
-   *
-   * @return data
-   */
-  public int[] getData() {
-    return data;
-  }
-
-  /**
-   * Compare method for ArrayWrapper class this will used to compare Two
-   * ArrayWrapper data object, basically it will compare two surrogate keys
-   * array to check which one is greater
-   *
-   * @param other ArrayWrapper Object
-   */
-  @Override public int compareTo(IntArrayWrapper other) {
-    for (int i = 0; i < data.length; i++) {
-      if (data[i] > other.data[i]) {
-        return 1;
-      } else if (data[i] < other.data[i]) {
-        return -1;
-      }
-    }
-    return 0;
-  }
-
-  public int getPrimaryKey() {
-    return primaryKey;
-  }
-
-  public void setPrimaryKey(int primaryKey) {
-    this.primaryKey = primaryKey;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/util/CarbonSchemaParser.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonSchemaParser.java b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonSchemaParser.java
index 6ff63e9..b356eb2 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/CarbonSchemaParser.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/CarbonSchemaParser.java
@@ -19,8 +19,16 @@
 
 package org.apache.carbondata.processing.util;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
 
 import org.apache.carbondata.core.carbon.CarbonDataLoadSchema;
 import org.apache.carbondata.core.carbon.CarbonDataLoadSchema.DimensionRelation;
@@ -30,7 +38,6 @@ import org.apache.carbondata.core.carbon.metadata.schema.table.CarbonTable;
 import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension;
 import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.core.keygenerator.KeyGenerator;
 import org.apache.carbondata.core.keygenerator.factory.KeyGeneratorFactory;
 import org.apache.carbondata.core.util.CarbonUtil;
 import org.apache.carbondata.processing.schema.metadata.ColumnSchemaDetails;
@@ -42,11 +49,6 @@ public final class CarbonSchemaParser {
    */
   public static final String QUOTES = "\"";
 
-  /**
-   * BACK_TICK
-   */
-  public static final String BACK_TICK = "`";
-
   private CarbonSchemaParser() {
 
   }
@@ -209,14 +211,6 @@ public final class CarbonSchemaParser {
 
     getQueryForDimension(dimensions, query, factTableName, isQuotesRequired, carbonDataLoadSchema);
 
-    //No properties in new Schema
-    //        if (checkIfDenormalized(carbonDataLoadSchema)) {
-    //            if (!isQuotesRequired) {
-    //                getPropetiesQuerypart(dimensions, query, factTableName, schema);
-    //            } else {
-    //                getPropetiesQuerypartWithQuotes(dimensions, query, factTableName, schema);
-    //            }
-    //        }
     if (!"select".equalsIgnoreCase(query.toString().trim())) {
       query.append(",");
     }
@@ -250,52 +244,6 @@ public final class CarbonSchemaParser {
     return query.toString();
   }
 
-  /**
-   * @param aggDim
-   * @param measures
-   * @param factTableName
-   * @param isQuotesRequired
-   * @param schemaInfo
-   * @return
-   */
-  public static String getTableInputSQLQueryForAGG(String[] aggDim, String[] measures,
-      String factTableName, boolean isQuotesRequired) {
-    StringBuilder queryBuilder = new StringBuilder("SELECT ");
-    queryBuilder.append(System.getProperty("line.separator"));
-    //query.append("\n");
-    for (int i = 0; i < aggDim.length; i++) {
-      if (isQuotesRequired) {
-        queryBuilder.append(QUOTES + aggDim[i] + QUOTES);
-      } else {
-        queryBuilder.append(aggDim[i]);
-      }
-      queryBuilder.append(",");
-      queryBuilder.append(System.getProperty("line.separator"));
-      //query.append("\n");
-    }
-
-    for (int i = 0; i < measures.length - 1; i++) {
-      if (isQuotesRequired) {
-        queryBuilder.append(QUOTES + measures[i] + QUOTES);
-      } else {
-        queryBuilder.append(measures[i]);
-      }
-      queryBuilder.append(",");
-      queryBuilder.append(System.getProperty("line.separator"));
-      //query.append("\n");
-    }
-    if (isQuotesRequired) {
-      queryBuilder.append(QUOTES + measures[measures.length - 1] + QUOTES);
-      queryBuilder.append(System.getProperty("line.separator"));
-      queryBuilder.append(" FROM " + QUOTES + factTableName + QUOTES);
-    } else {
-      queryBuilder.append(measures[measures.length - 1]);
-      queryBuilder.append(System.getProperty("line.separator"));
-      queryBuilder.append(" FROM " + factTableName);
-    }
-    return queryBuilder.toString();
-  }
-
   private static void getQueryForDimension(List<CarbonDimension> dimensions, StringBuilder query,
       String factTableName, boolean isQuotesRequired, CarbonDataLoadSchema carbonDataLoadSchema) {
     int counter = 0;
@@ -364,38 +312,6 @@ public final class CarbonSchemaParser {
   }
 
   /**
-   * @param dimensions
-   * @param dimString
-   * @param counter
-   * @param dimCardinalities
-   * @return
-   */
-  public static int getDimensionStringForAgg(String[] dimensions, StringBuilder dimString,
-      int counter, Map<String, String> dimCardinalities, String[] acutalDimension) {
-    //
-    int len = dimensions.length;
-    for (int i = 0; i < len - 1; i++) {
-      dimString.append(dimensions[i]);
-      dimString.append(CarbonCommonConstants.COLON_SPC_CHARACTER);
-      dimString.append(counter++);
-      dimString.append(CarbonCommonConstants.COLON_SPC_CHARACTER);
-      dimString.append(dimCardinalities.get(acutalDimension[i]));
-      dimString.append(CarbonCommonConstants.COLON_SPC_CHARACTER);
-      dimString.append("Y");
-      dimString.append(CarbonCommonConstants.COMA_SPC_CHARACTER);
-    }
-    //
-    dimString.append(dimensions[len - 1]);
-    dimString.append(CarbonCommonConstants.COLON_SPC_CHARACTER);
-    dimString.append(counter++);
-    dimString.append(CarbonCommonConstants.COLON_SPC_CHARACTER);
-    dimString.append(dimCardinalities.get(acutalDimension[len - 1]));
-    dimString.append(CarbonCommonConstants.COLON_SPC_CHARACTER);
-    dimString.append("Y");
-    return counter;
-  }
-
-  /**
    * Return mapping of Column name to cardinality
    */
 
@@ -438,49 +354,6 @@ public final class CarbonSchemaParser {
    * @param measures
    * @return
    */
-  public static String getMeasureStringForAgg(String[] measures, int counter) {
-    StringBuilder measureString = new StringBuilder();
-    int i = measures.length;
-    for (String measure : measures) {
-
-      measureString.append(measure + CarbonCommonConstants.COLON_SPC_CHARACTER + counter);
-      counter++;
-      if (i > 1) {
-        measureString.append(CarbonCommonConstants.COMA_SPC_CHARACTER);
-      }
-      i--;
-
-    }
-    return measureString.toString();
-  }
-
-  /**
-   * Get measure string from a array of Measure
-   *
-   * @param measures
-   * @return
-   */
-  public static String getStringWithSeperator(String[] measures, String seperator) {
-    StringBuilder measureString = new StringBuilder();
-    int i = measures.length;
-    for (String measure : measures) {
-
-      measureString.append(measure);
-      if (i > 1) {
-        measureString.append(seperator);
-      }
-      i--;
-
-    }
-    return measureString.toString();
-  }
-
-  /**
-   * Get measure string from a array of Measure
-   *
-   * @param measures
-   * @return
-   */
   public static String[] getMeasures(List<CarbonMeasure> measures) {
     String[] measuresStringArray = new String[measures.size()];
 
@@ -646,7 +519,7 @@ public final class CarbonSchemaParser {
     return prop;
   }
 
-  public static String getTableNameString(String factTableName, List<CarbonDimension> dimensions,
+  public static String getTableNameString(List<CarbonDimension> dimensions,
       CarbonDataLoadSchema carbonDataLoadSchema) {
     StringBuilder stringBuffer = new StringBuilder();
 
@@ -706,36 +579,6 @@ public final class CarbonSchemaParser {
 
   /**
    * @param dimensions
-   * @param dimCardinalities
-   * @return
-   */
-  public static String getMdkeySizeForAgg(String[] dimensions,
-      Map<String, String> dimCardinalities) {
-    int[] dims = new int[dimensions.length];
-    for (int i = 0; i < dimensions.length; i++) {
-      dims[i] = Integer.parseInt(dimCardinalities.get(dimensions[i]));
-    }
-    return KeyGeneratorFactory.getKeyGenerator(dims).getKeySizeInBytes() + "";
-
-  }
-
-  /**
-   * @param dimensions
-   * @param dimCardinalities
-   * @return
-   */
-  public static KeyGenerator getKeyGeneratorForAGG(String[] dimensions,
-      Map<String, String> dimCardinalities) {
-    int[] dims = new int[dimensions.length];
-    for (int i = 0; i < dimensions.length; i++) {
-      dims[i] = Integer.parseInt(dimCardinalities.get(dimensions[i]));
-    }
-    return KeyGeneratorFactory.getKeyGenerator(dims);
-
-  }
-
-  /**
-   * @param dimensions
    * @param schema
    * @return
    */
@@ -983,30 +826,6 @@ public final class CarbonSchemaParser {
   }
 
   /**
-   * Get Measure Name String
-   *
-   * @param table
-   * @return
-   */
-  public static String getMeasuresNamesStringForAgg(String[] measures) {
-    StringBuilder measureNames = new StringBuilder();
-
-    for (int i = 0; i < measures.length; i++) {
-      measureNames.append(measures[i]);
-      measureNames.append(CarbonCommonConstants.AMPERSAND_SPC_CHARACTER);
-    }
-
-    String measureNameString = measureNames.toString();
-
-    if (measureNameString.endsWith(CarbonCommonConstants.AMPERSAND_SPC_CHARACTER)) {
-      measureNameString = measureNameString.substring(0,
-          measureNameString.length() - CarbonCommonConstants.AMPERSAND_SPC_CHARACTER.length());
-    }
-
-    return measureNameString;
-  }
-
-  /**
    * Get Measure Aggregator array
    *
    * @param table
@@ -1066,28 +885,6 @@ public final class CarbonSchemaParser {
     return dimDataType;
   }
 
-  /**
-   * @param table
-   * @return
-   */
-  public static String getActualDimensionsForAggregate(String[] columns) {
-    //
-    StringBuilder actualDim = new StringBuilder();
-    for (String column : columns) {
-      actualDim.append(column);
-      actualDim.append(CarbonCommonConstants.AMPERSAND_SPC_CHARACTER);
-    }
-
-    String actualDimString = actualDim.toString();
-
-    if (actualDimString.endsWith(CarbonCommonConstants.AMPERSAND_SPC_CHARACTER)) {
-      actualDimString = actualDimString.substring(0,
-          actualDimString.length() - CarbonCommonConstants.AMPERSAND_SPC_CHARACTER.length());
-    }
-
-    return actualDimString;
-  }
-
   public static String getMeasuresDataType(List<CarbonMeasure> measures) {
     StringBuilder measureDataTypeString = new StringBuilder();
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/main/java/org/apache/carbondata/processing/util/RemoveDictionaryUtil.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/util/RemoveDictionaryUtil.java b/processing/src/main/java/org/apache/carbondata/processing/util/RemoveDictionaryUtil.java
index 0a5bbe2..769e5ae 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/util/RemoveDictionaryUtil.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/util/RemoveDictionaryUtil.java
@@ -273,15 +273,6 @@ public class RemoveDictionaryUtil {
   }
 
   /**
-   * @param row
-   * @return
-   */
-  public static Integer[] getCompleteDimensions(Object[] row) {
-
-    return (Integer[]) row[0];
-  }
-
-  /**
    * This will extract the high cardinality count from the string.
    */
   public static int extractNoDictionaryCount(String noDictionaryDim) {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/3fe69037/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java b/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
index f935ba4..95f3d35 100644
--- a/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
+++ b/processing/src/test/java/org/apache/carbondata/test/util/StoreCreator.java
@@ -479,20 +479,10 @@ public class StoreCreator {
     model.setQuoteCharacter(dataProcessTaskStatus.getQuoteCharacter());
     model.setCommentCharacter(dataProcessTaskStatus.getCommentCharacter());
     model.setDateFormat(dataProcessTaskStatus.getDateFormat());
-    if (null != loadMetadataDetails && !loadMetadataDetails.isEmpty()) {
-      model.setLoadNames(
-          CarbonDataProcessorUtil.getLoadNameFromLoadMetaDataDetails(loadMetadataDetails));
-      model.setModificationOrDeletionTime(CarbonDataProcessorUtil
-          .getModificationOrDeletionTimesFromLoadMetadataDetails(loadMetadataDetails));
-    }
-    boolean hdfsReadMode =
-        dataProcessTaskStatus.getCsvFilePath() != null && dataProcessTaskStatus.getCsvFilePath().startsWith("hdfs:");
-    int allocate = null != dataProcessTaskStatus.getCsvFilePath() ? 1 : dataProcessTaskStatus.getFilesToProcess().size();
     String outputLocation = CarbonProperties.getInstance()
         .getProperty("store_output_location", "../carbon-store/system/carbon/etl");
     GraphGenerator generator =
-        new GraphGenerator(model, hdfsReadMode, partitionID, factStoreLocation,
-            allocate, schema, "0", outputLocation);
+        new GraphGenerator(model, partitionID, factStoreLocation, schema, "0", outputLocation);
     generator.generateGraph();
   }
 


Mime
View raw message