carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gvram...@apache.org
Subject [01/22] incubator-carbondata git commit: Comments on test cases Handle Null Pointer in Spark2 Empty Table Test Case test cases were failing due to nullpointer. corrected that. fixing style issues. Handled review comments & clean up tasks IUD Rebase - 05/
Date Fri, 06 Jan 2017 13:57:01 GMT
Repository: incubator-carbondata
Updated Branches:
  refs/heads/master a2a081dfc -> b0750c192


Comments on test cases
Handle Null Pointer in Spark2 Empty Table Test Case
test cases were failing due to nullpointer. corrected that.
fixing style issues.
Handled review comments & clean up tasks
IUD Rebase - 05/01


Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/9b8090ba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/9b8090ba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/9b8090ba

Branch: refs/heads/master
Commit: 9b8090baeaf7a86de8c21043517a6dcc3fa5031d
Parents: c9c68a1
Author: ravikiran <ravikiran.sn042@gmail.com>
Authored: Thu Jan 5 19:56:45 2017 +0530
Committer: Venkata Ramana G <ramana.gollamudi@huawei.com>
Committed: Fri Jan 6 19:16:29 2017 +0530

----------------------------------------------------------------------
 .../BlockletLevelDeleteDeltaDataCache.java      | 19 +++-
 .../iudprocessor/iuddata/BlockMappingVO.java    | 17 ++++
 .../iuddata/BlockletDeleteDeltaCacheLoader.java | 21 +++++
 .../iuddata/DeleteDeltaCacheLoaderIntf.java     | 19 +++-
 .../iuddata/DeleteDeltaDataUtil.java            | 44 ---------
 .../iudprocessor/iuddata/RowCountDetailsVO.java | 33 ++++++-
 .../carbon/datastore/SegmentTaskIndexStore.java | 16 ++--
 .../impl/btree/AbstractBTreeLeafNode.java       |  4 +-
 .../datastore/impl/btree/BTreeNonLeafNode.java  |  2 -
 .../table/column/CarbonImplicitDimension.java   |  3 +
 .../core/carbon/path/CarbonTablePath.java       | 28 ------
 .../core/load/LoadMetadataDetails.java          |  4 +-
 .../reader/CarbonDeleteDeltaFileReaderImpl.java | 13 ++-
 .../reader/CarbonDeleteFilesDataReader.java     | 10 +-
 .../core/update/CarbonUpdateUtil.java           | 31 ++++++-
 .../core/update/DeleteDeltaBlockDetails.java    |  1 +
 .../core/update/DeleteDeltaBlockletDetails.java |  2 +-
 .../core/updatestatus/SegmentStatusManager.java |  9 --
 .../SegmentUpdateStatusManager.java             | 25 +++--
 .../carbondata/core/util/CarbonProperties.java  |  9 ++
 .../writer/CarbonDeleteDeltaWriterImpl.java     |  2 -
 .../impl/DictionaryBasedResultCollector.java    |  2 -
 .../collector/impl/RawBasedResultCollector.java |  2 -
 .../executor/impl/AbstractQueryExecutor.java    | 15 ++-
 .../scan/executor/infos/BlockExecutionInfo.java | 10 ++
 .../executer/ImplicitColumnFilterExecutor.java  | 43 ---------
 .../ImplicitIncludeFilterExecutorImpl.java      | 97 --------------------
 .../scan/scanner/AbstractBlockletScanner.java   |  7 +-
 .../scan/scanner/impl/FilterScanner.java        | 15 +--
 .../core/load/LoadMetadataDetailsUnitTest.java  | 12 ---
 .../carbondata/hadoop/CarbonInputFormat.java    | 48 ++++++----
 .../hadoop/util/BlockLevelTraverser.java        | 19 +++-
 .../hadoop/ft/CarbonInputMapperTest.java        | 92 +++++++++----------
 .../hadoop/test/util/StoreCreator.java          |  5 +-
 .../carbondata/spark/load/CarbonLoaderUtil.java | 23 -----
 .../spark/merger/CarbonDataMergerUtil.java      | 30 ------
 .../merger/CarbonDataMergerUtilResult.java      | 17 ++++
 .../carbondata/spark/rdd/CarbonMergerRDD.scala  |  4 +-
 .../spark/rdd/DataManagementFunc.scala          |  1 -
 .../carbondata/spark/util/CommonUtil.scala      |  2 +-
 .../spark/sql/CarbonDatasourceRelation.scala    |  2 +-
 .../execution/command/carbonTableSchema.scala   |  2 -
 .../FileStoreSurrogateKeyGenForCSV.java         |  4 +-
 43 files changed, 326 insertions(+), 438 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/main/java/org/apache/carbondata/common/iudprocessor/cache/BlockletLevelDeleteDeltaDataCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/common/iudprocessor/cache/BlockletLevelDeleteDeltaDataCache.java b/core/src/main/java/org/apache/carbondata/common/iudprocessor/cache/BlockletLevelDeleteDeltaDataCache.java
index 9ecd40f..acc00f0 100644
--- a/core/src/main/java/org/apache/carbondata/common/iudprocessor/cache/BlockletLevelDeleteDeltaDataCache.java
+++ b/core/src/main/java/org/apache/carbondata/common/iudprocessor/cache/BlockletLevelDeleteDeltaDataCache.java
@@ -1,9 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.carbondata.common.iudprocessor.cache;
 
 import org.roaringbitmap.RoaringBitmap;
 
 /**
- * Created by S71955 on 06-10-2016.
+ * This class maintains delete delta data cache of each blocklet along with the block timestamp
  */
 public class BlockletLevelDeleteDeltaDataCache {
   private RoaringBitmap deleteDelataDataCache;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/BlockMappingVO.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/BlockMappingVO.java b/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/BlockMappingVO.java
index 2be8b56..ab43b6b 100644
--- a/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/BlockMappingVO.java
+++ b/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/BlockMappingVO.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.carbondata.common.iudprocessor.iuddata;
 
 import java.util.Map;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/BlockletDeleteDeltaCacheLoader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/BlockletDeleteDeltaCacheLoader.java b/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/BlockletDeleteDeltaCacheLoader.java
index 1cd3459..5b380c2 100644
--- a/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/BlockletDeleteDeltaCacheLoader.java
+++ b/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/BlockletDeleteDeltaCacheLoader.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.carbondata.common.iudprocessor.iuddata;
 
 import org.apache.carbondata.common.iudprocessor.cache.BlockletLevelDeleteDeltaDataCache;
@@ -7,6 +24,10 @@ import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier;
 import org.apache.carbondata.core.carbon.datastore.DataRefNode;
 import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager;
 
+/**
+ * This class is responsible for loading delete delta file cache based on
+ * blocklet id of a particular block
+ */
 public class BlockletDeleteDeltaCacheLoader implements DeleteDeltaCacheLoaderIntf {
   private String blockletID;
   private DataRefNode blockletNode;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/DeleteDeltaCacheLoaderIntf.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/DeleteDeltaCacheLoaderIntf.java b/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/DeleteDeltaCacheLoaderIntf.java
index 79a3c14..2455910 100644
--- a/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/DeleteDeltaCacheLoaderIntf.java
+++ b/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/DeleteDeltaCacheLoaderIntf.java
@@ -1,7 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.carbondata.common.iudprocessor.iuddata;
 
 /**
- * Created by S71955 on 06-10-2016.
+ * This interface holds all methods required to load delete delta file data to cache
  */
 public interface DeleteDeltaCacheLoaderIntf {
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/DeleteDeltaDataUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/DeleteDeltaDataUtil.java b/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/DeleteDeltaDataUtil.java
deleted file mode 100644
index 32ac31d..0000000
--- a/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/DeleteDeltaDataUtil.java
+++ /dev/null
@@ -1,44 +0,0 @@
-package org.apache.carbondata.common.iudprocessor.iuddata;
-
-import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier;
-import org.apache.carbondata.core.carbon.metadata.CarbonMetadata;
-import org.apache.carbondata.core.carbon.metadata.schema.table.CarbonTable;
-import org.apache.carbondata.core.load.LoadMetadataDetails;
-import org.apache.carbondata.core.updatestatus.SegmentStatusManager;
-
-/**
- * Created by S71955 on 06-10-2016.
- */
-public class DeleteDeltaDataUtil {
-
-  private DeleteDeltaDataUtil() {
-
-  }
-
-  public static int transformDataToNumericType(String deleteDeltadata) {
-    return Integer.parseInt(
-        deleteDeltadata.substring(deleteDeltadata.lastIndexOf('.'), deleteDeltadata.length()));
-  }
-
-  /**
-   * This method will verify whether any segment is updated or not as part of
-   * delete/update query.
-   * @param segmentStatusManager
-   * @param tableIdentifier
-   * @return boolean
-   */
-  public static boolean isSegmentStatusUpdated(SegmentStatusManager segmentStatusManager,
-      AbsoluteTableIdentifier tableIdentifier, String segmentID) {
-    CarbonTable table = CarbonMetadata.getInstance()
-        .getCarbonTable(tableIdentifier.getCarbonTableIdentifier().getTableUniqueName());
-    LoadMetadataDetails[] loadMetadataDetails =
-        segmentStatusManager.readLoadMetadata(table.getMetaDataFilepath());
-    for (LoadMetadataDetails loadMetadataDetail : loadMetadataDetails) {
-      if (segmentID.equals(loadMetadataDetail.getLoadName()) && "TRUE"
-          .equalsIgnoreCase(loadMetadataDetail.getIsDeleted())) {
-        return true;
-      }
-    }
-    return false;
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/RowCountDetailsVO.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/RowCountDetailsVO.java b/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/RowCountDetailsVO.java
index d0c74c5..7b5cc31 100644
--- a/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/RowCountDetailsVO.java
+++ b/core/src/main/java/org/apache/carbondata/common/iudprocessor/iuddata/RowCountDetailsVO.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.carbondata.common.iudprocessor.iuddata;
 
 import java.io.Serializable;
@@ -26,13 +43,19 @@ public class RowCountDetailsVO implements Serializable {
     return deletedRowsInBlock;
   }
 
-  @Override public boolean equals(Object o) {
-    if (this == o) return true;
-    if (o == null || getClass() != o.getClass()) return false;
+  @Override public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (obj == null || getClass() != obj.getClass()) {
+      return false;
+    }
 
-    RowCountDetailsVO that = (RowCountDetailsVO) o;
+    RowCountDetailsVO that = (RowCountDetailsVO) obj;
 
-    if (totalNumberOfRows != that.totalNumberOfRows) return false;
+    if (totalNumberOfRows != that.totalNumberOfRows) {
+      return false;
+    }
     return deletedRowsInBlock == that.deletedRowsInBlock;
 
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/main/java/org/apache/carbondata/core/carbon/datastore/SegmentTaskIndexStore.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/SegmentTaskIndexStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/SegmentTaskIndexStore.java
index a0a367a..4d67dd6 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/SegmentTaskIndexStore.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/SegmentTaskIndexStore.java
@@ -84,7 +84,7 @@ public class SegmentTaskIndexStore
 
   @Override
   public SegmentTaskIndexWrapper get(TableSegmentUniqueIdentifier tableSegmentUniqueIdentifier)
-      throws CarbonUtilException {
+      throws IOException {
     SegmentTaskIndexWrapper segmentTaskIndexWrapper = null;
     try {
       segmentTaskIndexWrapper =
@@ -92,19 +92,15 @@ public class SegmentTaskIndexStore
               tableSegmentUniqueIdentifier.getAbsoluteTableIdentifier(),
               tableSegmentUniqueIdentifier);
     } catch (IndexBuilderException e) {
-      throw new CarbonUtilException(e.getMessage(), e);
+      throw new IOException(e.getMessage(), e);
     } catch (Throwable e) {
-      throw new CarbonUtilException("Problem in loading segment block.", e);
-    }
-
-    if (null != segmentTaskIndexWrapper) {
-      segmentTaskIndexWrapper.incrementAccessCount();
+      throw new IOException("Problem in loading segment block.", e);
     }
     return segmentTaskIndexWrapper;
   }
 
   @Override public List<SegmentTaskIndexWrapper> getAll(
-      List<TableSegmentUniqueIdentifier> tableSegmentUniqueIdentifiers) throws CarbonUtilException {
+      List<TableSegmentUniqueIdentifier> tableSegmentUniqueIdentifiers) throws IOException {
     List<SegmentTaskIndexWrapper> segmentTaskIndexWrappers =
         new ArrayList<>(tableSegmentUniqueIdentifiers.size());
     try {
@@ -115,7 +111,7 @@ public class SegmentTaskIndexStore
       for (SegmentTaskIndexWrapper segmentTaskIndexWrapper : segmentTaskIndexWrappers) {
         segmentTaskIndexWrapper.clear();
       }
-      throw new CarbonUtilException("Problem in loading segment blocks.", e);
+      throw new IOException("Problem in loading segment blocks.", e);
     }
     return segmentTaskIndexWrappers;
   }
@@ -146,7 +142,7 @@ public class SegmentTaskIndexStore
   }
 
   /**
-   *
+   * returns block timestamp value from the given task
    * @param taskKey
    * @param listOfUpdatedFactFiles
    * @return

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeLeafNode.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeLeafNode.java
index 80024ca..eef239e 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/AbstractBTreeLeafNode.java
@@ -27,7 +27,6 @@ import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChun
 import org.apache.carbondata.core.carbon.datastore.chunk.MeasureColumnDataChunk;
 import org.apache.carbondata.core.datastorage.store.FileHolder;
 
-
 /**
  * Non leaf node abstract class
  */
@@ -228,8 +227,9 @@ public abstract class AbstractBTreeLeafNode implements BTreeNode {
     // measure chunks
     return null;
   }
+
   /**
-   * @return the segmentProperties
+   * @param deleteDeltaDataCache
    */
   public void setDeleteDeltaDataCache(BlockletLevelDeleteDeltaDataCache deleteDeltaDataCache) {
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeNonLeafNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeNonLeafNode.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeNonLeafNode.java
index bddebf8..4449581 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeNonLeafNode.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/impl/btree/BTreeNonLeafNode.java
@@ -29,8 +29,6 @@ import org.apache.carbondata.core.carbon.datastore.chunk.MeasureColumnDataChunk;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.core.datastorage.store.FileHolder;
 
-
-
 /**
  * No leaf node of a b+tree class which will keep the matadata(start key) of the
  * leaf node

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonImplicitDimension.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonImplicitDimension.java b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonImplicitDimension.java
index 2ad5346..ad05702 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonImplicitDimension.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/metadata/schema/table/column/CarbonImplicitDimension.java
@@ -27,6 +27,9 @@ import java.util.UUID;
 import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
 import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
 
+/**
+ * This instance will be created for implicit column like tupleid.
+ */
 public class CarbonImplicitDimension extends CarbonDimension {
   /**
    * serialization version

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java b/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
index e57fae0..a5c9289 100644
--- a/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
+++ b/core/src/main/java/org/apache/carbondata/core/carbon/path/CarbonTablePath.java
@@ -29,8 +29,6 @@ import static org.apache.carbondata.core.constants.CarbonCommonConstants.INVALID
 
 import org.apache.hadoop.fs.Path;
 
-
-
 /**
  * Helps to get Table content paths.
  */
@@ -333,19 +331,6 @@ public class CarbonTablePath extends Path {
     }
   }
 
-  public CarbonFile[] getCarbonDeleteDeltaFile(final String segmentId, final String partitionId) {
-    String segmentDir = getSegmentDir(partitionId, segmentId);
-    CarbonFile carbonFile =
-        FileFactory.getCarbonFile(segmentDir, FileFactory.getFileType(segmentDir));
-
-    CarbonFile[] files = carbonFile.listFiles(new CarbonFileFilter() {
-      @Override public boolean accept(CarbonFile file) {
-        return file.getName().endsWith(DELETE_DELTA_FILE_EXT);
-      }
-    });
-    return files;
-  }
-
   /**
    * Gets absolute path of data file
    *
@@ -402,15 +387,6 @@ public class CarbonTablePath extends Path {
     return getFactDir() + File.separator + PARTITION_PREFIX + partitionId;
   }
 
-  private String getAggSegmentDir(String aggTableID, String partitionId, String segmentId) {
-    return getAggPartitionDir(aggTableID, partitionId) + File.separator + SEGMENT_PREFIX
-        + segmentId;
-  }
-
-  private String getAggPartitionDir(String aggTableID, String partitionId) {
-    return getAggregateTableDir(aggTableID) + File.separator + PARTITION_PREFIX + partitionId;
-  }
-
   private String getMetaDataDir() {
     return tablePath + File.separator + METADATA_DIR;
   }
@@ -419,10 +395,6 @@ public class CarbonTablePath extends Path {
     return tablePath + File.separator + FACT_DIR;
   }
 
-  private String getAggregateTableDir(String aggTableId) {
-    return tablePath + File.separator + AGGREGATE_TABLE_PREFIX + aggTableId;
-  }
-
   @Override public boolean equals(Object o) {
     if (!(o instanceof CarbonTablePath)) {
       return false;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/main/java/org/apache/carbondata/core/load/LoadMetadataDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/load/LoadMetadataDetails.java b/core/src/main/java/org/apache/carbondata/core/load/LoadMetadataDetails.java
index f067f07..2861b20 100644
--- a/core/src/main/java/org/apache/carbondata/core/load/LoadMetadataDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/load/LoadMetadataDetails.java
@@ -118,7 +118,6 @@ public class LoadMetadataDetails implements Serializable {
    * @param modificationOrdeletionTimesStamp the modificationOrdeletionTimesStamp to set
    */
   public void setModificationOrdeletionTimesStamp(long modificationOrdeletionTimesStamp) {
-
     this.modificationOrdeletionTimesStamp =
         getTimeStampConvertion(modificationOrdeletionTimesStamp);
   }
@@ -256,8 +255,7 @@ public class LoadMetadataDetails implements Serializable {
 
   /**
    * Return true if it is a major compacted segment.
-   *
-   * @return
+   * @return majorCompacted
    */
   public String isMajorCompacted() {
     return majorCompacted;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java
index 8697dbb..dac2d43 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteDeltaFileReaderImpl.java
@@ -47,17 +47,15 @@ public class CarbonDeleteDeltaFileReaderImpl implements CarbonDeleteDeltaFileRea
 
   private FileFactory.FileType fileType;
 
-  DataInputStream dataInputStream = null;
+  private DataInputStream dataInputStream = null;
 
-  BufferedReader buffReader = null;
+  private BufferedReader buffReader = null;
 
-  InputStreamReader inputStream = null;
+  private InputStreamReader inputStream = null;
 
   private static final int DEFAULT_BUFFER_SIZE = 258;
 
   /**
-   * Constructor
-   *
    * @param filePath
    * @param fileType
    */
@@ -91,6 +89,11 @@ public class CarbonDeleteDeltaFileReaderImpl implements CarbonDeleteDeltaFileRea
     return sw.toString();
   }
 
+  /**
+   * Reads delete delta file (json file) and returns DeleteDeltaBlockDetails
+   * @return DeleteDeltaBlockDetails
+   * @throws IOException
+   */
   @Override public DeleteDeltaBlockDetails readJson() throws IOException {
     Gson gsonObjectToRead = new Gson();
     DataInputStream dataInputStream = null;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteFilesDataReader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteFilesDataReader.java b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteFilesDataReader.java
index 38f9a0c..a0922ec 100644
--- a/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteFilesDataReader.java
+++ b/core/src/main/java/org/apache/carbondata/core/reader/CarbonDeleteFilesDataReader.java
@@ -53,9 +53,6 @@ public class CarbonDeleteFilesDataReader {
    */
   protected int thread_pool_size;
 
-  /**
-   * Constructor
-   */
   public CarbonDeleteFilesDataReader() {
     initThreadPoolSize();
   }
@@ -119,6 +116,13 @@ public class CarbonDeleteFilesDataReader {
 
   }
 
+  /**
+   * returns delete delta file details for the specified block name
+   * @param deltaFiles
+   * @param blockName
+   * @return DeleteDeltaBlockDetails
+   * @throws Exception
+   */
   public DeleteDeltaBlockDetails getCompactedDeleteDeltaFileFromBlock(List<String> deltaFiles,
       String blockName) throws Exception {
     // get the data.

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java b/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java
index 89fc56c..2c7abbf 100644
--- a/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/update/CarbonUpdateUtil.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 package org.apache.carbondata.core.update;
 
@@ -23,12 +39,11 @@ import org.apache.carbondata.core.updatestatus.SegmentStatusManager;
 import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager;
 import org.apache.carbondata.core.util.CarbonProperties;
 import org.apache.carbondata.core.util.CarbonUtil;
-import org.apache.carbondata.core.util.CarbonUtilException;
 import org.apache.carbondata.locks.ICarbonLock;
 
 
 /**
- *
+ * This class contains all update utility methods
  */
 public class CarbonUpdateUtil {
 
@@ -158,8 +173,12 @@ public class CarbonUpdateUtil {
   }
 
   /**
+   *
    * @param updatedSegmentsList
    * @param table
+   * @param updatedTimeStamp
+   * @param isTimestampUpdationRequired
+   * @param segmentsToBeDeleted
    * @return
    */
   public static boolean updateTableMetadataStatus(Set<String> updatedSegmentsList,
@@ -300,7 +319,9 @@ public class CarbonUpdateUtil {
       // deleting the files of a segment.
       try {
         CarbonUtil.deleteFoldersAndFilesSilent(toBeDeleted);
-      } catch (CarbonUtilException e) {
+      } catch (IOException e) {
+        LOGGER.error("Exception in deleting the delta files." + e);
+      } catch (InterruptedException e) {
         LOGGER.error("Exception in deleting the delta files." + e);
       }
     }
@@ -660,7 +681,9 @@ public class CarbonUpdateUtil {
       try {
         LOGGER.info("deleting the invalid file : " + invalidFile.getName());
         CarbonUtil.deleteFoldersAndFiles(invalidFile);
-      } catch (CarbonUtilException e) {
+      } catch (IOException e) {
+        LOGGER.error("error in clean up of compacted files." + e.getMessage());
+      } catch (InterruptedException e) {
         LOGGER.error("error in clean up of compacted files." + e.getMessage());
       }
     }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/main/java/org/apache/carbondata/core/update/DeleteDeltaBlockDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/update/DeleteDeltaBlockDetails.java b/core/src/main/java/org/apache/carbondata/core/update/DeleteDeltaBlockDetails.java
index f8f56fb..644c5c1 100644
--- a/core/src/main/java/org/apache/carbondata/core/update/DeleteDeltaBlockDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/update/DeleteDeltaBlockDetails.java
@@ -88,6 +88,7 @@ public class DeleteDeltaBlockDetails implements Serializable {
       blocklet.addDeletedRow(CarbonUpdateUtil.getIntegerValue(offset));
       return addBlockletDetails(blocklet);
     } catch (Exception e) {
+      LOGGER.debug(e.getMessage());
       throw e;
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/main/java/org/apache/carbondata/core/update/DeleteDeltaBlockletDetails.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/update/DeleteDeltaBlockletDetails.java b/core/src/main/java/org/apache/carbondata/core/update/DeleteDeltaBlockletDetails.java
index f2ee86c..d3976f4 100644
--- a/core/src/main/java/org/apache/carbondata/core/update/DeleteDeltaBlockletDetails.java
+++ b/core/src/main/java/org/apache/carbondata/core/update/DeleteDeltaBlockletDetails.java
@@ -27,7 +27,7 @@ import org.apache.carbondata.common.logging.LogService;
 import org.apache.carbondata.common.logging.LogServiceFactory;
 
 /**
- * This class stores the block details of delete delta file
+ * This class stores the blocklet details of delete delta file
  */
 public class DeleteDeltaBlockletDetails implements Serializable {
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java
index 1622f3c..cfba54d 100644
--- a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentStatusManager.java
@@ -129,16 +129,12 @@ public class SegmentStatusManager {
     LoadMetadataDetails[] loadFolderDetailsArray;
     try {
       if (FileFactory.isFileExist(dataPath, FileFactory.getFileType(dataPath))) {
-
         dataInputStream = fileOperation.openForRead();
-
         BufferedReader buffReader =
                 new BufferedReader(new InputStreamReader(dataInputStream, "UTF-8"));
-
         loadFolderDetailsArray = gsonObjectToRead.fromJson(buffReader, LoadMetadataDetails[].class);
         //just directly iterate Array
         List<LoadMetadataDetails> loadFolderDetails = Arrays.asList(loadFolderDetailsArray);
-
         for (LoadMetadataDetails loadMetadataDetails : loadFolderDetails) {
           if (CarbonCommonConstants.STORE_LOADSTATUS_SUCCESS
                   .equalsIgnoreCase(loadMetadataDetails.getLoadStatus())
@@ -173,7 +169,6 @@ public class SegmentStatusManager {
                   .equalsIgnoreCase(loadMetadataDetails.getLoadStatus()))) {
             listOfInvalidSegments.add(loadMetadataDetails.getLoadName());
           }
-
         }
       }
     } catch (IOException e) {
@@ -181,7 +176,6 @@ public class SegmentStatusManager {
       throw e;
     } finally {
       try {
-
         if (null != dataInputStream) {
           dataInputStream.close();
         }
@@ -189,7 +183,6 @@ public class SegmentStatusManager {
         LOG.error(e);
         throw e;
       }
-
     }
     return new ValidAndInvalidSegmentsInfo(listOfValidSegments, listOfValidUpdatedSegments,
             listOfInvalidSegments);
@@ -209,7 +202,6 @@ public class SegmentStatusManager {
     String metadataFileName = tableFolderPath + CarbonCommonConstants.FILE_SEPARATOR
         + CarbonCommonConstants.LOADMETADATA_FILENAME;
     LoadMetadataDetails[] listOfLoadFolderDetailsArray;
-
     AtomicFileOperations fileOperation =
         new AtomicFileOperationsImpl(metadataFileName, FileFactory.getFileType(metadataFileName));
 
@@ -485,7 +477,6 @@ public class SegmentStatusManager {
         invalidLoadIds.add(loadId);
         return invalidLoadIds;
       }
-
     }
     return invalidLoadIds;
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java
index a946383..3071a98 100644
--- a/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java
+++ b/core/src/main/java/org/apache/carbondata/core/updatestatus/SegmentUpdateStatusManager.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.carbondata.core.updatestatus;
 
 import java.io.*;
@@ -32,7 +49,7 @@ import org.apache.carbondata.locks.LockUsage;
 import com.google.gson.Gson;
 
 /**
- * Manages Segment & block status of carbon table
+ * Manages Segment & block status of carbon table for Delete operation
  */
 public class SegmentUpdateStatusManager {
 
@@ -49,8 +66,6 @@ public class SegmentUpdateStatusManager {
   private Map<String, SegmentUpdateDetails> blockAndDetailsMap;
 
   /**
-   * Constructor
-   *
    * @param absoluteTableIdentifier
    */
   public SegmentUpdateStatusManager(AbsoluteTableIdentifier absoluteTableIdentifier) {
@@ -680,10 +695,8 @@ public class SegmentUpdateStatusManager {
     return listOfSegmentUpdateDetailsArray;
   }
 
-
   /**
-   *
-   * @return
+   * @return updateStatusFileName
    */
   private String getUpdatedStatusIdentifier() {
     SegmentStatusManager ssm = new SegmentStatusManager(absoluteTableIdentifier);

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
index d0895df..8a5a1c6 100644
--- a/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
+++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonProperties.java
@@ -546,6 +546,11 @@ public final class CarbonProperties {
         String.valueOf(executorStartUpTimeOut));
     LOGGER.info("Executor start up wait time: " + executorStartUpTimeOut);
   }
+
+  /**
+   * Returns configured update deleta files value for IUD compaction
+   * @return numberOfDeltaFilesThreshold
+   */
   public int getNoUpdateDeltaFilesThresholdForIUDCompaction() {
     int numberOfDeltaFilesThreshold;
     try {
@@ -572,6 +577,10 @@ public final class CarbonProperties {
     return numberOfDeltaFilesThreshold;
   }
 
+  /**
+   * Returns configured delete deleta files value for IUD compaction
+   * @return numberOfDeltaFilesThreshold
+   */
   public int getNoDeleteDeltaFilesThresholdForIUDCompaction() {
     int numberOfDeltaFilesThreshold;
     try {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java
index 2ce4914..1683f24 100644
--- a/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/writer/CarbonDeleteDeltaWriterImpl.java
@@ -51,8 +51,6 @@ public class CarbonDeleteDeltaWriterImpl implements CarbonDeleteDeltaWriter {
   private DataOutputStream dataOutStream = null;
 
   /**
-   * Constructor
-   *
    * @param filePath
    * @param fileType
    */

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/main/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedResultCollector.java b/core/src/main/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedResultCollector.java
index 81764cb..cd21df8 100644
--- a/core/src/main/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/scan/collector/impl/DictionaryBasedResultCollector.java
@@ -25,8 +25,6 @@ import java.util.List;
 import java.util.Map;
 
 import org.apache.carbondata.common.iudprocessor.cache.BlockletLevelDeleteDeltaDataCache;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.carbon.metadata.datatype.DataType;
 import org.apache.carbondata.core.carbon.metadata.encoder.Encoding;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/main/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollector.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollector.java b/core/src/main/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollector.java
index 6bb4ca7..b61ef97 100644
--- a/core/src/main/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollector.java
+++ b/core/src/main/java/org/apache/carbondata/scan/collector/impl/RawBasedResultCollector.java
@@ -23,8 +23,6 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.carbondata.common.iudprocessor.cache.BlockletLevelDeleteDeltaDataCache;
-import org.apache.carbondata.common.logging.LogService;
-import org.apache.carbondata.common.logging.LogServiceFactory;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
 import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
 import org.apache.carbondata.scan.model.QueryMeasure;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/main/java/org/apache/carbondata/scan/executor/impl/AbstractQueryExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/impl/AbstractQueryExecutor.java b/core/src/main/java/org/apache/carbondata/scan/executor/impl/AbstractQueryExecutor.java
index 3c05117..9976db3 100644
--- a/core/src/main/java/org/apache/carbondata/scan/executor/impl/AbstractQueryExecutor.java
+++ b/core/src/main/java/org/apache/carbondata/scan/executor/impl/AbstractQueryExecutor.java
@@ -118,15 +118,11 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
     // remove the invalid table blocks, block which is deleted or compacted
     cache.removeTableBlocks(queryModel.getInvalidSegmentIds(),
         queryModel.getAbsoluteTableIdentifier());
-    try {
-      List<TableBlockUniqueIdentifier> tableBlockUniqueIdentifiers =
-          prepareTableBlockUniqueIdentifier(queryModel.getTableBlockInfos(),
-              queryModel.getAbsoluteTableIdentifier());
-      cache.removeTableBlocksIfHorizontalCompactionDone(queryModel);
-      queryProperties.dataBlocks = cache.getAll(tableBlockUniqueIdentifiers);
-    } catch (CarbonUtilException e) {
-      throw new QueryExecutionException(e);
-    }
+    List<TableBlockUniqueIdentifier> tableBlockUniqueIdentifiers =
+        prepareTableBlockUniqueIdentifier(queryModel.getTableBlockInfos(),
+            queryModel.getAbsoluteTableIdentifier());
+    cache.removeTableBlocksIfHorizontalCompactionDone(queryModel);
+    queryProperties.dataBlocks = cache.getAll(tableBlockUniqueIdentifiers);
     queryStatistic
         .addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_EXECUTOR, System.currentTimeMillis());
     queryProperties.queryStatisticsRecorder.recordStatistics(queryStatistic);
@@ -288,6 +284,7 @@ public abstract class AbstractQueryExecutor<E> implements QueryExecutor<E> {
         .setTotalNumberDimensionBlock(segmentProperties.getDimensionOrdinalToBlockMapping().size());
     blockExecutionInfo
         .setTotalNumberOfMeasureBlock(segmentProperties.getMeasuresOrdinalToBlockMapping().size());
+    blockExecutionInfo.setAbsoluteTableIdentifier(queryModel.getAbsoluteTableIdentifier());
     blockExecutionInfo.setComplexDimensionInfoMap(QueryUtil
         .getComplexDimensionsMap(updatedQueryDimension,
             segmentProperties.getDimensionOrdinalToBlockMapping(),

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/main/java/org/apache/carbondata/scan/executor/infos/BlockExecutionInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/executor/infos/BlockExecutionInfo.java b/core/src/main/java/org/apache/carbondata/scan/executor/infos/BlockExecutionInfo.java
index 524ed76..88b5ca3 100644
--- a/core/src/main/java/org/apache/carbondata/scan/executor/infos/BlockExecutionInfo.java
+++ b/core/src/main/java/org/apache/carbondata/scan/executor/infos/BlockExecutionInfo.java
@@ -70,6 +70,8 @@ public class BlockExecutionInfo {
    */
   private IndexKey endKey;
 
+  private String blockId;
+
   /**
    * masked byte for block which will be used to unpack the fixed length key,
    * this will be used for updating the older block key with new block key
@@ -591,4 +593,12 @@ public class BlockExecutionInfo {
   public void setVectorBatchCollector(boolean vectorBatchCollector) {
     this.vectorBatchCollector = vectorBatchCollector;
   }
+
+  public String getBlockId() {
+    return blockId;
+  }
+
+  public void setBlockId(String blockId) {
+    this.blockId = blockId;
+  }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/main/java/org/apache/carbondata/scan/filter/executer/ImplicitColumnFilterExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/ImplicitColumnFilterExecutor.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/ImplicitColumnFilterExecutor.java
deleted file mode 100644
index de0eb5f..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/ImplicitColumnFilterExecutor.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.filter.executer;
-
-import java.util.BitSet;
-
-import org.apache.carbondata.core.carbon.datastore.DataRefNode;
-
-/**
- * Implementation of this interface will involve block
- * and blocklet pruning based on block/blocklet id where
- * the filter values are present.
- */
-public interface ImplicitColumnFilterExecutor extends FilterExecuter {
-
-  /**
-   * This method will validate the block or blocklet id with the implicit
-   * column filter value list and decide whether the required block or
-   * blocklet has to be scanned for the data or not
-   *
-   * @param dataRefNode
-   * @param uniqueBlockPath
-   * @return
-   */
-  BitSet isFilterValuesPresentInBlockOrBlocklet(DataRefNode dataRefNode, String uniqueBlockPath);
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/main/java/org/apache/carbondata/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
deleted file mode 100644
index eaf45e9..0000000
--- a/core/src/main/java/org/apache/carbondata/scan/filter/executer/ImplicitIncludeFilterExecutorImpl.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.carbondata.scan.filter.executer;
-
-import java.util.BitSet;
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.carbondata.core.carbon.datastore.DataRefNode;
-import org.apache.carbondata.core.carbon.datastore.block.SegmentProperties;
-import org.apache.carbondata.core.carbon.path.CarbonTablePath;
-import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
-import org.apache.carbondata.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
-import org.apache.carbondata.scan.processor.BlocksChunkHolder;
-
-/**
- * This class will implement the blocklet and block pruning logic based
- * on the implicit column filter values
- */
-public class ImplicitIncludeFilterExecutorImpl implements ImplicitColumnFilterExecutor {
-
-  private final DimColumnResolvedFilterInfo dimColumnEvaluatorInfo;
-  private final SegmentProperties segmentProperties;
-  /**
-   * List with implicit column filter values
-   */
-  private Set<String> implicitFilterValueList;
-
-  public ImplicitIncludeFilterExecutorImpl(DimColumnResolvedFilterInfo dimColumnEvaluatorInfo,
-      SegmentProperties segmentProperties) {
-    this.dimColumnEvaluatorInfo = dimColumnEvaluatorInfo;
-    this.segmentProperties = segmentProperties;
-    this.implicitFilterValueList =
-        new HashSet(dimColumnEvaluatorInfo.getFilterValues().getImplicitColumnFilterList());
-  }
-
-
-  public BitSet applyFilter(BlocksChunkHolder blockChunkHolder)
-      throws FilterUnsupportedException {
-    return setBitSetForCompleteDimensionData(blockChunkHolder.getDataBlock().nodeSize());
-  }
-
-  public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
-    return null;
-  }
-
-  public BitSet isFilterValuesPresentInBlockOrBlocklet(DataRefNode dataRefNode,
-                                                       String uniqueBlockPath) {
-    BitSet bitSet = new BitSet(1);
-    boolean isScanRequired = false;
-    if (uniqueBlockPath.endsWith(".carbondata")) {
-      String shortBlockId = CarbonTablePath.getShortBlockId(uniqueBlockPath);
-      for (String blockletId : implicitFilterValueList) {
-        if (blockletId.startsWith(shortBlockId)) {
-          isScanRequired = true;
-          break;
-        }
-      }
-    } else if (implicitFilterValueList.contains(uniqueBlockPath)) {
-      isScanRequired = true;
-    }
-    if (isScanRequired) {
-      bitSet.set(0);
-    }
-    return bitSet;
-  }
-
-  /**
-   * For implicit column filtering, complete data need to be selected. As it is a special case
-   * no data need to be discarded, implicit filtering is only for slecting block and blocklets
-   *
-   * @param numberOfRows
-   * @return
-   */
-  private BitSet setBitSetForCompleteDimensionData(int numberOfRows) {
-    BitSet bitSet = new BitSet();
-    bitSet.set(0, numberOfRows, true);
-    return bitSet;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/main/java/org/apache/carbondata/scan/scanner/AbstractBlockletScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/scanner/AbstractBlockletScanner.java b/core/src/main/java/org/apache/carbondata/scan/scanner/AbstractBlockletScanner.java
index f97641e..8f7b269 100644
--- a/core/src/main/java/org/apache/carbondata/scan/scanner/AbstractBlockletScanner.java
+++ b/core/src/main/java/org/apache/carbondata/scan/scanner/AbstractBlockletScanner.java
@@ -26,7 +26,6 @@ import org.apache.carbondata.core.carbon.querystatistics.QueryStatistic;
 import org.apache.carbondata.core.carbon.querystatistics.QueryStatisticsConstants;
 import org.apache.carbondata.core.carbon.querystatistics.QueryStatisticsModel;
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
-import org.apache.carbondata.scan.executor.exception.QueryExecutionException;
 import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
 import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
 import org.apache.carbondata.scan.processor.BlocksChunkHolder;
@@ -73,10 +72,10 @@ public abstract class AbstractBlockletScanner implements BlockletScanner {
                     validScannedBlockletStatistic.getCount() + 1);
     queryStatisticsModel.getRecorder().recordStatistics(validScannedBlockletStatistic);
     scannedResult.reset();
-    scannedResult.setBlockletId(
-        blockExecutionInfo.getBlockId() + CarbonCommonConstants.FILE_SEPARATOR
-            + blocksChunkHolder.getDataBlock().nodeNumber());
     scannedResult.setNumberOfRows(blocksChunkHolder.getDataBlock().nodeSize());
+    scannedResult.setBlockletId(
+              blockExecutionInfo.getBlockId() + CarbonCommonConstants.FILE_SEPARATOR
+                      + blocksChunkHolder.getDataBlock().nodeNumber());
     scannedResult.setDimensionChunks(blocksChunkHolder.getDataBlock()
         .getDimensionChunks(blocksChunkHolder.getFileReader(),
             blockExecutionInfo.getAllSelectedDimensionBlocksIndexes()));

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/main/java/org/apache/carbondata/scan/scanner/impl/FilterScanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/scan/scanner/impl/FilterScanner.java b/core/src/main/java/org/apache/carbondata/scan/scanner/impl/FilterScanner.java
index 76c74ad..850a03f 100644
--- a/core/src/main/java/org/apache/carbondata/scan/scanner/impl/FilterScanner.java
+++ b/core/src/main/java/org/apache/carbondata/scan/scanner/impl/FilterScanner.java
@@ -35,7 +35,6 @@ import org.apache.carbondata.core.util.CarbonProperties;
 import org.apache.carbondata.scan.executor.infos.BlockExecutionInfo;
 import org.apache.carbondata.scan.expression.exception.FilterUnsupportedException;
 import org.apache.carbondata.scan.filter.executer.FilterExecuter;
-import org.apache.carbondata.scan.filter.executer.ImplicitColumnFilterExecutor;
 import org.apache.carbondata.scan.processor.BlocksChunkHolder;
 import org.apache.carbondata.scan.result.AbstractScannedResult;
 import org.apache.carbondata.scan.result.impl.FilterQueryScannedResult;
@@ -117,17 +116,9 @@ public class FilterScanner extends AbstractBlockletScanner {
             .getDataBlock().nodeNumber());
     // apply min max
     if (isMinMaxEnabled) {
-      BitSet bitSet = null;
-      // check for implicit include filter instance
-      if (filterExecuter instanceof ImplicitColumnFilterExecutor) {
-        bitSet = ((ImplicitColumnFilterExecutor) filterExecuter)
-            .isFilterValuesPresentInBlockOrBlocklet(blocksChunkHolder.getDataBlock(),
-                scannedResult.getBlockletId());
-      } else {
-        bitSet = this.filterExecuter
-            .isScanRequired(blocksChunkHolder.getDataBlock().getColumnsMaxValue(),
-                blocksChunkHolder.getDataBlock().getColumnsMinValue());
-      }
+      BitSet bitSet = this.filterExecuter
+          .isScanRequired(blocksChunkHolder.getDataBlock().getColumnsMaxValue(),
+              blocksChunkHolder.getDataBlock().getColumnsMinValue());
       if (bitSet.isEmpty()) {
         scannedResult.setNumberOfRows(0);
         scannedResult.setIndexes(new int[0]);

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/core/src/test/java/org/apache/carbondata/core/load/LoadMetadataDetailsUnitTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/carbondata/core/load/LoadMetadataDetailsUnitTest.java b/core/src/test/java/org/apache/carbondata/core/load/LoadMetadataDetailsUnitTest.java
index 02366e5..0dbac8f 100644
--- a/core/src/test/java/org/apache/carbondata/core/load/LoadMetadataDetailsUnitTest.java
+++ b/core/src/test/java/org/apache/carbondata/core/load/LoadMetadataDetailsUnitTest.java
@@ -98,18 +98,6 @@ public class LoadMetadataDetailsUnitTest {
     assertEquals(true, result);
   }
 
-//  @Test public void testGetTimeStampWithEmptyTimeStamp() throws Exception {
-//    loadMetadataDetails.setLoadStartTime(0);
-//    Long result = loadMetadataDetails.getLoadStartTimeAsLong();
-//    assertNull(result);
-//  }
-//
-//  @Test public void testGetTimeStampWithParserException() throws Exception {
-//    loadMetadataDetails.setLoadStartTime(0);
-//    Long result = loadMetadataDetails.getLoadStartTimeAsLong();
-//    assertNull(result);
-//  }
-
   @Test public void testGetTimeStampWithDate() throws Exception {
     String date = "01-01-2016 00:00:00:000";
     long longVal = loadMetadataDetails.getTimeStamp(date);

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
index 108aafa..87f2e0a 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/CarbonInputFormat.java
@@ -332,7 +332,7 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
       FilterExpressionProcessor filterExpressionProcessor,
       AbsoluteTableIdentifier absoluteTableIdentifier, FilterResolverIntf resolver,
       String segmentId, CacheClient cacheClient, SegmentUpdateStatusManager updateStatusManager)
-      throws IndexBuilderException, IOException, CarbonUtilException {
+      throws IOException {
     QueryStatisticsRecorder recorder = CarbonTimeStatisticsFactory.createDriverRecorder();
     QueryStatistic statistic = new QueryStatistic();
     Map<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> segmentIndexMap =
@@ -341,22 +341,24 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
 
     List<DataRefNode> resultFilterredBlocks = new LinkedList<DataRefNode>();
 
-    // build result
-    for (AbstractIndex abstractIndex : segmentIndexMap.values()) {
-      List<DataRefNode> filterredBlocks;
-      // if no filter is given get all blocks from Btree Index
-      if (null == resolver) {
-        filterredBlocks = getDataBlocksOfIndex(abstractIndex);
-      } else {
-        // apply filter and get matching blocks
-        filterredBlocks = filterExpressionProcessor.getFilterredBlocks(
-            abstractIndex.getDataRefNode(),
-            resolver,
-            abstractIndex,
-            absoluteTableIdentifier
-        );
+    if (null != segmentIndexMap) {
+      // build result
+      for (AbstractIndex abstractIndex : segmentIndexMap.values()) {
+        List<DataRefNode> filterredBlocks;
+        // if no filter is given get all blocks from Btree Index
+        if (null == resolver) {
+          filterredBlocks = getDataBlocksOfIndex(abstractIndex);
+        } else {
+          // apply filter and get matching blocks
+          filterredBlocks = filterExpressionProcessor.getFilterredBlocks(
+                  abstractIndex.getDataRefNode(),
+                  resolver,
+                  abstractIndex,
+                  absoluteTableIdentifier
+          );
+        }
+        resultFilterredBlocks.addAll(filterredBlocks);
       }
-      resultFilterredBlocks.addAll(filterredBlocks);
     }
     statistic
         .addStatistics(QueryStatisticsConstants.LOAD_BLOCKS_DRIVER, System.currentTimeMillis());
@@ -452,8 +454,7 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
    */
   private Map<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> getSegmentAbstractIndexs(
       JobContext job, AbsoluteTableIdentifier absoluteTableIdentifier, String segmentId,
-      CacheClient cacheClient, SegmentUpdateStatusManager updateStatusManager)
-      throws IOException, IndexBuilderException, CarbonUtilException {
+      CacheClient cacheClient, SegmentUpdateStatusManager updateStatusManager) throws IOException {
     Map<SegmentTaskIndexStore.TaskBucketHolder, AbstractIndex> segmentIndexMap = null;
     SegmentTaskIndexWrapper segmentTaskIndexWrapper = null;
     List<String> updatedTaskList = null;
@@ -498,9 +499,16 @@ public class CarbonInputFormat<T> extends FileInputFormat<Void, T> {
     return segmentIndexMap;
   }
 
+  /**
+   * Get the row count of the Block and mapping of segment and Block count.
+   * @param job
+   * @param absoluteTableIdentifier
+   * @return
+   * @throws IOException
+   * @throws KeyGenException
+   */
   public BlockMappingVO getBlockRowCount(JobContext job,
-                                         AbsoluteTableIdentifier absoluteTableIdentifier)
-          throws IOException, CarbonUtilException, IndexBuilderException, KeyGenException {
+      AbsoluteTableIdentifier absoluteTableIdentifier) throws IOException, KeyGenException {
     CacheClient cacheClient = new CacheClient(absoluteTableIdentifier.getStorePath());
     try {
       SegmentUpdateStatusManager updateStatusManager =

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
index d81aca8..145cd85 100644
--- a/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
+++ b/hadoop/src/main/java/org/apache/carbondata/hadoop/util/BlockLevelTraverser.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.carbondata.hadoop.util;
 
 import java.util.Map;
@@ -17,7 +34,7 @@ import org.apache.carbondata.scan.filter.FilterUtil;
 
 
 /**
- *
+ * Block level traverser
  */
 public class BlockLevelTraverser {
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonInputMapperTest.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonInputMapperTest.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonInputMapperTest.java
index 2ae8e23..31a7d04 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonInputMapperTest.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/ft/CarbonInputMapperTest.java
@@ -57,58 +57,58 @@ public class CarbonInputMapperTest extends TestCase {
   }
 
   @Test public void testInputFormatMapperReadAllRowsAndColumns() throws Exception {
-//    try {
-//      String outPath = "target/output";
-//      CarbonProjection carbonProjection = new CarbonProjection();
-//      carbonProjection.addColumn("ID");
-//      carbonProjection.addColumn("date");
-//      carbonProjection.addColumn("country");
-//      carbonProjection.addColumn("name");
-//      carbonProjection.addColumn("phonetype");
-//      carbonProjection.addColumn("serialname");
-//      carbonProjection.addColumn("salary");
-//      runJob(outPath, carbonProjection, null);
-//      Assert.assertEquals("Count lines are not matching", 1000, countTheLines(outPath));
-//      Assert.assertEquals("Column count are not matching", 7, countTheColumns(outPath));
-//    } catch (Exception e) {
-//      Assert.assertTrue("failed", false);
-//      e.printStackTrace();
-//      throw e;
-//    }
+    try {
+      String outPath = "target/output";
+      CarbonProjection carbonProjection = new CarbonProjection();
+      carbonProjection.addColumn("ID");
+      carbonProjection.addColumn("date");
+      carbonProjection.addColumn("country");
+      carbonProjection.addColumn("name");
+      carbonProjection.addColumn("phonetype");
+      carbonProjection.addColumn("serialname");
+      carbonProjection.addColumn("salary");
+      runJob(outPath, carbonProjection, null);
+      Assert.assertEquals("Count lines are not matching", 1000, countTheLines(outPath));
+      Assert.assertEquals("Column count are not matching", 7, countTheColumns(outPath));
+    } catch (Exception e) {
+      Assert.assertTrue("failed", false);
+      e.printStackTrace();
+      throw e;
+    }
   }
 
   @Test public void testInputFormatMapperReadAllRowsAndFewColumns() throws Exception {
-//    try {
-//      String outPath = "target/output2";
-//      CarbonProjection carbonProjection = new CarbonProjection();
-//      carbonProjection.addColumn("ID");
-//      carbonProjection.addColumn("country");
-//      carbonProjection.addColumn("salary");
-//      runJob(outPath, carbonProjection, null);
-//
-//      Assert.assertEquals("Count lines are not matching", 1000, countTheLines(outPath));
-//      Assert.assertEquals("Column count are not matching", 3, countTheColumns(outPath));
-//    } catch (Exception e) {
-//      Assert.assertTrue("failed", false);
-//    }
+    try {
+      String outPath = "target/output2";
+      CarbonProjection carbonProjection = new CarbonProjection();
+      carbonProjection.addColumn("ID");
+      carbonProjection.addColumn("country");
+      carbonProjection.addColumn("salary");
+      runJob(outPath, carbonProjection, null);
+
+      Assert.assertEquals("Count lines are not matching", 1000, countTheLines(outPath));
+      Assert.assertEquals("Column count are not matching", 3, countTheColumns(outPath));
+    } catch (Exception e) {
+      Assert.assertTrue("failed", false);
+    }
   }
 
   @Test public void testInputFormatMapperReadAllRowsAndFewColumnsWithFilter() throws Exception {
-//    try {
-//      String outPath = "target/output3";
-//      CarbonProjection carbonProjection = new CarbonProjection();
-//      carbonProjection.addColumn("ID");
-//      carbonProjection.addColumn("country");
-//      carbonProjection.addColumn("salary");
-//      Expression expression =
-//          new EqualToExpression(new ColumnExpression("country", DataType.STRING),
-//              new LiteralExpression("france", DataType.STRING));
-//      runJob(outPath, carbonProjection, expression);
-//      Assert.assertEquals("Count lines are not matching", 101, countTheLines(outPath));
-//      Assert.assertEquals("Column count are not matching", 3, countTheColumns(outPath));
-//    } catch (Exception e) {
-//      Assert.assertTrue("failed", false);
-//    }
+    try {
+      String outPath = "target/output3";
+      CarbonProjection carbonProjection = new CarbonProjection();
+      carbonProjection.addColumn("ID");
+      carbonProjection.addColumn("country");
+      carbonProjection.addColumn("salary");
+      Expression expression =
+          new EqualToExpression(new ColumnExpression("country", DataType.STRING),
+              new LiteralExpression("france", DataType.STRING));
+      runJob(outPath, carbonProjection, expression);
+      Assert.assertEquals("Count lines are not matching", 101, countTheLines(outPath));
+      Assert.assertEquals("Column count are not matching", 3, countTheColumns(outPath));
+    } catch (Exception e) {
+      Assert.assertTrue("failed", false);
+    }
   }
 
   private int countTheLines(String outPath) throws Exception {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
index 4b1a09b..04685e9 100644
--- a/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
+++ b/hadoop/src/test/java/org/apache/carbondata/hadoop/test/util/StoreCreator.java
@@ -19,6 +19,7 @@
 package org.apache.carbondata.hadoop.test.util;
 
 import com.google.gson.Gson;
+import org.apache.carbondata.core.update.CarbonUpdateUtil;
 import org.apache.hadoop.fs.Path;
 
 import org.apache.carbondata.core.cache.Cache;
@@ -407,7 +408,6 @@ public class StoreCreator {
   public static void writeLoadMetadata(CarbonDataLoadSchema schema, String databaseName,
       String tableName, List<LoadMetadataDetails> listOfLoadFolderDetails) throws IOException {
     LoadMetadataDetails loadMetadataDetails = new LoadMetadataDetails();
-   // loadMetadataDetails.setTimestamp(readCurrentTime());
     loadMetadataDetails.setLoadStatus("SUCCESS");
     loadMetadataDetails.setLoadName(String.valueOf(0));
     loadMetadataDetails.setLoadStartTime(loadMetadataDetails.getTimeStamp(readCurrentTime()));
@@ -469,7 +469,7 @@ public class StoreCreator {
     model.setTableName(dataProcessTaskStatus.getTableName());
     model.setTaskNo("1");
     model.setBlocksID(dataProcessTaskStatus.getBlocksID());
-    model.setFactTimeStamp(readCurrentTime());
+    model.setFactTimeStamp(System.currentTimeMillis() + "");
     model.setEscapeCharacter(dataProcessTaskStatus.getEscapeCharacter());
     model.setQuoteCharacter(dataProcessTaskStatus.getQuoteCharacter());
     model.setCommentCharacter(dataProcessTaskStatus.getCommentCharacter());
@@ -543,6 +543,7 @@ public class StoreCreator {
       this.databaseName = databaseName;
     }
 
+
   }
 
   public static void main(String[] args) {

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
index 6847537..f469983 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/load/CarbonLoaderUtil.java
@@ -70,30 +70,12 @@ import com.google.gson.Gson;
 import org.apache.spark.SparkConf;
 import org.apache.spark.util.Utils;
 
-
 public final class CarbonLoaderUtil {
 
   private static final LogService LOGGER =
       LogServiceFactory.getLogService(CarbonLoaderUtil.class.getName());
-  /**
-   * minimum no of blocklet required for distribution
-   */
-  private static int minBlockLetsReqForDistribution = 0;
-
-  static {
-    String property = CarbonProperties.getInstance()
-        .getProperty(CarbonCommonConstants.CARBON_BLOCKLETDISTRIBUTION_MIN_REQUIRED_SIZE);
-    try {
-      minBlockLetsReqForDistribution = Integer.parseInt(property);
-    } catch (NumberFormatException ne) {
-      LOGGER.info("Invalid configuration. Consisering the defaul");
-      minBlockLetsReqForDistribution =
-          CarbonCommonConstants.DEFAULT_CARBON_BLOCKLETDISTRIBUTION_MIN_REQUIRED_SIZE;
-    }
-  }
 
   private CarbonLoaderUtil() {
-
   }
 
   private static void generateGraph(IDataProcessStatus dataProcessTaskStatus, SchemaInfo info,
@@ -167,7 +149,6 @@ public final class CarbonLoaderUtil {
     dataProcessTaskStatus.setRddIteratorKey(loadModel.getRddIteratorKey());
     dataProcessTaskStatus.setDateFormat(loadModel.getDateFormat());
     SchemaInfo info = new SchemaInfo();
-
     info.setDatabaseName(databaseName);
     info.setTableName(tableName);
     info.setAutoAggregateRequest(loadModel.isAggLoadRequest());
@@ -367,15 +348,12 @@ public final class CarbonLoaderUtil {
 
     String tableStatusPath = carbonTablePath.getTableStatusFilePath();
     SegmentStatusManager segmentStatusManager = new SegmentStatusManager(absoluteTableIdentifier);
-
     ICarbonLock carbonLock = segmentStatusManager.getTableStatusLock();
-
     try {
       if (carbonLock.lockWithRetries()) {
         LOGGER.info(
             "Acquired lock for table" + loadModel.getDatabaseName() + "." + loadModel.getTableName()
                 + " for table status updation");
-
         LoadMetadataDetails[] listOfLoadFolderDetailsArray =
             SegmentStatusManager.readLoadMetadata(metaDataFilepath);
 
@@ -384,7 +362,6 @@ public final class CarbonLoaderUtil {
         loadMetadataDetails.setLoadStatus(loadStatus);
         loadMetadataDetails.setLoadName(String.valueOf(loadCount));
         loadMetadataDetails.setLoadStartTime(startLoadTime);
-
         List<LoadMetadataDetails> listOfLoadFolderDetails =
             new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java
index a42d8b0..5051625 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtil.java
@@ -852,36 +852,6 @@ public final class CarbonDataMergerUtil {
     return validAndInvalidSegments.getValidSegments();
   }
 
-  /**
-   * Combining the list of maps to one map.
-   *
-   * @param mapsOfNodeBlockMapping
-   * @return
-   */
-  public static Map<String, List<TableBlockInfo>> combineNodeBlockMaps(
-      List<Map<String, List<TableBlockInfo>>> mapsOfNodeBlockMapping) {
-
-    Map<String, List<TableBlockInfo>> combinedMap =
-        new HashMap<String, List<TableBlockInfo>>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
-    // traverse list of maps.
-    for (Map<String, List<TableBlockInfo>> eachMap : mapsOfNodeBlockMapping) {
-      // traverse inside each map.
-      for (Map.Entry<String, List<TableBlockInfo>> eachEntry : eachMap.entrySet()) {
-
-        String node = eachEntry.getKey();
-        List<TableBlockInfo> blocks = eachEntry.getValue();
-
-        // if already that node detail exist in the combined map.
-        if (null != combinedMap.get(node)) {
-          List<TableBlockInfo> blocksAlreadyPresent = combinedMap.get(node);
-          blocksAlreadyPresent.addAll(blocks);
-        } else { // if its not present in map then put to map.
-          combinedMap.put(node, blocks);
-        }
-      }
-    }
-    return combinedMap;
-  }
 
   /**
    * Removing the already merged segments from list.

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtilResult.java
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtilResult.java b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtilResult.java
index 4511f90..9ea6816 100644
--- a/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtilResult.java
+++ b/integration/spark-common/src/main/java/org/apache/carbondata/spark/merger/CarbonDataMergerUtilResult.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.apache.carbondata.spark.merger;
 
 import org.apache.carbondata.core.update.SegmentUpdateDetails;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
index baa333a..09c35ae 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonMergerRDD.scala
@@ -41,7 +41,7 @@ import org.apache.carbondata.core.carbon.path.CarbonTablePath
 import org.apache.carbondata.core.constants.CarbonCommonConstants
 import org.apache.carbondata.core.update.UpdateVO
 import org.apache.carbondata.core.updatestatus.SegmentUpdateStatusManager
-import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil, CarbonUtilException}
+import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil}
 import org.apache.carbondata.hadoop.{CarbonInputFormat, CarbonInputSplit, CarbonMultiBlockSplit}
 import org.apache.carbondata.hadoop.util.CarbonInputFormatUtil
 import org.apache.carbondata.processing.model.CarbonLoadModel
@@ -126,7 +126,7 @@ class CarbonMergerRDD[K, V](
             dataFileFooter = CarbonUtil
               .readMetadatFile(tableBlockInfoList.get(tableBlockInfoList.size() - 1))
           } catch {
-            case e: CarbonUtilException =>
+            case e: IOException =>
               logError("Exception in preparing the data file footer for compaction " + e.getMessage)
               throw e
           }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
index 4c76a25..33d9e65 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/DataManagementFunc.scala
@@ -372,7 +372,6 @@ object DataManagementFunc {
         LOGGER.audit(errorMsg)
         LOGGER.error(errorMsg)
         throw new Exception(errorMsg + " Please try after some time.")
-
       }
     } finally {
       CarbonLockUtil.fileUnlock(carbonCleanFilesLock, LockUsage.CLEAN_FILES_LOCK)

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
----------------------------------------------------------------------
diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
index b8cb4f7..410e4bd 100644
--- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
+++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/util/CommonUtil.scala
@@ -30,8 +30,8 @@ import org.apache.spark.util.FileUtils
 
 import org.apache.carbondata.common.logging.LogServiceFactory
 import org.apache.carbondata.core.constants.CarbonCommonConstants
-import org.apache.carbondata.core.util.CarbonProperties
 import org.apache.carbondata.core.updatestatus.SegmentStatusManager
+import org.apache.carbondata.core.util.CarbonProperties
 import org.apache.carbondata.hadoop.csv.CSVInputFormat
 import org.apache.carbondata.processing.model.CarbonLoadModel
 import org.apache.carbondata.spark.exception.MalformedCarbonCommandException

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
index f85e56b..5ea7129 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/CarbonDatasourceRelation.scala
@@ -206,7 +206,7 @@ case class CarbonRelation(
   }
 
   val dimensionsAttr = {
-    val sett = new LinkedHashSet(tableMeta.carbonTable
+    val sett = new java.util.LinkedHashSet(tableMeta.carbonTable
       .getDimensionByTableName(tableMeta.carbonTableIdentifier.getTableName).asScala.asJava)
     sett.asScala.toSeq.filter(dim => !dim.isInvisible ||
                                      (dim.isInvisible && dim.isInstanceOf[CarbonImplicitDimension]))

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
----------------------------------------------------------------------
diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
index f366608..2dbe817 100644
--- a/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
+++ b/integration/spark/src/main/scala/org/apache/spark/sql/execution/command/carbonTableSchema.scala
@@ -744,7 +744,6 @@ private[sql] case class ShowLoads(
     limit: Option[String],
     override val output: Seq[Attribute]) extends RunnableCommand {
 
-
   override def run(sqlContext: SQLContext): Seq[Row] = {
     Checker.validateTableExists(databaseNameOp, tableName, sqlContext)
     CarbonStore.showSegments(
@@ -753,7 +752,6 @@ private[sql] case class ShowLoads(
       limit
     )
   }
-
 }
 
 private[sql] case class DescribeCommandFormatted(

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/9b8090ba/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
----------------------------------------------------------------------
diff --git a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
index ee76f8f..f8d36d4 100644
--- a/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
+++ b/processing/src/main/java/org/apache/carbondata/processing/surrogatekeysgenerator/csvbased/FileStoreSurrogateKeyGenForCSV.java
@@ -93,8 +93,8 @@ public class FileStoreSurrogateKeyGenForCSV extends CarbonCSVBasedDimSurrogateKe
    * @param columnsInfo
    * @throws IOException
    */
-  public FileStoreSurrogateKeyGenForCSV(ColumnsInfo columnsInfo, String partitionID, int segmentId,
-      String taskNo) throws IOException {
+  public FileStoreSurrogateKeyGenForCSV(ColumnsInfo columnsInfo, String partitionID,
+      String segmentId, String taskNo) throws IOException {
     super(columnsInfo);
     populatePrimaryKeyarray(dimInsertFileNames, columnsInfo.getPrimaryKeyMap());
     this.partitionID = partitionID;


Mime
View raw message