carbondata-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gvram...@apache.org
Subject [05/14] incubator-carbondata git commit: Following changes are done as part of this commit.
Date Thu, 16 Mar 2017 09:28:48 GMT
http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/AggregatorInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/AggregatorInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/AggregatorInfo.java
deleted file mode 100644
index b992ea6..0000000
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/AggregatorInfo.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.carbondata.core.scan.executor.infos;
-
-import org.apache.carbondata.core.metadata.datatype.DataType;
-
-/**
- * Info class which store all the details
- * which is required during aggregation
- */
-public class AggregatorInfo {
-
-  /**
-   * selected query measure ordinal
-   * which will be used to read the measures chunk data
-   * this will be storing the index of the measure in measures chunk
-   */
-  private int[] measureOrdinals;
-
-  /**
-   * This parameter will be used to
-   * check whether particular measure is present
-   * in the table block, if not then its default value will be used
-   */
-  private boolean[] measureExists;
-
-  /**
-   * this default value will be used to when some measure is not present
-   * in the table block, in case of restructuring of the table if user is adding any
-   * measure then in older block that measure wont be present so for measure default value
-   * will be used to aggregate in the older table block query execution
-   */
-  private Object[] defaultValues;
-
-  /**
-   * Datatype of each measure;
-   */
-  private DataType[] measureDataTypes;
-
-  /**
-   * @return the measureOrdinal
-   */
-  public int[] getMeasureOrdinals() {
-    return measureOrdinals;
-  }
-
-  /**
-   * @param measureOrdinal the measureOrdinal to set
-   */
-  public void setMeasureOrdinals(int[] measureOrdinal) {
-    this.measureOrdinals = measureOrdinal;
-  }
-
-  /**
-   * @return the measureExists
-   */
-  public boolean[] getMeasureExists() {
-    return measureExists;
-  }
-
-  /**
-   * @param measureExists the measureExists to set
-   */
-  public void setMeasureExists(boolean[] measureExists) {
-    this.measureExists = measureExists;
-  }
-
-  /**
-   * @return the defaultValues
-   */
-  public Object[] getDefaultValues() {
-    return defaultValues;
-  }
-
-  /**
-   * @param defaultValues the defaultValues to set
-   */
-  public void setDefaultValues(Object[] defaultValues) {
-    this.defaultValues = defaultValues;
-  }
-
-  public DataType[] getMeasureDataTypes() {
-    return measureDataTypes;
-  }
-
-  public void setMeasureDataTypes(DataType[] measureDataTypes) {
-    this.measureDataTypes = measureDataTypes;
-  }
-}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
index 2dd6721..b294b58 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java
@@ -49,10 +49,10 @@ public class BlockExecutionInfo {
   private boolean isFixedKeyUpdateRequired;
 
   /**
-   * below to store all the information required for aggregation during query
+   * below to store all the information required for measures during query
    * execution
    */
-  private AggregatorInfo aggregatorInfo;
+  private MeasureInfo measureInfo;
 
   /**
    * this will be used to get the first tentative block from which query
@@ -71,13 +71,6 @@ public class BlockExecutionInfo {
   private String blockId;
 
   /**
-   * masked byte for block which will be used to unpack the fixed length key,
-   * this will be used for updating the older block key with new block key
-   * generator
-   */
-  private int[] maskedByteForBlock;
-
-  /**
    * total number of dimension in block
    */
   private int totalNumberDimensionBlock;
@@ -196,21 +189,43 @@ public class BlockExecutionInfo {
   }
 
   /**
-   * list of dimension selected for in query
+   * list of dimension present in the current block. This will be
+   * different in case of restructured block
    */
   private QueryDimension[] queryDimensions;
 
   /**
-   * list of measure selected in query
+   * list of dimension selected for in query
+   */
+  private QueryDimension[] actualQueryDimensions;
+
+  /**
+   * list of dimension present in the current block. This will be
+   * different in case of restructured block
    */
   private QueryMeasure[] queryMeasures;
 
   /**
+   * list of measure selected in query
+   */
+  private QueryMeasure[] actualQueryMeasures;
+
+  /**
+   * variable to maintain dimension existence and default value info
+   */
+  private DimensionInfo dimensionInfo;
+
+  /**
    * whether it needs to read data in vector/columnar format.
    */
   private boolean vectorBatchCollector;
 
   /**
+   * flag to specify that whether the current block is with latest schema or old schema
+   */
+  private boolean isRestructuredBlock;
+
+  /**
    * absolute table identifier
    */
   private AbsoluteTableIdentifier absoluteTableIdentifier;
@@ -247,15 +262,15 @@ public class BlockExecutionInfo {
   /**
    * @return the aggregatorInfos
    */
-  public AggregatorInfo getAggregatorInfo() {
-    return aggregatorInfo;
+  public MeasureInfo getMeasureInfo() {
+    return measureInfo;
   }
 
   /**
-   * @param aggregatorInfo the aggregatorInfos to set
+   * @param measureInfo the aggregatorInfos to set
    */
-  public void setAggregatorInfo(AggregatorInfo aggregatorInfo) {
-    this.aggregatorInfo = aggregatorInfo;
+  public void setMeasureInfo(MeasureInfo measureInfo) {
+    this.measureInfo = measureInfo;
   }
 
   /**
@@ -287,22 +302,6 @@ public class BlockExecutionInfo {
   }
 
   /**
-   * @return the maskedByteForBlock
-   */
-  public int[] getMaskedByteForBlock() {
-    return maskedByteForBlock;
-  }
-
-
-
-  /**
-   * @param maskedByteForBlock the maskedByteForBlock to set
-   */
-  public void setMaskedByteForBlock(int[] maskedByteForBlock) {
-    this.maskedByteForBlock = maskedByteForBlock;
-  }
-
-  /**
    * @return the totalNumberDimensionBlock
    */
   public int getTotalNumberDimensionBlock() {
@@ -359,20 +358,6 @@ public class BlockExecutionInfo {
   }
 
   /**
-   * @return the restructureInfos
-   */
-  public KeyStructureInfo getKeyStructureInfo() {
-    return keyStructureInfo;
-  }
-
-  /**
-   * @param keyStructureInfo the restructureInfos to set
-   */
-  public void setKeyStructureInfo(KeyStructureInfo keyStructureInfo) {
-    this.keyStructureInfo = keyStructureInfo;
-  }
-
-  /**
    * @return the firstDataBlock
    */
   public DataRefNode getFirstDataBlock() {
@@ -610,6 +595,38 @@ public class BlockExecutionInfo {
     this.blockId = blockId;
   }
 
+  public boolean isRestructuredBlock() {
+    return isRestructuredBlock;
+  }
+
+  public void setRestructuredBlock(boolean restructuredBlock) {
+    isRestructuredBlock = restructuredBlock;
+  }
+
+  public DimensionInfo getDimensionInfo() {
+    return dimensionInfo;
+  }
+
+  public void setDimensionInfo(DimensionInfo dimensionInfo) {
+    this.dimensionInfo = dimensionInfo;
+  }
+
+  public QueryDimension[] getActualQueryDimensions() {
+    return actualQueryDimensions;
+  }
+
+  public void setActualQueryDimensions(QueryDimension[] actualQueryDimensions) {
+    this.actualQueryDimensions = actualQueryDimensions;
+  }
+
+  public QueryMeasure[] getActualQueryMeasures() {
+    return actualQueryMeasures;
+  }
+
+  public void setActualQueryMeasures(QueryMeasure[] actualQueryMeasures) {
+    this.actualQueryMeasures = actualQueryMeasures;
+  }
+
   public int[] getProjectionListDimensionIndexes() {
     return projectionListDimensionIndexes;
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/DimensionInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/DimensionInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/DimensionInfo.java
new file mode 100644
index 0000000..3b2e2bd
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/DimensionInfo.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.carbondata.core.scan.executor.infos;
+
+/**
+ * This method will information about the query dimensions whether they exist in particular block
+ * and their default value
+ */
+public class DimensionInfo {
+
+  /**
+   * flag to check whether a given dimension exists in a given block
+   */
+  private boolean[] dimensionExists;
+
+  /**
+   * maintains default value for each dimension
+   */
+  private Object[] defaultValues;
+
+  /**
+   * @param dimensionExists
+   * @param defaultValues
+   */
+  public DimensionInfo(boolean[] dimensionExists, Object[] defaultValues) {
+    this.dimensionExists = dimensionExists;
+    this.defaultValues = defaultValues;
+  }
+
+  /**
+   * @return
+   */
+  public boolean[] getDimensionExists() {
+    return dimensionExists;
+  }
+
+  /**
+   * @return
+   */
+  public Object[] getDefaultValues() {
+    return defaultValues;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/MeasureInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/MeasureInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/MeasureInfo.java
new file mode 100644
index 0000000..5dd4fb8
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/MeasureInfo.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.scan.executor.infos;
+
+import org.apache.carbondata.core.metadata.datatype.DataType;
+
+/**
+ * Info class which store all the details
+ * which is required during aggregation
+ */
+public class MeasureInfo {
+
+  /**
+   * selected query measure ordinal
+   * which will be used to read the measures chunk data
+   * this will be storing the index of the measure in measures chunk
+   */
+  private int[] measureOrdinals;
+
+  /**
+   * This parameter will be used to
+   * check whether particular measure is present
+   * in the table block, if not then its default value will be used
+   */
+  private boolean[] measureExists;
+
+  /**
+   * this default value will be used to when some measure is not present
+   * in the table block, in case of restructuring of the table if user is adding any
+   * measure then in older block that measure wont be present so for measure default value
+   * will be used to aggregate in the older table block query execution
+   */
+  private Object[] defaultValues;
+
+  /**
+   * Datatype of each measure;
+   */
+  private DataType[] measureDataTypes;
+
+  /**
+   * @return the measureOrdinal
+   */
+  public int[] getMeasureOrdinals() {
+    return measureOrdinals;
+  }
+
+  /**
+   * @param measureOrdinal the measureOrdinal to set
+   */
+  public void setMeasureOrdinals(int[] measureOrdinal) {
+    this.measureOrdinals = measureOrdinal;
+  }
+
+  /**
+   * @return the measureExists
+   */
+  public boolean[] getMeasureExists() {
+    return measureExists;
+  }
+
+  /**
+   * @param measureExists the measureExists to set
+   */
+  public void setMeasureExists(boolean[] measureExists) {
+    this.measureExists = measureExists;
+  }
+
+  /**
+   * @return the defaultValues
+   */
+  public Object[] getDefaultValues() {
+    return defaultValues;
+  }
+
+  /**
+   * @param defaultValues the defaultValues to set
+   */
+  public void setDefaultValues(Object[] defaultValues) {
+    this.defaultValues = defaultValues;
+  }
+
+  public DataType[] getMeasureDataTypes() {
+    return measureDataTypes;
+  }
+
+  public void setMeasureDataTypes(DataType[] measureDataTypes) {
+    this.measureDataTypes = measureDataTypes;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
index ef6fb8a..2f47b6a 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java
@@ -208,6 +208,7 @@ public class QueryUtil {
    */
   public static int[] getDimensionsBlockIndexes(List<QueryDimension> queryDimensions,
       Map<Integer, Integer> dimensionOrdinalToBlockMapping,
+      Map<Integer, Integer> queryDimensionToCurrentBlockDimensionOrdinalMapping,
       List<CarbonDimension> customAggregationDimension, Set<CarbonDimension> filterDimensions,
       Set<Integer> allProjectionListDimensionIndexes) {
     // using set as in row group columns will point to same block
@@ -219,16 +220,17 @@ public class QueryUtil {
         continue;
       }
 
-      allProjectionListDimensionIndexes.add(
-          dimensionOrdinalToBlockMapping.get(queryDimensions.get(i).getDimension().getOrdinal()));
+      Integer dimensionOrdinal = queryDimensionToCurrentBlockDimensionOrdinalMapping
+          .get(queryDimensions.get(i).getDimension().getOrdinal());
+      allProjectionListDimensionIndexes.add(dimensionOrdinal);
       if (queryDimensions.get(i).getDimension().numberOfChild() > 0) {
         addChildrenBlockIndex(allProjectionListDimensionIndexes,
             queryDimensions.get(i).getDimension());
       }
 
-      if (!filterDimensionOrdinal.contains(queryDimensions.get(i).getDimension().getOrdinal())) {
+      if (!filterDimensionOrdinal.contains(dimensionOrdinal)) {
         blockIndex =
-            dimensionOrdinalToBlockMapping.get(queryDimensions.get(i).getDimension().getOrdinal());
+            dimensionOrdinalToBlockMapping.get(dimensionOrdinal);
         dimensionBlockIndex.add(blockIndex);
         if (queryDimensions.get(i).getDimension().numberOfChild() > 0) {
           addChildrenBlockIndex(dimensionBlockIndex, queryDimensions.get(i).getDimension());
@@ -249,6 +251,91 @@ public class QueryUtil {
   }
 
   /**
+   * This method will maintain a position mapping of query dimensions to current block dimensions
+   *
+   * @param queryDimensions
+   * @param currentBlockDimensions
+   * @return
+   */
+  public static Map<Integer, Integer> getQueryDimensionToCurrentBlockDimensionOrdinalMapping(
+      List<QueryDimension> queryDimensions, List<CarbonDimension> currentBlockDimensions) {
+    Map<Integer, Integer> queryToCurrentBlockDimensionOrdinals =
+        new HashMap<>(queryDimensions.size());
+    for (QueryDimension queryDimension : queryDimensions) {
+      if (queryDimension.getDimension().hasEncoding(Encoding.IMPLICIT)) {
+        continue;
+      }
+      int ordinalOfDimensionFromCurrentBlock =
+          getOrdinalOfDimensionFromCurrentBlock(currentBlockDimensions,
+              queryDimension.getDimension());
+      queryToCurrentBlockDimensionOrdinals
+          .put(queryDimension.getDimension().getOrdinal(), ordinalOfDimensionFromCurrentBlock);
+    }
+    return queryToCurrentBlockDimensionOrdinals;
+  }
+
+  /**
+   * This method will return the key ordinal of the query dimension from the current block
+   *
+   * @param blockDimensions
+   * @param queryDimension
+   * @return
+   */
+  public static int getKeyOrdinalOfDimensionFromCurrentBlock(List<CarbonDimension> blockDimensions,
+      CarbonDimension queryDimension) {
+    int keyOrdinalInCurrentDimensionBlock = -1;
+    for (CarbonDimension blockDimension : blockDimensions) {
+      if (queryDimension.getColumnId().equals(blockDimension.getColumnId())) {
+        keyOrdinalInCurrentDimensionBlock = blockDimension.getKeyOrdinal();
+        break;
+      }
+    }
+    return keyOrdinalInCurrentDimensionBlock;
+  }
+
+  /**
+   * This method will return the ordinal of the query dimension from the current block
+   *
+   * @param blockDimensions
+   * @param queryDimension
+   * @return
+   */
+  public static int getOrdinalOfDimensionFromCurrentBlock(List<CarbonDimension> blockDimensions,
+      CarbonDimension queryDimension) {
+    int keyOrdinalInCurrentDimensionBlock = -1;
+    for (CarbonDimension blockDimension : blockDimensions) {
+      if (queryDimension.getColumnId().equals(blockDimension.getColumnId())) {
+        keyOrdinalInCurrentDimensionBlock = blockDimension.getOrdinal();
+        break;
+      }
+    }
+    return keyOrdinalInCurrentDimensionBlock;
+  }
+
+  /**
+   * This method will maintain a position mapping of query measures to current block measures
+   *
+   * @param queryMeasures
+   * @param currentBlockMeasures
+   * @return
+   */
+  public static Map<Integer, Integer> getQueryMeasuresToCurrentBlockMeasuresOrdinalMapping(
+      List<QueryMeasure> queryMeasures, List<CarbonMeasure> currentBlockMeasures) {
+    Map<Integer, Integer> queryToCurrentBlockDimensionOrdinals =
+        new HashMap<>(queryMeasures.size());
+    for (QueryMeasure queryMeasure : queryMeasures) {
+      for (CarbonMeasure currentBlockMeasure : currentBlockMeasures) {
+        if (queryMeasure.getMeasure().getColumnId().equals(currentBlockMeasure.getColumnId())) {
+          queryToCurrentBlockDimensionOrdinals
+              .put(queryMeasure.getMeasure().getOrdinal(), currentBlockMeasure.getOrdinal());
+          break;
+        }
+      }
+    }
+    return queryToCurrentBlockDimensionOrdinals;
+  }
+
+  /**
    * Below method will be used to add the children block index
    * this will be basically for complex dimension which will have children
    *
@@ -400,18 +487,21 @@ public class QueryUtil {
    */
   public static int[] getMeasureBlockIndexes(List<QueryMeasure> queryMeasures,
       List<CarbonMeasure> expressionMeasure, Map<Integer, Integer> ordinalToBlockIndexMapping,
+      Map<Integer, Integer> queryMeasuresToCurrentBlockMeasuresOrdinalMapping,
       Set<CarbonMeasure> filterMeasures, List<Integer> allProjectionListMeasureIdexes) {
     Set<Integer> measureBlockIndex = new HashSet<Integer>();
     Set<Integer> filterMeasureOrdinal = getFilterMeasureOrdinal(filterMeasures);
     for (int i = 0; i < queryMeasures.size(); i++) {
+      Integer measureOrdinal = queryMeasuresToCurrentBlockMeasuresOrdinalMapping
+          .get(queryMeasures.get(i).getMeasure().getOrdinal());
       allProjectionListMeasureIdexes.add(queryMeasures.get(i).getMeasure().getOrdinal());
-      if (!filterMeasureOrdinal.contains(queryMeasures.get(i).getMeasure().getOrdinal())) {
-        measureBlockIndex
-            .add(ordinalToBlockIndexMapping.get(queryMeasures.get(i).getMeasure().getOrdinal()));
+      if (!filterMeasureOrdinal.contains(measureOrdinal)) {
+        measureBlockIndex.add(ordinalToBlockIndexMapping.get(measureOrdinal));
       }
     }
     for (int i = 0; i < expressionMeasure.size(); i++) {
-      measureBlockIndex.add(ordinalToBlockIndexMapping.get(expressionMeasure.get(i).getOrdinal()));
+      measureBlockIndex.add(queryMeasuresToCurrentBlockMeasuresOrdinalMapping
+          .get(ordinalToBlockIndexMapping.get(expressionMeasure.get(i).getOrdinal())));
     }
     int[] measureIndexes =
         ArrayUtils.toPrimitive(measureBlockIndex.toArray(new Integer[measureBlockIndex.size()]));
@@ -420,6 +510,54 @@ public class QueryUtil {
   }
 
   /**
+   * This method will create the updated list of filter measures present in the current block
+   *
+   * @param queryFilterMeasures
+   * @param currentBlockMeasures
+   * @return
+   */
+  public static Set<CarbonMeasure> getUpdatedFilterMeasures(Set<CarbonMeasure> queryFilterMeasures,
+      List<CarbonMeasure> currentBlockMeasures) {
+    if (!queryFilterMeasures.isEmpty()) {
+      Set<CarbonMeasure> updatedFilterMeasures = new HashSet<>(queryFilterMeasures.size());
+      for (CarbonMeasure queryMeasure : queryFilterMeasures) {
+        CarbonMeasure measureFromCurrentBlock =
+            CarbonUtil.getMeasureFromCurrentBlock(currentBlockMeasures, queryMeasure.getColumnId());
+        if (null != measureFromCurrentBlock) {
+          updatedFilterMeasures.add(measureFromCurrentBlock);
+        }
+      }
+      return updatedFilterMeasures;
+    } else {
+      return queryFilterMeasures;
+    }
+  }
+
+  /**
+   * This method will create the updated list of filter dimensions present in the current block
+   *
+   * @param queryFilterDimensions
+   * @param currentBlockDimensions
+   * @return
+   */
+  public static Set<CarbonDimension> getUpdatedFilterDimensions(
+      Set<CarbonDimension> queryFilterDimensions, List<CarbonDimension> currentBlockDimensions) {
+    if (!queryFilterDimensions.isEmpty()) {
+      Set<CarbonDimension> updatedFilterDimensions = new HashSet<>(queryFilterDimensions.size());
+      for (CarbonDimension queryDimension : queryFilterDimensions) {
+        CarbonDimension dimensionFromCurrentBlock =
+            CarbonUtil.getDimensionFromCurrentBlock(currentBlockDimensions, queryDimension);
+        if (null != dimensionFromCurrentBlock) {
+          updatedFilterDimensions.add(dimensionFromCurrentBlock);
+        }
+      }
+      return updatedFilterDimensions;
+    } else {
+      return queryFilterDimensions;
+    }
+  }
+
+  /**
    * Below method will be used to get mapping whether dimension is present in
    * order by or not
    *
@@ -608,17 +746,20 @@ public class QueryUtil {
    */
   public static void fillQueryDimensionsBlockIndexes(List<QueryDimension> queryDimensions,
       Map<Integer, Integer> columnOrdinalToBlockIndexMapping,
+      Map<Integer, Integer> queryDimensionToCurrentBlockDimensionOrdinalMapping,
       Set<Integer> dictionaryDimensionBlockIndex, List<Integer> noDictionaryDimensionBlockIndex) {
     for (QueryDimension queryDimension : queryDimensions) {
       if (CarbonUtil.hasEncoding(queryDimension.getDimension().getEncoder(), Encoding.DICTIONARY)
           && queryDimension.getDimension().numberOfChild() == 0) {
-        dictionaryDimensionBlockIndex
-            .add(columnOrdinalToBlockIndexMapping.get(queryDimension.getDimension().getOrdinal()));
+        dictionaryDimensionBlockIndex.add(columnOrdinalToBlockIndexMapping.get(
+            queryDimensionToCurrentBlockDimensionOrdinalMapping
+                .get(queryDimension.getDimension().getOrdinal())));
       } else if (
           !CarbonUtil.hasEncoding(queryDimension.getDimension().getEncoder(), Encoding.IMPLICIT)
               && queryDimension.getDimension().numberOfChild() == 0) {
-        noDictionaryDimensionBlockIndex
-            .add(columnOrdinalToBlockIndexMapping.get(queryDimension.getDimension().getOrdinal()));
+        noDictionaryDimensionBlockIndex.add(columnOrdinalToBlockIndexMapping.get(
+            queryDimensionToCurrentBlockDimensionOrdinalMapping
+                .get(queryDimension.getDimension().getOrdinal())));
       }
     }
   }
@@ -759,8 +900,8 @@ public class QueryUtil {
         if (filterDimension.hasEncoding(Encoding.IMPLICIT)) {
           continue;
         }
-        fillParentDetails(dimensionToBlockIndexMap, filterDimension, complexTypeMap,
-            eachComplexColumnValueSize, columnIdToDictionaryMap);
+        // fillParentDetails(dimensionToBlockIndexMap, filterDimension, complexTypeMap,
+        //     eachComplexColumnValueSize, columnIdToDictionaryMap);
       }
     }
     return complexTypeMap;

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
index 3549932..847a795 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java
@@ -16,16 +16,30 @@
  */
 package org.apache.carbondata.core.scan.executor.util;
 
+import java.math.BigDecimal;
+import java.math.RoundingMode;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.carbondata.core.constants.CarbonCommonConstants;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator;
+import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory;
+import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.metadata.encoder.Encoding;
 import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
 import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
-import org.apache.carbondata.core.scan.executor.infos.AggregatorInfo;
+import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema;
+import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo;
+import org.apache.carbondata.core.scan.executor.infos.DimensionInfo;
+import org.apache.carbondata.core.scan.executor.infos.MeasureInfo;
 import org.apache.carbondata.core.scan.model.QueryDimension;
 import org.apache.carbondata.core.scan.model.QueryMeasure;
+import org.apache.carbondata.core.util.CarbonUtil;
+import org.apache.carbondata.core.util.DataTypeUtil;
+
+import org.apache.commons.lang3.ArrayUtils;
+import org.apache.spark.unsafe.types.UTF8String;
 
 /**
  * Utility class for restructuring
@@ -38,37 +52,160 @@ public class RestructureUtil {
    * table blocks in that case we need to select only those dimension out of
    * query dimension which is present in the current table block
    *
+   * @param blockExecutionInfo
    * @param queryDimensions
    * @param tableBlockDimensions
+   * @param tableComplexDimension
    * @return list of query dimension which is present in the table block
    */
-  public static List<QueryDimension> getUpdatedQueryDimension(List<QueryDimension> queryDimensions,
+  public static List<QueryDimension> createDimensionInfoAndGetUpdatedQueryDimension(
+      BlockExecutionInfo blockExecutionInfo, List<QueryDimension> queryDimensions,
       List<CarbonDimension> tableBlockDimensions, List<CarbonDimension> tableComplexDimension) {
     List<QueryDimension> presentDimension =
         new ArrayList<QueryDimension>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
+    boolean[] isDimensionExists = new boolean[queryDimensions.size()];
+    Object[] defaultValues = new Object[queryDimensions.size()];
     // selecting only those dimension which is present in the query
+    int dimIndex = 0;
     for (QueryDimension queryDimension : queryDimensions) {
       if (queryDimension.getDimension().hasEncoding(Encoding.IMPLICIT)) {
         presentDimension.add(queryDimension);
+        isDimensionExists[dimIndex] = true;
       } else {
         for (CarbonDimension tableDimension : tableBlockDimensions) {
-          if (tableDimension.equals(queryDimension.getDimension())) {
+          if (tableDimension.getColumnId().equals(queryDimension.getDimension().getColumnId())) {
             presentDimension.add(queryDimension);
+            isDimensionExists[dimIndex] = true;
+            break;
           }
         }
+        // add default value only in case query dimension is not found in the current block
+        if (!isDimensionExists[dimIndex]) {
+          defaultValues[dimIndex] = validateAndGetDefaultValue(queryDimension.getDimension());
+          blockExecutionInfo.setRestructuredBlock(true);
+        }
       }
+      dimIndex++;
     }
-    for (QueryDimension queryDimimension : queryDimensions) {
+    dimIndex = 0;
+    for (QueryDimension queryDimension : queryDimensions) {
       for (CarbonDimension tableDimension : tableComplexDimension) {
-        if (tableDimension.equals(queryDimimension.getDimension())) {
-          presentDimension.add(queryDimimension);
+        if (tableDimension.getColumnId().equals(queryDimension.getDimension().getColumnId())) {
+          presentDimension.add(queryDimension);
+          isDimensionExists[dimIndex] = true;
+          break;
         }
       }
+      // add default value only in case query dimension is not found in the current block
+      if (!isDimensionExists[dimIndex]) {
+        defaultValues[dimIndex] =
+            validateAndGetDefaultValue(queryDimension.getDimension());
+      }
+      dimIndex++;
     }
+    DimensionInfo dimensionInfo =
+        new DimensionInfo(isDimensionExists, defaultValues);
+    blockExecutionInfo.setDimensionInfo(dimensionInfo);
     return presentDimension;
   }
 
   /**
+   * This method will validate and return the default value to be
+   * filled at the time of result preparation
+   *
+   * @param queryDimension
+   * @return
+   */
+  public static Object validateAndGetDefaultValue(CarbonDimension queryDimension) {
+    byte[] defaultValue = queryDimension.getDefaultValue();
+    Object defaultValueToBeConsidered = null;
+    if (CarbonUtil.hasEncoding(queryDimension.getEncoder(), Encoding.DICTIONARY)) {
+      // direct dictionary case
+      if (CarbonUtil.hasEncoding(queryDimension.getEncoder(), Encoding.DIRECT_DICTIONARY)) {
+        defaultValueToBeConsidered = getDirectDictionaryDefaultValue(queryDimension.getDataType(),
+            queryDimension.getDefaultValue());
+      } else {
+        // dictionary case
+        defaultValueToBeConsidered = getDictionaryDefaultValue(defaultValue);
+      }
+    } else {
+      // no dictionary
+      defaultValueToBeConsidered = getNoDictionaryDefaultValue(defaultValue);
+    }
+    return defaultValueToBeConsidered;
+  }
+
+  /**
+   * Method for computing default value for dictionary column
+   *
+   * @param defaultValue
+   * @return
+   */
+  private static Object getDictionaryDefaultValue(byte[] defaultValue) {
+    Object dictionaryDefaultValue = null;
+    // dictionary has 2 cases:
+    // 1. If default value is specified then its surrogate key will be 2
+    // 2.  If default value is not specified then its surrogate key will be
+    // 1 which is for member default value null
+    if (isDefaultValueNull(defaultValue)) {
+      dictionaryDefaultValue = new Integer(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY);
+    } else {
+      dictionaryDefaultValue =
+          new Integer(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY + 1);
+    }
+    return dictionaryDefaultValue;
+  }
+
+  /**
+   * Method for computing default value for direct dictionary
+   *
+   * @param dataType
+   * @param defaultValue
+   * @return
+   */
+  private static Object getDirectDictionaryDefaultValue(DataType dataType, byte[] defaultValue) {
+    Object directDictionaryDefaultValue = null;
+    if (!isDefaultValueNull(defaultValue)) {
+      DirectDictionaryGenerator directDictionaryGenerator =
+          DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(dataType);
+      if (directDictionaryGenerator != null) {
+        String value =
+            new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+        directDictionaryDefaultValue =
+            directDictionaryGenerator.getValueFromSurrogate(Integer.parseInt(value));
+      }
+    }
+    return directDictionaryDefaultValue;
+  }
+
+  /**
+   * Method for computing default value for no dictionary
+   *
+   * @param defaultValue
+   * @return
+   */
+  private static Object getNoDictionaryDefaultValue(byte[] defaultValue) {
+    Object noDictionaryDefaultValue = null;
+    if (!isDefaultValueNull(defaultValue)) {
+      noDictionaryDefaultValue = UTF8String.fromBytes(defaultValue);
+    }
+    return noDictionaryDefaultValue;
+  }
+
+  /**
+   * This method will validate whether a given value is empty or null
+   *
+   * @param defaultValue
+   * @return
+   */
+  private static boolean isDefaultValueNull(byte[] defaultValue) {
+    if (null == defaultValue) {
+      return true;
+    }
+    return false;
+  }
+
+  /**
    * Below method is to add dimension children for complex type dimension as
    * internally we are creating dimension column for each each complex
    * dimension so when complex query dimension request will come in the query,
@@ -102,37 +239,88 @@ public class RestructureUtil {
   }
 
   /**
-   * Below method will be used to get the aggregator info object
+   * Method for computing measure default value based on the data type
+   *
+   * @param columnSchema
+   * @param defaultValue
+   * @return
+   */
+  public static Object getMeasureDefaultValue(ColumnSchema columnSchema, byte[] defaultValue) {
+    Object measureDefaultValue = null;
+    if (!isDefaultValueNull(defaultValue)) {
+      String value = null;
+      switch (columnSchema.getDataType()) {
+        case SHORT:
+        case INT:
+        case LONG:
+          value =
+              new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+          measureDefaultValue = Long.parseLong(value);
+          break;
+        case DECIMAL:
+          BigDecimal decimal = DataTypeUtil.byteToBigDecimal(defaultValue);
+          if (columnSchema.getScale() > decimal.scale()) {
+            decimal = decimal.setScale(columnSchema.getScale(), RoundingMode.HALF_UP);
+          }
+          measureDefaultValue = decimal;
+          break;
+        default:
+          value =
+              new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+          Double parsedValue = Double.valueOf(value);
+          if (!Double.isInfinite(parsedValue) && !Double.isNaN(parsedValue)) {
+            measureDefaultValue = value;
+          }
+      }
+    }
+    return measureDefaultValue;
+  }
+
+  /**
+   * Below method will be used to prepare the measure info object
    * in this method some of the properties which will be extracted
    * from query measure and current block measures will be set
    *
+   * @param blockExecutionInfo
    * @param queryMeasures        measures present in query
    * @param currentBlockMeasures current block measures
-   * @return aggregator info
+   * @return measures present in the block
    */
-  public static AggregatorInfo getAggregatorInfos(List<QueryMeasure> queryMeasures,
+  public static List<QueryMeasure> createMeasureInfoAndGetUpdatedQueryMeasures(
+      BlockExecutionInfo blockExecutionInfo, List<QueryMeasure> queryMeasures,
       List<CarbonMeasure> currentBlockMeasures) {
-    AggregatorInfo aggregatorInfos = new AggregatorInfo();
+    MeasureInfo measureInfo = new MeasureInfo();
+    List<QueryMeasure> presentMeasure = new ArrayList<>(queryMeasures.size());
     int numberOfMeasureInQuery = queryMeasures.size();
-    int[] measureOrdinals = new int[numberOfMeasureInQuery];
+    List<Integer> measureOrdinalList = new ArrayList<>(numberOfMeasureInQuery);
     Object[] defaultValues = new Object[numberOfMeasureInQuery];
     boolean[] measureExistsInCurrentBlock = new boolean[numberOfMeasureInQuery];
     int index = 0;
     for (QueryMeasure queryMeasure : queryMeasures) {
-      measureOrdinals[index] = queryMeasure.getMeasure().getOrdinal();
       // if query measure exists in current dimension measures
       // then setting measure exists is true
       // otherwise adding a default value of a measure
-      if (currentBlockMeasures.contains(queryMeasure.getMeasure())) {
-        measureExistsInCurrentBlock[index] = true;
-      } else {
-        defaultValues[index] = queryMeasure.getMeasure().getDefaultValue();
+      for (CarbonMeasure carbonMeasure : currentBlockMeasures) {
+        if (carbonMeasure.getColumnId().equals(queryMeasure.getMeasure().getColumnId())) {
+          presentMeasure.add(queryMeasure);
+          measureOrdinalList.add(carbonMeasure.getOrdinal());
+          measureExistsInCurrentBlock[index] = true;
+          break;
+        }
+      }
+      if (!measureExistsInCurrentBlock[index]) {
+        defaultValues[index] = getMeasureDefaultValue(queryMeasure.getMeasure().getColumnSchema(),
+            queryMeasure.getMeasure().getDefaultValue());
+        blockExecutionInfo.setRestructuredBlock(true);
       }
       index++;
     }
-    aggregatorInfos.setDefaultValues(defaultValues);
-    aggregatorInfos.setMeasureOrdinals(measureOrdinals);
-    aggregatorInfos.setMeasureExists(measureExistsInCurrentBlock);
-    return aggregatorInfos;
+    int[] measureOrdinals =
+        ArrayUtils.toPrimitive(measureOrdinalList.toArray(new Integer[measureOrdinalList.size()]));
+    measureInfo.setDefaultValues(defaultValues);
+    measureInfo.setMeasureOrdinals(measureOrdinals);
+    measureInfo.setMeasureExists(measureExistsInCurrentBlock);
+    blockExecutionInfo.setMeasureInfo(measureInfo);
+    return presentMeasure;
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
index 520b460..deb07c7 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java
@@ -53,6 +53,7 @@ import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
 import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.metadata.encoder.Encoding;
 import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.scan.executor.util.QueryUtil;
 import org.apache.carbondata.core.scan.expression.ColumnExpression;
 import org.apache.carbondata.core.scan.expression.Expression;
 import org.apache.carbondata.core.scan.expression.ExpressionResult;
@@ -68,6 +69,8 @@ import org.apache.carbondata.core.scan.filter.executer.FilterExecuter;
 import org.apache.carbondata.core.scan.filter.executer.IncludeColGroupFilterExecuterImpl;
 import org.apache.carbondata.core.scan.filter.executer.IncludeFilterExecuterImpl;
 import org.apache.carbondata.core.scan.filter.executer.OrFilterExecuterImpl;
+import org.apache.carbondata.core.scan.filter.executer.RestructureExcludeFilterExecutorImpl;
+import org.apache.carbondata.core.scan.filter.executer.RestructureIncludeFilterExecutorImpl;
 import org.apache.carbondata.core.scan.filter.executer.RowLevelFilterExecuterImpl;
 import org.apache.carbondata.core.scan.filter.executer.RowLevelRangeTypeExecuterFacory;
 import org.apache.carbondata.core.scan.filter.intf.ExpressionType;
@@ -164,7 +167,20 @@ public final class FilterUtil {
       DimColumnResolvedFilterInfo dimColResolvedFilterInfo, SegmentProperties segmentProperties) {
 
     if (dimColResolvedFilterInfo.getDimension().isColumnar()) {
-      return new IncludeFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
+      CarbonDimension dimensionFromCurrentBlock = CarbonUtil
+          .getDimensionFromCurrentBlock(segmentProperties.getDimensions(),
+              dimColResolvedFilterInfo.getDimension());
+      if (null != dimensionFromCurrentBlock) {
+        // update dimension and column index according to the dimension position in current block
+        DimColumnResolvedFilterInfo dimColResolvedFilterInfoCopyObject =
+            dimColResolvedFilterInfo.getCopyObject();
+        dimColResolvedFilterInfoCopyObject.setDimension(dimensionFromCurrentBlock);
+        dimColResolvedFilterInfoCopyObject.setColumnIndex(dimensionFromCurrentBlock.getOrdinal());
+        return new IncludeFilterExecuterImpl(dimColResolvedFilterInfoCopyObject, segmentProperties);
+      } else {
+        return new RestructureIncludeFilterExecutorImpl(dimColResolvedFilterInfo,
+            segmentProperties);
+      }
     } else {
       return new IncludeColGroupFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
     }
@@ -181,7 +197,20 @@ public final class FilterUtil {
       DimColumnResolvedFilterInfo dimColResolvedFilterInfo, SegmentProperties segmentProperties) {
 
     if (dimColResolvedFilterInfo.getDimension().isColumnar()) {
-      return new ExcludeFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
+      CarbonDimension dimensionFromCurrentBlock = CarbonUtil
+          .getDimensionFromCurrentBlock(segmentProperties.getDimensions(),
+              dimColResolvedFilterInfo.getDimension());
+      if (null != dimensionFromCurrentBlock) {
+        // update dimension and column index according to the dimension position in current block
+        DimColumnResolvedFilterInfo dimColResolvedFilterInfoCopyObject =
+            dimColResolvedFilterInfo.getCopyObject();
+        dimColResolvedFilterInfoCopyObject.setDimension(dimensionFromCurrentBlock);
+        dimColResolvedFilterInfoCopyObject.setColumnIndex(dimensionFromCurrentBlock.getOrdinal());
+        return new ExcludeFilterExecuterImpl(dimColResolvedFilterInfoCopyObject, segmentProperties);
+      } else {
+        return new RestructureExcludeFilterExecutorImpl(dimColResolvedFilterInfo,
+            segmentProperties);
+      }
     } else {
       return new ExcludeColGroupFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties);
     }
@@ -571,6 +600,53 @@ public final class FilterUtil {
   }
 
   /**
+   * This method will check whether a default value for the non-existing column is present
+   * in the filter values list
+   *
+   * @param dimColumnEvaluatorInfo
+   * @return
+   */
+  public static boolean isDimensionDefaultValuePresentInFilterValues(
+      DimColumnResolvedFilterInfo dimColumnEvaluatorInfo) {
+    boolean isDefaultValuePresentInFilterValues = false;
+    DimColumnFilterInfo filterValues = dimColumnEvaluatorInfo.getFilterValues();
+    CarbonDimension dimension = dimColumnEvaluatorInfo.getDimension();
+    byte[] defaultValue = dimension.getDefaultValue();
+    if (!dimension.hasEncoding(Encoding.DICTIONARY)) {
+      // for no dictionary cases
+      // 3 cases: is NUll, is Not Null and filter on default value of newly added column
+      if (null == defaultValue) {
+        // default value for case where user gives is Null condition
+        defaultValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL
+            .getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+      }
+      List<byte[]> noDictionaryFilterValuesList = filterValues.getNoDictionaryFilterValuesList();
+      for (byte[] filterValue : noDictionaryFilterValuesList) {
+        int compare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(defaultValue, filterValue);
+        if (compare == 0) {
+          isDefaultValuePresentInFilterValues = true;
+          break;
+        }
+      }
+    } else {
+      // for dictionary and direct dictionary cases
+      // 3 cases: is NUll, is Not Null and filter on default value of newly added column
+      int defaultSurrogateValueToCompare = CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY;
+      if (null != defaultValue) {
+        defaultSurrogateValueToCompare++;
+      }
+      List<Integer> filterList = filterValues.getFilterList();
+      for (Integer filterValue : filterList) {
+        if (defaultSurrogateValueToCompare == filterValue) {
+          isDefaultValuePresentInFilterValues = true;
+          break;
+        }
+      }
+    }
+    return isDefaultValuePresentInFilterValues;
+  }
+
+  /**
    * Below method will be used to covert the filter surrogate keys
    * to mdkey
    *
@@ -590,13 +666,14 @@ public final class FilterUtil {
     int[] keys = new int[blockLevelKeyGenerator.getDimCount()];
     List<byte[]> filterValuesList = new ArrayList<byte[]>(20);
     Arrays.fill(keys, 0);
-    int[] rangesForMaskedByte =
-        getRangesForMaskedByte((carbonDimension.getKeyOrdinal()), blockLevelKeyGenerator);
+    int keyOrdinalOfDimensionFromCurrentBlock = carbonDimension.getKeyOrdinal();
     if (null != dimColumnFilterInfo) {
+      int[] rangesForMaskedByte =
+          getRangesForMaskedByte(keyOrdinalOfDimensionFromCurrentBlock, blockLevelKeyGenerator);
       for (Integer surrogate : dimColumnFilterInfo.getFilterList()) {
         try {
-          if (surrogate <= dimColumnsCardinality[carbonDimension.getKeyOrdinal()]) {
-            keys[carbonDimension.getKeyOrdinal()] = surrogate;
+          if (surrogate <= dimColumnsCardinality[keyOrdinalOfDimensionFromCurrentBlock]) {
+            keys[keyOrdinalOfDimensionFromCurrentBlock] = surrogate;
             filterValuesList
                 .add(getMaskedKey(rangesForMaskedByte, blockLevelKeyGenerator.generateKey(keys)));
           } else {
@@ -647,12 +724,12 @@ public final class FilterUtil {
    * @return long[] start key
    */
   public static void getStartKey(Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter,
-      long[] startKey, List<long[]> startKeyList) {
+      SegmentProperties segmentProperties, long[] startKey, List<long[]> startKeyList) {
     for (int i = 0; i < startKey.length; i++) {
       // The min surrogate key is 1, set it as the init value for starkey of each column level
       startKey[i] = 1;
     }
-    getStartKeyWithFilter(dimensionFilter, startKey, startKeyList);
+    getStartKeyWithFilter(dimensionFilter, segmentProperties, startKey, startKeyList);
   }
 
   /**
@@ -673,6 +750,7 @@ public final class FilterUtil {
    */
   public static void getStartKeyForNoDictionaryDimension(
       DimColumnResolvedFilterInfo dimColResolvedFilterInfo,
+      SegmentProperties segmentProperties,
       SortedMap<Integer, byte[]> setOfStartKeyByteArray) {
     Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter =
         dimColResolvedFilterInfo.getDimensionResolvedFilterInstance();
@@ -692,18 +770,25 @@ public final class FilterUtil {
         if (isExcludePresent) {
           continue;
         }
+        // in case of restructure scenarios it can happen that the filter dimension is not
+        // present in the current block. In those cases no need to determine the key
+        CarbonDimension dimensionFromCurrentBlock = CarbonUtil
+            .getDimensionFromCurrentBlock(segmentProperties.getDimensions(), entry.getKey());
+        if (null == dimensionFromCurrentBlock) {
+          continue;
+        }
         // step 2
         byte[] noDictionaryStartKey =
             listOfDimColFilterInfo.get(0).getNoDictionaryFilterValuesList().get(0);
         if (setOfStartKeyByteArray.isEmpty()) {
-          setOfStartKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryStartKey);
-        } else if (null == setOfStartKeyByteArray.get(entry.getKey().getOrdinal())) {
-          setOfStartKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryStartKey);
+          setOfStartKeyByteArray.put(dimensionFromCurrentBlock.getOrdinal(), noDictionaryStartKey);
+        } else if (null == setOfStartKeyByteArray.get(dimensionFromCurrentBlock.getOrdinal())) {
+          setOfStartKeyByteArray.put(dimensionFromCurrentBlock.getOrdinal(), noDictionaryStartKey);
 
         } else if (ByteUtil.UnsafeComparer.INSTANCE
-            .compareTo(setOfStartKeyByteArray.get(entry.getKey().getOrdinal()),
+            .compareTo(setOfStartKeyByteArray.get(dimensionFromCurrentBlock.getOrdinal()),
                 noDictionaryStartKey) > 0) {
-          setOfStartKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryStartKey);
+          setOfStartKeyByteArray.put(dimensionFromCurrentBlock.getOrdinal(), noDictionaryStartKey);
         }
       }
     }
@@ -727,6 +812,7 @@ public final class FilterUtil {
    */
   public static void getEndKeyForNoDictionaryDimension(
       DimColumnResolvedFilterInfo dimColResolvedFilterInfo,
+      SegmentProperties segmentProperties,
       SortedMap<Integer, byte[]> setOfEndKeyByteArray) {
 
     Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter =
@@ -747,18 +833,25 @@ public final class FilterUtil {
         if (isExcludePresent) {
           continue;
         }
+        // in case of restructure scenarios it can happen that the filter dimension is not
+        // present in the current block. In those cases no need to determine the key
+        CarbonDimension dimensionFromCurrentBlock = CarbonUtil
+            .getDimensionFromCurrentBlock(segmentProperties.getDimensions(), entry.getKey());
+        if (null == dimensionFromCurrentBlock) {
+          continue;
+        }
         // step 2
         byte[] noDictionaryEndKey = listOfDimColFilterInfo.get(0).getNoDictionaryFilterValuesList()
             .get(listOfDimColFilterInfo.get(0).getNoDictionaryFilterValuesList().size() - 1);
         if (setOfEndKeyByteArray.isEmpty()) {
-          setOfEndKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryEndKey);
-        } else if (null == setOfEndKeyByteArray.get(entry.getKey().getOrdinal())) {
-          setOfEndKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryEndKey);
+          setOfEndKeyByteArray.put(dimensionFromCurrentBlock.getOrdinal(), noDictionaryEndKey);
+        } else if (null == setOfEndKeyByteArray.get(dimensionFromCurrentBlock.getOrdinal())) {
+          setOfEndKeyByteArray.put(dimensionFromCurrentBlock.getOrdinal(), noDictionaryEndKey);
 
         } else if (ByteUtil.UnsafeComparer.INSTANCE
-            .compareTo(setOfEndKeyByteArray.get(entry.getKey().getOrdinal()), noDictionaryEndKey)
-            < 0) {
-          setOfEndKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryEndKey);
+            .compareTo(setOfEndKeyByteArray.get(dimensionFromCurrentBlock.getOrdinal()),
+                noDictionaryEndKey) < 0) {
+          setOfEndKeyByteArray.put(dimensionFromCurrentBlock.getOrdinal(), noDictionaryEndKey);
         }
 
       }
@@ -794,8 +887,8 @@ public final class FilterUtil {
    * @param startKey
    */
   private static void getStartKeyWithFilter(
-      Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter, long[] startKey,
-      List<long[]> startKeyList) {
+      Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter,
+      SegmentProperties segmentProperties, long[] startKey, List<long[]> startKeyList) {
     for (Map.Entry<CarbonDimension, List<DimColumnFilterInfo>> entry : dimensionFilter.entrySet()) {
       List<DimColumnFilterInfo> values = entry.getValue();
       if (null == values || !entry.getKey().hasEncoding(Encoding.DICTIONARY)) {
@@ -810,9 +903,17 @@ public final class FilterUtil {
       if (isExcludePresent) {
         continue;
       }
+      int keyOrdinalOfDimensionFromCurrentBlock = QueryUtil
+          .getKeyOrdinalOfDimensionFromCurrentBlock(segmentProperties.getDimensions(),
+              entry.getKey());
+      // if key ordinal is -1 that means this dimension does not exist in the current block.
+      // Applicable for restructure scenarios
+      if (keyOrdinalOfDimensionFromCurrentBlock == -1) {
+        continue;
+      }
       for (DimColumnFilterInfo info : values) {
-        if (startKey[entry.getKey().getKeyOrdinal()] < info.getFilterList().get(0)) {
-          startKey[entry.getKey().getKeyOrdinal()] = info.getFilterList().get(0);
+        if (startKey[keyOrdinalOfDimensionFromCurrentBlock] < info.getFilterList().get(0)) {
+          startKey[keyOrdinalOfDimensionFromCurrentBlock] = info.getFilterList().get(0);
         }
       }
       long[] newStartKey = new long[startKey.length];
@@ -831,7 +932,7 @@ public final class FilterUtil {
       endKey[i] = getMaxValue(updatedDimListBasedOnKeyGenerator.get(i),
           segmentProperties.getDimColumnsCardinality());
     }
-    getEndKeyWithFilter(dimensionFilter, endKey, endKeyList);
+    getEndKeyWithFilter(dimensionFilter, segmentProperties, endKey, endKeyList);
 
   }
 
@@ -850,8 +951,8 @@ public final class FilterUtil {
   }
 
   private static void getEndKeyWithFilter(
-      Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter, long[] endKey,
-      List<long[]> endKeyList) {
+      Map<CarbonDimension, List<DimColumnFilterInfo>> dimensionFilter,
+      SegmentProperties segmentProperties, long[] endKey, List<long[]> endKeyList) {
     for (Map.Entry<CarbonDimension, List<DimColumnFilterInfo>> entry : dimensionFilter.entrySet()) {
       List<DimColumnFilterInfo> values = entry.getValue();
       if (null == values || !entry.getKey().hasEncoding(Encoding.DICTIONARY)) {
@@ -866,11 +967,18 @@ public final class FilterUtil {
       if (isExcludeFilterPresent) {
         continue;
       }
-
+      int keyOrdinalOfDimensionFromCurrentBlock = QueryUtil
+          .getKeyOrdinalOfDimensionFromCurrentBlock(segmentProperties.getDimensions(),
+              entry.getKey());
+      // if key ordinal is -1 that means this dimension does not exist in the current block.
+      // Applicable for restructure scenarios
+      if (keyOrdinalOfDimensionFromCurrentBlock == -1) {
+        continue;
+      }
       for (DimColumnFilterInfo info : values) {
-        if (endKey[entry.getKey().getKeyOrdinal()] > info.getFilterList()
+        if (endKey[keyOrdinalOfDimensionFromCurrentBlock] > info.getFilterList()
             .get(info.getFilterList().size() - 1)) {
-          endKey[entry.getKey().getKeyOrdinal()] =
+          endKey[keyOrdinalOfDimensionFromCurrentBlock] =
               info.getFilterList().get(info.getFilterList().size() - 1);
         }
       }
@@ -1291,7 +1399,8 @@ public final class FilterUtil {
     traverseResolverTreeAndPopulateStartAndEndKeys(filterResolverTree.getLeft(),
         segmentProperties, startKeys, setOfStartKeyByteArray, endKeys, setOfEndKeyByteArray,
         startKeyList, endKeyList);
-    filterResolverTree.getStartKey(startKeys, setOfStartKeyByteArray, startKeyList);
+    filterResolverTree
+        .getStartKey(segmentProperties, startKeys, setOfStartKeyByteArray, startKeyList);
     filterResolverTree.getEndKey(segmentProperties, endKeys, setOfEndKeyByteArray,
         endKeyList);
 

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
new file mode 100644
index 0000000..68f0dfe
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.scan.filter.executer;
+
+import java.io.IOException;
+import java.util.BitSet;
+
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.util.BitSetGroup;
+
+public class RestructureExcludeFilterExecutorImpl implements FilterExecuter {
+
+  protected DimColumnResolvedFilterInfo dimColEvaluatorInfo;
+  protected SegmentProperties segmentProperties;
+
+  /**
+   * flag to check whether filter values contain the default value applied on the dimension column
+   * which does not exist in the current block
+   */
+  protected boolean isDefaultValuePresentInFilterValues;
+
+  public RestructureExcludeFilterExecutorImpl(DimColumnResolvedFilterInfo dimColEvaluatorInfo,
+      SegmentProperties segmentProperties) {
+    this.dimColEvaluatorInfo = dimColEvaluatorInfo;
+    this.segmentProperties = segmentProperties;
+    isDefaultValuePresentInFilterValues =
+        FilterUtil.isDimensionDefaultValuePresentInFilterValues(dimColEvaluatorInfo);
+  }
+
+  @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder) throws IOException {
+    int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
+    BitSetGroup bitSetGroup = new BitSetGroup(1);
+    BitSet bitSet = new BitSet(numberOfRows);
+    bitSet.set(0, numberOfRows, !isDefaultValuePresentInFilterValues);
+    bitSetGroup.setBitSet(bitSet, 0);
+    return bitSetGroup;
+  }
+
+  @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
+    BitSet bitSet = new BitSet(1);
+    bitSet.flip(0, 1);
+    return bitSet;
+  }
+
+  @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
new file mode 100644
index 0000000..c1ba2af
--- /dev/null
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.carbondata.core.scan.filter.executer;
+
+import java.io.IOException;
+import java.util.BitSet;
+
+import org.apache.carbondata.core.datastore.block.SegmentProperties;
+import org.apache.carbondata.core.scan.filter.FilterUtil;
+import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
+import org.apache.carbondata.core.scan.processor.BlocksChunkHolder;
+import org.apache.carbondata.core.util.BitSetGroup;
+
+public class RestructureIncludeFilterExecutorImpl implements FilterExecuter {
+
+  protected DimColumnResolvedFilterInfo dimColumnEvaluatorInfo;
+  protected SegmentProperties segmentProperties;
+
+  /**
+   * flag to check whether filter values contain the default value applied on the dimension column
+   * which does not exist in the current block
+   */
+  protected boolean isDefaultValuePresentInFilterValues;
+
+  public RestructureIncludeFilterExecutorImpl(DimColumnResolvedFilterInfo dimColumnEvaluatorInfo,
+      SegmentProperties segmentProperties) {
+    this.dimColumnEvaluatorInfo = dimColumnEvaluatorInfo;
+    this.segmentProperties = segmentProperties;
+    isDefaultValuePresentInFilterValues =
+        FilterUtil.isDimensionDefaultValuePresentInFilterValues(dimColumnEvaluatorInfo);
+  }
+
+  @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder) throws IOException {
+    int numberOfRows = blockChunkHolder.getDataBlock().nodeSize();
+    BitSetGroup bitSetGroup = new BitSetGroup(1);
+    BitSet bitSet = new BitSet(numberOfRows);
+    bitSet.set(0, numberOfRows, isDefaultValuePresentInFilterValues);
+    bitSetGroup.setBitSet(bitSet, 0);
+    return bitSetGroup;
+  }
+
+  public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) {
+    BitSet bitSet = new BitSet(1);
+    bitSet.set(0, isDefaultValuePresentInFilterValues);
+    return bitSet;
+  }
+
+  @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
+
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
index 7595ab6..b48382e 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java
@@ -19,7 +19,10 @@ package org.apache.carbondata.core.scan.filter.executer;
 import java.io.ByteArrayOutputStream;
 import java.io.DataOutputStream;
 import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.RoundingMode;
 import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.BitSet;
@@ -40,8 +43,11 @@ import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionary
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
 import org.apache.carbondata.core.metadata.datatype.DataType;
 import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure;
 import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo;
 import org.apache.carbondata.core.scan.executor.util.QueryUtil;
+import org.apache.carbondata.core.scan.executor.util.RestructureUtil;
 import org.apache.carbondata.core.scan.expression.Expression;
 import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException;
 import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
@@ -68,29 +74,85 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
   /**
    * it has index at which given dimension is stored in file
    */
-  private int[] blocksIndex;
+  protected int[] dimensionBlocksIndex;
+
+  /**
+   * it has index at which given measure is stored in file
+   */
+  protected int[] measureBlocksIndex;
 
   private Map<Integer, GenericQueryType> complexDimensionInfoMap;
 
+  /**
+   * flag to check whether the filter dimension is present in current block list of dimensions.
+   * Applicable for restructure scenarios
+   */
+  protected boolean[] isDimensionPresentInCurrentBlock;
+
+  /**
+   * flag to check whether the filter measure is present in current block list of measures.
+   * Applicable for restructure scenarios
+   */
+  protected boolean[] isMeasurePresentInCurrentBlock;
+
   public RowLevelFilterExecuterImpl(List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
       List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
       AbsoluteTableIdentifier tableIdentifier, SegmentProperties segmentProperties,
       Map<Integer, GenericQueryType> complexDimensionInfoMap) {
     this.dimColEvaluatorInfoList = dimColEvaluatorInfoList;
     this.segmentProperties = segmentProperties;
-    this.blocksIndex = new int[dimColEvaluatorInfoList.size()];
-    for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
-      this.blocksIndex[i] = segmentProperties.getDimensionOrdinalToBlockMapping()
-          .get(dimColEvaluatorInfoList.get(i).getColumnIndex());
-    }
+    this.dimensionBlocksIndex = new int[dimColEvaluatorInfoList.size()];
+    this.isDimensionPresentInCurrentBlock = new boolean[dimColEvaluatorInfoList.size()];
     if (null == msrColEvalutorInfoList) {
       this.msrColEvalutorInfoList = new ArrayList<MeasureColumnResolvedFilterInfo>(20);
     } else {
       this.msrColEvalutorInfoList = msrColEvalutorInfoList;
     }
+    this.measureBlocksIndex = new int[msrColEvalutorInfoList.size()];
+    this.isMeasurePresentInCurrentBlock = new boolean[msrColEvalutorInfoList.size()];
     this.exp = exp;
     this.tableIdentifier = tableIdentifier;
     this.complexDimensionInfoMap = complexDimensionInfoMap;
+    initDimensionBlockIndexes();
+    initMeasureBlockIndexes();
+  }
+
+  /**
+   * This method will initialize the dimension info for the current block to be
+   * used for filtering the data
+   */
+  private void initDimensionBlockIndexes() {
+    for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
+      // find the dimension in the current block dimensions list
+      CarbonDimension dimensionFromCurrentBlock = CarbonUtil
+          .getDimensionFromCurrentBlock(segmentProperties.getDimensions(),
+              dimColEvaluatorInfoList.get(i).getDimension());
+      if (null != dimensionFromCurrentBlock) {
+        dimColEvaluatorInfoList.get(i).setColumnIndex(dimensionFromCurrentBlock.getOrdinal());
+        this.dimensionBlocksIndex[i] = segmentProperties.getDimensionOrdinalToBlockMapping()
+            .get(dimensionFromCurrentBlock.getOrdinal());
+        isDimensionPresentInCurrentBlock[i] = true;
+      }
+    }
+  }
+
+  /**
+   * This method will initialize the measure info for the current block to be
+   * used for filtering the data
+   */
+  private void initMeasureBlockIndexes() {
+    for (int i = 0; i < msrColEvalutorInfoList.size(); i++) {
+      // find the measure in the current block measures list
+      CarbonMeasure measureFromCurrentBlock = CarbonUtil
+          .getMeasureFromCurrentBlock(segmentProperties.getMeasures(),
+              msrColEvalutorInfoList.get(i).getCarbonColumn().getColumnId());
+      if (null != measureFromCurrentBlock) {
+        msrColEvalutorInfoList.get(i).setColumnIndex(measureFromCurrentBlock.getOrdinal());
+        this.measureBlocksIndex[i] = segmentProperties.getMeasuresOrdinalToBlockMapping()
+            .get(measureFromCurrentBlock.getOrdinal());
+        isMeasurePresentInCurrentBlock[i] = true;
+      }
+    }
   }
 
   @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
@@ -102,13 +164,20 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
     int pageNumbers = 0;
 
     if (dimColEvaluatorInfoList.size() > 0) {
-      pageNumbers = blockChunkHolder.getDimensionRawDataChunk()[blocksIndex[0]].getPagesCount();
-      numberOfRows = blockChunkHolder.getDimensionRawDataChunk()[blocksIndex[0]].getRowCount();
+      if (isDimensionPresentInCurrentBlock[0]) {
+        pageNumbers =
+            blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[0]].getPagesCount();
+        numberOfRows =
+            blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[0]].getRowCount();
+      }
     }
     if (msrColEvalutorInfoList.size() > 0) {
-      int columnIndex = msrColEvalutorInfoList.get(0).getColumnIndex();
-      pageNumbers = blockChunkHolder.getMeasureRawDataChunk()[columnIndex].getPagesCount();
-      numberOfRows = blockChunkHolder.getMeasureRawDataChunk()[columnIndex].getRowCount();
+      if (isMeasurePresentInCurrentBlock[0]) {
+        pageNumbers =
+            blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]].getPagesCount();
+        numberOfRows =
+            blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]].getRowCount();
+      }
     }
     BitSetGroup bitSetGroup = new BitSetGroup(pageNumbers);
     for (int i = 0; i < pageNumbers; i++) {
@@ -152,13 +221,21 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
     String memberString;
     for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
       DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = dimColEvaluatorInfoList.get(i);
+      // if filter dimension is not present in the current add its default value
+      if (!isDimensionPresentInCurrentBlock[i]) {
+        // fill default value here
+        record[dimColumnEvaluatorInfo.getRowIndex()] =
+            getDimensionDefaultValue(dimColumnEvaluatorInfo);
+        continue;
+      }
       if (dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.ARRAY
           && dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.STRUCT) {
         if (!dimColumnEvaluatorInfo.isDimensionExistsInCurrentSilce()) {
-          record[dimColumnEvaluatorInfo.getRowIndex()] = dimColumnEvaluatorInfo.getDefaultValue();
+          record[dimColumnEvaluatorInfo.getRowIndex()] =
+              dimColumnEvaluatorInfo.getDimension().getDefaultValue();
         }
         DimensionColumnDataChunk columnDataChunk =
-            blockChunkHolder.getDimensionRawDataChunk()[blocksIndex[i]]
+            blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[i]]
                 .convertToDimColDataChunk(pageIndex);
         if (!dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)
             && columnDataChunk instanceof VariableLengthDimensionDataChunk) {
@@ -178,7 +255,7 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
           }
         } else {
           int dictionaryValue = readSurrogatesFromColumnBlock(blockChunkHolder, index, pageIndex,
-              dimColumnEvaluatorInfo, blocksIndex[i]);
+              dimColumnEvaluatorInfo, dimensionBlocksIndex[i]);
           if (dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY)
               && !dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
             memberString =
@@ -196,7 +273,7 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
         }
       } else {
         try {
-          GenericQueryType complexType = complexDimensionInfoMap.get(blocksIndex[i]);
+          GenericQueryType complexType = complexDimensionInfoMap.get(dimensionBlocksIndex[i]);
           ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
           DataOutputStream dataOutputStream = new DataOutputStream(byteStream);
           complexType.parseBlocksAndReturnComplexColumnByteArray(
@@ -211,8 +288,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
     }
 
     DataType msrType;
-
-    for (MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo : msrColEvalutorInfoList) {
+    for (int i = 0; i < msrColEvalutorInfoList.size(); i++) {
+      MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo = msrColEvalutorInfoList.get(i);
       switch (msrColumnEvalutorInfo.getType()) {
         case INT:
         case LONG:
@@ -224,10 +301,19 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
         default:
           msrType = DataType.DOUBLE;
       }
-      // if measure doesnt exist then set the default value.
+      // add default value for the measure in case filter measure is not present
+      // in the current block measure list
+      if (!isMeasurePresentInCurrentBlock[i]) {
+        byte[] defaultValue = msrColumnEvalutorInfo.getCarbonColumn().getDefaultValue();
+        record[msrColumnEvalutorInfo.getRowIndex()] = RestructureUtil
+            .getMeasureDefaultValue(msrColumnEvalutorInfo.getCarbonColumn().getColumnSchema(),
+                defaultValue);
+        continue;
+      }
+
       Object msrValue;
       MeasureColumnDataChunk measureColumnDataChunk =
-          blockChunkHolder.getMeasureRawDataChunk()[msrColumnEvalutorInfo.getColumnIndex()]
+          blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]]
               .convertToMeasureColDataChunk(pageIndex);
       switch (msrType) {
         case INT:
@@ -236,8 +322,17 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
               measureColumnDataChunk.getMeasureDataHolder().getReadableLongValueByIndex(index);
           break;
         case DECIMAL:
-          msrValue = measureColumnDataChunk.getMeasureDataHolder()
-              .getReadableBigDecimalValueByIndex(index);
+          BigDecimal bigDecimalValue =
+              measureColumnDataChunk.getMeasureDataHolder()
+                  .getReadableBigDecimalValueByIndex(index);
+          if (null != bigDecimalValue
+              && msrColumnEvalutorInfo.getCarbonColumn().getColumnSchema().getScale()
+              > bigDecimalValue.scale()) {
+            bigDecimalValue = bigDecimalValue
+                .setScale(msrColumnEvalutorInfo.getCarbonColumn().getColumnSchema().getScale(),
+                    RoundingMode.HALF_UP);
+          }
+          msrValue = bigDecimalValue;
           break;
         default:
           msrValue =
@@ -250,6 +345,28 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
   }
 
   /**
+   * This method will compute the default value for a dimension
+   *
+   * @param dimColumnEvaluatorInfo
+   * @return
+   */
+  private Object getDimensionDefaultValue(DimColumnResolvedFilterInfo dimColumnEvaluatorInfo) {
+    Object dimensionDefaultValue = null;
+    CarbonDimension dimension = dimColumnEvaluatorInfo.getDimension();
+    if (dimension.hasEncoding(Encoding.DICTIONARY) && !dimension
+        .hasEncoding(Encoding.DIRECT_DICTIONARY)) {
+      byte[] defaultValue = dimension.getDefaultValue();
+      if (null != defaultValue) {
+        dimensionDefaultValue =
+            new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET));
+      }
+    } else {
+      dimensionDefaultValue = RestructureUtil.validateAndGetDefaultValue(dimension);
+    }
+    return dimensionDefaultValue;
+  }
+
+  /**
    * method will read the actual data from the direct dictionary generator
    * by passing direct dictionary value.
    *
@@ -329,9 +446,9 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
       byte[] colData = chunk.getChunkData(index);
       long[] result = keyStructureInfo.getKeyGenerator().getKeyArray(colData);
       int colGroupId =
-          QueryUtil.getColumnGroupId(segmentProperties, dimColumnEvaluatorInfo.getColumnIndex());
+          QueryUtil.getColumnGroupId(segmentProperties, dimensionBlocksIndex[0]);
       int dictionaryValue = (int) result[segmentProperties
-          .getColumnGroupMdKeyOrdinal(colGroupId, dimColumnEvaluatorInfo.getColumnIndex())];
+          .getColumnGroupMdKeyOrdinal(colGroupId, dimensionBlocksIndex[0])];
       return dictionaryValue;
     } catch (KeyGenException e) {
       LOGGER.error(e);
@@ -346,30 +463,41 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter {
     return bitSet;
   }
 
+  /**
+   * This method will set the bitset to true by default for a given number of rows
+   *
+   * @param numberOfRows
+   * @return
+   */
+  protected BitSetGroup getDefaultBitSetGroup(int numberOfRows) {
+    BitSetGroup bitSetGroup = new BitSetGroup(1);
+    BitSet bitSet = new BitSet(numberOfRows);
+    bitSet.set(0, numberOfRows, true);
+    bitSetGroup.setBitSet(bitSet, 0);
+    return bitSetGroup;
+  }
+
   @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
     for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) {
       DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = dimColEvaluatorInfoList.get(i);
       if (dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.ARRAY
           && dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.STRUCT) {
-        if (null == blockChunkHolder.getDimensionRawDataChunk()[blocksIndex[i]]) {
-          blockChunkHolder.getDimensionRawDataChunk()[blocksIndex[i]] =
+        if (null == blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[i]]) {
+          blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[i]] =
               blockChunkHolder.getDataBlock()
-                  .getDimensionChunk(blockChunkHolder.getFileReader(), blocksIndex[i]);
+                  .getDimensionChunk(blockChunkHolder.getFileReader(), dimensionBlocksIndex[i]);
         }
       } else {
-        GenericQueryType complexType = complexDimensionInfoMap.get(blocksIndex[i]);
+        GenericQueryType complexType = complexDimensionInfoMap.get(dimensionBlocksIndex[i]);
         complexType.fillRequiredBlockData(blockChunkHolder);
       }
     }
 
     if (null != msrColEvalutorInfoList) {
-      for (MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo : msrColEvalutorInfoList) {
-        if (null == blockChunkHolder.getMeasureRawDataChunk()[msrColumnEvalutorInfo
-            .getColumnIndex()]) {
-          blockChunkHolder.getMeasureRawDataChunk()[msrColumnEvalutorInfo.getColumnIndex()] =
-              blockChunkHolder.getDataBlock().getMeasureChunk(blockChunkHolder.getFileReader(),
-                  msrColumnEvalutorInfo.getColumnIndex());
-        }
+      if (null == blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]]) {
+        blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]] =
+            blockChunkHolder.getDataBlock()
+                .getMeasureChunk(blockChunkHolder.getFileReader(), measureBlocksIndex[0]);
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
index 9f28d7c..e748141 100644
--- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
+++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java
@@ -26,6 +26,7 @@ import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk;
 import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk;
 import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier;
 import org.apache.carbondata.core.metadata.encoder.Encoding;
+import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension;
 import org.apache.carbondata.core.scan.expression.Expression;
 import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException;
 import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo;
@@ -38,6 +39,11 @@ import org.apache.carbondata.core.util.CarbonUtil;
 public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecuterImpl {
   private byte[][] filterRangeValues;
 
+  /**
+   * flag to check whether default values is present in the filter value list
+   */
+  private boolean isDefaultValuePresentInFilter;
+
   public RowLevelRangeGrtThanFiterExecuterImpl(
       List<DimColumnResolvedFilterInfo> dimColEvaluatorInfoList,
       List<MeasureColumnResolvedFilterInfo> msrColEvalutorInfoList, Expression exp,
@@ -46,44 +52,71 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
     super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties,
         null);
     this.filterRangeValues = filterRangeValues;
+    checkIfDefaultValueIsPresentInFilterList();
+  }
+
+  /**
+   * This method will check whether default value is present in the given filter values
+   */
+  private void checkIfDefaultValueIsPresentInFilterList() {
+    if (!this.isDimensionPresentInCurrentBlock[0]) {
+      CarbonDimension dimension = this.dimColEvaluatorInfoList.get(0).getDimension();
+      byte[] defaultValue = dimension.getDefaultValue();
+      if (null != defaultValue) {
+        for (int k = 0; k < filterRangeValues.length; k++) {
+          int maxCompare =
+              ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterRangeValues[k], defaultValue);
+          if (maxCompare < 0) {
+            isDefaultValuePresentInFilter = true;
+            break;
+          }
+        }
+      }
+    }
   }
 
   @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) {
     BitSet bitSet = new BitSet(1);
-    byte[][] filterValues = this.filterRangeValues;
-    int columnIndex = this.dimColEvaluatorInfoList.get(0).getColumnIndex();
-    boolean isScanRequired = isScanRequired(blockMaxValue[columnIndex], filterValues);
+    boolean isScanRequired =
+        isScanRequired(blockMaxValue[dimensionBlocksIndex[0]], filterRangeValues);
     if (isScanRequired) {
       bitSet.set(0);
     }
     return bitSet;
-
   }
 
   private boolean isScanRequired(byte[] blockMaxValue, byte[][] filterValues) {
     boolean isScanRequired = false;
-    for (int k = 0; k < filterValues.length; k++) {
-      // filter value should be in range of max and min value i.e
-      // max>filtervalue>min
-      // so filter-max should be negative
-      int maxCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue);
-      // if any filter value is in range than this block needs to be
-      // scanned means always less than block max range.
-      if (maxCompare < 0) {
-        isScanRequired = true;
-        break;
+    if (isDimensionPresentInCurrentBlock[0]) {
+      for (int k = 0; k < filterValues.length; k++) {
+        // filter value should be in range of max and min value i.e
+        // max>filtervalue>min
+        // so filter-max should be negative
+        int maxCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue);
+        // if any filter value is in range than this block needs to be
+        // scanned means always less than block max range.
+        if (maxCompare < 0) {
+          isScanRequired = true;
+          break;
+        }
       }
+    } else {
+      isScanRequired = isDefaultValuePresentInFilter;
     }
     return isScanRequired;
   }
 
   @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder)
       throws FilterUnsupportedException, IOException {
+    // select all rows if dimension does not exists in the current block
+    if (!isDimensionPresentInCurrentBlock[0]) {
+      return getDefaultBitSetGroup(blockChunkHolder.getDataBlock().nodeSize());
+    }
     if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
       return super.applyFilter(blockChunkHolder);
     }
     int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
-        .get(dimColEvaluatorInfoList.get(0).getColumnIndex());
+        .get(dimensionBlocksIndex[0]);
     if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
       blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
           .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
@@ -238,14 +271,15 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute
   }
 
   @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException {
-    if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
-      super.readBlocks(blockChunkHolder);
-    }
-    int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping()
-        .get(dimColEvaluatorInfoList.get(0).getColumnIndex());
-    if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
-      blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
-          .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+    if (isDimensionPresentInCurrentBlock[0]) {
+      if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) {
+        super.readBlocks(blockChunkHolder);
+      }
+      int blockIndex = dimensionBlocksIndex[0];
+      if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) {
+        blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock()
+            .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex);
+      }
     }
   }
 }


Mime
View raw message