Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 8C3A5200C39 for ; Thu, 16 Mar 2017 10:28:57 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 8AD41160B8E; Thu, 16 Mar 2017 09:28:57 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 405FA160B78 for ; Thu, 16 Mar 2017 10:28:55 +0100 (CET) Received: (qmail 52329 invoked by uid 500); 16 Mar 2017 09:28:54 -0000 Mailing-List: contact commits-help@carbondata.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@carbondata.incubator.apache.org Delivered-To: mailing list commits@carbondata.incubator.apache.org Received: (qmail 52320 invoked by uid 99); 16 Mar 2017 09:28:54 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd1-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 16 Mar 2017 09:28:54 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd1-us-west.apache.org (ASF Mail Server at spamd1-us-west.apache.org) with ESMTP id D6026C1809 for ; Thu, 16 Mar 2017 09:28:53 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd1-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -3.569 X-Spam-Level: X-Spam-Status: No, score=-3.569 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RP_MATCHES_RCVD=-0.001, SPF_NEUTRAL=0.652] autolearn=disabled Received: from mx1-lw-eu.apache.org ([10.40.0.8]) by localhost (spamd1-us-west.apache.org [10.40.0.7]) (amavisd-new, port 10024) with ESMTP id tfTL1pZ6Het9 for ; Thu, 16 Mar 2017 09:28:48 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-lw-eu.apache.org (ASF Mail Server at mx1-lw-eu.apache.org) with SMTP id BD95A5FC7B for ; Thu, 16 Mar 2017 09:28:45 +0000 (UTC) Received: (qmail 52078 invoked by uid 99); 16 Mar 2017 09:28:45 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 16 Mar 2017 09:28:44 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id BC228E1835; Thu, 16 Mar 2017 09:28:44 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: gvramana@apache.org To: commits@carbondata.incubator.apache.org Date: Thu, 16 Mar 2017 09:28:48 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [05/14] incubator-carbondata git commit: Following changes are done as part of this commit. archived-at: Thu, 16 Mar 2017 09:28:57 -0000 http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/AggregatorInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/AggregatorInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/AggregatorInfo.java deleted file mode 100644 index b992ea6..0000000 --- a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/AggregatorInfo.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.carbondata.core.scan.executor.infos; - -import org.apache.carbondata.core.metadata.datatype.DataType; - -/** - * Info class which store all the details - * which is required during aggregation - */ -public class AggregatorInfo { - - /** - * selected query measure ordinal - * which will be used to read the measures chunk data - * this will be storing the index of the measure in measures chunk - */ - private int[] measureOrdinals; - - /** - * This parameter will be used to - * check whether particular measure is present - * in the table block, if not then its default value will be used - */ - private boolean[] measureExists; - - /** - * this default value will be used to when some measure is not present - * in the table block, in case of restructuring of the table if user is adding any - * measure then in older block that measure wont be present so for measure default value - * will be used to aggregate in the older table block query execution - */ - private Object[] defaultValues; - - /** - * Datatype of each measure; - */ - private DataType[] measureDataTypes; - - /** - * @return the measureOrdinal - */ - public int[] getMeasureOrdinals() { - return measureOrdinals; - } - - /** - * @param measureOrdinal the measureOrdinal to set - */ - public void setMeasureOrdinals(int[] measureOrdinal) { - this.measureOrdinals = measureOrdinal; - } - - /** - * @return the measureExists - */ - public boolean[] getMeasureExists() { - return measureExists; - } - - /** - * @param measureExists the measureExists to set - */ - public void setMeasureExists(boolean[] measureExists) { - this.measureExists = measureExists; - } - - /** - * @return the defaultValues - */ - public Object[] getDefaultValues() { - return defaultValues; - } - - /** - * @param defaultValues the defaultValues to set - */ - public void setDefaultValues(Object[] defaultValues) { - this.defaultValues = defaultValues; - } - - public DataType[] getMeasureDataTypes() { - return measureDataTypes; - } - - public void setMeasureDataTypes(DataType[] measureDataTypes) { - this.measureDataTypes = measureDataTypes; - } -} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java index 2dd6721..b294b58 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/BlockExecutionInfo.java @@ -49,10 +49,10 @@ public class BlockExecutionInfo { private boolean isFixedKeyUpdateRequired; /** - * below to store all the information required for aggregation during query + * below to store all the information required for measures during query * execution */ - private AggregatorInfo aggregatorInfo; + private MeasureInfo measureInfo; /** * this will be used to get the first tentative block from which query @@ -71,13 +71,6 @@ public class BlockExecutionInfo { private String blockId; /** - * masked byte for block which will be used to unpack the fixed length key, - * this will be used for updating the older block key with new block key - * generator - */ - private int[] maskedByteForBlock; - - /** * total number of dimension in block */ private int totalNumberDimensionBlock; @@ -196,21 +189,43 @@ public class BlockExecutionInfo { } /** - * list of dimension selected for in query + * list of dimension present in the current block. This will be + * different in case of restructured block */ private QueryDimension[] queryDimensions; /** - * list of measure selected in query + * list of dimension selected for in query + */ + private QueryDimension[] actualQueryDimensions; + + /** + * list of dimension present in the current block. This will be + * different in case of restructured block */ private QueryMeasure[] queryMeasures; /** + * list of measure selected in query + */ + private QueryMeasure[] actualQueryMeasures; + + /** + * variable to maintain dimension existence and default value info + */ + private DimensionInfo dimensionInfo; + + /** * whether it needs to read data in vector/columnar format. */ private boolean vectorBatchCollector; /** + * flag to specify that whether the current block is with latest schema or old schema + */ + private boolean isRestructuredBlock; + + /** * absolute table identifier */ private AbsoluteTableIdentifier absoluteTableIdentifier; @@ -247,15 +262,15 @@ public class BlockExecutionInfo { /** * @return the aggregatorInfos */ - public AggregatorInfo getAggregatorInfo() { - return aggregatorInfo; + public MeasureInfo getMeasureInfo() { + return measureInfo; } /** - * @param aggregatorInfo the aggregatorInfos to set + * @param measureInfo the aggregatorInfos to set */ - public void setAggregatorInfo(AggregatorInfo aggregatorInfo) { - this.aggregatorInfo = aggregatorInfo; + public void setMeasureInfo(MeasureInfo measureInfo) { + this.measureInfo = measureInfo; } /** @@ -287,22 +302,6 @@ public class BlockExecutionInfo { } /** - * @return the maskedByteForBlock - */ - public int[] getMaskedByteForBlock() { - return maskedByteForBlock; - } - - - - /** - * @param maskedByteForBlock the maskedByteForBlock to set - */ - public void setMaskedByteForBlock(int[] maskedByteForBlock) { - this.maskedByteForBlock = maskedByteForBlock; - } - - /** * @return the totalNumberDimensionBlock */ public int getTotalNumberDimensionBlock() { @@ -359,20 +358,6 @@ public class BlockExecutionInfo { } /** - * @return the restructureInfos - */ - public KeyStructureInfo getKeyStructureInfo() { - return keyStructureInfo; - } - - /** - * @param keyStructureInfo the restructureInfos to set - */ - public void setKeyStructureInfo(KeyStructureInfo keyStructureInfo) { - this.keyStructureInfo = keyStructureInfo; - } - - /** * @return the firstDataBlock */ public DataRefNode getFirstDataBlock() { @@ -610,6 +595,38 @@ public class BlockExecutionInfo { this.blockId = blockId; } + public boolean isRestructuredBlock() { + return isRestructuredBlock; + } + + public void setRestructuredBlock(boolean restructuredBlock) { + isRestructuredBlock = restructuredBlock; + } + + public DimensionInfo getDimensionInfo() { + return dimensionInfo; + } + + public void setDimensionInfo(DimensionInfo dimensionInfo) { + this.dimensionInfo = dimensionInfo; + } + + public QueryDimension[] getActualQueryDimensions() { + return actualQueryDimensions; + } + + public void setActualQueryDimensions(QueryDimension[] actualQueryDimensions) { + this.actualQueryDimensions = actualQueryDimensions; + } + + public QueryMeasure[] getActualQueryMeasures() { + return actualQueryMeasures; + } + + public void setActualQueryMeasures(QueryMeasure[] actualQueryMeasures) { + this.actualQueryMeasures = actualQueryMeasures; + } + public int[] getProjectionListDimensionIndexes() { return projectionListDimensionIndexes; } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/DimensionInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/DimensionInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/DimensionInfo.java new file mode 100644 index 0000000..3b2e2bd --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/DimensionInfo.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.core.scan.executor.infos; + +/** + * This method will information about the query dimensions whether they exist in particular block + * and their default value + */ +public class DimensionInfo { + + /** + * flag to check whether a given dimension exists in a given block + */ + private boolean[] dimensionExists; + + /** + * maintains default value for each dimension + */ + private Object[] defaultValues; + + /** + * @param dimensionExists + * @param defaultValues + */ + public DimensionInfo(boolean[] dimensionExists, Object[] defaultValues) { + this.dimensionExists = dimensionExists; + this.defaultValues = defaultValues; + } + + /** + * @return + */ + public boolean[] getDimensionExists() { + return dimensionExists; + } + + /** + * @return + */ + public Object[] getDefaultValues() { + return defaultValues; + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/MeasureInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/MeasureInfo.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/MeasureInfo.java new file mode 100644 index 0000000..5dd4fb8 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/infos/MeasureInfo.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.scan.executor.infos; + +import org.apache.carbondata.core.metadata.datatype.DataType; + +/** + * Info class which store all the details + * which is required during aggregation + */ +public class MeasureInfo { + + /** + * selected query measure ordinal + * which will be used to read the measures chunk data + * this will be storing the index of the measure in measures chunk + */ + private int[] measureOrdinals; + + /** + * This parameter will be used to + * check whether particular measure is present + * in the table block, if not then its default value will be used + */ + private boolean[] measureExists; + + /** + * this default value will be used to when some measure is not present + * in the table block, in case of restructuring of the table if user is adding any + * measure then in older block that measure wont be present so for measure default value + * will be used to aggregate in the older table block query execution + */ + private Object[] defaultValues; + + /** + * Datatype of each measure; + */ + private DataType[] measureDataTypes; + + /** + * @return the measureOrdinal + */ + public int[] getMeasureOrdinals() { + return measureOrdinals; + } + + /** + * @param measureOrdinal the measureOrdinal to set + */ + public void setMeasureOrdinals(int[] measureOrdinal) { + this.measureOrdinals = measureOrdinal; + } + + /** + * @return the measureExists + */ + public boolean[] getMeasureExists() { + return measureExists; + } + + /** + * @param measureExists the measureExists to set + */ + public void setMeasureExists(boolean[] measureExists) { + this.measureExists = measureExists; + } + + /** + * @return the defaultValues + */ + public Object[] getDefaultValues() { + return defaultValues; + } + + /** + * @param defaultValues the defaultValues to set + */ + public void setDefaultValues(Object[] defaultValues) { + this.defaultValues = defaultValues; + } + + public DataType[] getMeasureDataTypes() { + return measureDataTypes; + } + + public void setMeasureDataTypes(DataType[] measureDataTypes) { + this.measureDataTypes = measureDataTypes; + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java index ef6fb8a..2f47b6a 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/QueryUtil.java @@ -208,6 +208,7 @@ public class QueryUtil { */ public static int[] getDimensionsBlockIndexes(List queryDimensions, Map dimensionOrdinalToBlockMapping, + Map queryDimensionToCurrentBlockDimensionOrdinalMapping, List customAggregationDimension, Set filterDimensions, Set allProjectionListDimensionIndexes) { // using set as in row group columns will point to same block @@ -219,16 +220,17 @@ public class QueryUtil { continue; } - allProjectionListDimensionIndexes.add( - dimensionOrdinalToBlockMapping.get(queryDimensions.get(i).getDimension().getOrdinal())); + Integer dimensionOrdinal = queryDimensionToCurrentBlockDimensionOrdinalMapping + .get(queryDimensions.get(i).getDimension().getOrdinal()); + allProjectionListDimensionIndexes.add(dimensionOrdinal); if (queryDimensions.get(i).getDimension().numberOfChild() > 0) { addChildrenBlockIndex(allProjectionListDimensionIndexes, queryDimensions.get(i).getDimension()); } - if (!filterDimensionOrdinal.contains(queryDimensions.get(i).getDimension().getOrdinal())) { + if (!filterDimensionOrdinal.contains(dimensionOrdinal)) { blockIndex = - dimensionOrdinalToBlockMapping.get(queryDimensions.get(i).getDimension().getOrdinal()); + dimensionOrdinalToBlockMapping.get(dimensionOrdinal); dimensionBlockIndex.add(blockIndex); if (queryDimensions.get(i).getDimension().numberOfChild() > 0) { addChildrenBlockIndex(dimensionBlockIndex, queryDimensions.get(i).getDimension()); @@ -249,6 +251,91 @@ public class QueryUtil { } /** + * This method will maintain a position mapping of query dimensions to current block dimensions + * + * @param queryDimensions + * @param currentBlockDimensions + * @return + */ + public static Map getQueryDimensionToCurrentBlockDimensionOrdinalMapping( + List queryDimensions, List currentBlockDimensions) { + Map queryToCurrentBlockDimensionOrdinals = + new HashMap<>(queryDimensions.size()); + for (QueryDimension queryDimension : queryDimensions) { + if (queryDimension.getDimension().hasEncoding(Encoding.IMPLICIT)) { + continue; + } + int ordinalOfDimensionFromCurrentBlock = + getOrdinalOfDimensionFromCurrentBlock(currentBlockDimensions, + queryDimension.getDimension()); + queryToCurrentBlockDimensionOrdinals + .put(queryDimension.getDimension().getOrdinal(), ordinalOfDimensionFromCurrentBlock); + } + return queryToCurrentBlockDimensionOrdinals; + } + + /** + * This method will return the key ordinal of the query dimension from the current block + * + * @param blockDimensions + * @param queryDimension + * @return + */ + public static int getKeyOrdinalOfDimensionFromCurrentBlock(List blockDimensions, + CarbonDimension queryDimension) { + int keyOrdinalInCurrentDimensionBlock = -1; + for (CarbonDimension blockDimension : blockDimensions) { + if (queryDimension.getColumnId().equals(blockDimension.getColumnId())) { + keyOrdinalInCurrentDimensionBlock = blockDimension.getKeyOrdinal(); + break; + } + } + return keyOrdinalInCurrentDimensionBlock; + } + + /** + * This method will return the ordinal of the query dimension from the current block + * + * @param blockDimensions + * @param queryDimension + * @return + */ + public static int getOrdinalOfDimensionFromCurrentBlock(List blockDimensions, + CarbonDimension queryDimension) { + int keyOrdinalInCurrentDimensionBlock = -1; + for (CarbonDimension blockDimension : blockDimensions) { + if (queryDimension.getColumnId().equals(blockDimension.getColumnId())) { + keyOrdinalInCurrentDimensionBlock = blockDimension.getOrdinal(); + break; + } + } + return keyOrdinalInCurrentDimensionBlock; + } + + /** + * This method will maintain a position mapping of query measures to current block measures + * + * @param queryMeasures + * @param currentBlockMeasures + * @return + */ + public static Map getQueryMeasuresToCurrentBlockMeasuresOrdinalMapping( + List queryMeasures, List currentBlockMeasures) { + Map queryToCurrentBlockDimensionOrdinals = + new HashMap<>(queryMeasures.size()); + for (QueryMeasure queryMeasure : queryMeasures) { + for (CarbonMeasure currentBlockMeasure : currentBlockMeasures) { + if (queryMeasure.getMeasure().getColumnId().equals(currentBlockMeasure.getColumnId())) { + queryToCurrentBlockDimensionOrdinals + .put(queryMeasure.getMeasure().getOrdinal(), currentBlockMeasure.getOrdinal()); + break; + } + } + } + return queryToCurrentBlockDimensionOrdinals; + } + + /** * Below method will be used to add the children block index * this will be basically for complex dimension which will have children * @@ -400,18 +487,21 @@ public class QueryUtil { */ public static int[] getMeasureBlockIndexes(List queryMeasures, List expressionMeasure, Map ordinalToBlockIndexMapping, + Map queryMeasuresToCurrentBlockMeasuresOrdinalMapping, Set filterMeasures, List allProjectionListMeasureIdexes) { Set measureBlockIndex = new HashSet(); Set filterMeasureOrdinal = getFilterMeasureOrdinal(filterMeasures); for (int i = 0; i < queryMeasures.size(); i++) { + Integer measureOrdinal = queryMeasuresToCurrentBlockMeasuresOrdinalMapping + .get(queryMeasures.get(i).getMeasure().getOrdinal()); allProjectionListMeasureIdexes.add(queryMeasures.get(i).getMeasure().getOrdinal()); - if (!filterMeasureOrdinal.contains(queryMeasures.get(i).getMeasure().getOrdinal())) { - measureBlockIndex - .add(ordinalToBlockIndexMapping.get(queryMeasures.get(i).getMeasure().getOrdinal())); + if (!filterMeasureOrdinal.contains(measureOrdinal)) { + measureBlockIndex.add(ordinalToBlockIndexMapping.get(measureOrdinal)); } } for (int i = 0; i < expressionMeasure.size(); i++) { - measureBlockIndex.add(ordinalToBlockIndexMapping.get(expressionMeasure.get(i).getOrdinal())); + measureBlockIndex.add(queryMeasuresToCurrentBlockMeasuresOrdinalMapping + .get(ordinalToBlockIndexMapping.get(expressionMeasure.get(i).getOrdinal()))); } int[] measureIndexes = ArrayUtils.toPrimitive(measureBlockIndex.toArray(new Integer[measureBlockIndex.size()])); @@ -420,6 +510,54 @@ public class QueryUtil { } /** + * This method will create the updated list of filter measures present in the current block + * + * @param queryFilterMeasures + * @param currentBlockMeasures + * @return + */ + public static Set getUpdatedFilterMeasures(Set queryFilterMeasures, + List currentBlockMeasures) { + if (!queryFilterMeasures.isEmpty()) { + Set updatedFilterMeasures = new HashSet<>(queryFilterMeasures.size()); + for (CarbonMeasure queryMeasure : queryFilterMeasures) { + CarbonMeasure measureFromCurrentBlock = + CarbonUtil.getMeasureFromCurrentBlock(currentBlockMeasures, queryMeasure.getColumnId()); + if (null != measureFromCurrentBlock) { + updatedFilterMeasures.add(measureFromCurrentBlock); + } + } + return updatedFilterMeasures; + } else { + return queryFilterMeasures; + } + } + + /** + * This method will create the updated list of filter dimensions present in the current block + * + * @param queryFilterDimensions + * @param currentBlockDimensions + * @return + */ + public static Set getUpdatedFilterDimensions( + Set queryFilterDimensions, List currentBlockDimensions) { + if (!queryFilterDimensions.isEmpty()) { + Set updatedFilterDimensions = new HashSet<>(queryFilterDimensions.size()); + for (CarbonDimension queryDimension : queryFilterDimensions) { + CarbonDimension dimensionFromCurrentBlock = + CarbonUtil.getDimensionFromCurrentBlock(currentBlockDimensions, queryDimension); + if (null != dimensionFromCurrentBlock) { + updatedFilterDimensions.add(dimensionFromCurrentBlock); + } + } + return updatedFilterDimensions; + } else { + return queryFilterDimensions; + } + } + + /** * Below method will be used to get mapping whether dimension is present in * order by or not * @@ -608,17 +746,20 @@ public class QueryUtil { */ public static void fillQueryDimensionsBlockIndexes(List queryDimensions, Map columnOrdinalToBlockIndexMapping, + Map queryDimensionToCurrentBlockDimensionOrdinalMapping, Set dictionaryDimensionBlockIndex, List noDictionaryDimensionBlockIndex) { for (QueryDimension queryDimension : queryDimensions) { if (CarbonUtil.hasEncoding(queryDimension.getDimension().getEncoder(), Encoding.DICTIONARY) && queryDimension.getDimension().numberOfChild() == 0) { - dictionaryDimensionBlockIndex - .add(columnOrdinalToBlockIndexMapping.get(queryDimension.getDimension().getOrdinal())); + dictionaryDimensionBlockIndex.add(columnOrdinalToBlockIndexMapping.get( + queryDimensionToCurrentBlockDimensionOrdinalMapping + .get(queryDimension.getDimension().getOrdinal()))); } else if ( !CarbonUtil.hasEncoding(queryDimension.getDimension().getEncoder(), Encoding.IMPLICIT) && queryDimension.getDimension().numberOfChild() == 0) { - noDictionaryDimensionBlockIndex - .add(columnOrdinalToBlockIndexMapping.get(queryDimension.getDimension().getOrdinal())); + noDictionaryDimensionBlockIndex.add(columnOrdinalToBlockIndexMapping.get( + queryDimensionToCurrentBlockDimensionOrdinalMapping + .get(queryDimension.getDimension().getOrdinal()))); } } } @@ -759,8 +900,8 @@ public class QueryUtil { if (filterDimension.hasEncoding(Encoding.IMPLICIT)) { continue; } - fillParentDetails(dimensionToBlockIndexMap, filterDimension, complexTypeMap, - eachComplexColumnValueSize, columnIdToDictionaryMap); + // fillParentDetails(dimensionToBlockIndexMap, filterDimension, complexTypeMap, + // eachComplexColumnValueSize, columnIdToDictionaryMap); } } return complexTypeMap; http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java index 3549932..847a795 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/executor/util/RestructureUtil.java @@ -16,16 +16,30 @@ */ package org.apache.carbondata.core.scan.executor.util; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryGenerator; +import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionaryKeyGeneratorFactory; +import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.encoder.Encoding; import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure; -import org.apache.carbondata.core.scan.executor.infos.AggregatorInfo; +import org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema; +import org.apache.carbondata.core.scan.executor.infos.BlockExecutionInfo; +import org.apache.carbondata.core.scan.executor.infos.DimensionInfo; +import org.apache.carbondata.core.scan.executor.infos.MeasureInfo; import org.apache.carbondata.core.scan.model.QueryDimension; import org.apache.carbondata.core.scan.model.QueryMeasure; +import org.apache.carbondata.core.util.CarbonUtil; +import org.apache.carbondata.core.util.DataTypeUtil; + +import org.apache.commons.lang3.ArrayUtils; +import org.apache.spark.unsafe.types.UTF8String; /** * Utility class for restructuring @@ -38,37 +52,160 @@ public class RestructureUtil { * table blocks in that case we need to select only those dimension out of * query dimension which is present in the current table block * + * @param blockExecutionInfo * @param queryDimensions * @param tableBlockDimensions + * @param tableComplexDimension * @return list of query dimension which is present in the table block */ - public static List getUpdatedQueryDimension(List queryDimensions, + public static List createDimensionInfoAndGetUpdatedQueryDimension( + BlockExecutionInfo blockExecutionInfo, List queryDimensions, List tableBlockDimensions, List tableComplexDimension) { List presentDimension = new ArrayList(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); + boolean[] isDimensionExists = new boolean[queryDimensions.size()]; + Object[] defaultValues = new Object[queryDimensions.size()]; // selecting only those dimension which is present in the query + int dimIndex = 0; for (QueryDimension queryDimension : queryDimensions) { if (queryDimension.getDimension().hasEncoding(Encoding.IMPLICIT)) { presentDimension.add(queryDimension); + isDimensionExists[dimIndex] = true; } else { for (CarbonDimension tableDimension : tableBlockDimensions) { - if (tableDimension.equals(queryDimension.getDimension())) { + if (tableDimension.getColumnId().equals(queryDimension.getDimension().getColumnId())) { presentDimension.add(queryDimension); + isDimensionExists[dimIndex] = true; + break; } } + // add default value only in case query dimension is not found in the current block + if (!isDimensionExists[dimIndex]) { + defaultValues[dimIndex] = validateAndGetDefaultValue(queryDimension.getDimension()); + blockExecutionInfo.setRestructuredBlock(true); + } } + dimIndex++; } - for (QueryDimension queryDimimension : queryDimensions) { + dimIndex = 0; + for (QueryDimension queryDimension : queryDimensions) { for (CarbonDimension tableDimension : tableComplexDimension) { - if (tableDimension.equals(queryDimimension.getDimension())) { - presentDimension.add(queryDimimension); + if (tableDimension.getColumnId().equals(queryDimension.getDimension().getColumnId())) { + presentDimension.add(queryDimension); + isDimensionExists[dimIndex] = true; + break; } } + // add default value only in case query dimension is not found in the current block + if (!isDimensionExists[dimIndex]) { + defaultValues[dimIndex] = + validateAndGetDefaultValue(queryDimension.getDimension()); + } + dimIndex++; } + DimensionInfo dimensionInfo = + new DimensionInfo(isDimensionExists, defaultValues); + blockExecutionInfo.setDimensionInfo(dimensionInfo); return presentDimension; } /** + * This method will validate and return the default value to be + * filled at the time of result preparation + * + * @param queryDimension + * @return + */ + public static Object validateAndGetDefaultValue(CarbonDimension queryDimension) { + byte[] defaultValue = queryDimension.getDefaultValue(); + Object defaultValueToBeConsidered = null; + if (CarbonUtil.hasEncoding(queryDimension.getEncoder(), Encoding.DICTIONARY)) { + // direct dictionary case + if (CarbonUtil.hasEncoding(queryDimension.getEncoder(), Encoding.DIRECT_DICTIONARY)) { + defaultValueToBeConsidered = getDirectDictionaryDefaultValue(queryDimension.getDataType(), + queryDimension.getDefaultValue()); + } else { + // dictionary case + defaultValueToBeConsidered = getDictionaryDefaultValue(defaultValue); + } + } else { + // no dictionary + defaultValueToBeConsidered = getNoDictionaryDefaultValue(defaultValue); + } + return defaultValueToBeConsidered; + } + + /** + * Method for computing default value for dictionary column + * + * @param defaultValue + * @return + */ + private static Object getDictionaryDefaultValue(byte[] defaultValue) { + Object dictionaryDefaultValue = null; + // dictionary has 2 cases: + // 1. If default value is specified then its surrogate key will be 2 + // 2. If default value is not specified then its surrogate key will be + // 1 which is for member default value null + if (isDefaultValueNull(defaultValue)) { + dictionaryDefaultValue = new Integer(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY); + } else { + dictionaryDefaultValue = + new Integer(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY + 1); + } + return dictionaryDefaultValue; + } + + /** + * Method for computing default value for direct dictionary + * + * @param dataType + * @param defaultValue + * @return + */ + private static Object getDirectDictionaryDefaultValue(DataType dataType, byte[] defaultValue) { + Object directDictionaryDefaultValue = null; + if (!isDefaultValueNull(defaultValue)) { + DirectDictionaryGenerator directDictionaryGenerator = + DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(dataType); + if (directDictionaryGenerator != null) { + String value = + new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)); + directDictionaryDefaultValue = + directDictionaryGenerator.getValueFromSurrogate(Integer.parseInt(value)); + } + } + return directDictionaryDefaultValue; + } + + /** + * Method for computing default value for no dictionary + * + * @param defaultValue + * @return + */ + private static Object getNoDictionaryDefaultValue(byte[] defaultValue) { + Object noDictionaryDefaultValue = null; + if (!isDefaultValueNull(defaultValue)) { + noDictionaryDefaultValue = UTF8String.fromBytes(defaultValue); + } + return noDictionaryDefaultValue; + } + + /** + * This method will validate whether a given value is empty or null + * + * @param defaultValue + * @return + */ + private static boolean isDefaultValueNull(byte[] defaultValue) { + if (null == defaultValue) { + return true; + } + return false; + } + + /** * Below method is to add dimension children for complex type dimension as * internally we are creating dimension column for each each complex * dimension so when complex query dimension request will come in the query, @@ -102,37 +239,88 @@ public class RestructureUtil { } /** - * Below method will be used to get the aggregator info object + * Method for computing measure default value based on the data type + * + * @param columnSchema + * @param defaultValue + * @return + */ + public static Object getMeasureDefaultValue(ColumnSchema columnSchema, byte[] defaultValue) { + Object measureDefaultValue = null; + if (!isDefaultValueNull(defaultValue)) { + String value = null; + switch (columnSchema.getDataType()) { + case SHORT: + case INT: + case LONG: + value = + new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)); + measureDefaultValue = Long.parseLong(value); + break; + case DECIMAL: + BigDecimal decimal = DataTypeUtil.byteToBigDecimal(defaultValue); + if (columnSchema.getScale() > decimal.scale()) { + decimal = decimal.setScale(columnSchema.getScale(), RoundingMode.HALF_UP); + } + measureDefaultValue = decimal; + break; + default: + value = + new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)); + Double parsedValue = Double.valueOf(value); + if (!Double.isInfinite(parsedValue) && !Double.isNaN(parsedValue)) { + measureDefaultValue = value; + } + } + } + return measureDefaultValue; + } + + /** + * Below method will be used to prepare the measure info object * in this method some of the properties which will be extracted * from query measure and current block measures will be set * + * @param blockExecutionInfo * @param queryMeasures measures present in query * @param currentBlockMeasures current block measures - * @return aggregator info + * @return measures present in the block */ - public static AggregatorInfo getAggregatorInfos(List queryMeasures, + public static List createMeasureInfoAndGetUpdatedQueryMeasures( + BlockExecutionInfo blockExecutionInfo, List queryMeasures, List currentBlockMeasures) { - AggregatorInfo aggregatorInfos = new AggregatorInfo(); + MeasureInfo measureInfo = new MeasureInfo(); + List presentMeasure = new ArrayList<>(queryMeasures.size()); int numberOfMeasureInQuery = queryMeasures.size(); - int[] measureOrdinals = new int[numberOfMeasureInQuery]; + List measureOrdinalList = new ArrayList<>(numberOfMeasureInQuery); Object[] defaultValues = new Object[numberOfMeasureInQuery]; boolean[] measureExistsInCurrentBlock = new boolean[numberOfMeasureInQuery]; int index = 0; for (QueryMeasure queryMeasure : queryMeasures) { - measureOrdinals[index] = queryMeasure.getMeasure().getOrdinal(); // if query measure exists in current dimension measures // then setting measure exists is true // otherwise adding a default value of a measure - if (currentBlockMeasures.contains(queryMeasure.getMeasure())) { - measureExistsInCurrentBlock[index] = true; - } else { - defaultValues[index] = queryMeasure.getMeasure().getDefaultValue(); + for (CarbonMeasure carbonMeasure : currentBlockMeasures) { + if (carbonMeasure.getColumnId().equals(queryMeasure.getMeasure().getColumnId())) { + presentMeasure.add(queryMeasure); + measureOrdinalList.add(carbonMeasure.getOrdinal()); + measureExistsInCurrentBlock[index] = true; + break; + } + } + if (!measureExistsInCurrentBlock[index]) { + defaultValues[index] = getMeasureDefaultValue(queryMeasure.getMeasure().getColumnSchema(), + queryMeasure.getMeasure().getDefaultValue()); + blockExecutionInfo.setRestructuredBlock(true); } index++; } - aggregatorInfos.setDefaultValues(defaultValues); - aggregatorInfos.setMeasureOrdinals(measureOrdinals); - aggregatorInfos.setMeasureExists(measureExistsInCurrentBlock); - return aggregatorInfos; + int[] measureOrdinals = + ArrayUtils.toPrimitive(measureOrdinalList.toArray(new Integer[measureOrdinalList.size()])); + measureInfo.setDefaultValues(defaultValues); + measureInfo.setMeasureOrdinals(measureOrdinals); + measureInfo.setMeasureExists(measureExistsInCurrentBlock); + blockExecutionInfo.setMeasureInfo(measureInfo); + return presentMeasure; } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java index 520b460..deb07c7 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/FilterUtil.java @@ -53,6 +53,7 @@ import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.encoder.Encoding; import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; +import org.apache.carbondata.core.scan.executor.util.QueryUtil; import org.apache.carbondata.core.scan.expression.ColumnExpression; import org.apache.carbondata.core.scan.expression.Expression; import org.apache.carbondata.core.scan.expression.ExpressionResult; @@ -68,6 +69,8 @@ import org.apache.carbondata.core.scan.filter.executer.FilterExecuter; import org.apache.carbondata.core.scan.filter.executer.IncludeColGroupFilterExecuterImpl; import org.apache.carbondata.core.scan.filter.executer.IncludeFilterExecuterImpl; import org.apache.carbondata.core.scan.filter.executer.OrFilterExecuterImpl; +import org.apache.carbondata.core.scan.filter.executer.RestructureExcludeFilterExecutorImpl; +import org.apache.carbondata.core.scan.filter.executer.RestructureIncludeFilterExecutorImpl; import org.apache.carbondata.core.scan.filter.executer.RowLevelFilterExecuterImpl; import org.apache.carbondata.core.scan.filter.executer.RowLevelRangeTypeExecuterFacory; import org.apache.carbondata.core.scan.filter.intf.ExpressionType; @@ -164,7 +167,20 @@ public final class FilterUtil { DimColumnResolvedFilterInfo dimColResolvedFilterInfo, SegmentProperties segmentProperties) { if (dimColResolvedFilterInfo.getDimension().isColumnar()) { - return new IncludeFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties); + CarbonDimension dimensionFromCurrentBlock = CarbonUtil + .getDimensionFromCurrentBlock(segmentProperties.getDimensions(), + dimColResolvedFilterInfo.getDimension()); + if (null != dimensionFromCurrentBlock) { + // update dimension and column index according to the dimension position in current block + DimColumnResolvedFilterInfo dimColResolvedFilterInfoCopyObject = + dimColResolvedFilterInfo.getCopyObject(); + dimColResolvedFilterInfoCopyObject.setDimension(dimensionFromCurrentBlock); + dimColResolvedFilterInfoCopyObject.setColumnIndex(dimensionFromCurrentBlock.getOrdinal()); + return new IncludeFilterExecuterImpl(dimColResolvedFilterInfoCopyObject, segmentProperties); + } else { + return new RestructureIncludeFilterExecutorImpl(dimColResolvedFilterInfo, + segmentProperties); + } } else { return new IncludeColGroupFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties); } @@ -181,7 +197,20 @@ public final class FilterUtil { DimColumnResolvedFilterInfo dimColResolvedFilterInfo, SegmentProperties segmentProperties) { if (dimColResolvedFilterInfo.getDimension().isColumnar()) { - return new ExcludeFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties); + CarbonDimension dimensionFromCurrentBlock = CarbonUtil + .getDimensionFromCurrentBlock(segmentProperties.getDimensions(), + dimColResolvedFilterInfo.getDimension()); + if (null != dimensionFromCurrentBlock) { + // update dimension and column index according to the dimension position in current block + DimColumnResolvedFilterInfo dimColResolvedFilterInfoCopyObject = + dimColResolvedFilterInfo.getCopyObject(); + dimColResolvedFilterInfoCopyObject.setDimension(dimensionFromCurrentBlock); + dimColResolvedFilterInfoCopyObject.setColumnIndex(dimensionFromCurrentBlock.getOrdinal()); + return new ExcludeFilterExecuterImpl(dimColResolvedFilterInfoCopyObject, segmentProperties); + } else { + return new RestructureExcludeFilterExecutorImpl(dimColResolvedFilterInfo, + segmentProperties); + } } else { return new ExcludeColGroupFilterExecuterImpl(dimColResolvedFilterInfo, segmentProperties); } @@ -571,6 +600,53 @@ public final class FilterUtil { } /** + * This method will check whether a default value for the non-existing column is present + * in the filter values list + * + * @param dimColumnEvaluatorInfo + * @return + */ + public static boolean isDimensionDefaultValuePresentInFilterValues( + DimColumnResolvedFilterInfo dimColumnEvaluatorInfo) { + boolean isDefaultValuePresentInFilterValues = false; + DimColumnFilterInfo filterValues = dimColumnEvaluatorInfo.getFilterValues(); + CarbonDimension dimension = dimColumnEvaluatorInfo.getDimension(); + byte[] defaultValue = dimension.getDefaultValue(); + if (!dimension.hasEncoding(Encoding.DICTIONARY)) { + // for no dictionary cases + // 3 cases: is NUll, is Not Null and filter on default value of newly added column + if (null == defaultValue) { + // default value for case where user gives is Null condition + defaultValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL + .getBytes(Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)); + } + List noDictionaryFilterValuesList = filterValues.getNoDictionaryFilterValuesList(); + for (byte[] filterValue : noDictionaryFilterValuesList) { + int compare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(defaultValue, filterValue); + if (compare == 0) { + isDefaultValuePresentInFilterValues = true; + break; + } + } + } else { + // for dictionary and direct dictionary cases + // 3 cases: is NUll, is Not Null and filter on default value of newly added column + int defaultSurrogateValueToCompare = CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY; + if (null != defaultValue) { + defaultSurrogateValueToCompare++; + } + List filterList = filterValues.getFilterList(); + for (Integer filterValue : filterList) { + if (defaultSurrogateValueToCompare == filterValue) { + isDefaultValuePresentInFilterValues = true; + break; + } + } + } + return isDefaultValuePresentInFilterValues; + } + + /** * Below method will be used to covert the filter surrogate keys * to mdkey * @@ -590,13 +666,14 @@ public final class FilterUtil { int[] keys = new int[blockLevelKeyGenerator.getDimCount()]; List filterValuesList = new ArrayList(20); Arrays.fill(keys, 0); - int[] rangesForMaskedByte = - getRangesForMaskedByte((carbonDimension.getKeyOrdinal()), blockLevelKeyGenerator); + int keyOrdinalOfDimensionFromCurrentBlock = carbonDimension.getKeyOrdinal(); if (null != dimColumnFilterInfo) { + int[] rangesForMaskedByte = + getRangesForMaskedByte(keyOrdinalOfDimensionFromCurrentBlock, blockLevelKeyGenerator); for (Integer surrogate : dimColumnFilterInfo.getFilterList()) { try { - if (surrogate <= dimColumnsCardinality[carbonDimension.getKeyOrdinal()]) { - keys[carbonDimension.getKeyOrdinal()] = surrogate; + if (surrogate <= dimColumnsCardinality[keyOrdinalOfDimensionFromCurrentBlock]) { + keys[keyOrdinalOfDimensionFromCurrentBlock] = surrogate; filterValuesList .add(getMaskedKey(rangesForMaskedByte, blockLevelKeyGenerator.generateKey(keys))); } else { @@ -647,12 +724,12 @@ public final class FilterUtil { * @return long[] start key */ public static void getStartKey(Map> dimensionFilter, - long[] startKey, List startKeyList) { + SegmentProperties segmentProperties, long[] startKey, List startKeyList) { for (int i = 0; i < startKey.length; i++) { // The min surrogate key is 1, set it as the init value for starkey of each column level startKey[i] = 1; } - getStartKeyWithFilter(dimensionFilter, startKey, startKeyList); + getStartKeyWithFilter(dimensionFilter, segmentProperties, startKey, startKeyList); } /** @@ -673,6 +750,7 @@ public final class FilterUtil { */ public static void getStartKeyForNoDictionaryDimension( DimColumnResolvedFilterInfo dimColResolvedFilterInfo, + SegmentProperties segmentProperties, SortedMap setOfStartKeyByteArray) { Map> dimensionFilter = dimColResolvedFilterInfo.getDimensionResolvedFilterInstance(); @@ -692,18 +770,25 @@ public final class FilterUtil { if (isExcludePresent) { continue; } + // in case of restructure scenarios it can happen that the filter dimension is not + // present in the current block. In those cases no need to determine the key + CarbonDimension dimensionFromCurrentBlock = CarbonUtil + .getDimensionFromCurrentBlock(segmentProperties.getDimensions(), entry.getKey()); + if (null == dimensionFromCurrentBlock) { + continue; + } // step 2 byte[] noDictionaryStartKey = listOfDimColFilterInfo.get(0).getNoDictionaryFilterValuesList().get(0); if (setOfStartKeyByteArray.isEmpty()) { - setOfStartKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryStartKey); - } else if (null == setOfStartKeyByteArray.get(entry.getKey().getOrdinal())) { - setOfStartKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryStartKey); + setOfStartKeyByteArray.put(dimensionFromCurrentBlock.getOrdinal(), noDictionaryStartKey); + } else if (null == setOfStartKeyByteArray.get(dimensionFromCurrentBlock.getOrdinal())) { + setOfStartKeyByteArray.put(dimensionFromCurrentBlock.getOrdinal(), noDictionaryStartKey); } else if (ByteUtil.UnsafeComparer.INSTANCE - .compareTo(setOfStartKeyByteArray.get(entry.getKey().getOrdinal()), + .compareTo(setOfStartKeyByteArray.get(dimensionFromCurrentBlock.getOrdinal()), noDictionaryStartKey) > 0) { - setOfStartKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryStartKey); + setOfStartKeyByteArray.put(dimensionFromCurrentBlock.getOrdinal(), noDictionaryStartKey); } } } @@ -727,6 +812,7 @@ public final class FilterUtil { */ public static void getEndKeyForNoDictionaryDimension( DimColumnResolvedFilterInfo dimColResolvedFilterInfo, + SegmentProperties segmentProperties, SortedMap setOfEndKeyByteArray) { Map> dimensionFilter = @@ -747,18 +833,25 @@ public final class FilterUtil { if (isExcludePresent) { continue; } + // in case of restructure scenarios it can happen that the filter dimension is not + // present in the current block. In those cases no need to determine the key + CarbonDimension dimensionFromCurrentBlock = CarbonUtil + .getDimensionFromCurrentBlock(segmentProperties.getDimensions(), entry.getKey()); + if (null == dimensionFromCurrentBlock) { + continue; + } // step 2 byte[] noDictionaryEndKey = listOfDimColFilterInfo.get(0).getNoDictionaryFilterValuesList() .get(listOfDimColFilterInfo.get(0).getNoDictionaryFilterValuesList().size() - 1); if (setOfEndKeyByteArray.isEmpty()) { - setOfEndKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryEndKey); - } else if (null == setOfEndKeyByteArray.get(entry.getKey().getOrdinal())) { - setOfEndKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryEndKey); + setOfEndKeyByteArray.put(dimensionFromCurrentBlock.getOrdinal(), noDictionaryEndKey); + } else if (null == setOfEndKeyByteArray.get(dimensionFromCurrentBlock.getOrdinal())) { + setOfEndKeyByteArray.put(dimensionFromCurrentBlock.getOrdinal(), noDictionaryEndKey); } else if (ByteUtil.UnsafeComparer.INSTANCE - .compareTo(setOfEndKeyByteArray.get(entry.getKey().getOrdinal()), noDictionaryEndKey) - < 0) { - setOfEndKeyByteArray.put(entry.getKey().getOrdinal(), noDictionaryEndKey); + .compareTo(setOfEndKeyByteArray.get(dimensionFromCurrentBlock.getOrdinal()), + noDictionaryEndKey) < 0) { + setOfEndKeyByteArray.put(dimensionFromCurrentBlock.getOrdinal(), noDictionaryEndKey); } } @@ -794,8 +887,8 @@ public final class FilterUtil { * @param startKey */ private static void getStartKeyWithFilter( - Map> dimensionFilter, long[] startKey, - List startKeyList) { + Map> dimensionFilter, + SegmentProperties segmentProperties, long[] startKey, List startKeyList) { for (Map.Entry> entry : dimensionFilter.entrySet()) { List values = entry.getValue(); if (null == values || !entry.getKey().hasEncoding(Encoding.DICTIONARY)) { @@ -810,9 +903,17 @@ public final class FilterUtil { if (isExcludePresent) { continue; } + int keyOrdinalOfDimensionFromCurrentBlock = QueryUtil + .getKeyOrdinalOfDimensionFromCurrentBlock(segmentProperties.getDimensions(), + entry.getKey()); + // if key ordinal is -1 that means this dimension does not exist in the current block. + // Applicable for restructure scenarios + if (keyOrdinalOfDimensionFromCurrentBlock == -1) { + continue; + } for (DimColumnFilterInfo info : values) { - if (startKey[entry.getKey().getKeyOrdinal()] < info.getFilterList().get(0)) { - startKey[entry.getKey().getKeyOrdinal()] = info.getFilterList().get(0); + if (startKey[keyOrdinalOfDimensionFromCurrentBlock] < info.getFilterList().get(0)) { + startKey[keyOrdinalOfDimensionFromCurrentBlock] = info.getFilterList().get(0); } } long[] newStartKey = new long[startKey.length]; @@ -831,7 +932,7 @@ public final class FilterUtil { endKey[i] = getMaxValue(updatedDimListBasedOnKeyGenerator.get(i), segmentProperties.getDimColumnsCardinality()); } - getEndKeyWithFilter(dimensionFilter, endKey, endKeyList); + getEndKeyWithFilter(dimensionFilter, segmentProperties, endKey, endKeyList); } @@ -850,8 +951,8 @@ public final class FilterUtil { } private static void getEndKeyWithFilter( - Map> dimensionFilter, long[] endKey, - List endKeyList) { + Map> dimensionFilter, + SegmentProperties segmentProperties, long[] endKey, List endKeyList) { for (Map.Entry> entry : dimensionFilter.entrySet()) { List values = entry.getValue(); if (null == values || !entry.getKey().hasEncoding(Encoding.DICTIONARY)) { @@ -866,11 +967,18 @@ public final class FilterUtil { if (isExcludeFilterPresent) { continue; } - + int keyOrdinalOfDimensionFromCurrentBlock = QueryUtil + .getKeyOrdinalOfDimensionFromCurrentBlock(segmentProperties.getDimensions(), + entry.getKey()); + // if key ordinal is -1 that means this dimension does not exist in the current block. + // Applicable for restructure scenarios + if (keyOrdinalOfDimensionFromCurrentBlock == -1) { + continue; + } for (DimColumnFilterInfo info : values) { - if (endKey[entry.getKey().getKeyOrdinal()] > info.getFilterList() + if (endKey[keyOrdinalOfDimensionFromCurrentBlock] > info.getFilterList() .get(info.getFilterList().size() - 1)) { - endKey[entry.getKey().getKeyOrdinal()] = + endKey[keyOrdinalOfDimensionFromCurrentBlock] = info.getFilterList().get(info.getFilterList().size() - 1); } } @@ -1291,7 +1399,8 @@ public final class FilterUtil { traverseResolverTreeAndPopulateStartAndEndKeys(filterResolverTree.getLeft(), segmentProperties, startKeys, setOfStartKeyByteArray, endKeys, setOfEndKeyByteArray, startKeyList, endKeyList); - filterResolverTree.getStartKey(startKeys, setOfStartKeyByteArray, startKeyList); + filterResolverTree + .getStartKey(segmentProperties, startKeys, setOfStartKeyByteArray, startKeyList); filterResolverTree.getEndKey(segmentProperties, endKeys, setOfEndKeyByteArray, endKeyList); http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java new file mode 100644 index 0000000..68f0dfe --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureExcludeFilterExecutorImpl.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.scan.filter.executer; + +import java.io.IOException; +import java.util.BitSet; + +import org.apache.carbondata.core.datastore.block.SegmentProperties; +import org.apache.carbondata.core.scan.filter.FilterUtil; +import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo; +import org.apache.carbondata.core.scan.processor.BlocksChunkHolder; +import org.apache.carbondata.core.util.BitSetGroup; + +public class RestructureExcludeFilterExecutorImpl implements FilterExecuter { + + protected DimColumnResolvedFilterInfo dimColEvaluatorInfo; + protected SegmentProperties segmentProperties; + + /** + * flag to check whether filter values contain the default value applied on the dimension column + * which does not exist in the current block + */ + protected boolean isDefaultValuePresentInFilterValues; + + public RestructureExcludeFilterExecutorImpl(DimColumnResolvedFilterInfo dimColEvaluatorInfo, + SegmentProperties segmentProperties) { + this.dimColEvaluatorInfo = dimColEvaluatorInfo; + this.segmentProperties = segmentProperties; + isDefaultValuePresentInFilterValues = + FilterUtil.isDimensionDefaultValuePresentInFilterValues(dimColEvaluatorInfo); + } + + @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder) throws IOException { + int numberOfRows = blockChunkHolder.getDataBlock().nodeSize(); + BitSetGroup bitSetGroup = new BitSetGroup(1); + BitSet bitSet = new BitSet(numberOfRows); + bitSet.set(0, numberOfRows, !isDefaultValuePresentInFilterValues); + bitSetGroup.setBitSet(bitSet, 0); + return bitSetGroup; + } + + @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) { + BitSet bitSet = new BitSet(1); + bitSet.flip(0, 1); + return bitSet; + } + + @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException { + + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java new file mode 100644 index 0000000..c1ba2af --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RestructureIncludeFilterExecutorImpl.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.scan.filter.executer; + +import java.io.IOException; +import java.util.BitSet; + +import org.apache.carbondata.core.datastore.block.SegmentProperties; +import org.apache.carbondata.core.scan.filter.FilterUtil; +import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo; +import org.apache.carbondata.core.scan.processor.BlocksChunkHolder; +import org.apache.carbondata.core.util.BitSetGroup; + +public class RestructureIncludeFilterExecutorImpl implements FilterExecuter { + + protected DimColumnResolvedFilterInfo dimColumnEvaluatorInfo; + protected SegmentProperties segmentProperties; + + /** + * flag to check whether filter values contain the default value applied on the dimension column + * which does not exist in the current block + */ + protected boolean isDefaultValuePresentInFilterValues; + + public RestructureIncludeFilterExecutorImpl(DimColumnResolvedFilterInfo dimColumnEvaluatorInfo, + SegmentProperties segmentProperties) { + this.dimColumnEvaluatorInfo = dimColumnEvaluatorInfo; + this.segmentProperties = segmentProperties; + isDefaultValuePresentInFilterValues = + FilterUtil.isDimensionDefaultValuePresentInFilterValues(dimColumnEvaluatorInfo); + } + + @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder) throws IOException { + int numberOfRows = blockChunkHolder.getDataBlock().nodeSize(); + BitSetGroup bitSetGroup = new BitSetGroup(1); + BitSet bitSet = new BitSet(numberOfRows); + bitSet.set(0, numberOfRows, isDefaultValuePresentInFilterValues); + bitSetGroup.setBitSet(bitSet, 0); + return bitSetGroup; + } + + public BitSet isScanRequired(byte[][] blkMaxVal, byte[][] blkMinVal) { + BitSet bitSet = new BitSet(1); + bitSet.set(0, isDefaultValuePresentInFilterValues); + return bitSet; + } + + @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException { + + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java index 7595ab6..b48382e 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecuterImpl.java @@ -19,7 +19,10 @@ package org.apache.carbondata.core.scan.filter.executer; import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; import java.io.IOException; +import java.math.BigDecimal; +import java.math.RoundingMode; import java.nio.ByteBuffer; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; @@ -40,8 +43,11 @@ import org.apache.carbondata.core.keygenerator.directdictionary.DirectDictionary import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.metadata.datatype.DataType; import org.apache.carbondata.core.metadata.encoder.Encoding; +import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; +import org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure; import org.apache.carbondata.core.scan.executor.infos.KeyStructureInfo; import org.apache.carbondata.core.scan.executor.util.QueryUtil; +import org.apache.carbondata.core.scan.executor.util.RestructureUtil; import org.apache.carbondata.core.scan.expression.Expression; import org.apache.carbondata.core.scan.expression.exception.FilterIllegalMemberException; import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException; @@ -68,29 +74,85 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter { /** * it has index at which given dimension is stored in file */ - private int[] blocksIndex; + protected int[] dimensionBlocksIndex; + + /** + * it has index at which given measure is stored in file + */ + protected int[] measureBlocksIndex; private Map complexDimensionInfoMap; + /** + * flag to check whether the filter dimension is present in current block list of dimensions. + * Applicable for restructure scenarios + */ + protected boolean[] isDimensionPresentInCurrentBlock; + + /** + * flag to check whether the filter measure is present in current block list of measures. + * Applicable for restructure scenarios + */ + protected boolean[] isMeasurePresentInCurrentBlock; + public RowLevelFilterExecuterImpl(List dimColEvaluatorInfoList, List msrColEvalutorInfoList, Expression exp, AbsoluteTableIdentifier tableIdentifier, SegmentProperties segmentProperties, Map complexDimensionInfoMap) { this.dimColEvaluatorInfoList = dimColEvaluatorInfoList; this.segmentProperties = segmentProperties; - this.blocksIndex = new int[dimColEvaluatorInfoList.size()]; - for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) { - this.blocksIndex[i] = segmentProperties.getDimensionOrdinalToBlockMapping() - .get(dimColEvaluatorInfoList.get(i).getColumnIndex()); - } + this.dimensionBlocksIndex = new int[dimColEvaluatorInfoList.size()]; + this.isDimensionPresentInCurrentBlock = new boolean[dimColEvaluatorInfoList.size()]; if (null == msrColEvalutorInfoList) { this.msrColEvalutorInfoList = new ArrayList(20); } else { this.msrColEvalutorInfoList = msrColEvalutorInfoList; } + this.measureBlocksIndex = new int[msrColEvalutorInfoList.size()]; + this.isMeasurePresentInCurrentBlock = new boolean[msrColEvalutorInfoList.size()]; this.exp = exp; this.tableIdentifier = tableIdentifier; this.complexDimensionInfoMap = complexDimensionInfoMap; + initDimensionBlockIndexes(); + initMeasureBlockIndexes(); + } + + /** + * This method will initialize the dimension info for the current block to be + * used for filtering the data + */ + private void initDimensionBlockIndexes() { + for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) { + // find the dimension in the current block dimensions list + CarbonDimension dimensionFromCurrentBlock = CarbonUtil + .getDimensionFromCurrentBlock(segmentProperties.getDimensions(), + dimColEvaluatorInfoList.get(i).getDimension()); + if (null != dimensionFromCurrentBlock) { + dimColEvaluatorInfoList.get(i).setColumnIndex(dimensionFromCurrentBlock.getOrdinal()); + this.dimensionBlocksIndex[i] = segmentProperties.getDimensionOrdinalToBlockMapping() + .get(dimensionFromCurrentBlock.getOrdinal()); + isDimensionPresentInCurrentBlock[i] = true; + } + } + } + + /** + * This method will initialize the measure info for the current block to be + * used for filtering the data + */ + private void initMeasureBlockIndexes() { + for (int i = 0; i < msrColEvalutorInfoList.size(); i++) { + // find the measure in the current block measures list + CarbonMeasure measureFromCurrentBlock = CarbonUtil + .getMeasureFromCurrentBlock(segmentProperties.getMeasures(), + msrColEvalutorInfoList.get(i).getCarbonColumn().getColumnId()); + if (null != measureFromCurrentBlock) { + msrColEvalutorInfoList.get(i).setColumnIndex(measureFromCurrentBlock.getOrdinal()); + this.measureBlocksIndex[i] = segmentProperties.getMeasuresOrdinalToBlockMapping() + .get(measureFromCurrentBlock.getOrdinal()); + isMeasurePresentInCurrentBlock[i] = true; + } + } } @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder) @@ -102,13 +164,20 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter { int pageNumbers = 0; if (dimColEvaluatorInfoList.size() > 0) { - pageNumbers = blockChunkHolder.getDimensionRawDataChunk()[blocksIndex[0]].getPagesCount(); - numberOfRows = blockChunkHolder.getDimensionRawDataChunk()[blocksIndex[0]].getRowCount(); + if (isDimensionPresentInCurrentBlock[0]) { + pageNumbers = + blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[0]].getPagesCount(); + numberOfRows = + blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[0]].getRowCount(); + } } if (msrColEvalutorInfoList.size() > 0) { - int columnIndex = msrColEvalutorInfoList.get(0).getColumnIndex(); - pageNumbers = blockChunkHolder.getMeasureRawDataChunk()[columnIndex].getPagesCount(); - numberOfRows = blockChunkHolder.getMeasureRawDataChunk()[columnIndex].getRowCount(); + if (isMeasurePresentInCurrentBlock[0]) { + pageNumbers = + blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]].getPagesCount(); + numberOfRows = + blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]].getRowCount(); + } } BitSetGroup bitSetGroup = new BitSetGroup(pageNumbers); for (int i = 0; i < pageNumbers; i++) { @@ -152,13 +221,21 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter { String memberString; for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) { DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = dimColEvaluatorInfoList.get(i); + // if filter dimension is not present in the current add its default value + if (!isDimensionPresentInCurrentBlock[i]) { + // fill default value here + record[dimColumnEvaluatorInfo.getRowIndex()] = + getDimensionDefaultValue(dimColumnEvaluatorInfo); + continue; + } if (dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.ARRAY && dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.STRUCT) { if (!dimColumnEvaluatorInfo.isDimensionExistsInCurrentSilce()) { - record[dimColumnEvaluatorInfo.getRowIndex()] = dimColumnEvaluatorInfo.getDefaultValue(); + record[dimColumnEvaluatorInfo.getRowIndex()] = + dimColumnEvaluatorInfo.getDimension().getDefaultValue(); } DimensionColumnDataChunk columnDataChunk = - blockChunkHolder.getDimensionRawDataChunk()[blocksIndex[i]] + blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[i]] .convertToDimColDataChunk(pageIndex); if (!dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY) && columnDataChunk instanceof VariableLengthDimensionDataChunk) { @@ -178,7 +255,7 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter { } } else { int dictionaryValue = readSurrogatesFromColumnBlock(blockChunkHolder, index, pageIndex, - dimColumnEvaluatorInfo, blocksIndex[i]); + dimColumnEvaluatorInfo, dimensionBlocksIndex[i]); if (dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DICTIONARY) && !dimColumnEvaluatorInfo.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) { memberString = @@ -196,7 +273,7 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter { } } else { try { - GenericQueryType complexType = complexDimensionInfoMap.get(blocksIndex[i]); + GenericQueryType complexType = complexDimensionInfoMap.get(dimensionBlocksIndex[i]); ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); DataOutputStream dataOutputStream = new DataOutputStream(byteStream); complexType.parseBlocksAndReturnComplexColumnByteArray( @@ -211,8 +288,8 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter { } DataType msrType; - - for (MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo : msrColEvalutorInfoList) { + for (int i = 0; i < msrColEvalutorInfoList.size(); i++) { + MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo = msrColEvalutorInfoList.get(i); switch (msrColumnEvalutorInfo.getType()) { case INT: case LONG: @@ -224,10 +301,19 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter { default: msrType = DataType.DOUBLE; } - // if measure doesnt exist then set the default value. + // add default value for the measure in case filter measure is not present + // in the current block measure list + if (!isMeasurePresentInCurrentBlock[i]) { + byte[] defaultValue = msrColumnEvalutorInfo.getCarbonColumn().getDefaultValue(); + record[msrColumnEvalutorInfo.getRowIndex()] = RestructureUtil + .getMeasureDefaultValue(msrColumnEvalutorInfo.getCarbonColumn().getColumnSchema(), + defaultValue); + continue; + } + Object msrValue; MeasureColumnDataChunk measureColumnDataChunk = - blockChunkHolder.getMeasureRawDataChunk()[msrColumnEvalutorInfo.getColumnIndex()] + blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]] .convertToMeasureColDataChunk(pageIndex); switch (msrType) { case INT: @@ -236,8 +322,17 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter { measureColumnDataChunk.getMeasureDataHolder().getReadableLongValueByIndex(index); break; case DECIMAL: - msrValue = measureColumnDataChunk.getMeasureDataHolder() - .getReadableBigDecimalValueByIndex(index); + BigDecimal bigDecimalValue = + measureColumnDataChunk.getMeasureDataHolder() + .getReadableBigDecimalValueByIndex(index); + if (null != bigDecimalValue + && msrColumnEvalutorInfo.getCarbonColumn().getColumnSchema().getScale() + > bigDecimalValue.scale()) { + bigDecimalValue = bigDecimalValue + .setScale(msrColumnEvalutorInfo.getCarbonColumn().getColumnSchema().getScale(), + RoundingMode.HALF_UP); + } + msrValue = bigDecimalValue; break; default: msrValue = @@ -250,6 +345,28 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter { } /** + * This method will compute the default value for a dimension + * + * @param dimColumnEvaluatorInfo + * @return + */ + private Object getDimensionDefaultValue(DimColumnResolvedFilterInfo dimColumnEvaluatorInfo) { + Object dimensionDefaultValue = null; + CarbonDimension dimension = dimColumnEvaluatorInfo.getDimension(); + if (dimension.hasEncoding(Encoding.DICTIONARY) && !dimension + .hasEncoding(Encoding.DIRECT_DICTIONARY)) { + byte[] defaultValue = dimension.getDefaultValue(); + if (null != defaultValue) { + dimensionDefaultValue = + new String(defaultValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)); + } + } else { + dimensionDefaultValue = RestructureUtil.validateAndGetDefaultValue(dimension); + } + return dimensionDefaultValue; + } + + /** * method will read the actual data from the direct dictionary generator * by passing direct dictionary value. * @@ -329,9 +446,9 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter { byte[] colData = chunk.getChunkData(index); long[] result = keyStructureInfo.getKeyGenerator().getKeyArray(colData); int colGroupId = - QueryUtil.getColumnGroupId(segmentProperties, dimColumnEvaluatorInfo.getColumnIndex()); + QueryUtil.getColumnGroupId(segmentProperties, dimensionBlocksIndex[0]); int dictionaryValue = (int) result[segmentProperties - .getColumnGroupMdKeyOrdinal(colGroupId, dimColumnEvaluatorInfo.getColumnIndex())]; + .getColumnGroupMdKeyOrdinal(colGroupId, dimensionBlocksIndex[0])]; return dictionaryValue; } catch (KeyGenException e) { LOGGER.error(e); @@ -346,30 +463,41 @@ public class RowLevelFilterExecuterImpl implements FilterExecuter { return bitSet; } + /** + * This method will set the bitset to true by default for a given number of rows + * + * @param numberOfRows + * @return + */ + protected BitSetGroup getDefaultBitSetGroup(int numberOfRows) { + BitSetGroup bitSetGroup = new BitSetGroup(1); + BitSet bitSet = new BitSet(numberOfRows); + bitSet.set(0, numberOfRows, true); + bitSetGroup.setBitSet(bitSet, 0); + return bitSetGroup; + } + @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException { for (int i = 0; i < dimColEvaluatorInfoList.size(); i++) { DimColumnResolvedFilterInfo dimColumnEvaluatorInfo = dimColEvaluatorInfoList.get(i); if (dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.ARRAY && dimColumnEvaluatorInfo.getDimension().getDataType() != DataType.STRUCT) { - if (null == blockChunkHolder.getDimensionRawDataChunk()[blocksIndex[i]]) { - blockChunkHolder.getDimensionRawDataChunk()[blocksIndex[i]] = + if (null == blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[i]]) { + blockChunkHolder.getDimensionRawDataChunk()[dimensionBlocksIndex[i]] = blockChunkHolder.getDataBlock() - .getDimensionChunk(blockChunkHolder.getFileReader(), blocksIndex[i]); + .getDimensionChunk(blockChunkHolder.getFileReader(), dimensionBlocksIndex[i]); } } else { - GenericQueryType complexType = complexDimensionInfoMap.get(blocksIndex[i]); + GenericQueryType complexType = complexDimensionInfoMap.get(dimensionBlocksIndex[i]); complexType.fillRequiredBlockData(blockChunkHolder); } } if (null != msrColEvalutorInfoList) { - for (MeasureColumnResolvedFilterInfo msrColumnEvalutorInfo : msrColEvalutorInfoList) { - if (null == blockChunkHolder.getMeasureRawDataChunk()[msrColumnEvalutorInfo - .getColumnIndex()]) { - blockChunkHolder.getMeasureRawDataChunk()[msrColumnEvalutorInfo.getColumnIndex()] = - blockChunkHolder.getDataBlock().getMeasureChunk(blockChunkHolder.getFileReader(), - msrColumnEvalutorInfo.getColumnIndex()); - } + if (null == blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]]) { + blockChunkHolder.getMeasureRawDataChunk()[measureBlocksIndex[0]] = + blockChunkHolder.getDataBlock() + .getMeasureChunk(blockChunkHolder.getFileReader(), measureBlocksIndex[0]); } } } http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/70256e77/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java index 9f28d7c..e748141 100644 --- a/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java +++ b/core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelRangeGrtThanFiterExecuterImpl.java @@ -26,6 +26,7 @@ import org.apache.carbondata.core.datastore.chunk.impl.DimensionRawColumnChunk; import org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionDataChunk; import org.apache.carbondata.core.metadata.AbsoluteTableIdentifier; import org.apache.carbondata.core.metadata.encoder.Encoding; +import org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension; import org.apache.carbondata.core.scan.expression.Expression; import org.apache.carbondata.core.scan.expression.exception.FilterUnsupportedException; import org.apache.carbondata.core.scan.filter.resolver.resolverinfo.DimColumnResolvedFilterInfo; @@ -38,6 +39,11 @@ import org.apache.carbondata.core.util.CarbonUtil; public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecuterImpl { private byte[][] filterRangeValues; + /** + * flag to check whether default values is present in the filter value list + */ + private boolean isDefaultValuePresentInFilter; + public RowLevelRangeGrtThanFiterExecuterImpl( List dimColEvaluatorInfoList, List msrColEvalutorInfoList, Expression exp, @@ -46,44 +52,71 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute super(dimColEvaluatorInfoList, msrColEvalutorInfoList, exp, tableIdentifier, segmentProperties, null); this.filterRangeValues = filterRangeValues; + checkIfDefaultValueIsPresentInFilterList(); + } + + /** + * This method will check whether default value is present in the given filter values + */ + private void checkIfDefaultValueIsPresentInFilterList() { + if (!this.isDimensionPresentInCurrentBlock[0]) { + CarbonDimension dimension = this.dimColEvaluatorInfoList.get(0).getDimension(); + byte[] defaultValue = dimension.getDefaultValue(); + if (null != defaultValue) { + for (int k = 0; k < filterRangeValues.length; k++) { + int maxCompare = + ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterRangeValues[k], defaultValue); + if (maxCompare < 0) { + isDefaultValuePresentInFilter = true; + break; + } + } + } + } } @Override public BitSet isScanRequired(byte[][] blockMaxValue, byte[][] blockMinValue) { BitSet bitSet = new BitSet(1); - byte[][] filterValues = this.filterRangeValues; - int columnIndex = this.dimColEvaluatorInfoList.get(0).getColumnIndex(); - boolean isScanRequired = isScanRequired(blockMaxValue[columnIndex], filterValues); + boolean isScanRequired = + isScanRequired(blockMaxValue[dimensionBlocksIndex[0]], filterRangeValues); if (isScanRequired) { bitSet.set(0); } return bitSet; - } private boolean isScanRequired(byte[] blockMaxValue, byte[][] filterValues) { boolean isScanRequired = false; - for (int k = 0; k < filterValues.length; k++) { - // filter value should be in range of max and min value i.e - // max>filtervalue>min - // so filter-max should be negative - int maxCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue); - // if any filter value is in range than this block needs to be - // scanned means always less than block max range. - if (maxCompare < 0) { - isScanRequired = true; - break; + if (isDimensionPresentInCurrentBlock[0]) { + for (int k = 0; k < filterValues.length; k++) { + // filter value should be in range of max and min value i.e + // max>filtervalue>min + // so filter-max should be negative + int maxCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue); + // if any filter value is in range than this block needs to be + // scanned means always less than block max range. + if (maxCompare < 0) { + isScanRequired = true; + break; + } } + } else { + isScanRequired = isDefaultValuePresentInFilter; } return isScanRequired; } @Override public BitSetGroup applyFilter(BlocksChunkHolder blockChunkHolder) throws FilterUnsupportedException, IOException { + // select all rows if dimension does not exists in the current block + if (!isDimensionPresentInCurrentBlock[0]) { + return getDefaultBitSetGroup(blockChunkHolder.getDataBlock().nodeSize()); + } if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) { return super.applyFilter(blockChunkHolder); } int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping() - .get(dimColEvaluatorInfoList.get(0).getColumnIndex()); + .get(dimensionBlocksIndex[0]); if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) { blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock() .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex); @@ -238,14 +271,15 @@ public class RowLevelRangeGrtThanFiterExecuterImpl extends RowLevelFilterExecute } @Override public void readBlocks(BlocksChunkHolder blockChunkHolder) throws IOException { - if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) { - super.readBlocks(blockChunkHolder); - } - int blockIndex = segmentProperties.getDimensionOrdinalToBlockMapping() - .get(dimColEvaluatorInfoList.get(0).getColumnIndex()); - if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) { - blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock() - .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex); + if (isDimensionPresentInCurrentBlock[0]) { + if (!dimColEvaluatorInfoList.get(0).getDimension().hasEncoding(Encoding.DICTIONARY)) { + super.readBlocks(blockChunkHolder); + } + int blockIndex = dimensionBlocksIndex[0]; + if (null == blockChunkHolder.getDimensionRawDataChunk()[blockIndex]) { + blockChunkHolder.getDimensionRawDataChunk()[blockIndex] = blockChunkHolder.getDataBlock() + .getDimensionChunk(blockChunkHolder.getFileReader(), blockIndex); + } } } }