Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 536B2200B76 for ; Mon, 15 Aug 2016 09:09:29 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 51C23160A6C; Mon, 15 Aug 2016 07:09:29 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id E743C160ABD for ; Mon, 15 Aug 2016 09:09:26 +0200 (CEST) Received: (qmail 33406 invoked by uid 500); 15 Aug 2016 07:09:25 -0000 Mailing-List: contact commits-help@carbondata.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@carbondata.incubator.apache.org Delivered-To: mailing list commits@carbondata.incubator.apache.org Received: (qmail 33364 invoked by uid 99); 15 Aug 2016 07:09:25 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd4-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 15 Aug 2016 07:09:25 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd4-us-west.apache.org (ASF Mail Server at spamd4-us-west.apache.org) with ESMTP id 37A24C0439 for ; Mon, 15 Aug 2016 07:09:25 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd4-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -0.546 X-Spam-Level: X-Spam-Status: No, score=-0.546 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RP_MATCHES_RCVD=-1.426, URIBL_SBL=4, URIBL_SBL_A=0.1] autolearn=disabled Received: from mx2-lw-eu.apache.org ([10.40.0.8]) by localhost (spamd4-us-west.apache.org [10.40.0.11]) (amavisd-new, port 10024) with ESMTP id z_UWe7NjbnXE for ; Mon, 15 Aug 2016 07:09:06 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx2-lw-eu.apache.org (ASF Mail Server at mx2-lw-eu.apache.org) with SMTP id 1E49460CE6 for ; Mon, 15 Aug 2016 07:08:48 +0000 (UTC) Received: (qmail 31332 invoked by uid 99); 15 Aug 2016 07:08:48 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 15 Aug 2016 07:08:48 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id CC92DED321; Mon, 15 Aug 2016 07:08:47 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: chenliang613@apache.org To: commits@carbondata.incubator.apache.org Date: Mon, 15 Aug 2016 07:09:32 -0000 Message-Id: In-Reply-To: <77c7cb8c3d174e35994262977707e15a@git.apache.org> References: <77c7cb8c3d174e35994262977707e15a@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [47/52] [partial] incubator-carbondata git commit: Renamed packages to org.apache.carbondata and fixed errors archived-at: Mon, 15 Aug 2016 07:09:29 -0000 http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/BlockIndexStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/BlockIndexStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/BlockIndexStore.java new file mode 100644 index 0000000..be48ce5 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/BlockIndexStore.java @@ -0,0 +1,309 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.carbon.datastore; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +import org.apache.carbondata.common.logging.LogService; +import org.apache.carbondata.common.logging.LogServiceFactory; +import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier; +import org.apache.carbondata.core.carbon.datastore.block.AbstractIndex; +import org.apache.carbondata.core.carbon.datastore.block.BlockIndex; +import org.apache.carbondata.core.carbon.datastore.block.TableBlockInfo; +import org.apache.carbondata.core.carbon.datastore.exception.IndexBuilderException; +import org.apache.carbondata.core.carbon.metadata.blocklet.DataFileFooter; +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.util.CarbonProperties; +import org.apache.carbondata.core.util.CarbonUtil; +import org.apache.carbondata.core.util.CarbonUtilException; + +/** + * Singleton Class to handle loading, unloading,clearing,storing of the table + * blocks + */ +public class BlockIndexStore { + + private static final LogService LOGGER = + LogServiceFactory.getLogService(BlockIndexStore.class.getName()); + /** + * singleton instance + */ + private static final BlockIndexStore CARBONTABLEBLOCKSINSTANCE = new BlockIndexStore(); + + /** + * map to hold the table and its list of blocks + */ + private Map> tableBlocksMap; + + /** + * map of block info to lock object map, while loading the btree this will be filled + * and removed after loading the tree for that particular block info, this will be useful + * while loading the tree concurrently so only block level lock will be applied another + * block can be loaded concurrently + */ + private Map blockInfoLock; + + /** + * table and its lock object to this will be useful in case of concurrent + * query scenario when more than one query comes for same table and in that + * case it will ensure that only one query will able to load the blocks + */ + private Map tableLockMap; + + private BlockIndexStore() { + tableBlocksMap = + new ConcurrentHashMap>( + CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); + tableLockMap = new ConcurrentHashMap( + CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); + blockInfoLock = new ConcurrentHashMap(); + } + + /** + * Return the instance of this class + * + * @return singleton instance + */ + public static BlockIndexStore getInstance() { + return CARBONTABLEBLOCKSINSTANCE; + } + + /** + * below method will be used to load the block which are not loaded and to + * get the loaded blocks if all the blocks which are passed is loaded then + * it will not load , else it will load. + * + * @param tableBlocksInfos list of blocks to be loaded + * @param absoluteTableIdentifier absolute Table Identifier to identify the table + * @throws IndexBuilderException + */ + public List loadAndGetBlocks(List tableBlocksInfos, + AbsoluteTableIdentifier absoluteTableIdentifier) throws IndexBuilderException { + AbstractIndex[] loadedBlock = new AbstractIndex[tableBlocksInfos.size()]; + addTableLockObject(absoluteTableIdentifier); + // sort the block info + // so block will be loaded in sorted order this will be required for + // query execution + Collections.sort(tableBlocksInfos); + // get the instance + Object lockObject = tableLockMap.get(absoluteTableIdentifier); + Map tableBlockMapTemp = null; + int numberOfCores = 1; + try { + numberOfCores = Integer.parseInt(CarbonProperties.getInstance() + .getProperty(CarbonCommonConstants.NUM_CORES, + CarbonCommonConstants.NUM_CORES_DEFAULT_VAL)); + } catch (NumberFormatException e) { + numberOfCores = Integer.parseInt(CarbonCommonConstants.NUM_CORES_DEFAULT_VAL); + } + ExecutorService executor = Executors.newFixedThreadPool(numberOfCores); + // Acquire the lock to ensure only one query is loading the table blocks + // if same block is assigned to both the queries + synchronized (lockObject) { + tableBlockMapTemp = tableBlocksMap.get(absoluteTableIdentifier); + // if it is loading for first time + if (null == tableBlockMapTemp) { + tableBlockMapTemp = new ConcurrentHashMap(); + tableBlocksMap.put(absoluteTableIdentifier, tableBlockMapTemp); + } + } + AbstractIndex tableBlock = null; + List> blocksList = new ArrayList>(); + int counter = -1; + for (TableBlockInfo blockInfo : tableBlocksInfos) { + counter++; + // if table block is already loaded then do not load + // that block + tableBlock = tableBlockMapTemp.get(blockInfo); + // if block is not loaded + if (null == tableBlock) { + // check any lock object is present in + // block info lock map + Object blockInfoLockObject = blockInfoLock.get(blockInfo); + // if lock object is not present then acquire + // the lock in block info lock and add a lock object in the map for + // particular block info, added double checking mechanism to add the lock + // object so in case of concurrent query we for same block info only one lock + // object will be added + if (null == blockInfoLockObject) { + synchronized (blockInfoLock) { + // again checking the block info lock, to check whether lock object is present + // or not if now also not present then add a lock object + blockInfoLockObject = blockInfoLock.get(blockInfo); + if (null == blockInfoLockObject) { + blockInfoLockObject = new Object(); + blockInfoLock.put(blockInfo, blockInfoLockObject); + } + } + } + //acquire the lock for particular block info + synchronized (blockInfoLockObject) { + // check again whether block is present or not to avoid the + // same block is loaded + //more than once in case of concurrent query + tableBlock = tableBlockMapTemp.get(blockInfo); + // if still block is not present then load the block + if (null == tableBlock) { + blocksList.add(executor.submit(new BlockLoaderThread(blockInfo, tableBlockMapTemp))); + } + } + } else { + // if blocks is already loaded then directly set the block at particular position + //so block will be present in sorted order + loadedBlock[counter] = tableBlock; + } + } + // shutdown the executor gracefully and wait until all the task is finished + executor.shutdown(); + try { + executor.awaitTermination(1, TimeUnit.HOURS); + } catch (InterruptedException e) { + throw new IndexBuilderException(e); + } + // fill the block which were not loaded before to loaded blocks array + fillLoadedBlocks(loadedBlock, blocksList); + return Arrays.asList(loadedBlock); + } + + /** + * Below method will be used to fill the loaded blocks to the array + * which will be used for query execution + * + * @param loadedBlockArray array of blocks which will be filled + * @param blocksList blocks loaded in thread + * @throws IndexBuilderException in case of any failure + */ + private void fillLoadedBlocks(AbstractIndex[] loadedBlockArray, + List> blocksList) throws IndexBuilderException { + int blockCounter = 0; + for (int i = 0; i < loadedBlockArray.length; i++) { + if (null == loadedBlockArray[i]) { + try { + loadedBlockArray[i] = blocksList.get(blockCounter++).get(); + } catch (InterruptedException | ExecutionException e) { + throw new IndexBuilderException(e); + } + } + + } + } + + private AbstractIndex loadBlock(Map tableBlockMapTemp, + TableBlockInfo blockInfo) throws CarbonUtilException { + AbstractIndex tableBlock; + DataFileFooter footer; + // getting the data file meta data of the block + footer = CarbonUtil.readMetadatFile(blockInfo.getFilePath(), blockInfo.getBlockOffset(), + blockInfo.getBlockLength()); + tableBlock = new BlockIndex(); + footer.setTableBlockInfo(blockInfo); + // building the block + tableBlock.buildIndex(Arrays.asList(footer)); + tableBlockMapTemp.put(blockInfo, tableBlock); + // finally remove the lock object from block info lock as once block is loaded + // it will not come inside this if condition + blockInfoLock.remove(blockInfo); + return tableBlock; + } + + /** + * Method to add table level lock if lock is not present for the table + * + * @param absoluteTableIdentifier + */ + private synchronized void addTableLockObject(AbsoluteTableIdentifier absoluteTableIdentifier) { + // add the instance to lock map if it is not present + if (null == tableLockMap.get(absoluteTableIdentifier)) { + tableLockMap.put(absoluteTableIdentifier, new Object()); + } + } + + /** + * This will be used to remove a particular blocks useful in case of + * deletion of some of the blocks in case of retention or may be some other + * scenario + * + * @param removeTableBlocksInfos blocks to be removed + * @param absoluteTableIdentifier absolute table identifier + */ + public void removeTableBlocks(List removeTableBlocksInfos, + AbsoluteTableIdentifier absoluteTableIdentifier) { + // get the lock object if lock object is not present then it is not + // loaded at all + // we can return from here + Object lockObject = tableLockMap.get(absoluteTableIdentifier); + if (null == lockObject) { + return; + } + Map map = tableBlocksMap.get(absoluteTableIdentifier); + // if there is no loaded blocks then return + if (null == map) { + return; + } + for (TableBlockInfo blockInfos : removeTableBlocksInfos) { + map.remove(blockInfos); + } + } + + /** + * remove all the details of a table this will be used in case of drop table + * + * @param absoluteTableIdentifier absolute table identifier to find the table + */ + public void clear(AbsoluteTableIdentifier absoluteTableIdentifier) { + // removing all the details of table + tableLockMap.remove(absoluteTableIdentifier); + tableBlocksMap.remove(absoluteTableIdentifier); + } + + /** + * Thread class which will be used to load the blocks + */ + private class BlockLoaderThread implements Callable { + /** + * table block info to block index map + */ + private Map tableBlockMap; + + // block info + private TableBlockInfo blockInfo; + + private BlockLoaderThread(TableBlockInfo blockInfo, + Map tableBlockMap) { + this.tableBlockMap = tableBlockMap; + this.blockInfo = blockInfo; + } + + @Override public AbstractIndex call() throws Exception { + // load and return the loaded blocks + return loadBlock(tableBlockMap, blockInfo); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/BtreeBuilder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/BtreeBuilder.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/BtreeBuilder.java new file mode 100644 index 0000000..fb59607 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/BtreeBuilder.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.carbon.datastore; + +/** + * Below interface will be used to build the index + * in some data structure + */ +public interface BtreeBuilder { + + /** + * Below method will be used to store the leaf collection in some data structure + */ + void build(BTreeBuilderInfo blocksBuilderInfos); + + /** + * below method to get the first data block + * + * @return data block + */ + DataRefNode get(); +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/DataRefNode.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/DataRefNode.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/DataRefNode.java new file mode 100644 index 0000000..e81a9a6 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/DataRefNode.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.carbon.datastore; + +import org.apache.carbondata.core.carbon.datastore.chunk.DimensionColumnDataChunk; +import org.apache.carbondata.core.carbon.datastore.chunk.MeasureColumnDataChunk; +import org.apache.carbondata.core.datastorage.store.FileHolder; + +/** + * Interface data block reference + */ +public interface DataRefNode { + + /** + * Method to get the next block this can be used while scanning when + * iterator of this class can be used iterate over blocks + * + * @return next block + */ + DataRefNode getNextDataRefNode(); + + /** + * to get the number of keys tuples present in the block + * + * @return number of keys in the block + */ + int nodeSize(); + + /** + * Method can be used to get the block index .This can be used when multiple + * thread can be used scan group of blocks in that can we can assign the + * some of the blocks to one thread and some to other + * + * @return block number + */ + long nodeNumber(); + + /** + * This method will be used to get the max value of all the columns this can + * be used in case of filter query + * + * @param max value of all the columns + */ + byte[][] getColumnsMaxValue(); + + /** + * This method will be used to get the min value of all the columns this can + * be used in case of filter query + * + * @param min value of all the columns + */ + byte[][] getColumnsMinValue(); + + /** + * Below method will be used to get the dimension chunks + * + * @param fileReader file reader to read the chunks from file + * @param blockIndexes indexes of the blocks need to be read + * @return dimension data chunks + */ + DimensionColumnDataChunk[] getDimensionChunks(FileHolder fileReader, int[] blockIndexes); + + /** + * Below method will be used to get the dimension chunk + * + * @param fileReader file reader to read the chunk from file + * @param blockIndex block index to be read + * @return dimension data chunk + */ + DimensionColumnDataChunk getDimensionChunk(FileHolder fileReader, int blockIndexes); + + /** + * Below method will be used to get the measure chunk + * + * @param fileReader file reader to read the chunk from file + * @param blockIndexes block indexes to be read from file + * @return measure column data chunk + */ + MeasureColumnDataChunk[] getMeasureChunks(FileHolder fileReader, int[] blockIndexes); + + /** + * Below method will be used to read the measure chunk + * + * @param fileReader file read to read the file chunk + * @param blockIndex block index to be read from file + * @return measure data chunk + */ + MeasureColumnDataChunk getMeasureChunk(FileHolder fileReader, int blockIndex); +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/DataRefNodeFinder.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/DataRefNodeFinder.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/DataRefNodeFinder.java new file mode 100644 index 0000000..78592f7 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/DataRefNodeFinder.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.carbon.datastore; + +/** + * Below Interface is to search a block + */ +public interface DataRefNodeFinder { + + /** + * Below method will be used to get the first tentative block which matches with + * the search key + * + * @param dataBlocks complete data blocks present + * @param serachKey key to be search + * @return data block + */ + DataRefNode findFirstDataBlock(DataRefNode dataBlocks, IndexKey searchKey); + + /** + * Below method will be used to get the last tentative block which matches with + * the search key + * + * @param dataBlocks complete data blocks present + * @param serachKey key to be search + * @return data block + */ + DataRefNode findLastDataBlock(DataRefNode dataBlocks, IndexKey searchKey); +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/IndexKey.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/IndexKey.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/IndexKey.java new file mode 100644 index 0000000..cefd32c --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/IndexKey.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.carbon.datastore; + +/** + * Index class to store the index of the segment blocklet infos + */ +public class IndexKey { + + /** + * key which is generated from key generator + */ + private byte[] dictionaryKeys; + + /** + * key which was no generated using key generator + * + * <2nd Key ByteArray> + */ + private byte[] noDictionaryKeys; + + public IndexKey(byte[] dictionaryKeys, byte[] noDictionaryKeys) { + this.dictionaryKeys = dictionaryKeys; + this.noDictionaryKeys = noDictionaryKeys; + if (null == dictionaryKeys) { + this.dictionaryKeys = new byte[0]; + } + if (null == noDictionaryKeys) { + this.noDictionaryKeys = new byte[0]; + } + } + + /** + * @return the dictionaryKeys + */ + public byte[] getDictionaryKeys() { + return dictionaryKeys; + } + + /** + * @return the noDictionaryKeys + */ + public byte[] getNoDictionaryKeys() { + return noDictionaryKeys; + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/SegmentTaskIndexStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/SegmentTaskIndexStore.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/SegmentTaskIndexStore.java new file mode 100644 index 0000000..50d462a --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/SegmentTaskIndexStore.java @@ -0,0 +1,334 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.carbon.datastore; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.carbondata.common.logging.LogService; +import org.apache.carbondata.common.logging.LogServiceFactory; +import org.apache.carbondata.core.carbon.AbsoluteTableIdentifier; +import org.apache.carbondata.core.carbon.datastore.block.AbstractIndex; +import org.apache.carbondata.core.carbon.datastore.block.SegmentTaskIndex; +import org.apache.carbondata.core.carbon.datastore.block.TableBlockInfo; +import org.apache.carbondata.core.carbon.datastore.exception.IndexBuilderException; +import org.apache.carbondata.core.carbon.metadata.blocklet.DataFileFooter; +import org.apache.carbondata.core.carbon.path.CarbonTablePath.DataFileUtil; +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.util.CarbonUtil; +import org.apache.carbondata.core.util.CarbonUtilException; + +/** + * Singleton Class to handle loading, unloading,clearing,storing of the table + * blocks + */ +public class SegmentTaskIndexStore { + + private static final LogService LOGGER = + LogServiceFactory.getLogService(SegmentTaskIndexStore.class.getName()); + /** + * singleton instance + */ + private static final SegmentTaskIndexStore SEGMENTTASKINDEXSTORE = new SegmentTaskIndexStore(); + + /** + * mapping of table identifier to map of segmentId_taskId to table segment + * reason of so many map as each segment can have multiple data file and + * each file will have its own btree + */ + private Map>> tableSegmentMap; + + /** + * map of block info to lock object map, while loading the btree this will be filled + * and removed after loading the tree for that particular block info, this will be useful + * while loading the tree concurrently so only block level lock will be applied another + * block can be loaded concurrently + */ + private Map segmentLockMap; + + /** + * table and its lock object to this will be useful in case of concurrent + * query scenario when more than one query comes for same table and in that + * case it will ensure that only one query will able to load the blocks + */ + private Map tableLockMap; + + private SegmentTaskIndexStore() { + tableSegmentMap = + new ConcurrentHashMap>>( + CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); + tableLockMap = new ConcurrentHashMap( + CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); + segmentLockMap = new ConcurrentHashMap(); + } + + /** + * Return the instance of this class + * + * @return singleton instance + */ + public static SegmentTaskIndexStore getInstance() { + return SEGMENTTASKINDEXSTORE; + } + + /** + * Below method will be used to load the segment of segments + * One segment may have multiple task , so table segment will be loaded + * based on task id and will return the map of taksId to table segment + * map + * + * @param segmentToTableBlocksInfos segment id to block info + * @param absoluteTableIdentifier absolute table identifier + * @return map of taks id to segment mapping + * @throws IndexBuilderException + */ + public Map loadAndGetTaskIdToSegmentsMap( + Map> segmentToTableBlocksInfos, + AbsoluteTableIdentifier absoluteTableIdentifier) throws IndexBuilderException { + // task id to segment map + Map taskIdToTableSegmentMap = + new HashMap(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); + addLockObject(absoluteTableIdentifier); + Iterator>> iteratorOverSegmentBlocksInfos = + segmentToTableBlocksInfos.entrySet().iterator(); + Map> tableSegmentMapTemp = + addTableSegmentMap(absoluteTableIdentifier); + Map taskIdToSegmentIndexMap = null; + String segmentId = null; + String taskId = null; + try { + while (iteratorOverSegmentBlocksInfos.hasNext()) { + // segment id to table block mapping + Entry> next = iteratorOverSegmentBlocksInfos.next(); + // group task id to table block info mapping for the segment + Map> taskIdToTableBlockInfoMap = + mappedAndGetTaskIdToTableBlockInfo(segmentToTableBlocksInfos); + // get the existing map of task id to table segment map + segmentId = next.getKey(); + // check if segment is already loaded, if segment is already loaded + //no need to load the segment block + taskIdToSegmentIndexMap = tableSegmentMapTemp.get(segmentId); + if (taskIdToSegmentIndexMap == null) { + // get the segment loader lock object this is to avoid + // same segment is getting loaded multiple times + // in case of concurrent query + Object segmentLoderLockObject = segmentLockMap.get(segmentId); + if (null == segmentLoderLockObject) { + segmentLoderLockObject = addAndGetSegmentLock(segmentId); + } + // acquire lock to lod the segment + synchronized (segmentLoderLockObject) { + taskIdToSegmentIndexMap = tableSegmentMapTemp.get(segmentId); + if (null == taskIdToSegmentIndexMap) { + // creating a map of take if to table segment + taskIdToSegmentIndexMap = new HashMap(); + Iterator>> iterator = + taskIdToTableBlockInfoMap.entrySet().iterator(); + while (iterator.hasNext()) { + Entry> taskToBlockInfoList = iterator.next(); + taskId = taskToBlockInfoList.getKey(); + taskIdToSegmentIndexMap.put(taskId, + loadBlocks(taskId, taskToBlockInfoList.getValue(), absoluteTableIdentifier)); + } + tableSegmentMapTemp.put(next.getKey(), taskIdToSegmentIndexMap); + // removing from segment lock map as once segment is loaded + //if concurrent query is coming for same segment + // it will wait on the lock so after this segment will be already + //loaded so lock is not required, that is why removing the + // the lock object as it wont be useful + segmentLockMap.remove(segmentId); + } + } + taskIdToTableSegmentMap.putAll(taskIdToSegmentIndexMap); + } + } + } catch (CarbonUtilException e) { + LOGGER.error("Problem while loading the segment"); + throw new IndexBuilderException(e); + } + return taskIdToTableSegmentMap; + } + + /** + * Below method will be used to get the segment level lock object + * + * @param segmentId + * @return lock object + */ + private synchronized Object addAndGetSegmentLock(String segmentId) { + // get the segment lock object if it is present then return + // otherwise add the new lock and return + Object segmentLoderLockObject = segmentLockMap.get(segmentId); + if (null == segmentLoderLockObject) { + segmentLoderLockObject = new Object(); + segmentLockMap.put(segmentId, segmentLoderLockObject); + } + return segmentLoderLockObject; + } + + /** + * Below code is to add table lock map which will be used to + * add + * + * @param absoluteTableIdentifier + */ + private synchronized void addLockObject(AbsoluteTableIdentifier absoluteTableIdentifier) { + // add the instance to lock map if it is not present + if (null == tableLockMap.get(absoluteTableIdentifier)) { + tableLockMap.put(absoluteTableIdentifier, new Object()); + } + } + + /** + * Below method will be used to get the table segment map + * if table segment is not present then it will add and return + * + * @param absoluteTableIdentifier + * @return table segment map + */ + private Map> addTableSegmentMap( + AbsoluteTableIdentifier absoluteTableIdentifier) { + // get the instance of lock object + Object lockObject = tableLockMap.get(absoluteTableIdentifier); + Map> tableSegmentMapTemp = + tableSegmentMap.get(absoluteTableIdentifier); + if (null == tableSegmentMapTemp) { + synchronized (lockObject) { + // segment id to task id to table segment map + tableSegmentMapTemp = tableSegmentMap.get(absoluteTableIdentifier); + if (null == tableSegmentMapTemp) { + tableSegmentMapTemp = new ConcurrentHashMap>(); + tableSegmentMap.put(absoluteTableIdentifier, tableSegmentMapTemp); + } + } + } + return tableSegmentMapTemp; + } + + /** + * Below method will be used to load the blocks + * + * @param tableBlockInfoList + * @return loaded segment + * @throws CarbonUtilException + */ + private AbstractIndex loadBlocks(String taskId, List tableBlockInfoList, + AbsoluteTableIdentifier tableIdentifier) throws CarbonUtilException { + // all the block of one task id will be loaded together + // so creating a list which will have all the data file meta data to of one task + List footerList = + CarbonUtil.readCarbonIndexFile(taskId, tableBlockInfoList, tableIdentifier); + AbstractIndex segment = new SegmentTaskIndex(); + // file path of only first block is passed as it all table block info path of + // same task id will be same + segment.buildIndex(footerList); + return segment; + } + + /** + * Below method will be used to get the task id to all the table block info belongs to + * that task id mapping + * + * @param segmentToTableBlocksInfos segment if to table blocks info map + * @return task id to table block info mapping + */ + private Map> mappedAndGetTaskIdToTableBlockInfo( + Map> segmentToTableBlocksInfos) { + Map> taskIdToTableBlockInfoMap = + new HashMap>(); + Iterator>> iterator = + segmentToTableBlocksInfos.entrySet().iterator(); + while (iterator.hasNext()) { + Entry> next = iterator.next(); + List value = next.getValue(); + for (TableBlockInfo blockInfo : value) { + String taskNo = DataFileUtil.getTaskNo(blockInfo.getFilePath()); + List list = taskIdToTableBlockInfoMap.get(taskNo); + if (null == list) { + list = new ArrayList(); + taskIdToTableBlockInfoMap.put(taskNo, list); + } + list.add(blockInfo); + } + + } + return taskIdToTableBlockInfoMap; + } + + /** + * remove all the details of a table this will be used in case of drop table + * + * @param absoluteTableIdentifier absolute table identifier to find the table + */ + public void clear(AbsoluteTableIdentifier absoluteTableIdentifier) { + // removing all the details of table + tableLockMap.remove(absoluteTableIdentifier); + tableSegmentMap.remove(absoluteTableIdentifier); + } + + /** + * Below method will be used to remove the segment block based on + * segment id is passed + * + * @param segmentToBeRemoved segment to be removed + * @param absoluteTableIdentifier absoluteTableIdentifier + */ + public void removeTableBlocks(List segmentToBeRemoved, + AbsoluteTableIdentifier absoluteTableIdentifier) { + // get the lock object if lock object is not present then it is not + // loaded at all + // we can return from here + Object lockObject = tableLockMap.get(absoluteTableIdentifier); + if (null == lockObject) { + return; + } + // Acquire the lock and remove only those instance which was loaded + Map> map = tableSegmentMap.get(absoluteTableIdentifier); + // if there is no loaded blocks then return + if (null == map) { + return; + } + for (String segmentId : segmentToBeRemoved) { + map.remove(segmentId); + } + } + + /** + * Below method will be used to check if segment blocks + * is already loaded or not + * + * @param absoluteTableIdentifier + * @param segmentId + * @return is loaded then return the loaded blocks otherwise null + */ + public Map getSegmentBTreeIfExists( + AbsoluteTableIdentifier absoluteTableIdentifier, String segmentId) { + Map> tableSegment = + tableSegmentMap.get(absoluteTableIdentifier); + if (null == tableSegment) { + return null; + } + return tableSegment.get(segmentId); + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/AbstractIndex.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/AbstractIndex.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/AbstractIndex.java new file mode 100644 index 0000000..7e1ed8c --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/AbstractIndex.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.carbon.datastore.block; + +import java.util.List; + +import org.apache.carbondata.core.carbon.datastore.DataRefNode; +import org.apache.carbondata.core.carbon.metadata.blocklet.DataFileFooter; + +public abstract class AbstractIndex { + + /** + * vo class which will hold the RS information of the block + */ + protected SegmentProperties segmentProperties; + + /** + * data block + */ + protected DataRefNode dataRefNode; + + /** + * total number of row present in the block + */ + protected long totalNumberOfRows; + + /** + * @return the totalNumberOfRows + */ + public long getTotalNumberOfRows() { + return totalNumberOfRows; + } + + /** + * @return the segmentProperties + */ + public SegmentProperties getSegmentProperties() { + return segmentProperties; + } + + /** + * @return the dataBlock + */ + public DataRefNode getDataRefNode() { + return dataRefNode; + } + + /** + * Below method will be used to load the data block + * + * @param blockInfo block detail + */ + public abstract void buildIndex(List footerList); +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/BlockIndex.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/BlockIndex.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/BlockIndex.java new file mode 100644 index 0000000..cfdb127 --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/BlockIndex.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.carbon.datastore.block; + +import java.util.List; + +import org.apache.carbondata.core.carbon.datastore.BTreeBuilderInfo; +import org.apache.carbondata.core.carbon.datastore.BtreeBuilder; +import org.apache.carbondata.core.carbon.datastore.impl.btree.BlockletBTreeBuilder; +import org.apache.carbondata.core.carbon.metadata.blocklet.DataFileFooter; + +/** + * Class which is responsible for loading the b+ tree block. This class will + * persist all the detail of a table block + */ +public class BlockIndex extends AbstractIndex { + + /** + * Below method will be used to load the data block + * + * @param blockInfo block detail + */ + public void buildIndex(List footerList) { + // create a metadata details + // this will be useful in query handling + segmentProperties = new SegmentProperties(footerList.get(0).getColumnInTable(), + footerList.get(0).getSegmentInfo().getColumnCardinality()); + // create a segment builder info + BTreeBuilderInfo indexBuilderInfo = + new BTreeBuilderInfo(footerList, segmentProperties.getDimensionColumnsValueSize()); + BtreeBuilder blocksBuilder = new BlockletBTreeBuilder(); + // load the metadata + blocksBuilder.build(indexBuilderInfo); + dataRefNode = blocksBuilder.get(); + totalNumberOfRows = footerList.get(0).getNumberOfRows(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/Distributable.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/Distributable.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/Distributable.java new file mode 100644 index 0000000..817aafc --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/Distributable.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.core.carbon.datastore.block; + +/** + * Abstract class which is maintains the locations of node. + */ +public abstract class Distributable implements Comparable { + + public abstract String[] getLocations(); +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/SegmentProperties.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/SegmentProperties.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/SegmentProperties.java new file mode 100644 index 0000000..816ca3a --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/SegmentProperties.java @@ -0,0 +1,748 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.carbon.datastore.block; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import org.apache.carbondata.core.carbon.metadata.datatype.DataType; +import org.apache.carbondata.core.carbon.metadata.encoder.Encoding; +import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonDimension; +import org.apache.carbondata.core.carbon.metadata.schema.table.column.CarbonMeasure; +import org.apache.carbondata.core.carbon.metadata.schema.table.column.ColumnSchema; +import org.apache.carbondata.core.constants.CarbonCommonConstants; +import org.apache.carbondata.core.datastorage.store.columnar.ColumnGroupModel; +import org.apache.carbondata.core.keygenerator.KeyGenerator; +import org.apache.carbondata.core.keygenerator.columnar.ColumnarSplitter; +import org.apache.carbondata.core.keygenerator.columnar.impl.MultiDimKeyVarLengthVariableSplitGenerator; +import org.apache.carbondata.core.keygenerator.mdkey.MultiDimKeyVarLengthGenerator; +import org.apache.carbondata.core.util.CarbonUtil; + +import org.apache.commons.lang3.ArrayUtils; + +/** + * This class contains all the details about the restructuring information of + * the block. This will be used during query execution to handle restructure + * information + */ +public class SegmentProperties { + + /** + * key generator of the block which was used to generate the mdkey for + * normal dimension. this will be required to + */ + private KeyGenerator dimensionKeyGenerator; + + /** + * list of dimension present in the block + */ + private List dimensions; + + /** + * list of dimension present in the block + */ + private List complexDimensions; + + /** + * list of measure present in the block + */ + private List measures; + + /** + * cardinality of dimension columns participated in key generator + */ + private int[] dimColumnsCardinality; + + /** + * cardinality of complex dimension + */ + private int[] complexDimColumnCardinality; + + /** + * mapping of dimension column to block in a file this will be used for + * reading the blocks from file + */ + private Map dimensionOrdinalToBlockMapping; + + /** + * a block can have multiple columns. This will have block index as key + * and all dimension participated in that block as values + */ + private Map> blockTodimensionOrdinalMapping; + + /** + * mapping of measure column to block to in file this will be used while + * reading the block in a file + */ + private Map measuresOrdinalToBlockMapping; + + /** + * size of the each dimension column value in a block this can be used when + * we need to do copy a cell value to create a tuple.for no dictionary + * column this value will be -1. for dictionary column we size of the value + * will be fixed. + */ + private int[] eachDimColumnValueSize; + + /** + * size of the each dimension column value in a block this can be used when + * we need to do copy a cell value to create a tuple.for no dictionary + * column this value will be -1. for dictionary column we size of the value + * will be fixed. + */ + private int[] eachComplexDimColumnValueSize; + + /** + * below mapping will have mapping of the column group to dimensions ordinal + * for example if 3 dimension present in the columngroupid 0 and its ordinal in + * 2,3,4 then map will contain 0,{2,3,4} + */ + private Map columnGroupAndItsKeygenartor; + + /** + * column group key generator dimension index will not be same as dimension ordinal + * This will have mapping with ordinal and keygenerator or mdkey index + */ + private Map> columnGroupOrdinalToMdkeymapping; + + /** + * this will be used to split the fixed length key + * this will all the information about how key was created + * and how to split the key based on group + */ + private ColumnarSplitter fixedLengthKeySplitter; + + /** + * to store the number of no dictionary dimension + * this will be used during query execution for creating + * start and end key. Purpose of storing this value here is + * so during query execution no need to calculate every time + */ + private int numberOfNoDictionaryDimension; + + /** + * column group model + */ + private ColumnGroupModel colGroupModel; + + public SegmentProperties(List columnsInTable, int[] columnCardinality) { + dimensions = new ArrayList(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); + complexDimensions = + new ArrayList(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); + measures = new ArrayList(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); + fillDimensionAndMeasureDetails(columnsInTable, columnCardinality); + dimensionOrdinalToBlockMapping = + new HashMap(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); + blockTodimensionOrdinalMapping = + new HashMap>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); + measuresOrdinalToBlockMapping = + new HashMap(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); + intialiseColGroups(columnsInTable); + fillOrdinalToBlockMappingForDimension(); + fillOrdinalToBlockIndexMappingForMeasureColumns(); + fillColumnGroupAndItsCardinality(columnCardinality); + fillKeyGeneratorDetails(); + } + + /** + * it fills column groups + * e.g {{1},{2,3,4},{5},{6},{7,8,9}} + * + * @param columnsInTable + */ + private void intialiseColGroups(List columnsInTable) { + // StringBuffer columnGroups = new StringBuffer(); + List> colGrpList = new ArrayList>(); + List group = new ArrayList(); + for (int i = 0; i < dimensions.size(); i++) { + CarbonDimension dimension = dimensions.get(i); + if (!dimension.hasEncoding(Encoding.DICTIONARY)) { + continue; + } + group.add(dimension.getOrdinal()); + // columnGroups.append(dimension.getOrdinal()); + if (i < dimensions.size() - 1) { + int currGroupOrdinal = dimension.columnGroupId(); + int nextGroupOrdinal = dimensions.get(i + 1).columnGroupId(); + if (!(currGroupOrdinal == nextGroupOrdinal && currGroupOrdinal != -1)) { + colGrpList.add(group); + group = new ArrayList(); + } + } else { + colGrpList.add(group); + } + + } + int[][] colGroups = new int[colGrpList.size()][]; + for (int i = 0; i < colGroups.length; i++) { + colGroups[i] = new int[colGrpList.get(i).size()]; + for (int j = 0; j < colGroups[i].length; j++) { + colGroups[i][j] = colGrpList.get(i).get(j); + } + } + this.colGroupModel = CarbonUtil.getColGroupModel(colGroups); + } + + /** + * below method is to fill the dimension and its mapping to file blocks all + * the column will point to same column group + */ + private void fillOrdinalToBlockMappingForDimension() { + int blockOrdinal = -1; + CarbonDimension dimension = null; + int index = 0; + int prvcolumnGroupId = -1; + while (index < dimensions.size()) { + dimension = dimensions.get(index); + // if column id is same as previous one then block index will be + // same + if (dimension.isColumnar() || dimension.columnGroupId() != prvcolumnGroupId) { + blockOrdinal++; + } + dimensionOrdinalToBlockMapping.put(dimension.getOrdinal(), blockOrdinal); + prvcolumnGroupId = dimension.columnGroupId(); + index++; + } + index = 0; + // complex dimension will be stored at last + while (index < complexDimensions.size()) { + dimension = complexDimensions.get(index); + dimensionOrdinalToBlockMapping.put(dimension.getOrdinal(), ++blockOrdinal); + blockOrdinal = fillComplexDimensionChildBlockIndex(blockOrdinal, dimension); + index++; + } + fillBlockToDimensionOrdinalMapping(); + } + + /** + * + */ + private void fillBlockToDimensionOrdinalMapping() { + Set> blocks = dimensionOrdinalToBlockMapping.entrySet(); + Iterator> blockItr = blocks.iterator(); + while (blockItr.hasNext()) { + Entry block = blockItr.next(); + Set dimensionOrdinals = blockTodimensionOrdinalMapping.get(block.getValue()); + if (dimensionOrdinals == null) { + dimensionOrdinals = new HashSet(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); + blockTodimensionOrdinalMapping.put(block.getValue(), dimensionOrdinals); + } + dimensionOrdinals.add(block.getKey()); + } + } + + /** + * Below method will be used to add the complex dimension child + * block index.It is a recursive method which will be get the children + * add the block index + * + * @param blockOrdinal start block ordinal + * @param dimension parent dimension + * @return last block index + */ + private int fillComplexDimensionChildBlockIndex(int blockOrdinal, CarbonDimension dimension) { + for (int i = 0; i < dimension.numberOfChild(); i++) { + dimensionOrdinalToBlockMapping + .put(dimension.getListOfChildDimensions().get(i).getOrdinal(), ++blockOrdinal); + if (dimension.getListOfChildDimensions().get(i).numberOfChild() > 0) { + blockOrdinal = fillComplexDimensionChildBlockIndex(blockOrdinal, + dimension.getListOfChildDimensions().get(i)); + } + } + return blockOrdinal; + } + + /** + * Below method will be used to fill the mapping + * of measure ordinal to its block index mapping in + * file + */ + private void fillOrdinalToBlockIndexMappingForMeasureColumns() { + int blockOrdinal = 0; + int index = 0; + while (index < measures.size()) { + measuresOrdinalToBlockMapping.put(measures.get(index).getOrdinal(), blockOrdinal); + blockOrdinal++; + index++; + } + } + + /** + * below method will fill dimension and measure detail of the block. + * + * @param columnsInTable + * @param columnCardinality + */ + private void fillDimensionAndMeasureDetails(List columnsInTable, + int[] columnCardinality) { + ColumnSchema columnSchema = null; + // ordinal will be required to read the data from file block + int dimensonOrdinal = 0; + int measureOrdinal = -1; + // table ordinal is actually a schema ordinal this is required as + // cardinality array + // which is stored in segment info contains -1 if that particular column + // is n + int tableOrdinal = -1; + // creating a list as we do not know how many dimension not participated + // in the mdkey + List cardinalityIndexForNormalDimensionColumn = + new ArrayList(columnsInTable.size()); + // creating a list as we do not know how many dimension not participated + // in the mdkey + List cardinalityIndexForComplexDimensionColumn = + new ArrayList(columnsInTable.size()); + boolean isComplexDimensionStarted = false; + CarbonDimension carbonDimension = null; + // to store the position of dimension in surrogate key array which is + // participating in mdkey + int keyOrdinal = 0; + int previousColumnGroup = -1; + // to store the ordinal of the column group ordinal + int columnGroupOrdinal = 0; + int counter = 0; + int complexTypeOrdinal = 0; + while (counter < columnsInTable.size()) { + columnSchema = columnsInTable.get(counter); + if (columnSchema.isDimensionColumn()) { + tableOrdinal++; + // not adding the cardinality of the non dictionary + // column as it was not the part of mdkey + if (CarbonUtil.hasEncoding(columnSchema.getEncodingList(), Encoding.DICTIONARY) + && !isComplexDimensionStarted && columnSchema.getNumberOfChild() == 0) { + cardinalityIndexForNormalDimensionColumn.add(tableOrdinal); + if (columnSchema.isColumnar()) { + // if it is a columnar dimension participated in mdkey then added + // key ordinal and dimension ordinal + carbonDimension = + new CarbonDimension(columnSchema, dimensonOrdinal++, keyOrdinal++, -1, -1); + } else { + // if not columnnar then it is a column group dimension + + // below code to handle first dimension of the column group + // in this case ordinal of the column group will be 0 + if (previousColumnGroup != columnSchema.getColumnGroupId()) { + columnGroupOrdinal = 0; + carbonDimension = new CarbonDimension(columnSchema, dimensonOrdinal++, keyOrdinal++, + columnGroupOrdinal++, -1); + } + // if previous dimension column group id is same as current then + // then its belongs to same row group + else { + carbonDimension = new CarbonDimension(columnSchema, dimensonOrdinal++, keyOrdinal++, + columnGroupOrdinal++, -1); + } + previousColumnGroup = columnSchema.getColumnGroupId(); + } + } + // as complex type will be stored at last so once complex type started all the dimension + // will be added to complex type + else if (isComplexDimensionStarted || CarbonUtil.hasDataType(columnSchema.getDataType(), + new DataType[] { DataType.ARRAY, DataType.STRUCT })) { + cardinalityIndexForComplexDimensionColumn.add(tableOrdinal); + carbonDimension = + new CarbonDimension(columnSchema, dimensonOrdinal++, -1, -1, complexTypeOrdinal++); + carbonDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild()); + complexDimensions.add(carbonDimension); + isComplexDimensionStarted = true; + int previouseOrdinal = dimensonOrdinal; + dimensonOrdinal = + readAllComplexTypeChildrens(dimensonOrdinal, columnSchema.getNumberOfChild(), + columnsInTable, carbonDimension, complexTypeOrdinal); + int numberOfChildrenDimensionAdded = dimensonOrdinal - previouseOrdinal; + for (int i = 0; i < numberOfChildrenDimensionAdded; i++) { + cardinalityIndexForComplexDimensionColumn.add(++tableOrdinal); + } + counter = dimensonOrdinal; + complexTypeOrdinal = carbonDimension.getListOfChildDimensions() + .get(carbonDimension.getListOfChildDimensions().size() - 1).getComplexTypeOrdinal(); + complexTypeOrdinal++; + continue; + } else { + // for no dictionary dimension + carbonDimension = new CarbonDimension(columnSchema, dimensonOrdinal++, -1, -1, -1); + numberOfNoDictionaryDimension++; + } + dimensions.add(carbonDimension); + } else { + measures.add(new CarbonMeasure(columnSchema, ++measureOrdinal)); + } + counter++; + } + dimColumnsCardinality = new int[cardinalityIndexForNormalDimensionColumn.size()]; + complexDimColumnCardinality = new int[cardinalityIndexForComplexDimensionColumn.size()]; + int index = 0; + // filling the cardinality of the dimension column to create the key + // generator + for (Integer cardinalityArrayIndex : cardinalityIndexForNormalDimensionColumn) { + dimColumnsCardinality[index++] = columnCardinality[cardinalityArrayIndex]; + } + index = 0; + // filling the cardinality of the complex dimension column to create the + // key generator + for (Integer cardinalityArrayIndex : cardinalityIndexForComplexDimensionColumn) { + complexDimColumnCardinality[index++] = columnCardinality[cardinalityArrayIndex]; + } + } + + /** + * Read all primitive/complex children and set it as list of child carbon dimension to parent + * dimension + * + * @param dimensionOrdinal + * @param childCount + * @param listOfColumns + * @param parentDimension + * @return + */ + private int readAllComplexTypeChildrens(int dimensionOrdinal, int childCount, + List listOfColumns, CarbonDimension parentDimension, + int complexDimensionOrdianl) { + for (int i = 0; i < childCount; i++) { + ColumnSchema columnSchema = listOfColumns.get(dimensionOrdinal); + if (columnSchema.isDimensionColumn()) { + if (columnSchema.getNumberOfChild() > 0) { + CarbonDimension complexDimension = + new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1, + complexDimensionOrdianl++); + complexDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild()); + parentDimension.getListOfChildDimensions().add(complexDimension); + dimensionOrdinal = + readAllComplexTypeChildrens(dimensionOrdinal, columnSchema.getNumberOfChild(), + listOfColumns, complexDimension, complexDimensionOrdianl); + } else { + parentDimension.getListOfChildDimensions().add( + new CarbonDimension(columnSchema, dimensionOrdinal++, -1, -1, + complexDimensionOrdianl++)); + } + } + } + return dimensionOrdinal; + } + + /** + * Below method will fill the key generator detail of both the type of key + * generator. This will be required for during both query execution and data + * loading. + */ + private void fillKeyGeneratorDetails() { + // create a dimension partitioner list + // this list will contain information about how dimension value are + // stored + // it is stored in group or individually + List dimensionPartitionList = + new ArrayList(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); + List isDictionaryColumn = + new ArrayList(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); + int prvcolumnGroupId = -1; + int counter = 0; + while (counter < dimensions.size()) { + CarbonDimension carbonDimension = dimensions.get(counter); + // if dimension is not a part of mdkey then no need to add + if (!carbonDimension.getEncoder().contains(Encoding.DICTIONARY)) { + isDictionaryColumn.add(false); + counter++; + continue; + } + // columnar column is stored individually + // so add one + if (carbonDimension.isColumnar()) { + dimensionPartitionList.add(1); + isDictionaryColumn.add(true); + } + // if in a group then need to add how many columns a selected in + // group + if (!carbonDimension.isColumnar() && carbonDimension.columnGroupId() == prvcolumnGroupId) { + // incrementing the previous value of the list as it is in same column group + dimensionPartitionList.set(dimensionPartitionList.size() - 1, + dimensionPartitionList.get(dimensionPartitionList.size() - 1) + 1); + } else if (!carbonDimension.isColumnar()) { + dimensionPartitionList.add(1); + isDictionaryColumn.add(true); + } + prvcolumnGroupId = carbonDimension.columnGroupId(); + counter++; + } + // get the partitioner + int[] dimensionPartitions = ArrayUtils + .toPrimitive(dimensionPartitionList.toArray(new Integer[dimensionPartitionList.size()])); + // get the bit length of each column + int[] bitLength = CarbonUtil.getDimensionBitLength(dimColumnsCardinality, dimensionPartitions); + // create a key generator + this.dimensionKeyGenerator = new MultiDimKeyVarLengthGenerator(bitLength); + this.fixedLengthKeySplitter = + new MultiDimKeyVarLengthVariableSplitGenerator(bitLength, dimensionPartitions); + // get the size of each value in file block + int[] dictionayDimColumnValueSize = fixedLengthKeySplitter.getBlockKeySize(); + int index = -1; + this.eachDimColumnValueSize = new int[isDictionaryColumn.size()]; + for (int i = 0; i < eachDimColumnValueSize.length; i++) { + if (!isDictionaryColumn.get(i)) { + eachDimColumnValueSize[i] = -1; + continue; + } + eachDimColumnValueSize[i] = dictionayDimColumnValueSize[++index]; + } + if (complexDimensions.size() > 0) { + int[] complexDimesionParition = new int[complexDimColumnCardinality.length]; + // as complex dimension will be stored in column format add one + Arrays.fill(complexDimesionParition, 1); + bitLength = + CarbonUtil.getDimensionBitLength(complexDimColumnCardinality, complexDimesionParition); + for (int i = 0; i < bitLength.length; i++) { + if (complexDimColumnCardinality[i] == 0) { + bitLength[i] = 64; + } + } + ColumnarSplitter keySplitter = + new MultiDimKeyVarLengthVariableSplitGenerator(bitLength, complexDimesionParition); + eachComplexDimColumnValueSize = keySplitter.getBlockKeySize(); + } else { + eachComplexDimColumnValueSize = new int[0]; + } + } + + /** + * Below method will be used to create a mapping of column group and its column cardinality this + * mapping will have column group id to cardinality of the dimension present in + * the column group.This mapping will be used during query execution, to create + * a mask key for the column group dimension which will be used in aggregation + * and filter query as column group dimension will be stored at the bit level + */ + private void fillColumnGroupAndItsCardinality(int[] cardinality) { + // mapping of the column group and its ordinal + Map> columnGroupAndOrdinalMapping = + new HashMap>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE); + // to store a column group + List currentColumnGroup = null; + // current index + int index = 0; + // previous column group to check all the column of column id has bee selected + int prvColumnGroupId = -1; + while (index < dimensions.size()) { + // if dimension group id is not zero and it is same as the previous + // column id + // then we need to add ordinal of that column as it belongs to same + // column group + if (!dimensions.get(index).isColumnar() + && dimensions.get(index).columnGroupId() == prvColumnGroupId + && null != currentColumnGroup) { + currentColumnGroup.add(index); + } + // if column is not a columnar then new column group has come + // so we need to create a list of new column id group and add the + // ordinal + else if (!dimensions.get(index).isColumnar()) { + currentColumnGroup = new ArrayList(); + columnGroupAndOrdinalMapping.put(dimensions.get(index).columnGroupId(), currentColumnGroup); + currentColumnGroup.add(index); + } + // update the column id every time,this is required to group the + // columns + // of the same column group + prvColumnGroupId = dimensions.get(index).columnGroupId(); + index++; + } + // Initializing the map + this.columnGroupAndItsKeygenartor = + new HashMap(columnGroupAndOrdinalMapping.size()); + this.columnGroupOrdinalToMdkeymapping = new HashMap<>(columnGroupAndOrdinalMapping.size()); + int[] columnGroupCardinality = null; + index = 0; + Iterator>> iterator = + columnGroupAndOrdinalMapping.entrySet().iterator(); + while (iterator.hasNext()) { + Entry> next = iterator.next(); + List currentGroupOrdinal = next.getValue(); + Map colGrpOrdinalMdkeyMapping = new HashMap<>(currentGroupOrdinal.size()); + // create the cardinality array + columnGroupCardinality = new int[currentGroupOrdinal.size()]; + for (int i = 0; i < columnGroupCardinality.length; i++) { + // fill the cardinality + columnGroupCardinality[i] = cardinality[currentGroupOrdinal.get(i)]; + colGrpOrdinalMdkeyMapping.put(currentGroupOrdinal.get(i), i); + } + this.columnGroupAndItsKeygenartor.put(next.getKey(), new MultiDimKeyVarLengthGenerator( + CarbonUtil.getDimensionBitLength(columnGroupCardinality, + new int[] { columnGroupCardinality.length }))); + this.columnGroupOrdinalToMdkeymapping.put(next.getKey(), colGrpOrdinalMdkeyMapping); + } + } + + /** + * Below method is to get the value of each dimension column. As this method + * will be used only once so we can merge both the dimension and complex + * dimension array. Complex dimension will be store at last so first copy + * the normal dimension the copy the complex dimension size. If we store + * this value as a class variable unnecessarily we will waste some space + * + * @return each dimension value size + */ + public int[] getDimensionColumnsValueSize() { + int[] dimensionValueSize = + new int[eachDimColumnValueSize.length + eachComplexDimColumnValueSize.length]; + System + .arraycopy(eachDimColumnValueSize, 0, dimensionValueSize, 0, eachDimColumnValueSize.length); + System.arraycopy(eachComplexDimColumnValueSize, 0, dimensionValueSize, + eachDimColumnValueSize.length, eachComplexDimColumnValueSize.length); + return dimensionValueSize; + } + + /** + * @return the dimensionKeyGenerator + */ + public KeyGenerator getDimensionKeyGenerator() { + return dimensionKeyGenerator; + } + + /** + * @return the dimensions + */ + public List getDimensions() { + return dimensions; + } + + /** + * @return the complexDimensions + */ + public List getComplexDimensions() { + return complexDimensions; + } + + /** + * @return the measures + */ + public List getMeasures() { + return measures; + } + + /** + * @return the dimColumnsCardinality + */ + public int[] getDimColumnsCardinality() { + return dimColumnsCardinality; + } + + /** + * @return the complexDimColumnCardinality + */ + public int[] getComplexDimColumnCardinality() { + return complexDimColumnCardinality; + } + + /** + * @return the dimensionOrdinalToBlockMapping + */ + public Map getDimensionOrdinalToBlockMapping() { + return dimensionOrdinalToBlockMapping; + } + + /** + * @return the measuresOrdinalToBlockMapping + */ + public Map getMeasuresOrdinalToBlockMapping() { + return measuresOrdinalToBlockMapping; + } + + /** + * @return the eachDimColumnValueSize + */ + public int[] getEachDimColumnValueSize() { + return eachDimColumnValueSize; + } + + /** + * @return the eachComplexDimColumnValueSize + */ + public int[] getEachComplexDimColumnValueSize() { + return eachComplexDimColumnValueSize; + } + + /** + * @return the fixedLengthKeySplitter + */ + public ColumnarSplitter getFixedLengthKeySplitter() { + return fixedLengthKeySplitter; + } + + /** + * @return the columnGroupAndItsKeygenartor + */ + public Map getColumnGroupAndItsKeygenartor() { + return columnGroupAndItsKeygenartor; + } + + /** + * @return the numberOfNoDictionaryDimension + */ + public int getNumberOfNoDictionaryDimension() { + return numberOfNoDictionaryDimension; + } + + /** + * @return + */ + public int[][] getColumnGroups() { + return colGroupModel.getColumnGroup(); + } + + /** + * @return colGroupModel + */ + public ColumnGroupModel getColumnGroupModel() { + return this.colGroupModel; + } + + /** + * get mdkey ordinal for given dimension ordinal of given column group + * + * @param colGrpId + * @param ordinal + * @return mdkeyordinal + */ + public int getColumnGroupMdKeyOrdinal(int colGrpId, int ordinal) { + return columnGroupOrdinalToMdkeymapping.get(colGrpId).get(ordinal); + } + + /** + * It returns no of column availble in given column group + * + * @param colGrpId + * @return no of column in given column group + */ + public int getNoOfColumnsInColumnGroup(int colGrpId) { + return columnGroupOrdinalToMdkeymapping.get(colGrpId).size(); + } + + /** + * @param blockIndex + * @return It returns all dimension present in given block index + */ + public Set getDimensionOrdinalForBlock(int blockIndex) { + return blockTodimensionOrdinalMapping.get(blockIndex); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/SegmentTaskIndex.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/SegmentTaskIndex.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/SegmentTaskIndex.java new file mode 100644 index 0000000..ce7a63a --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/SegmentTaskIndex.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.carbon.datastore.block; + +import java.util.List; + +import org.apache.carbondata.core.carbon.datastore.BTreeBuilderInfo; +import org.apache.carbondata.core.carbon.datastore.BtreeBuilder; +import org.apache.carbondata.core.carbon.datastore.impl.btree.BlockBTreeBuilder; +import org.apache.carbondata.core.carbon.metadata.blocklet.DataFileFooter; + +/** + * Class which is responsible for loading the b+ tree block. This class will + * persist all the detail of a table segment + */ +public class SegmentTaskIndex extends AbstractIndex { + + /** + * Below method is store the blocks in some data structure + * + * @param blockInfo block detail + */ + public void buildIndex(List footerList) { + // create a metadata details + // this will be useful in query handling + // all the data file metadata will have common segment properties we + // can use first one to get create the segment properties + segmentProperties = new SegmentProperties(footerList.get(0).getColumnInTable(), + footerList.get(0).getSegmentInfo().getColumnCardinality()); + // create a segment builder info + // in case of segment create we do not need any file path and each column value size + // as Btree will be build as per min max and start key + BTreeBuilderInfo btreeBuilderInfo = new BTreeBuilderInfo(footerList, null); + BtreeBuilder blocksBuilder = new BlockBTreeBuilder(); + // load the metadata + blocksBuilder.build(btreeBuilderInfo); + dataRefNode = blocksBuilder.get(); + for (DataFileFooter footer : footerList) { + totalNumberOfRows += footer.getNumberOfRows(); + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/cd6a4ff3/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/TableBlockInfo.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/TableBlockInfo.java b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/TableBlockInfo.java new file mode 100644 index 0000000..06166fd --- /dev/null +++ b/core/src/main/java/org/apache/carbondata/core/carbon/datastore/block/TableBlockInfo.java @@ -0,0 +1,204 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.core.carbon.datastore.block; + +import java.io.Serializable; +import java.util.Arrays; + +import org.apache.carbondata.core.carbon.path.CarbonTablePath; +import org.apache.carbondata.core.carbon.path.CarbonTablePath.DataFileUtil; +import org.apache.carbondata.core.datastorage.store.impl.FileFactory; + +/** + * class will be used to pass the block detail detail will be passed form driver + * to all the executor to load the b+ tree + */ +public class TableBlockInfo extends Distributable + implements Serializable, Comparable { + + /** + * serialization id + */ + private static final long serialVersionUID = -6502868998599821172L; + + /** + * full qualified file path of the block + */ + private String filePath; + + /** + * block offset in the file + */ + private long blockOffset; + + /** + * length of the block + */ + private long blockLength; + + /** + * id of the segment this will be used to sort the blocks + */ + private String segmentId; + + private String[] locations; + + + public TableBlockInfo(String filePath, long blockOffset, String segmentId, String[] locations, + long blockLength) { + this.filePath = FileFactory.getUpdatedFilePath(filePath); + this.blockOffset = blockOffset; + this.segmentId = segmentId; + this.locations = locations; + this.blockLength = blockLength; + } + + /** + * @return the filePath + */ + public String getFilePath() { + return filePath; + } + + /** + * @return the blockOffset + */ + public long getBlockOffset() { + return blockOffset; + } + + + /** + * @return the segmentId + */ + public String getSegmentId() { + return segmentId; + } + + /** + * @return the blockLength + */ + public long getBlockLength() { + return blockLength; + } + + /* + * (non-Javadoc) + * + * @see java.lang.Object#equals(java.lang.Object) + */ + @Override public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (!(obj instanceof TableBlockInfo)) { + return false; + } + TableBlockInfo other = (TableBlockInfo) obj; + if (!segmentId.equals(other.segmentId)) { + return false; + } + if (blockOffset != other.blockOffset) { + return false; + } + if (blockLength != other.blockLength) { + return false; + } + + if (filePath == null) { + if (other.filePath != null) { + return false; + } + } else if (!filePath.equals(other.filePath)) { + return false; + } + return true; + } + + /** + * Below method will used to compare to TableBlockInfos object this will + * used for sorting Comparison logic is: 1. compare segment id if segment id + * is same 2. compare task id if task id is same 3. compare offsets of the + * block + */ + @Override public int compareTo(Distributable other) { + + int compareResult = 0; + // get the segment id + // converr seg ID to double. + + double seg1 = Double.parseDouble(segmentId); + double seg2 = Double.parseDouble(((TableBlockInfo) other).segmentId); + if (seg1 - seg2 < 0) { + return -1; + } + if (seg1 - seg2 > 0) { + return 1; + } + + // Comparing the time task id of the file to other + // if both the task id of the file is same then we need to compare the + // offset of + // the file + if (CarbonTablePath.isCarbonDataFile(filePath)) { + int firstTaskId = Integer.parseInt(DataFileUtil.getTaskNo(filePath)); + int otherTaskId = Integer.parseInt(DataFileUtil.getTaskNo(((TableBlockInfo) other).filePath)); + if (firstTaskId != otherTaskId) { + return firstTaskId - otherTaskId; + } + // compare the part no of both block info + int firstPartNo = Integer.parseInt(DataFileUtil.getPartNo(filePath)); + int SecondPartNo = + Integer.parseInt(DataFileUtil.getPartNo(((TableBlockInfo) other).filePath)); + compareResult = firstPartNo - SecondPartNo; + } else { + compareResult = filePath.compareTo(((TableBlockInfo) other).getFilePath()); + } + if (compareResult != 0) { + return compareResult; + } + //compare result is not 0 then return + // if part no is also same then compare the offset and length of the block + if (blockOffset + blockLength + < ((TableBlockInfo) other).blockOffset + ((TableBlockInfo) other).blockLength) { + return -1; + } else if (blockOffset + blockLength + > ((TableBlockInfo) other).blockOffset + ((TableBlockInfo) other).blockLength) { + return 1; + } + return 0; + } + + @Override public int hashCode() { + int result = filePath.hashCode(); + result = 31 * result + (int) (blockOffset ^ (blockOffset >>> 32)); + result = 31 * result + (int) (blockLength ^ (blockLength >>> 32)); + result = 31 * result + segmentId.hashCode(); + result = 31 * result + Arrays.hashCode(locations); + return result; + } + + @Override public String[] getLocations() { + return locations; + } + +}