Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id DCBD8200C81 for ; Fri, 12 May 2017 01:43:23 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id DB600160BC7; Thu, 11 May 2017 23:43:23 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 6AC44160BD2 for ; Fri, 12 May 2017 01:43:21 +0200 (CEST) Received: (qmail 33745 invoked by uid 500); 11 May 2017 23:43:20 -0000 Mailing-List: contact commits-help@asterixdb.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@asterixdb.apache.org Delivered-To: mailing list commits@asterixdb.apache.org Received: (qmail 33068 invoked by uid 99); 11 May 2017 23:43:17 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 11 May 2017 23:43:17 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 744ADDFFB4; Thu, 11 May 2017 23:43:17 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: amoudi@apache.org To: commits@asterixdb.apache.org Date: Thu, 11 May 2017 23:43:40 -0000 Message-Id: <2336d15150b54523a3cd30cb145fac76@git.apache.org> In-Reply-To: <5f594fe565684258b006e5f3b566db33@git.apache.org> References: <5f594fe565684258b006e5f3b566db33@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [25/25] asterixdb git commit: Separate index build from index access archived-at: Thu, 11 May 2017 23:43:24 -0000 Separate index build from index access This change separates index build from index access. All indexes now have a single dataflow helper which uses the index path to locate the resource on the nc to read the resource from memory or disk. Existing resource metadata and dataflow helpers were combined into resource builders eliminating lots of duplicated code. Change-Id: Ie4ea3aaa63dff8d246fa43ca7c7359729bc8cf47 Reviewed-on: https://asterix-gerrit.ics.uci.edu/1728 Integration-Tests: Ian Maxon Tested-by: Ian Maxon Reviewed-by: Yingyi Bu Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/735532e4 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/735532e4 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/735532e4 Branch: refs/heads/master Commit: 735532e43aceb1fd806676ec4f537911ae29de4d Parents: 4671f71 Author: Abdullah Alamoudi Authored: Thu May 11 13:58:53 2017 -0700 Committer: abdullah alamoudi Committed: Thu May 11 16:42:33 2017 -0700 ---------------------------------------------------------------------- asterixdb/asterix-algebra/pom.xml | 4 - .../physical/InvertedIndexPOperator.java | 112 +--- .../rules/RemoveUnusedOneToOneEquiJoinRule.java | 3 +- .../optimizer/rules/UnnestToDataScanRule.java | 17 +- .../am/AbstractIntroduceAccessMethodRule.java | 14 +- .../optimizer/rules/am/AccessMethodUtils.java | 72 +-- .../am/IntroduceLSMComponentFilterRule.java | 28 +- .../rules/am/OptimizableOperatorSubTree.java | 22 +- .../LangExpressionToPlanTranslator.java | 4 +- .../AppRuntimeContextProviderForRecovery.java | 4 +- .../api/http/server/ConnectorApiServlet.java | 3 +- .../asterix/app/nc/NCAppRuntimeContext.java | 4 +- .../apache/asterix/app/nc/RecoveryManager.java | 13 +- .../asterix/app/nc/TransactionSubsystem.java | 2 +- .../asterix/app/translator/QueryTranslator.java | 22 +- .../asterix/file/StorageComponentProvider.java | 6 - .../hyracks/bootstrap/NCApplication.java | 2 +- .../apache/asterix/utils/FeedOperations.java | 9 +- .../app/bootstrap/TestNodeController.java | 188 ++---- .../asterix/test/dataflow/LogMarkerTest.java | 5 +- .../asterix/test/logging/CheckpointingTest.java | 5 +- .../common/api/IDatasetLifecycleManager.java | 12 +- .../common/api/INcApplicationContext.java | 4 +- .../AsterixVirtualBufferCacheProvider.java | 28 +- .../common/context/BaseOperationTracker.java | 4 +- .../context/CorrelatedPrefixMergePolicy.java | 17 +- .../CorrelatedPrefixMergePolicyFactory.java | 38 +- .../common/context/DatasetLifecycleManager.java | 51 +- .../asterix/common/context/DatasetResource.java | 2 +- .../context/IStorageComponentProvider.java | 7 - .../context/PrimaryIndexOperationTracker.java | 8 +- .../common/dataflow/DatasetLocalResource.java | 65 ++ .../common/dataflow/ICcApplicationContext.java | 33 +- .../LSMInsertDeleteOperatorNodePushable.java | 20 +- ...rtedIndexInsertDeleteOperatorDescriptor.java | 73 --- .../LSMTreeInsertDeleteOperatorDescriptor.java | 52 +- .../asterix/common/exceptions/ErrorCode.java | 2 + .../asterix/common/metadata/IDataset.java | 12 + .../IAppRuntimeContextProvider.java | 2 +- .../common/transactions/IResourceFactory.java | 32 - .../asterix/common/transactions/Resource.java | 88 --- .../common/transactions/ResourceFactory.java | 50 -- .../asterix/common/utils/StoragePathUtil.java | 6 + .../main/resources/asx_errormsg/en.properties | 2 + .../indexing/ExternalFileIndexAccessor.java | 32 +- .../indexing/FileIndexTupleTranslator.java | 21 +- .../indexing/FilesIndexDescription.java | 61 +- .../external/indexing/IndexingConstants.java | 34 +- ...xternalDatasetIndexesOperatorDescriptor.java | 28 +- .../ExternalBTreeSearchOperatorDescriptor.java | 37 +- ...ExternalBTreeSearchOperatorNodePushable.java | 32 +- ...alDatasetIndexesAbortOperatorDescriptor.java | 21 +- ...lDatasetIndexesCommitOperatorDescriptor.java | 27 +- ...DatasetIndexesRecoverOperatorDescriptor.java | 22 +- ...ernalFilesIndexCreateOperatorDescriptor.java | 111 ++++ ...ilesIndexModificationOperatorDescriptor.java | 124 ++++ .../ExternalFilesIndexOperatorDescriptor.java | 162 ----- ...ExternalIndexBulkLoadOperatorDescriptor.java | 51 ++ ...ternalIndexBulkLoadOperatorNodePushable.java | 46 ++ ...ternalIndexBulkModifyOperatorDescriptor.java | 39 +- ...rnalIndexBulkModifyOperatorNodePushable.java | 18 +- .../ExternalLookupOperatorDescriptor.java | 36 +- .../ExternalRTreeSearchOperatorDescriptor.java | 31 +- ...ExternalRTreeSearchOperatorNodePushable.java | 32 +- .../ExternalScanOperatorDescriptor.java | 2 +- .../FeedCollectOperatorDescriptor.java | 4 +- .../operators/FeedIntakeOperatorDescriptor.java | 4 +- .../operators/FeedMetaComputeNodePushable.java | 4 +- .../operators/FeedMetaOperatorDescriptor.java | 11 +- .../operators/FeedMetaStoreNodePushable.java | 16 +- .../operators/IndexInfoOperatorDescriptor.java | 130 ---- .../apache/asterix/external/util/FeedUtils.java | 4 +- .../apache/asterix/metadata/MetadataNode.java | 10 +- .../IIndexDataflowHelperFactoryProvider.java | 65 -- .../metadata/api/IResourceFactoryProvider.java | 65 ++ .../metadata/bootstrap/MetadataBootstrap.java | 84 +-- .../BTreeDataflowHelperFactoryProvider.java | 93 --- .../declared/BTreeResourceFactoryProvider.java | 194 ++++++ .../metadata/declared/LoadableDataSource.java | 3 +- .../metadata/declared/MetadataProvider.java | 594 +++---------------- .../asterix/metadata/entities/Dataset.java | 134 ++++- .../asterix/metadata/utils/DatasetUtil.java | 190 ++---- .../utils/ExternalIndexingOperations.java | 254 +++----- .../asterix/metadata/utils/IndexUtil.java | 25 +- ...ertedIndexDataflowHelperFactoryProvider.java | 117 ---- .../InvertedIndexResourceFactoryProvider.java | 248 ++++++++ .../RTreeDataflowHelperFactoryProvider.java | 172 ------ .../utils/RTreeResourceFactoryProvider.java | 258 ++++++++ .../utils/SecondaryBTreeOperationsHelper.java | 76 +-- .../utils/SecondaryIndexOperationsHelper.java | 70 +-- .../SecondaryInvertedIndexOperationsHelper.java | 122 +--- .../utils/SecondaryRTreeOperationsHelper.java | 113 +--- .../SecondaryTreeIndexOperationsHelper.java | 58 +- .../utils/SplitsAndConstraintsUtil.java | 83 +-- .../BinaryComparatorFactoryProvider.java | 1 + .../storage/ReplicaResourcesManager.java | 4 +- .../runtime/base/AsterixTupleFilterFactory.java | 3 +- ...SMInvertedIndexUpsertOperatorDescriptor.java | 68 --- .../LSMPrimaryUpsertOperatorDescriptor.java | 73 +++ .../LSMPrimaryUpsertOperatorNodePushable.java | 34 +- .../LSMSecondaryUpsertOperatorDescriptor.java | 54 ++ .../LSMSecondaryUpsertOperatorNodePushable.java | 22 +- .../LSMTreeUpsertOperatorDescriptor.java | 88 --- .../runtime/utils/CcApplicationContext.java | 6 - .../utils/RuntimeComponentsProvider.java | 40 +- asterixdb/asterix-transactions/pom.xml | 16 - ...tractIndexModificationOperationCallback.java | 2 +- .../LockThenSearchOperationCallback.java | 2 +- .../LockThenSearchOperationCallbackFactory.java | 3 +- ...maryIndexInstantSearchOperationCallback.java | 2 +- ...exInstantSearchOperationCallbackFactory.java | 2 +- ...dexModificationOperationCallbackFactory.java | 14 +- .../PrimaryIndexOperationTrackerFactory.java | 10 +- .../PrimaryIndexSearchOperationCallback.java | 2 +- ...maryIndexSearchOperationCallbackFactory.java | 2 +- ...dexModificationOperationCallbackFactory.java | 12 +- .../SecondaryIndexSearchOperationCallback.java | 2 +- ...daryIndexSearchOperationCallbackFactory.java | 2 +- ...dexModificationOperationCallbackFactory.java | 12 +- ...dexModificationOperationCallbackFactory.java | 15 +- .../UpsertOperationCallbackFactory.java | 12 +- .../resource/DatasetLocalResourceFactory.java | 45 ++ .../ExternalBTreeLocalResourceMetadata.java | 66 --- ...ternalBTreeLocalResourceMetadataFactory.java | 52 -- ...rnalBTreeWithBuddyLocalResourceMetadata.java | 81 --- ...eeWithBuddyLocalResourceMetadataFactory.java | 61 -- .../ExternalRTreeLocalResourceMetadata.java | 74 --- ...ternalRTreeLocalResourceMetadataFactory.java | 59 -- .../resource/LSMBTreeLocalResourceMetadata.java | 94 --- .../LSMBTreeLocalResourceMetadataFactory.java | 68 --- .../LSMInvertedIndexLocalResourceMetadata.java | 112 ---- ...vertedIndexLocalResourceMetadataFactory.java | 82 --- .../resource/LSMRTreeLocalResourceMetadata.java | 100 ---- .../LSMRTreeLocalResourceMetadataFactory.java | 81 --- .../PersistentLocalResourceFactory.java | 41 -- .../PersistentLocalResourceFactoryProvider.java | 40 -- .../PersistentLocalResourceRepository.java | 100 ++-- ...ersistentLocalResourceRepositoryFactory.java | 2 +- .../recovery/AbstractCheckpointManager.java | 2 + .../service/transaction/TransactionContext.java | 2 +- .../locking/TestRuntimeContextProvider.java | 2 +- .../algebra/metadata/IMetadataProvider.java | 4 +- .../data/IBinaryComparatorFactoryProvider.java | 17 +- .../meta/AlgebricksMetaOperatorDescriptor.java | 6 +- .../operators/std/SplitOperatorDescriptor.java | 2 +- .../api/context/IHyracksCommonContext.java | 2 +- .../api/context/IHyracksTaskContext.java | 18 +- .../hyracks/api/dataflow/value/IComparator.java | 23 - .../api/dataflow/value/IComparatorFactory.java | 25 - .../hyracks/api/dataflow/value/ITypeTraits.java | 13 +- .../hyracks/client/dataset/HyracksDataset.java | 2 +- .../org/apache/hyracks/control/nc/Task.java | 2 +- .../nc/dataset/DatasetPartitionWriter.java | 2 +- .../partitions/MaterializedPartitionWriter.java | 8 +- .../MaterializingPipelinedPartition.java | 6 +- .../std/base/AbstractOperatorDescriptor.java | 6 +- .../AbstractReplicateOperatorDescriptor.java | 2 +- ...tDeserializedFileScanOperatorDescriptor.java | 6 +- .../AbstractFileWriteOperatorDescriptor.java | 2 +- .../std/file/FileRemoveOperatorDescriptor.java | 2 +- .../std/file/FileScanOperatorDescriptor.java | 4 +- .../file/FrameFileWriterOperatorDescriptor.java | 2 +- .../file/PlainFileWriterOperatorDescriptor.java | 2 +- .../DeserializedPreclusteredGroupOperator.java | 137 ----- .../ExternalGroupBuildOperatorNodePushable.java | 2 +- .../ExternalGroupOperatorDescriptor.java | 4 +- .../ExternalGroupWriteOperatorNodePushable.java | 2 +- .../PreclusteredGroupOperatorDescriptor.java | 4 +- .../sort/ExternalSortGroupByRunGenerator.java | 2 +- .../sort/ExternalSortGroupByRunMerger.java | 2 +- .../intersect/IntersectOperatorDescriptor.java | 2 +- .../join/HybridHashJoinOperatorDescriptor.java | 6 +- .../InMemoryHashJoinOperatorDescriptor.java | 4 +- .../dataflow/std/join/NestedLoopJoin.java | 2 +- .../join/NestedLoopJoinOperatorDescriptor.java | 2 +- .../std/join/OptimizedHybridHashJoin.java | 2 +- ...timizedHybridHashJoinOperatorDescriptor.java | 2 +- .../DeserializedMapperOperatorDescriptor.java | 2 +- .../ConstantTupleSourceOperatorDescriptor.java | 2 +- .../std/misc/IdentityOperatorDescriptor.java | 2 +- .../std/misc/LimitOperatorDescriptor.java | 4 +- .../std/misc/MaterializerTaskState.java | 2 +- .../misc/MaterializingOperatorDescriptor.java | 2 +- .../std/misc/SplitVectorOperatorDescriptor.java | 2 +- .../sort/AbstractSorterOperatorDescriptor.java | 2 +- .../sort/ExternalSortOperatorDescriptor.java | 4 +- .../std/sort/ExternalSortRunGenerator.java | 2 +- .../std/sort/ExternalSortRunMerger.java | 2 +- .../dataflow/std/sort/HeapSortRunGenerator.java | 2 +- .../std/sort/HybridTopKSortRunGenerator.java | 2 +- .../sort/InMemorySortOperatorDescriptor.java | 4 +- .../std/sort/TopKSorterOperatorDescriptor.java | 4 +- .../std/union/UnionAllOperatorDescriptor.java | 2 +- .../btree/client/InsertPipelineExample.java | 58 +- .../client/PrimaryIndexBulkLoadExample.java | 46 +- .../btree/client/PrimaryIndexSearchExample.java | 17 +- .../client/SecondaryIndexBulkLoadExample.java | 33 +- .../client/SecondaryIndexSearchExample.java | 33 +- .../btree/helper/BTreeHelperStorageManager.java | 19 +- .../btree/helper/DataGenOperatorDescriptor.java | 4 +- .../helper/IndexLifecycleManagerProvider.java | 33 -- .../examples/btree/helper/RuntimeContext.java | 18 +- .../am/btree/AbstractBTreeOperatorTest.java | 141 ++--- .../tests/am/btree/BTreeOperatorTestHelper.java | 32 - .../BTreePrimaryIndexScanOperatorTest.java | 41 +- .../BTreePrimaryIndexSearchOperatorTest.java | 42 +- .../BTreePrimaryIndexStatsOperatorTest.java | 29 +- .../BTreeSecondaryIndexInsertOperatorTest.java | 43 +- .../BTreeSecondaryIndexSearchOperatorTest.java | 43 +- .../BTreeSecondaryIndexUpsertOperatorTest.java | 43 +- .../am/common/ITreeIndexOperatorTestHelper.java | 5 +- .../am/common/LSMTreeOperatorTestHelper.java | 22 +- .../lsm/btree/LSMBTreeOperatorTestHelper.java | 41 +- .../LSMBTreePrimaryIndexScanOperatorTest.java | 21 +- .../LSMBTreePrimaryIndexSearchOperatorTest.java | 68 +-- ...SMBTreeSecondaryIndexInsertOperatorTest.java | 21 +- ...SMBTreeSecondaryIndexSearchOperatorTest.java | 62 +- .../lsm/rtree/LSMRTreeOperatorTestHelper.java | 36 +- ...SMRTreeSecondaryIndexInsertOperatorTest.java | 15 +- ...SMRTreeSecondaryIndexSearchOperatorTest.java | 15 +- ...eWithAntiMatterTuplesOperatorTestHelper.java | 36 +- ...rTuplesSecondaryIndexInsertOperatorTest.java | 15 +- ...rTuplesSecondaryIndexSearchOperatorTest.java | 39 +- .../am/rtree/AbstractRTreeOperatorTest.java | 164 +++-- .../tests/am/rtree/RTreeOperatorTestHelper.java | 18 +- .../RTreeSecondaryIndexInsertOperatorTest.java | 39 +- .../RTreeSecondaryIndexScanOperatorTest.java | 42 +- .../RTreeSecondaryIndexSearchOperatorTest.java | 39 +- .../RTreeSecondaryIndexStatsOperatorTest.java | 31 +- .../tests/integration/JobFailureTest.java | 2 +- .../dataflow/HDFSReadOperatorDescriptor.java | 2 +- .../dataflow/HDFSReadOperatorDescriptor.java | 2 +- .../hyracks-storage-am-bloomfilter/pom.xml | 5 - .../am/bloomfilter/impls/BloomFilter.java | 2 +- .../hyracks/hyracks-storage-am-btree/pom.xml | 9 +- .../storage/am/btree/api/IBTreeLeafFrame.java | 2 +- .../am/btree/api/IPrefixSlotManager.java | 2 +- .../compressors/FieldPrefixCompressor.java | 2 +- .../am/btree/dataflow/BTreeDataflowHelper.java | 51 -- .../dataflow/BTreeDataflowHelperFactory.java | 43 -- .../am/btree/dataflow/BTreeResource.java | 67 +++ .../am/btree/dataflow/BTreeResourceFactory.java | 51 ++ .../dataflow/BTreeSearchOperatorDescriptor.java | 74 +-- .../BTreeSearchOperatorNodePushable.java | 23 +- .../BTreeUpdateSearchOperatorDescriptor.java | 30 +- .../BTreeUpdateSearchOperatorNodePushable.java | 19 +- .../frames/BTreeFieldPrefixNSMLeafFrame.java | 2 +- .../am/btree/frames/BTreeNSMInteriorFrame.java | 2 +- .../am/btree/frames/BTreeNSMLeafFrame.java | 2 +- .../am/btree/frames/OrderedSlotManager.java | 2 +- .../hyracks/storage/am/btree/impls/BTree.java | 19 +- .../btree/impls/BTreeCountingSearchCursor.java | 6 +- .../am/btree/impls/BTreeCursorInitialState.java | 8 +- .../storage/am/btree/impls/BTreeOpContext.java | 8 +- .../am/btree/impls/BTreeRangeSearchCursor.java | 10 +- .../am/btree/impls/FieldPrefixSlotManager.java | 2 +- .../storage/am/btree/impls/RangePredicate.java | 2 +- .../storage/am/btree/util/BTreeUtils.java | 2 +- .../storage/am/btree/test/FramewriterTest.java | 47 +- .../am/common/api/ICursorInitialState.java | 37 -- .../hyracks/storage/am/common/api/IIndex.java | 134 ----- .../storage/am/common/api/IIndexAccessor.java | 105 ---- .../storage/am/common/api/IIndexBuilder.java | 35 ++ .../am/common/api/IIndexBuilderFactory.java | 29 + .../storage/am/common/api/IIndexBulkLoader.java | 51 -- .../storage/am/common/api/IIndexCursor.java | 81 --- .../am/common/api/IIndexDataflowHelper.java | 39 +- .../api/IIndexLifecycleManagerProvider.java | 31 - .../api/IModificationOperationCallback.java | 52 -- .../IModificationOperationCallbackFactory.java | 3 +- .../common/api/IResourceLifecycleManager.java | 83 --- .../am/common/api/ISearchOperationCallback.java | 79 --- .../api/ISearchOperationCallbackFactory.java | 1 + .../storage/am/common/api/ISearchPredicate.java | 40 -- .../storage/am/common/api/ISlotManager.java | 2 +- .../storage/am/common/api/ITreeIndex.java | 1 + .../am/common/api/ITreeIndexAccessor.java | 1 + .../storage/am/common/api/ITreeIndexCursor.java | 1 + .../storage/am/common/api/ITreeIndexFrame.java | 7 +- .../common/api/ITreeIndexFrameCompressor.java | 2 +- .../am/common/api/ITupleFilterFactory.java | 3 +- .../storage/am/common/build/IndexBuilder.java | 97 +++ .../am/common/build/IndexBuilderFactory.java | 53 ++ .../AbstractIndexOperatorDescriptor.java | 144 ----- .../AbstractTreeIndexOperatorDescriptor.java | 85 --- .../dataflow/IIndexDataflowHelperFactory.java | 4 +- .../dataflow/IIndexOperatorDescriptor.java | 60 -- .../IndexBulkLoadOperatorNodePushable.java | 24 +- .../dataflow/IndexCreateOperatorDescriptor.java | 46 ++ .../IndexCreateOperatorNodePushable.java | 14 +- .../am/common/dataflow/IndexDataflowHelper.java | 125 ++-- .../dataflow/IndexDataflowHelperFactory.java | 46 ++ .../dataflow/IndexDropOperatorDescriptor.java | 31 +- .../dataflow/IndexDropOperatorNodePushable.java | 4 +- ...xInsertUpdateDeleteOperatorNodePushable.java | 34 +- .../common/dataflow/IndexLifecycleManager.java | 4 +- .../IndexSearchOperatorNodePushable.java | 54 +- .../TreeIndexBulkLoadOperatorDescriptor.java | 43 +- .../TreeIndexCreateOperatorDescriptor.java | 59 -- .../dataflow/TreeIndexDataflowHelper.java | 42 -- ...reeIndexDiskOrderScanOperatorDescriptor.java | 30 +- ...eIndexDiskOrderScanOperatorNodePushable.java | 34 +- ...dexInsertUpdateDeleteOperatorDescriptor.java | 41 +- .../TreeIndexStatsOperatorDescriptor.java | 33 +- .../TreeIndexStatsOperatorNodePushable.java | 36 +- .../common/impls/AbstractSearchPredicate.java | 2 +- .../am/common/impls/AbstractTreeIndex.java | 4 +- .../am/common/impls/NoOpOperationCallback.java | 4 +- .../impls/NoOpOperationCallbackFactory.java | 6 +- .../impls/TreeIndexDiskOrderScanCursor.java | 4 +- .../am/common/ophelpers/MultiComparator.java | 133 ----- .../am/common/ophelpers/SingleComparator.java | 42 -- .../am/common/util/IndexFileNameUtil.java | 6 +- .../dataflow/ExternalBTreeDataflowHelper.java | 82 --- .../ExternalBTreeDataflowHelperFactory.java | 56 -- .../dataflow/ExternalBTreeLocalResource.java | 66 +++ .../ExternalBTreeLocalResourceFactory.java | 59 ++ .../ExternalBTreeWithBuddyDataflowHelper.java | 92 --- ...rnalBTreeWithBuddyDataflowHelperFactory.java | 57 -- .../ExternalBTreeWithBuddyLocalResource.java | 69 +++ ...ernalBTreeWithBuddyLocalResourceFactory.java | 59 ++ .../btree/dataflow/LSMBTreeDataflowHelper.java | 84 --- .../dataflow/LSMBTreeDataflowHelperFactory.java | 65 -- .../btree/dataflow/LSMBTreeLocalResource.java | 80 +++ .../dataflow/LSMBTreeLocalResourceFactory.java | 68 +++ .../am/lsm/btree/impls/ExternalBTree.java | 29 +- .../lsm/btree/impls/ExternalBTreeOpContext.java | 8 +- .../lsm/btree/impls/ExternalBTreeWithBuddy.java | 26 +- .../impls/ExternalBTreeWithBuddyOpContext.java | 8 +- .../storage/am/lsm/btree/impls/LSMBTree.java | 14 +- .../btree/impls/LSMBTreeCursorInitialState.java | 8 +- .../am/lsm/btree/impls/LSMBTreeOpContext.java | 8 +- .../btree/impls/LSMBTreePointSearchCursor.java | 6 +- .../btree/impls/LSMBTreeRangeSearchCursor.java | 8 +- .../lsm/btree/impls/LSMBTreeSearchCursor.java | 4 +- .../impls/LSMBTreeWithBuddyAbstractCursor.java | 6 +- .../LSMBTreeWithBuddyCursorInitialState.java | 6 +- .../impls/LSMBTreeWithBuddySearchCursor.java | 4 +- .../impls/LSMBTreeWithBuddySortedCursor.java | 4 +- .../btree/impls/LSMBuddyBTreeMergeCursor.java | 8 +- .../am/lsm/btree/utils/LSMBTreeUtil.java | 29 +- .../am/lsm/common/api/ILSMComponentFilter.java | 2 +- .../storage/am/lsm/common/api/ILSMHarness.java | 4 +- .../api/ILSMIOOperationCallbackProvider.java | 24 - .../api/ILSMIOOperationSchedulerProvider.java | 4 +- .../storage/am/lsm/common/api/ILSMIndex.java | 10 +- .../am/lsm/common/api/ILSMIndexAccessor.java | 2 +- .../common/api/ILSMIndexOperationContext.java | 6 +- .../lsm/common/api/ILSMMergePolicyFactory.java | 11 +- .../am/lsm/common/api/ILSMOperationTracker.java | 6 +- .../storage/am/lsm/common/api/ITwoPCIndex.java | 9 +- .../common/api/IVirtualBufferCacheProvider.java | 6 +- .../AbstractLSMIndexDataflowHelper.java | 85 --- .../AbstractLSMIndexDataflowHelperFactory.java | 66 --- .../LSMIndexCompactOperatorNodePushable.java | 16 +- ...xInsertUpdateDeleteOperatorNodePushable.java | 16 +- .../LSMTreeIndexCompactOperatorDescriptor.java | 29 +- ...dexInsertUpdateDeleteOperatorDescriptor.java | 42 +- .../am/lsm/common/dataflow/LsmResource.java | 101 ++++ .../lsm/common/dataflow/LsmResourceFactory.java | 74 +++ .../am/lsm/common/impls/AbstractLSMIndex.java | 9 +- .../lsm/common/impls/ConstantMergePolicy.java | 2 +- .../impls/ConstantMergePolicyFactory.java | 19 +- .../lsm/common/impls/ExternalIndexHarness.java | 4 +- .../am/lsm/common/impls/IndexFactory.java | 2 +- .../am/lsm/common/impls/LSMComponentFilter.java | 2 +- .../common/impls/LSMComponentFilterManager.java | 2 +- .../storage/am/lsm/common/impls/LSMHarness.java | 4 +- .../lsm/common/impls/LSMIndexSearchCursor.java | 10 +- .../lsm/common/impls/LSMTreeIndexAccessor.java | 4 +- .../lsm/common/impls/NoMergePolicyFactory.java | 16 +- .../common/impls/NoOpIOOperationCallback.java | 66 --- .../impls/NoOpIOOperationCallbackFactory.java | 66 +++ .../impls/NoOpOperationTrackerFactory.java | 4 +- .../common/impls/PrefixMergePolicyFactory.java | 20 +- .../impls/SynchronousSchedulerProvider.java | 4 +- .../lsm/common/impls/ThreadCountingTracker.java | 6 +- .../lsm/invertedindex/api/IInvertedIndex.java | 2 +- .../api/IInvertedIndexAccessor.java | 6 +- .../api/IInvertedIndexOperatorDescriptor.java | 38 -- .../invertedindex/api/IInvertedListCursor.java | 2 +- ...tractLSMInvertedIndexOperatorDescriptor.java | 97 --- .../BinaryTokenizerOperatorDescriptor.java | 4 +- ...InvertedIndexBulkLoadOperatorDescriptor.java | 73 --- .../LSMInvertedIndexCompactOperator.java | 62 -- ...SMInvertedIndexCreateOperatorDescriptor.java | 64 -- .../LSMInvertedIndexDataflowHelper.java | 95 --- .../LSMInvertedIndexDataflowHelperFactory.java | 70 --- .../LSMInvertedIndexDropOperatorDescriptor.java | 58 -- ...InvertedIndexInsertUpdateDeleteOperator.java | 76 --- .../dataflow/LSMInvertedIndexLocalResource.java | 110 ++++ .../LSMInvertedIndexLocalResourceFactory.java | 81 +++ ...SMInvertedIndexSearchOperatorDescriptor.java | 76 +-- ...InvertedIndexSearchOperatorNodePushable.java | 39 +- ...rtitionedLSMInvertedIndexDataflowHelper.java | 94 --- ...edLSMInvertedIndexDataflowHelperFactory.java | 69 --- .../invertedindex/impls/LSMInvertedIndex.java | 16 +- .../impls/LSMInvertedIndexAccessor.java | 4 +- ...nvertedIndexDeletedKeysBTreeMergeCursor.java | 10 +- .../impls/LSMInvertedIndexMergeOperation.java | 2 +- .../impls/LSMInvertedIndexOpContext.java | 10 +- .../LSMInvertedIndexRangeSearchCursor.java | 10 +- ...ertedIndexRangeSearchCursorInitialState.java | 10 +- .../impls/LSMInvertedIndexSearchCursor.java | 12 +- ...SMInvertedIndexSearchCursorInitialState.java | 8 +- .../inmemory/InMemoryInvertedIndex.java | 13 +- .../inmemory/InMemoryInvertedIndexAccessor.java | 4 +- .../InMemoryInvertedIndexOpContext.java | 2 +- .../inmemory/InMemoryInvertedListCursor.java | 4 +- .../PartitionedInMemoryInvertedIndex.java | 6 +- .../FixedSizeElementInvertedListCursor.java | 2 +- .../ondisk/OnDiskInvertedIndex.java | 21 +- .../ondisk/OnDiskInvertedIndexOpContext.java | 6 +- .../OnDiskInvertedIndexRangeSearchCursor.java | 8 +- .../ondisk/OnDiskInvertedIndexSearchCursor.java | 6 +- .../ondisk/PartitionedOnDiskInvertedIndex.java | 6 +- .../search/AbstractTOccurrenceSearcher.java | 2 +- .../search/InvertedIndexSearchPredicate.java | 2 +- .../search/InvertedListMerger.java | 2 +- .../AbstractLSMRTreeDataflowHelper.java | 109 ---- .../dataflow/ExternalRTreeDataflowHelper.java | 108 ---- .../ExternalRTreeDataflowHelperFactory.java | 62 -- .../dataflow/ExternalRTreeLocalResource.java | 78 +++ .../ExternalRTreeLocalResourceFactory.java | 67 +++ .../rtree/dataflow/LSMRTreeDataflowHelper.java | 93 --- .../dataflow/LSMRTreeDataflowHelperFactory.java | 82 --- .../rtree/dataflow/LSMRTreeLocalResource.java | 94 +++ .../dataflow/LSMRTreeLocalResourceFactory.java | 83 +++ .../LSMRTreeWithAntiMatterLocalResource.java | 89 +++ ...RTreeWithAntiMatterLocalResourceFactory.java | 78 +++ ...RTreeWithAntiMatterTuplesDataflowHelper.java | 75 --- ...thAntiMatterTuplesDataflowHelperFactory.java | 79 --- .../am/lsm/rtree/impls/AbstractLSMRTree.java | 8 +- .../am/lsm/rtree/impls/ExternalRTree.java | 24 +- .../lsm/rtree/impls/ExternalRTreeOpContext.java | 8 +- .../storage/am/lsm/rtree/impls/LSMRTree.java | 12 +- .../lsm/rtree/impls/LSMRTreeAbstractCursor.java | 6 +- .../rtree/impls/LSMRTreeCursorInitialState.java | 6 +- .../LSMRTreeDeletedKeysBTreeMergeCursor.java | 8 +- .../am/lsm/rtree/impls/LSMRTreeOpContext.java | 8 +- .../lsm/rtree/impls/LSMRTreeSearchCursor.java | 4 +- .../lsm/rtree/impls/LSMRTreeSortedCursor.java | 9 +- .../impls/LSMRTreeWithAntiMatterTuples.java | 16 +- ...LSMRTreeWithAntiMatterTuplesFlushCursor.java | 6 +- ...SMRTreeWithAntiMatterTuplesSearchCursor.java | 8 +- .../am/lsm/rtree/impls/TreeTupleSorter.java | 6 +- .../am/lsm/rtree/utils/LSMRTreeUtils.java | 26 +- .../storage/am/rtree/api/IRTreeFrame.java | 2 +- .../am/rtree/api/IRTreeInteriorFrame.java | 2 +- .../storage/am/rtree/api/IRTreeLeafFrame.java | 2 +- .../storage/am/rtree/api/IRTreePolicy.java | 2 +- .../am/rtree/dataflow/RTreeDataflowHelper.java | 57 -- .../dataflow/RTreeDataflowHelperFactory.java | 50 -- .../am/rtree/dataflow/RTreeResource.java | 72 +++ .../am/rtree/dataflow/RTreeResourceFactory.java | 58 ++ .../dataflow/RTreeSearchOperatorDescriptor.java | 70 +-- .../RTreeSearchOperatorNodePushable.java | 21 +- .../am/rtree/frames/RStarTreePolicy.java | 2 +- .../am/rtree/frames/RTreeComputationUtils.java | 2 +- .../storage/am/rtree/frames/RTreeNSMFrame.java | 2 +- .../am/rtree/frames/RTreeNSMInteriorFrame.java | 2 +- .../am/rtree/frames/RTreeNSMLeafFrame.java | 2 +- .../storage/am/rtree/frames/RTreePolicy.java | 2 +- .../hyracks/storage/am/rtree/impls/RTree.java | 17 +- .../am/rtree/impls/RTreeCursorInitialState.java | 6 +- .../storage/am/rtree/impls/RTreeOpContext.java | 4 +- .../am/rtree/impls/RTreeSearchCursor.java | 6 +- .../storage/am/rtree/impls/SearchPredicate.java | 2 +- .../am/rtree/impls/UnorderedSlotManager.java | 2 +- .../storage/am/rtree/util/RTreeUtils.java | 2 +- .../hyracks/hyracks-storage-common/pom.xml | 5 + .../storage/common/ICursorInitialState.java | 36 ++ .../apache/hyracks/storage/common/IIndex.java | 139 +++++ .../hyracks/storage/common/IIndexAccessor.java | 105 ++++ .../storage/common/IIndexBulkLoader.java | 51 ++ .../hyracks/storage/common/IIndexCursor.java | 81 +++ .../common/ILocalResourceRepository.java | 32 + .../common/IModificationOperationCallback.java | 52 ++ .../hyracks/storage/common/IResource.java | 31 + .../storage/common/IResourceFactory.java | 34 ++ .../common/IResourceLifecycleManager.java | 82 +++ .../common/ISearchOperationCallback.java | 79 +++ .../storage/common/ISearchPredicate.java | 40 ++ .../hyracks/storage/common/IStorageManager.java | 28 +- .../hyracks/storage/common/LocalResource.java | 105 ++++ .../hyracks/storage/common/MultiComparator.java | 134 +++++ .../storage/common/SingleComparator.java | 43 ++ .../TransientLocalResourceRepository.java | 66 +++ .../common/file/ILocalResourceFactory.java | 24 - .../file/ILocalResourceFactoryProvider.java | 25 - .../common/file/ILocalResourceRepository.java | 32 - .../file/ILocalResourceRepositoryFactory.java | 1 + .../storage/common/file/LocalResource.java | 123 ---- .../file/NoOpLocalResourceFactoryProvider.java | 31 - .../common/file/ResourceIdFactoryProvider.java | 1 + .../file/TransientLocalResourceFactory.java | 27 - .../TransientLocalResourceFactoryProvider.java | 29 - .../file/TransientLocalResourceRepository.java | 66 --- ...TransientLocalResourceRepositoryFactory.java | 2 + .../hyracks/hyracks-test-support/pom.xml | 6 - ...stractModificationOperationCallbackTest.java | 4 +- .../am/btree/AbstractOperationCallbackTest.java | 4 +- .../AbstractSearchOperationCallbackTest.java | 6 +- .../am/btree/OrderedIndexExamplesTest.java | 8 +- .../am/btree/OrderedIndexMultiThreadTest.java | 2 +- .../am/btree/OrderedIndexTestContext.java | 2 +- .../storage/am/btree/OrderedIndexTestUtils.java | 6 +- .../am/common/AbstractIndexLifecycleTest.java | 2 +- .../am/common/AbstractIndexTestWorker.java | 6 +- .../storage/am/common/IIndexTestContext.java | 4 +- .../am/common/IIndexTestWorkerFactory.java | 2 +- .../am/common/ITreeIndexTestContext.java | 2 +- .../am/common/IndexMultiThreadTestDriver.java | 2 +- .../storage/am/common/IndexTestContext.java | 4 +- .../am/common/TestOperationCallback.java | 4 +- .../storage/am/common/TreeIndexTestUtils.java | 4 +- .../am/rtree/AbstractRTreeExamplesTest.java | 6 +- .../storage/am/rtree/RTreeTestUtils.java | 4 +- .../TestIndexLifecycleManagerProvider.java | 35 -- .../test/support/TestStorageManager.java | 26 +- .../TestStorageManagerComponentHolder.java | 24 +- .../hyracks/test/support/TestTaskContext.java | 5 +- .../apache/hyracks/test/support/TestUtils.java | 4 +- .../support/TestVirtualBufferCacheProvider.java | 10 +- .../hyracks-storage-am-bloomfilter-test/pom.xml | 5 - .../storage/am/bloomfilter/BloomFilterTest.java | 2 +- .../util/BloomFilterTestHarness.java | 6 +- .../storage/am/btree/BTreeSearchCursorTest.java | 2 +- .../storage/am/btree/FieldPrefixNSMTest.java | 2 +- .../am/btree/multithread/BTreeTestWorker.java | 4 +- .../multithread/BTreeTestWorkerFactory.java | 2 +- .../storage/am/btree/util/BTreeTestHarness.java | 10 +- .../am/lsm/btree/LSMBTreeExamplesTest.java | 2 +- .../btree/LSMBTreeFilterMergeTestDriver.java | 25 +- .../am/lsm/btree/LSMBTreeMergeTestDriver.java | 8 +- ...MBTreeModificationOperationCallbackTest.java | 14 +- .../LSMBTreeSearchOperationCallbackTest.java | 8 +- .../btree/multithread/LSMBTreeTestWorker.java | 11 +- .../multithread/LSMBTreeTestWorkerFactory.java | 2 +- .../am/lsm/btree/perf/BTreeBulkLoadRunner.java | 2 +- .../storage/am/lsm/btree/perf/BTreeRunner.java | 4 +- .../btree/perf/ConcurrentSkipListRunner.java | 2 +- .../am/lsm/btree/perf/InMemorySortRunner.java | 2 +- .../am/lsm/btree/perf/LSMTreeRunner.java | 12 +- .../am/lsm/btree/util/LSMBTreeTestHarness.java | 15 +- .../am/lsm/common/LSMIndexFileManagerTest.java | 2 +- .../LSMInvertedIndexMergeTest.java | 6 +- .../PartitionedLSMInvertedIndexMergeTest.java | 6 +- .../common/AbstractInvertedIndexDeleteTest.java | 2 +- .../common/AbstractInvertedIndexLoadTest.java | 2 +- .../common/AbstractInvertedIndexSearchTest.java | 2 +- .../common/AbstractInvertedIndexTest.java | 2 +- .../common/LSMInvertedIndexTestHarness.java | 12 +- .../LSMInvertedIndexMultiThreadTestDriver.java | 2 +- .../multithread/LSMInvertedIndexTestWorker.java | 9 +- .../LSMInvertedIndexWorkerFactory.java | 2 +- .../util/LSMInvertedIndexTestContext.java | 2 +- .../util/LSMInvertedIndexTestUtils.java | 6 +- .../lsm/rtree/AbstractLSMRTreeExamplesTest.java | 2 +- .../am/lsm/rtree/LSMRTreeMergeTestDriver.java | 4 +- .../multithread/AbstractLSMRTreeTestWorker.java | 4 +- .../rtree/multithread/LSMRTreeTestWorker.java | 9 +- .../multithread/LSMRTreeTestWorkerFactory.java | 2 +- .../LSMRTreeWithAntiMatterTuplesTestWorker.java | 8 +- ...eeWithAntiMatterTuplesTestWorkerFactory.java | 2 +- .../am/lsm/rtree/util/LSMRTreeTestHarness.java | 12 +- .../storage/am/rtree/RTreeSearchCursorTest.java | 2 +- .../am/rtree/multithread/RTreeTestWorker.java | 6 +- .../multithread/RTreeTestWorkerFactory.java | 2 +- .../am/rtree/utils/RTreeTestHarness.java | 4 +- .../common/BufferCacheRegressionTest.java | 5 +- .../hyracks/storage/common/BufferCacheTest.java | 15 +- 572 files changed, 7413 insertions(+), 10646 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/asterixdb/asterix-algebra/pom.xml ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/pom.xml b/asterixdb/asterix-algebra/pom.xml index e042d97..04f8d63 100644 --- a/asterixdb/asterix-algebra/pom.xml +++ b/asterixdb/asterix-algebra/pom.xml @@ -276,10 +276,6 @@ org.apache.hyracks - hyracks-storage-am-lsm-common - - - org.apache.hyracks hyracks-api http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java index 4e3ae55..50c762e 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/InvertedIndexPOperator.java @@ -18,25 +18,16 @@ */ package org.apache.asterix.algebra.operators.physical; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import org.apache.asterix.common.dataflow.ICcApplicationContext; import org.apache.asterix.metadata.MetadataException; import org.apache.asterix.metadata.MetadataManager; import org.apache.asterix.metadata.declared.DataSourceId; import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.asterix.metadata.entities.Dataset; import org.apache.asterix.metadata.entities.Index; -import org.apache.asterix.metadata.utils.DatasetUtil; import org.apache.asterix.om.base.IAObject; import org.apache.asterix.om.constants.AsterixConstantValue; import org.apache.asterix.om.functions.BuiltinFunctions; -import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; -import org.apache.asterix.om.types.IAType; -import org.apache.asterix.om.utils.NonTaggedFormatUtil; import org.apache.asterix.optimizer.rules.am.InvertedIndexAccessMethod; import org.apache.asterix.optimizer.rules.am.InvertedIndexAccessMethod.SearchModifierType; import org.apache.asterix.optimizer.rules.am.InvertedIndexJobGenParams; @@ -49,7 +40,6 @@ import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator; import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag; import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag; -import org.apache.hyracks.algebricks.core.algebra.base.LogicalVariable; import org.apache.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag; import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression; import org.apache.hyracks.algebricks.core.algebra.expressions.IAlgebricksConstantValue; @@ -57,21 +47,16 @@ import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvir import org.apache.hyracks.algebricks.core.algebra.metadata.IDataSourceIndex; import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestMapOperator; import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema; -import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities; import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain; import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext; import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenHelper; import org.apache.hyracks.api.dataflow.IOperatorDescriptor; -import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; -import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.api.dataflow.value.RecordDescriptor; import org.apache.hyracks.api.job.JobSpecification; -import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory; -import org.apache.hyracks.data.std.primitive.ShortPointable; import org.apache.hyracks.dataflow.std.file.IFileSplitProvider; import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory; +import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelperFactory; import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation; -import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexSearchModifierFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.dataflow.LSMInvertedIndexSearchOperatorDescriptor; import org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers.IBinaryTokenizerFactory; @@ -154,87 +139,15 @@ public class InvertedIndexPOperator extends IndexSearchPOperator { throws AlgebricksException { try { IAObject simThresh = ((AsterixConstantValue) similarityThreshold).getObject(); - IAType itemType = MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(), - dataset.getItemTypeDataverseName(), dataset.getItemTypeName()).getDatatype(); - int numPrimaryKeys = DatasetUtil.getPartitioningKeys(dataset).size(); + int numPrimaryKeys = dataset.getPrimaryKeys().size(); Index secondaryIndex = MetadataManager.INSTANCE.getIndex(metadataProvider.getMetadataTxnContext(), dataset.getDataverseName(), dataset.getDatasetName(), indexName); if (secondaryIndex == null) { throw new AlgebricksException( "Code generation error: no index " + indexName + " for dataset " + datasetName); } - List> secondaryKeyFieldEntries = secondaryIndex.getKeyFieldNames(); - List secondaryKeyTypeEntries = secondaryIndex.getKeyFieldTypes(); - int numSecondaryKeys = secondaryKeyFieldEntries.size(); - if (numSecondaryKeys != 1) { - throw new AlgebricksException( - "Cannot use " + numSecondaryKeys + " fields as a key for an inverted index. " - + "There can be only one field as a key for the inverted index index."); - } - if (itemType.getTypeTag() != ATypeTag.OBJECT) { - throw new AlgebricksException("Only record types can be indexed."); - } - ARecordType recordType = (ARecordType) itemType; - Pair keyPairType = Index.getNonNullableOpenFieldType(secondaryKeyTypeEntries.get(0), - secondaryKeyFieldEntries.get(0), recordType); - IAType secondaryKeyType = keyPairType.first; - if (secondaryKeyType == null) { - throw new AlgebricksException( - "Could not find field " + secondaryKeyFieldEntries.get(0) + " in the schema."); - } - - // TODO: For now we assume the type of the generated tokens is the - // same as the indexed field. - // We need a better way of expressing this because tokens may be - // hashed, or an inverted-index may index a list type, etc. - int numTokenKeys = (!isPartitioned) ? numSecondaryKeys : numSecondaryKeys + 1; - ITypeTraits[] tokenTypeTraits = new ITypeTraits[numTokenKeys]; - IBinaryComparatorFactory[] tokenComparatorFactories = new IBinaryComparatorFactory[numTokenKeys]; - for (int i = 0; i < numSecondaryKeys; i++) { - tokenComparatorFactories[i] = NonTaggedFormatUtil.getTokenBinaryComparatorFactory(secondaryKeyType); - tokenTypeTraits[i] = NonTaggedFormatUtil.getTokenTypeTrait(secondaryKeyType); - } - if (isPartitioned) { - // The partitioning field is hardcoded to be a short *without* an Asterix type tag. - tokenComparatorFactories[numSecondaryKeys] = - PointableBinaryComparatorFactory.of(ShortPointable.FACTORY); - tokenTypeTraits[numSecondaryKeys] = ShortPointable.TYPE_TRAITS; - } - IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(unnestMap); - List outputVars = unnestMap.getVariables(); - if (retainInput) { - outputVars = new ArrayList<>(); - VariableUtilities.getLiveVariables(unnestMap, outputVars); - } RecordDescriptor outputRecDesc = JobGenHelper.mkRecordDescriptor(typeEnv, opSchema, context); - - int start = outputRecDesc.getFieldCount() - numPrimaryKeys; - IBinaryComparatorFactory[] invListsComparatorFactories = JobGenHelper - .variablesToAscBinaryComparatorFactories(outputVars, start, numPrimaryKeys, typeEnv, context); - ITypeTraits[] invListsTypeTraits = - JobGenHelper.variablesToTypeTraits(outputVars, start, numPrimaryKeys, typeEnv, context); - ITypeTraits[] filterTypeTraits = DatasetUtil.computeFilterTypeTraits(dataset, recordType); - int[] filterFields; - int[] invertedIndexFields; - int[] filterFieldsForNonBulkLoadOps; - int[] invertedIndexFieldsForNonBulkLoadOps; - if (filterTypeTraits != null) { - filterFields = new int[1]; - filterFields[0] = numTokenKeys + numPrimaryKeys; - invertedIndexFields = new int[numTokenKeys + numPrimaryKeys]; - for (int k = 0; k < invertedIndexFields.length; k++) { - invertedIndexFields[k] = k; - } - - filterFieldsForNonBulkLoadOps = new int[1]; - filterFieldsForNonBulkLoadOps[0] = numPrimaryKeys + numSecondaryKeys; - invertedIndexFieldsForNonBulkLoadOps = new int[numPrimaryKeys + numSecondaryKeys]; - for (int k = 0; k < invertedIndexFieldsForNonBulkLoadOps.length; k++) { - invertedIndexFieldsForNonBulkLoadOps[k] = k; - } - } - ICcApplicationContext appContext = (ICcApplicationContext) context.getAppContext(); Pair secondarySplitsAndConstraint = metadataProvider.getSplitProviderAndConstraints(dataset, indexName); // TODO: Here we assume there is only one search key field. @@ -244,25 +157,16 @@ public class InvertedIndexPOperator extends IndexSearchPOperator { InvertedIndexAccessMethod.getSearchModifierFactory(searchModifierType, simThresh, secondaryIndex); IBinaryTokenizerFactory queryTokenizerFactory = InvertedIndexAccessMethod .getBinaryTokenizerFactory(searchModifierType, searchKeyType, secondaryIndex); - ARecordType metaType = dataset.hasMetaPart() - ? (ARecordType) metadataProvider - .findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName()).getType() - : null; - Pair> compactionInfo = - DatasetUtil.getMergePolicyFactory(dataset, metadataProvider.getMetadataTxnContext()); - IIndexDataflowHelperFactory dataflowHelperFactory = dataset.getIndexDataflowHelperFactory(metadataProvider, - secondaryIndex, recordType, metaType, compactionInfo.first, compactionInfo.second); + IIndexDataflowHelperFactory dataflowHelperFactory = + new IndexDataflowHelperFactory(metadataProvider.getStorageComponentProvider().getStorageManager(), + secondarySplitsAndConstraint.first); LSMInvertedIndexSearchOperatorDescriptor invIndexSearchOp = new LSMInvertedIndexSearchOperatorDescriptor( - jobSpec, queryField, appContext.getStorageManager(), secondarySplitsAndConstraint.first, - appContext.getIndexLifecycleManagerProvider(), tokenTypeTraits, tokenComparatorFactories, - invListsTypeTraits, invListsComparatorFactories, dataflowHelperFactory, queryTokenizerFactory, - searchModifierFactory, outputRecDesc, retainInput, retainMissing, context.getMissingWriterFactory(), + jobSpec, outputRecDesc, queryField, dataflowHelperFactory, queryTokenizerFactory, + searchModifierFactory, retainInput, retainMissing, context.getMissingWriterFactory(), dataset.getSearchCallbackFactory(metadataProvider.getStorageComponentProvider(), secondaryIndex, ((JobEventListenerFactory) jobSpec.getJobletEventListenerFactory()).getJobId(), IndexOperation.SEARCH, null), - minFilterFieldIndexes, maxFilterFieldIndexes, - metadataProvider.getStorageComponentProvider().getMetadataPageManagerFactory(), - isFullTextSearchQuery); + minFilterFieldIndexes, maxFilterFieldIndexes, isFullTextSearchQuery, numPrimaryKeys, false); return new Pair<>(invIndexSearchOp, secondarySplitsAndConstraint.second); } catch (MetadataException e) { throw new AlgebricksException(e); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/RemoveUnusedOneToOneEquiJoinRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/RemoveUnusedOneToOneEquiJoinRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/RemoveUnusedOneToOneEquiJoinRule.java index a20a33d..6a70786 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/RemoveUnusedOneToOneEquiJoinRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/RemoveUnusedOneToOneEquiJoinRule.java @@ -26,7 +26,6 @@ import java.util.Set; import org.apache.asterix.metadata.declared.DatasetDataSource; import org.apache.asterix.metadata.entities.InternalDatasetDetails; -import org.apache.asterix.metadata.utils.DatasetUtil; import org.apache.commons.lang3.mutable.Mutable; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; @@ -231,7 +230,7 @@ public class RemoveUnusedOneToOneEquiJoinRule implements IAlgebraicRewriteRule { DatasetDataSource datasetDataSource = (DatasetDataSource) dataScan.getDataSource(); pkVars.clear(); if (datasetDataSource.getDataset().getDatasetDetails() instanceof InternalDatasetDetails) { - int numPKs = DatasetUtil.getPartitioningKeys(datasetDataSource.getDataset()).size(); + int numPKs = datasetDataSource.getDataset().getPrimaryKeys().size(); for (int i = 0; i < numPKs; i++) { pkVars.add(dataScan.getVariables().get(i)); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java index 46b421b..7f35d08 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java @@ -37,7 +37,6 @@ import org.apache.asterix.metadata.entities.FeedConnection; import org.apache.asterix.metadata.entities.FeedPolicyEntity; import org.apache.asterix.metadata.entities.InternalDatasetDetails; import org.apache.asterix.metadata.feeds.BuiltinFeedPolicies; -import org.apache.asterix.metadata.utils.DatasetUtil; import org.apache.asterix.om.base.AString; import org.apache.asterix.om.constants.AsterixConstantValue; import org.apache.asterix.om.functions.BuiltinFunctions; @@ -124,7 +123,7 @@ public class UnnestToDataScanRule implements IAlgebraicRewriteRule { DataSourceId asid = new DataSourceId(dataverseName, datasetName); List variables = new ArrayList<>(); if (dataset.getDatasetType() == DatasetType.INTERNAL) { - int numPrimaryKeys = DatasetUtil.getPartitioningKeys(dataset).size(); + int numPrimaryKeys = dataset.getPrimaryKeys().size(); for (int i = 0; i < numPrimaryKeys; i++) { variables.add(context.newVar()); } @@ -144,8 +143,8 @@ public class UnnestToDataScanRule implements IAlgebraicRewriteRule { // Adds equivalence classes --- one equivalent class between a primary key // variable and a record field-access expression. IAType[] schemaTypes = dataSource.getSchemaTypes(); - ARecordType recordType = (ARecordType) (hasMeta ? schemaTypes[schemaTypes.length - 2] - : schemaTypes[schemaTypes.length - 1]); + ARecordType recordType = + (ARecordType) (hasMeta ? schemaTypes[schemaTypes.length - 2] : schemaTypes[schemaTypes.length - 1]); ARecordType metaRecordType = (ARecordType) (hasMeta ? schemaTypes[schemaTypes.length - 1] : null); EquivalenceClassUtils.addEquivalenceClassesForPrimaryIndexAccess(scan, variables, recordType, metaRecordType, dataset, context); @@ -198,7 +197,7 @@ public class UnnestToDataScanRule implements IAlgebraicRewriteRule { private void addPrimaryKey(List scanVariables, DataSource dataSource, IOptimizationContext context) { List primaryKey = dataSource.getPrimaryKeyVariables(scanVariables); - List tail = new ArrayList(); + List tail = new ArrayList<>(); tail.addAll(scanVariables); FunctionalDependency pk = new FunctionalDependency(primaryKey, tail); context.addPrimaryKey(pk); @@ -215,8 +214,8 @@ public class UnnestToDataScanRule implements IAlgebraicRewriteRule { Dataset dataset = metadataProvider.findDataset(aqlId.getDataverseName(), targetDataset); ARecordType feedOutputType = (ARecordType) metadataProvider.findType(aqlId.getDataverseName(), outputType); Feed sourceFeed = metadataProvider.findFeed(aqlId.getDataverseName(), sourceFeedName); - FeedConnection feedConnection = metadataProvider.findFeedConnection(aqlId.getDataverseName(), sourceFeedName, - targetDataset); + FeedConnection feedConnection = + metadataProvider.findFeedConnection(aqlId.getDataverseName(), sourceFeedName, targetDataset); ARecordType metaType = null; // Does dataset have meta? if (dataset.hasMetaPart()) { @@ -263,8 +262,8 @@ public class UnnestToDataScanRule implements IAlgebraicRewriteRule { } FeedDataSource feedDataSource = new FeedDataSource(sourceFeed, aqlId, targetDataset, feedOutputType, metaType, pkTypes, partitioningKeys, keyAccessScalarFunctionCallExpression, sourceFeed.getFeedId(), - FeedRuntimeType.valueOf(subscriptionLocation), locations.split(","), - context.getComputationNodeDomain(), feedConnection); + FeedRuntimeType.valueOf(subscriptionLocation), locations.split(","), context.getComputationNodeDomain(), + feedConnection); feedDataSource.getProperties().put(BuiltinFeedPolicies.CONFIG_FEED_POLICY_KEY, feedPolicy); return feedDataSource; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java index 86d2b9f..8df1756 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AbstractIntroduceAccessMethodRule.java @@ -30,7 +30,6 @@ import org.apache.asterix.common.config.DatasetConfig.IndexType; import org.apache.asterix.dataflow.data.common.ExpressionTypeComputer; import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.asterix.metadata.entities.Index; -import org.apache.asterix.metadata.utils.DatasetUtil; import org.apache.asterix.metadata.utils.MetadataUtil; import org.apache.asterix.om.base.AOrderedList; import org.apache.asterix.om.base.AString; @@ -643,7 +642,7 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew List> subTreePKs = null; if (!fromAdditionalDataSource) { - subTreePKs = DatasetUtil.getPartitioningKeys(subTree.getDataset()); + subTreePKs = subTree.getDataset().getPrimaryKeys(); // Check whether this variable is PK, not a record variable. if (varIndex <= subTreePKs.size() - 1) { fieldName = subTreePKs.get(varIndex); @@ -654,8 +653,7 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew // Need to check additional dataset one by one for (int i = 0; i < subTree.getIxJoinOuterAdditionalDatasets().size(); i++) { if (subTree.getIxJoinOuterAdditionalDatasets().get(i) != null) { - subTreePKs = - DatasetUtil.getPartitioningKeys(subTree.getIxJoinOuterAdditionalDatasets().get(i)); + subTreePKs = subTree.getIxJoinOuterAdditionalDatasets().get(i).getPrimaryKeys(); // Check whether this variable is PK, not a record variable. if (subTreePKs.contains(var) && varIndex <= subTreePKs.size() - 1) { @@ -698,10 +696,10 @@ public abstract class AbstractIntroduceAccessMethodRule implements IAlgebraicRew * * @throws AlgebricksException */ - protected List getFieldNameFromSubTree(IOptimizableFuncExpr optFuncExpr, - OptimizableOperatorSubTree subTree, int opIndex, int assignVarIndex, ARecordType recordType, - int funcVarIndex, ILogicalExpression parentFuncExpr, LogicalVariable recordVar, ARecordType metaType, - LogicalVariable metaVar) throws AlgebricksException { + protected List getFieldNameFromSubTree(IOptimizableFuncExpr optFuncExpr, OptimizableOperatorSubTree subTree, + int opIndex, int assignVarIndex, ARecordType recordType, int funcVarIndex, + ILogicalExpression parentFuncExpr, LogicalVariable recordVar, ARecordType metaType, LogicalVariable metaVar) + throws AlgebricksException { // Get expression corresponding to opVar at varIndex. AbstractLogicalExpression expr = null; AbstractFunctionCallExpression childFuncExpr = null; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java index c3b6473..59cb373 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/AccessMethodUtils.java @@ -36,7 +36,6 @@ import org.apache.asterix.metadata.declared.DataSourceId; import org.apache.asterix.metadata.entities.Dataset; import org.apache.asterix.metadata.entities.ExternalDatasetDetails; import org.apache.asterix.metadata.entities.Index; -import org.apache.asterix.metadata.utils.DatasetUtil; import org.apache.asterix.metadata.utils.KeyFieldTypeUtil; import org.apache.asterix.om.base.ABoolean; import org.apache.asterix.om.base.AInt32; @@ -190,8 +189,7 @@ public class AccessMethodUtils { private static void constructNewOptFuncExprAndAddToAnalysisCtx(AbstractFunctionCallExpression funcExpr, LogicalVariable fieldVar, ILogicalExpression expression, IAType expressionType, AccessMethodAnalysisContext analysisCtx) { - OptimizableFuncExpr newOptFuncExpr = - new OptimizableFuncExpr(funcExpr, fieldVar, expression, expressionType); + OptimizableFuncExpr newOptFuncExpr = new OptimizableFuncExpr(funcExpr, fieldVar, expression, expressionType); addNewOptFuncExprToAnalysisCtx(funcExpr, newOptFuncExpr, analysisCtx); } @@ -217,8 +215,7 @@ public class AccessMethodUtils { /** * Fetches each element and calls the check for the type and value in the given list using the given cursor. */ - private static void checkEachElementInFTSearchListPredicate(IACursor oListCursor) - throws AlgebricksException { + private static void checkEachElementInFTSearchListPredicate(IACursor oListCursor) throws AlgebricksException { String argValue; IAObject element; while (oListCursor.next()) { @@ -331,7 +328,7 @@ public class AccessMethodUtils { numPrimaryKeys = IndexingConstants .getRIDSize(((ExternalDatasetDetails) dataset.getDatasetDetails()).getProperties()); } else { - numPrimaryKeys = DatasetUtil.getPartitioningKeys(dataset).size(); + numPrimaryKeys = dataset.getPrimaryKeys().size(); } int numSecondaryKeys = KeyFieldTypeUtil.getNumSecondaryKeys(index, recordType, metaRecordType); int numVars = (primaryKeysOnly) ? numPrimaryKeys : numPrimaryKeys + numSecondaryKeys; @@ -347,7 +344,7 @@ public class AccessMethodUtils { numPrimaryKeys = IndexingConstants .getRIDSize(((ExternalDatasetDetails) dataset.getDatasetDetails()).getProperties()); } else { - numPrimaryKeys = DatasetUtil.getPartitioningKeys(dataset).size(); + numPrimaryKeys = dataset.getPrimaryKeys().size(); } List primaryKeyVars = new ArrayList<>(); List sourceVars = null; @@ -365,7 +362,7 @@ public class AccessMethodUtils { public static List getPrimaryKeyVarsFromPrimaryUnnestMap(Dataset dataset, ILogicalOperator unnestMapOp) { - int numPrimaryKeys = DatasetUtil.getPartitioningKeys(dataset).size(); + int numPrimaryKeys = dataset.getPrimaryKeys().size(); List primaryKeyVars = new ArrayList<>(); List sourceVars = null; @@ -495,8 +492,8 @@ public class AccessMethodUtils { secondaryIndexOutputTypes); // An index search is expressed as an unnest over an index-search function. IFunctionInfo secondaryIndexSearch = FunctionUtil.getFunctionInfo(BuiltinFunctions.INDEX_SEARCH); - UnnestingFunctionCallExpression secondaryIndexSearchFunc = new UnnestingFunctionCallExpression( - secondaryIndexSearch, secondaryIndexFuncArgs); + UnnestingFunctionCallExpression secondaryIndexSearchFunc = + new UnnestingFunctionCallExpression(secondaryIndexSearch, secondaryIndexFuncArgs); secondaryIndexSearchFunc.setReturnsUniqueValues(true); // This is the operator that jobgen will be looking for. It contains an unnest function that has all necessary arguments to determine // which index to use, which variables contain the index-search keys, what is the original dataset, etc. @@ -532,17 +529,15 @@ public class AccessMethodUtils { Dataset dataset, ARecordType recordType, ARecordType metaRecordType, ILogicalOperator inputOp, IOptimizationContext context, boolean sortPrimaryKeys, boolean retainInput, boolean retainNull, boolean requiresBroadcast) throws AlgebricksException { - List primaryKeyVars = AccessMethodUtils.getPrimaryKeyVarsFromSecondaryUnnestMap(dataset, - inputOp); + List primaryKeyVars = + AccessMethodUtils.getPrimaryKeyVarsFromSecondaryUnnestMap(dataset, inputOp); // Optionally add a sort on the primary-index keys before searching the primary index. OrderOperator order = null; if (sortPrimaryKeys) { order = new OrderOperator(); for (LogicalVariable pkVar : primaryKeyVars) { - Mutable vRef = new MutableObject<>( - new VariableReferenceExpression(pkVar)); - order.getOrderExpressions() - .add(new Pair<>(OrderOperator.ASC_ORDER, vRef)); + Mutable vRef = new MutableObject<>(new VariableReferenceExpression(pkVar)); + order.getOrderExpressions().add(new Pair<>(OrderOperator.ASC_ORDER, vRef)); } // The secondary-index search feeds into the sort. order.getInputs().add(new MutableObject<>(inputOp)); @@ -568,8 +563,8 @@ public class AccessMethodUtils { appendPrimaryIndexTypes(dataset, recordType, metaRecordType, primaryIndexOutputTypes); // An index search is expressed as an unnest over an index-search function. IFunctionInfo primaryIndexSearch = FunctionUtil.getFunctionInfo(BuiltinFunctions.INDEX_SEARCH); - AbstractFunctionCallExpression primaryIndexSearchFunc = new ScalarFunctionCallExpression(primaryIndexSearch, - primaryIndexFuncArgs); + AbstractFunctionCallExpression primaryIndexSearchFunc = + new ScalarFunctionCallExpression(primaryIndexSearch, primaryIndexFuncArgs); // This is the operator that jobgen will be looking for. It contains an unnest function that has all necessary arguments to determine // which index to use, which variables contain the index-search keys, what is the original dataset, etc. AbstractUnnestMapOperator primaryIndexUnnestOp = null; @@ -613,14 +608,14 @@ public class AccessMethodUtils { if (selectOp.getCondition().getValue().getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) { if (((AbstractFunctionCallExpression) selectOp.getCondition().getValue()).getFunctionIdentifier() .equals(AlgebricksBuiltinFunctions.NOT)) { - ScalarFunctionCallExpression notFuncExpr = (ScalarFunctionCallExpression) selectOp - .getCondition().getValue(); + ScalarFunctionCallExpression notFuncExpr = + (ScalarFunctionCallExpression) selectOp.getCondition().getValue(); if (notFuncExpr.getArguments().get(0).getValue() .getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) { if (((AbstractFunctionCallExpression) notFuncExpr.getArguments().get(0).getValue()) .getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.IS_MISSING)) { - isNullFuncExpr = (ScalarFunctionCallExpression) notFuncExpr.getArguments().get(0) - .getValue(); + isNullFuncExpr = + (ScalarFunctionCallExpression) notFuncExpr.getArguments().get(0).getValue(); if (isNullFuncExpr.getArguments().get(0).getValue() .getExpressionTag() == LogicalExpressionTag.VARIABLE) { foundSelectNonNull = true; @@ -661,43 +656,40 @@ public class AccessMethodUtils { } private static void appendExternalRecPrimaryKeys(Dataset dataset, List target) throws AsterixException { - int numPrimaryKeys = IndexingConstants - .getRIDSize(((ExternalDatasetDetails) dataset.getDatasetDetails()).getProperties()); + int numPrimaryKeys = + IndexingConstants.getRIDSize(((ExternalDatasetDetails) dataset.getDatasetDetails()).getProperties()); for (int i = 0; i < numPrimaryKeys; i++) { target.add(IndexingConstants.getFieldType(i)); } } private static void writeVarList(List varList, List> funcArgs) { - Mutable numKeysRef = new MutableObject<>( - new ConstantExpression(new AsterixConstantValue(new AInt32(varList.size())))); + Mutable numKeysRef = + new MutableObject<>(new ConstantExpression(new AsterixConstantValue(new AInt32(varList.size())))); funcArgs.add(numKeysRef); for (LogicalVariable keyVar : varList) { - Mutable keyVarRef = new MutableObject<>( - new VariableReferenceExpression(keyVar)); + Mutable keyVarRef = new MutableObject<>(new VariableReferenceExpression(keyVar)); funcArgs.add(keyVarRef); } } private static void addStringArg(String argument, List> funcArgs) { - Mutable stringRef = new MutableObject<>( - new ConstantExpression(new AsterixConstantValue(new AString(argument)))); + Mutable stringRef = + new MutableObject<>(new ConstantExpression(new AsterixConstantValue(new AString(argument)))); funcArgs.add(stringRef); } public static UnnestMapOperator createExternalDataLookupUnnestMap(AbstractDataSourceOperator dataSourceOp, Dataset dataset, ARecordType recordType, ILogicalOperator inputOp, IOptimizationContext context, boolean retainInput, boolean retainNull) throws AlgebricksException { - List primaryKeyVars = AccessMethodUtils.getPrimaryKeyVarsFromSecondaryUnnestMap(dataset, - inputOp); + List primaryKeyVars = + AccessMethodUtils.getPrimaryKeyVarsFromSecondaryUnnestMap(dataset, inputOp); // add a sort on the RID fields before fetching external data. OrderOperator order = new OrderOperator(); for (LogicalVariable pkVar : primaryKeyVars) { - Mutable vRef = new MutableObject<>( - new VariableReferenceExpression(pkVar)); - order.getOrderExpressions() - .add(new Pair<>(OrderOperator.ASC_ORDER, vRef)); + Mutable vRef = new MutableObject<>(new VariableReferenceExpression(pkVar)); + order.getOrderExpressions().add(new Pair<>(OrderOperator.ASC_ORDER, vRef)); } // The secondary-index search feeds into the sort. order.getInputs().add(new MutableObject<>(inputOp)); @@ -719,8 +711,8 @@ public class AccessMethodUtils { appendExternalRecTypes(dataset, recordType, outputTypes); IFunctionInfo externalLookup = FunctionUtil.getFunctionInfo(BuiltinFunctions.EXTERNAL_LOOKUP); - AbstractFunctionCallExpression externalLookupFunc = new ScalarFunctionCallExpression(externalLookup, - externalLookupArgs); + AbstractFunctionCallExpression externalLookupFunc = + new ScalarFunctionCallExpression(externalLookup, externalLookupArgs); UnnestMapOperator unnestOp = new UnnestMapOperator(externalUnnestVars, new MutableObject(externalLookupFunc), outputTypes, retainInput); // Fed by the order operator or the secondaryIndexUnnestOp. @@ -731,8 +723,8 @@ public class AccessMethodUtils { //set the physical operator DataSourceId dataSourceId = new DataSourceId(dataset.getDataverseName(), dataset.getDatasetName()); - unnestOp.setPhysicalOperator(new ExternalDataLookupPOperator(dataSourceId, dataset, recordType, - primaryKeyVars, false, retainInput, retainNull)); + unnestOp.setPhysicalOperator(new ExternalDataLookupPOperator(dataSourceId, dataset, recordType, primaryKeyVars, + false, retainInput, retainNull)); return unnestOp; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java index c7401b2..83b277d 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java @@ -129,11 +129,11 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule { List> assignKeyExprList = new ArrayList<>(); for (IOptimizableFuncExpr optFuncExpr : optFuncExprs) { - ComparisonKind ck = AlgebricksBuiltinFunctions - .getComparisonType(optFuncExpr.getFuncExpr().getFunctionIdentifier()); + ComparisonKind ck = + AlgebricksBuiltinFunctions.getComparisonType(optFuncExpr.getFuncExpr().getFunctionIdentifier()); ILogicalExpression searchKeyExpr = optFuncExpr.getConstantExpr(0); LogicalVariable var = context.newVar(); - assignKeyExprList.add(new MutableObject(searchKeyExpr)); + assignKeyExprList.add(new MutableObject<>(searchKeyExpr)); assignKeyVarList.add(var); if (ck == ComparisonKind.GE || ck == ComparisonKind.GT) { minFilterVars.add(var); @@ -177,7 +177,7 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule { dataSourceScanOp.setAdditionalFilteringExpressions(additionalFilteringExpressions); assignOp.getInputs() - .add(new MutableObject(dataSourceScanOp.getInputs().get(0).getValue())); + .add(new MutableObject<>(dataSourceScanOp.getInputs().get(0).getValue())); dataSourceScanOp.getInputs().get(0).setValue(assignOp); } } else if (descendantOp.getOperatorTag() == LogicalOperatorTag.UNNEST_MAP) { @@ -195,8 +195,8 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule { List minFilterVars = new ArrayList<>(); List maxFilterVars = new ArrayList<>(); - AssignOperator assignOp = createAssignOperator(optFuncExprs, minFilterVars, maxFilterVars, - context); + AssignOperator assignOp = + createAssignOperator(optFuncExprs, minFilterVars, maxFilterVars, context); unnestMapOp.setMinFilterVars(minFilterVars); unnestMapOp.setMaxFilterVars(maxFilterVars); @@ -208,7 +208,7 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule { } unnestMapOp.setAdditionalFilteringExpressions(additionalFilteringExpressions); assignOp.getInputs() - .add(new MutableObject(unnestMapOp.getInputs().get(0).getValue())); + .add(new MutableObject<>(unnestMapOp.getInputs().get(0).getValue())); unnestMapOp.getInputs().get(0).setValue(assignOp); } } @@ -246,8 +246,7 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule { } else { throw new AlgebricksException("Unexpected function for Unnest Map: " + fid); } - return ((MetadataProvider) context.getMetadataProvider()).findDataset(dataverseName, - datasetName); + return ((MetadataProvider) context.getMetadataProvider()).findDataset(dataverseName, datasetName); } } if (descendantOp.getInputs().isEmpty()) { @@ -318,8 +317,8 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule { if (funcVarIndex == -1) { continue; } - List fieldName = getFieldNameFromSubAssignTree(optFuncExpr, descendantOp, varIndex, - recType).second; + List fieldName = + getFieldNameFromSubAssignTree(optFuncExpr, descendantOp, varIndex, recType).second; if (fieldName == null) { return false; } @@ -336,7 +335,7 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule { continue; } // The variable value is one of the partitioning fields. - List fieldName = DatasetUtil.getPartitioningKeys(dataset).get(varIndex); + List fieldName = dataset.getPrimaryKeys().get(varIndex); if (fieldName == null) { return false; } @@ -379,7 +378,7 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule { int numSecondaryKeys = KeyFieldTypeUtil.getNumSecondaryKeys(index, recType, metaRecType); List fieldName; if (varIndex >= numSecondaryKeys) { - fieldName = DatasetUtil.getPartitioningKeys(dataset).get(varIndex - numSecondaryKeys); + fieldName = dataset.getPrimaryKeys().get(varIndex - numSecondaryKeys); } else { fieldName = index.getKeyFieldNames().get(varIndex); } @@ -411,8 +410,7 @@ public class IntroduceLSMComponentFilterRule implements IAlgebraicRewriteRule { } AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr; FunctionIdentifier funcIdent = funcExpr.getFunctionIdentifier(); - if (funcIdent == BuiltinFunctions.FIELD_ACCESS_BY_NAME - || funcIdent == BuiltinFunctions.FIELD_ACCESS_BY_INDEX) { + if (funcIdent == BuiltinFunctions.FIELD_ACCESS_BY_NAME || funcIdent == BuiltinFunctions.FIELD_ACCESS_BY_INDEX) { //get the variable from here. Figure out which input it came from. Go to that input!!! ArrayList usedVars = new ArrayList<>(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java index 18358c6..2534680 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java @@ -26,7 +26,6 @@ import org.apache.asterix.common.exceptions.ErrorCode; import org.apache.asterix.metadata.declared.DataSource; import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.asterix.metadata.entities.Dataset; -import org.apache.asterix.metadata.utils.DatasetUtil; import org.apache.asterix.om.functions.BuiltinFunctions; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; @@ -84,8 +83,7 @@ public class OptimizableOperatorSubTree { private List ixJoinOuterAdditionalDatasets = null; private List ixJoinOuterAdditionalRecordTypes = null; - public boolean initFromSubTree(Mutable subTreeOpRef) - throws AlgebricksException { + public boolean initFromSubTree(Mutable subTreeOpRef) throws AlgebricksException { reset(); rootRef = subTreeOpRef; root = subTreeOpRef.getValue(); @@ -226,8 +224,8 @@ public class OptimizableOperatorSubTree { Dataset ds = null; ARecordType rType = null; - List> sourceOpRefs = new ArrayList>(); - List dsTypes = new ArrayList(); + List> sourceOpRefs = new ArrayList<>(); + List dsTypes = new ArrayList<>(); sourceOpRefs.add(getDataSourceRef()); dsTypes.add(getDataSourceType()); @@ -247,8 +245,7 @@ public class OptimizableOperatorSubTree { IDataSource datasource = dataSourceScan.getDataSource(); if (datasource instanceof DataSource) { byte dsType = ((DataSource) datasource).getDatasourceType(); - if (dsType != DataSource.Type.INTERNAL_DATASET - && dsType != DataSource.Type.EXTERNAL_DATASET) { + if (dsType != DataSource.Type.INTERNAL_DATASET && dsType != DataSource.Type.EXTERNAL_DATASET) { return false; } } @@ -376,12 +373,12 @@ public class OptimizableOperatorSubTree { */ public void getPrimaryKeyVars(Mutable dataSourceRefToFetch, List target) throws AlgebricksException { - Mutable dataSourceRefToFetchKey = (dataSourceRefToFetch == null) ? dataSourceRef - : dataSourceRefToFetch; + Mutable dataSourceRefToFetchKey = + (dataSourceRefToFetch == null) ? dataSourceRef : dataSourceRefToFetch; switch (dataSourceType) { case DATASOURCE_SCAN: DataSourceScanOperator dataSourceScan = (DataSourceScanOperator) getDataSourceRef().getValue(); - int numPrimaryKeys = DatasetUtil.getPartitioningKeys(getDataset()).size(); + int numPrimaryKeys = dataset.getPrimaryKeys().size(); for (int i = 0; i < numPrimaryKeys; i++) { target.add(dataSourceScan.getVariables().get(i)); } @@ -400,6 +397,7 @@ public class OptimizableOperatorSubTree { } } + public List getDataSourceVariables() throws AlgebricksException { switch (getDataSourceType()) { case DATASOURCE_SCAN: @@ -421,8 +419,8 @@ public class OptimizableOperatorSubTree { case DATASOURCE_SCAN: case EXTERNAL_SCAN: case PRIMARY_INDEX_LOOKUP: - AbstractScanOperator scanOp = (AbstractScanOperator) getIxJoinOuterAdditionalDataSourceRefs() - .get(idx).getValue(); + AbstractScanOperator scanOp = + (AbstractScanOperator) getIxJoinOuterAdditionalDataSourceRefs().get(idx).getValue(); return scanOp.getVariables(); case COLLECTION_SCAN: return new ArrayList<>(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java index 2662be8..d5e01a4 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java @@ -211,7 +211,7 @@ class LangExpressionToPlanTranslator metadataProvider.findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName()); DatasetDataSource targetDatasource = validateDatasetInfo(metadataProvider, stmt.getDataverseName(), stmt.getDatasetName()); - List> partitionKeys = DatasetUtil.getPartitioningKeys(targetDatasource.getDataset()); + List> partitionKeys = targetDatasource.getDataset().getPrimaryKeys(); if (dataset.hasMetaPart()) { throw new AlgebricksException( dataset.getDatasetName() + ": load dataset is not supported on Datasets with Meta records"); @@ -361,7 +361,7 @@ class LangExpressionToPlanTranslator ArrayList vars = new ArrayList<>(); ArrayList> exprs = new ArrayList<>(); List> varRefsForLoading = new ArrayList<>(); - List> partitionKeys = DatasetUtil.getPartitioningKeys(targetDatasource.getDataset()); + List> partitionKeys = targetDatasource.getDataset().getPrimaryKeys(); int numOfPrimaryKeys = partitionKeys.size(); for (int i = 0; i < numOfPrimaryKeys; i++) { if (keySourceIndicator == null || keySourceIndicator.get(i).intValue() == 0) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AppRuntimeContextProviderForRecovery.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AppRuntimeContextProviderForRecovery.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AppRuntimeContextProviderForRecovery.java index 6a3472d..397b459 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AppRuntimeContextProviderForRecovery.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AppRuntimeContextProviderForRecovery.java @@ -27,9 +27,9 @@ import org.apache.asterix.common.transactions.ITransactionSubsystem; import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler; import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker; +import org.apache.hyracks.storage.common.ILocalResourceRepository; import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.file.IFileMapProvider; -import org.apache.hyracks.storage.common.file.ILocalResourceRepository; public class AppRuntimeContextProviderForRecovery implements IAppRuntimeContextProvider { @@ -76,7 +76,7 @@ public class AppRuntimeContextProviderForRecovery implements IAppRuntimeContextP @Override public IIOManager getIOManager() { - return asterixAppRuntimeContext.getIOManager(); + return asterixAppRuntimeContext.getIoManager(); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/735532e4/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ConnectorApiServlet.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ConnectorApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ConnectorApiServlet.java index cb3c063..421ee0e 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ConnectorApiServlet.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ConnectorApiServlet.java @@ -34,7 +34,6 @@ import org.apache.asterix.metadata.MetadataManager; import org.apache.asterix.metadata.MetadataTransactionContext; import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.asterix.metadata.entities.Dataset; -import org.apache.asterix.metadata.utils.DatasetUtil; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.utils.FlushDatasetUtil; import org.apache.hyracks.api.client.IHyracksClientConnection; @@ -110,7 +109,7 @@ public class ConnectorApiServlet extends AbstractServlet { metadataProvider.splitsForDataset(mdTxnCtx, dataverseName, datasetName, datasetName, temp); ARecordType recordType = (ARecordType) metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName()); - List> primaryKeys = DatasetUtil.getPartitioningKeys(dataset); + List> primaryKeys = dataset.getPrimaryKeys(); StringBuilder pkStrBuf = new StringBuilder(); for (List keys : primaryKeys) { for (String key : keys) {