Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id E0D84200CBB for ; Tue, 20 Jun 2017 00:46:42 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id DF696160BE4; Mon, 19 Jun 2017 22:46:42 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 66F93160BE1 for ; Tue, 20 Jun 2017 00:46:40 +0200 (CEST) Received: (qmail 99553 invoked by uid 500); 19 Jun 2017 22:46:39 -0000 Mailing-List: contact commits-help@asterixdb.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@asterixdb.apache.org Delivered-To: mailing list commits@asterixdb.apache.org Received: (qmail 99395 invoked by uid 99); 19 Jun 2017 22:46:39 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 19 Jun 2017 22:46:39 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id D36CBE93E1; Mon, 19 Jun 2017 22:46:38 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: buyingyi@apache.org To: commits@asterixdb.apache.org Date: Mon, 19 Jun 2017 22:46:43 -0000 Message-Id: <2314762eb4a14805a5482326d5b8b57f@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [6/7] asterixdb git commit: Revert "ASTERIXDB-1945 [STO] Cleanup Buffer Cache API" archived-at: Mon, 19 Jun 2017 22:46:43 -0000 http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/build/IndexBuilder.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/build/IndexBuilder.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/build/IndexBuilder.java index 9f648c5..f1ae6b0 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/build/IndexBuilder.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/build/IndexBuilder.java @@ -19,13 +19,10 @@ package org.apache.hyracks.storage.am.common.build; import java.io.IOException; -import java.util.logging.Level; -import java.util.logging.Logger; import org.apache.hyracks.api.application.INCServiceContext; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; -import org.apache.hyracks.api.util.IoUtil; import org.apache.hyracks.storage.am.common.api.IIndexBuilder; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame; import org.apache.hyracks.storage.common.IIndex; @@ -38,7 +35,6 @@ import org.apache.hyracks.storage.common.LocalResource; import org.apache.hyracks.storage.common.file.IResourceIdFactory; public class IndexBuilder implements IIndexBuilder { - private static final Logger LOGGER = Logger.getLogger(IndexBuilder.class.getName()); protected final INCServiceContext ctx; protected final IStorageManager storageManager; @@ -85,22 +81,8 @@ public class IndexBuilder implements IIndexBuilder { //The reason for this is to handle many cases such as: //1. Crash while delete index is running (we don't do global cleanup on restart) //2. Node leaves and then join with old data - LOGGER.log(Level.WARNING, - "Removing existing index on index create for the index: " + resourceRef.getRelativePath()); lcManager.unregister(resourceRef.getRelativePath()); - index.destroy(); } else { - if (resourceRef.getFile().exists()) { - // Index is not registered but the index file exists - // This is another big problem that we need to disallow soon - // We can only disallow this if we have a global cleanup after crash - // on reboot - LOGGER.log(Level.WARNING, - "Deleting " + resourceRef.getRelativePath() - + " on index create. The index is not registered" - + " but the file exists in the filesystem"); - IoUtil.delete(resourceRef); - } index = resource.createInstance(ctx); } index.create(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java index cc3bfe7..c00cecb 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java @@ -37,6 +37,7 @@ import org.apache.hyracks.storage.am.common.util.TreeIndexStatsGatherer; import org.apache.hyracks.storage.common.IStorageManager; import org.apache.hyracks.storage.common.LocalResource; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.storage.common.file.IFileMapProvider; public class TreeIndexStatsOperatorNodePushable extends AbstractUnaryOutputSourceOperatorNodePushable { private final IHyracksTaskContext ctx; @@ -68,11 +69,14 @@ public class TreeIndexStatsOperatorNodePushable extends AbstractUnaryOutputSourc try { writer.open(); IBufferCache bufferCache = storageManager.getBufferCache(ctx.getJobletContext().getServiceContext()); + IFileMapProvider fileMapProvider = + storageManager.getFileMapProvider(ctx.getJobletContext().getServiceContext()); LocalResource resource = treeIndexHelper.getResource(); IIOManager ioManager = ctx.getIoManager(); FileReference fileRef = ioManager.resolve(resource.getPath()); + int indexFileId = fileMapProvider.lookupFileId(fileRef); TreeIndexStatsGatherer statsGatherer = new TreeIndexStatsGatherer(bufferCache, treeIndex.getPageManager(), - fileRef, treeIndex.getRootPageId()); + indexFileId, treeIndex.getRootPageId()); TreeIndexStats stats = statsGatherer.gatherStats(treeIndex.getLeafFrameFactory().createFrame(), treeIndex.getInteriorFrameFactory().createFrame(), treeIndex.getPageManager().createMetadataFrame()); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java index 8ffd0eb..163a3d1 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java @@ -40,6 +40,7 @@ import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.buffercache.ICachedPage; import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue; import org.apache.hyracks.storage.common.file.BufferedFileHandle; +import org.apache.hyracks.storage.common.file.IFileMapProvider; public abstract class AbstractTreeIndex implements ITreeIndex { @@ -48,6 +49,7 @@ public abstract class AbstractTreeIndex implements ITreeIndex { protected int rootPage = 1; protected final IBufferCache bufferCache; + protected final IFileMapProvider fileMapProvider; protected final IPageManager freePageManager; protected final ITreeIndexFrameFactory interiorFrameFactory; @@ -57,16 +59,21 @@ public abstract class AbstractTreeIndex implements ITreeIndex { protected final int fieldCount; protected FileReference file; - private int fileId = -1; + protected int fileId = -1; protected boolean isActive = false; + //hasEverBeenActivated is to stop the throwing of an exception of deactivating an index that + //was never activated or failed to activate in try/finally blocks, as there's no way to know if + //an index is activated or not from the outside. + protected boolean hasEverBeenActivated = false; protected int bulkloadLeafStart = 0; - public AbstractTreeIndex(IBufferCache bufferCache, IPageManager freePageManager, + public AbstractTreeIndex(IBufferCache bufferCache, IFileMapProvider fileMapProvider, IPageManager freePageManager, ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory, IBinaryComparatorFactory[] cmpFactories, int fieldCount, FileReference file) { this.bufferCache = bufferCache; + this.fileMapProvider = fileMapProvider; this.freePageManager = freePageManager; this.interiorFrameFactory = interiorFrameFactory; this.leafFrameFactory = leafFrameFactory; @@ -80,8 +87,9 @@ public abstract class AbstractTreeIndex implements ITreeIndex { if (isActive) { throw HyracksDataException.create(ErrorCode.CANNOT_CREATE_ACTIVE_INDEX); } - fileId = bufferCache.createFile(file); - bufferCache.openFile(fileId); + synchronized (fileMapProvider) { + fileId = createAndOpen(bufferCache, fileMapProvider, file); + } freePageManager.open(fileId); freePageManager.init(interiorFrameFactory, leafFrameFactory); setRootPage(); @@ -89,6 +97,27 @@ public abstract class AbstractTreeIndex implements ITreeIndex { bufferCache.closeFile(fileId); } + public static int createAndOpen(IBufferCache bufferCache, IFileMapProvider fileMapProvider, FileReference file) + throws HyracksDataException { + int fileId; + boolean fileIsMapped = fileMapProvider.isMapped(file); + if (!fileIsMapped) { + bufferCache.createFile(file); + } + fileId = fileMapProvider.lookupFileId(file); + try { + // Also creates the file if it doesn't exist yet. + bufferCache.openFile(fileId); + } catch (HyracksDataException e) { + // Revert state of buffer cache since file failed to open. + if (!fileIsMapped) { + bufferCache.deleteFile(fileId, false); + } + throw e; + } + return fileId; + } + private void setRootPage() throws HyracksDataException { rootPage = freePageManager.getRootPageId(); bulkloadLeafStart = freePageManager.getBulkLoadLeaf(); @@ -99,35 +128,49 @@ public abstract class AbstractTreeIndex implements ITreeIndex { if (isActive) { throw HyracksDataException.create(ErrorCode.CANNOT_ACTIVATE_ACTIVE_INDEX); } - if (fileId >= 0) { - bufferCache.openFile(fileId); - } else { - fileId = bufferCache.openFile(file); + boolean fileIsMapped = false; + synchronized (fileMapProvider) { + fileIsMapped = fileMapProvider.isMapped(file); + if (!fileIsMapped) { + bufferCache.createFile(file); + } + fileId = fileMapProvider.lookupFileId(file); + try { + // Also creates the file if it doesn't exist yet. + bufferCache.openFile(fileId); + } catch (HyracksDataException e) { + // Revert state of buffer cache since file failed to open. + if (!fileIsMapped) { + bufferCache.deleteFile(fileId, false); + } + throw e; + } } freePageManager.open(fileId); setRootPage(); // TODO: Should probably have some way to check that the tree is physically consistent // or that the file we just opened actually is a tree isActive = true; + hasEverBeenActivated = true; } @Override public synchronized void deactivate() throws HyracksDataException { - if (!isActive) { + if (!isActive && hasEverBeenActivated) { throw HyracksDataException.create(ErrorCode.CANNOT_DEACTIVATE_INACTIVE_INDEX); } - freePageManager.close(); - bufferCache.closeFile(fileId); + if (isActive) { + freePageManager.close(); + bufferCache.closeFile(fileId); + } + isActive = false; } - public void purge() throws HyracksDataException { - if (isActive) { - throw HyracksDataException.create(ErrorCode.CANNOT_PURGE_ACTIVE_INDEX); - } + public synchronized void deactivateCloseHandle() throws HyracksDataException { + deactivate(); bufferCache.purgeHandle(fileId); - // after purging, the fileId has no mapping and no meaning - fileId = -1; + } @Override @@ -135,7 +178,13 @@ public abstract class AbstractTreeIndex implements ITreeIndex { if (isActive) { throw HyracksDataException.create(ErrorCode.CANNOT_DESTROY_ACTIVE_INDEX); } - bufferCache.deleteFile(file); + + if (fileId == -1) { + return; + } + bufferCache.deleteFile(fileId, false); + file.delete(); + fileId = -1; } @Override @@ -235,7 +284,7 @@ public abstract class AbstractTreeIndex implements ITreeIndex { queue = bufferCache.createFIFOQueue(); if (!isEmptyTree(leafFrame)) { - throw HyracksDataException.create(ErrorCode.CANNOT_BULK_LOAD_NON_EMPTY_TREE); + throw new HyracksDataException("Cannot bulk-load a non-empty tree."); } this.cmp = MultiComparator.create(cmpFactories); @@ -263,6 +312,9 @@ public abstract class AbstractTreeIndex implements ITreeIndex { pagesToWrite = new ArrayList<>(); } + @Override + public abstract void add(ITupleReference tuple) throws HyracksDataException; + protected void handleException() throws HyracksDataException { // Unlatch and unpin pages that weren't in the queue to avoid leaking memory. for (NodeFrontier nodeFrontier : nodeFrontiers) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java index 959f27a..2b47238 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java @@ -22,7 +22,7 @@ import java.util.ArrayList; import java.util.Random; import org.apache.hyracks.api.exceptions.HyracksDataException; -import org.apache.hyracks.api.io.FileReference; +import org.apache.hyracks.util.MathUtil; import org.apache.hyracks.storage.am.common.api.IMetadataPageManager; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame; import org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrame; @@ -30,25 +30,25 @@ import org.apache.hyracks.storage.common.arraylist.IntArrayList; import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.buffercache.ICachedPage; import org.apache.hyracks.storage.common.file.BufferedFileHandle; -import org.apache.hyracks.util.MathUtil; public class TreeIndexBufferCacheWarmup { private final IBufferCache bufferCache; private final IMetadataPageManager freePageManager; - private final FileReference fileRef; + private final int fileId; private final ArrayList pagesByLevel = new ArrayList<>(); private final Random rnd = new Random(); - public TreeIndexBufferCacheWarmup(IBufferCache bufferCache, IMetadataPageManager freePageManager, - FileReference fileRef) { + public TreeIndexBufferCacheWarmup(IBufferCache bufferCache, + IMetadataPageManager freePageManager, int fileId) { this.bufferCache = bufferCache; this.freePageManager = freePageManager; - this.fileRef = fileRef; + this.fileId = fileId; } - public void warmup(ITreeIndexFrame frame, ITreeIndexMetadataFrame metaFrame, int[] warmupTreeLevels, + public void warmup(ITreeIndexFrame frame, + ITreeIndexMetadataFrame metaFrame, int[] warmupTreeLevels, int[] warmupRepeats) throws HyracksDataException { - int fileId = bufferCache.openFile(fileRef); + bufferCache.openFile(fileId); // scan entire file to determine pages in each level int maxPageId = freePageManager.getMaxPageId(metaFrame); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java index 5410282..7e14b4a 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java @@ -19,7 +19,6 @@ package org.apache.hyracks.storage.am.common.util; import org.apache.hyracks.api.exceptions.HyracksDataException; -import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.storage.am.common.api.IPageManager; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame; import org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrame; @@ -32,21 +31,25 @@ public class TreeIndexStatsGatherer { private final TreeIndexStats treeIndexStats = new TreeIndexStats(); private final IBufferCache bufferCache; private final IPageManager freePageManager; - private final FileReference fileRef; + private final int fileId; private final int rootPage; - public TreeIndexStatsGatherer(IBufferCache bufferCache, IPageManager freePageManager, FileReference fileRef, - int rootPage) { + public TreeIndexStatsGatherer(IBufferCache bufferCache, + IPageManager freePageManager, int fileId, int rootPage) { this.bufferCache = bufferCache; this.freePageManager = freePageManager; - this.fileRef = fileRef; + this.fileId = fileId; this.rootPage = rootPage; } - public TreeIndexStats gatherStats(ITreeIndexFrame leafFrame, ITreeIndexFrame interiorFrame, - ITreeIndexMetadataFrame metaFrame) throws HyracksDataException { - int fileId = bufferCache.openFile(fileRef); + public TreeIndexStats gatherStats(ITreeIndexFrame leafFrame, + ITreeIndexFrame interiorFrame, ITreeIndexMetadataFrame metaFrame) + throws HyracksDataException { + + bufferCache.openFile(fileId); + treeIndexStats.begin(); + int maxPageId = freePageManager.getMaxPageId(metaFrame); for (int pageId = 0; pageId <= maxPageId; pageId++) { ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResource.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResource.java index f1e9cf1..6860b97 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResource.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResource.java @@ -57,9 +57,9 @@ public class ExternalBTreeLocalResource extends LSMBTreeLocalResource { public ILSMIndex createInstance(INCServiceContext serviceCtx) throws HyracksDataException { IIOManager ioManager = serviceCtx.getIoManager(); FileReference file = ioManager.resolve(path); - return LSMBTreeUtil.createExternalBTree(ioManager, file, storageManager.getBufferCache(serviceCtx), typeTraits, - cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, - mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx), + return LSMBTreeUtil.createExternalBTree(ioManager, file, storageManager.getBufferCache(serviceCtx), + storageManager.getFileMapProvider(serviceCtx), typeTraits, cmpFactories, bloomFilterKeyFields, + bloomFilterFalsePositiveRate, mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx), opTrackerProvider.getOperationTracker(serviceCtx), ioSchedulerProvider.getIoScheduler(serviceCtx), ioOpCallbackFactory.createIoOpCallback(), durable, metadataPageManagerFactory); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResource.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResource.java index 317a479..3819066 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResource.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResource.java @@ -61,7 +61,7 @@ public class ExternalBTreeWithBuddyLocalResource extends LSMBTreeLocalResource { IIOManager ioManager = serviceCtx.getIoManager(); FileReference file = ioManager.resolve(path); return LSMBTreeUtil.createExternalBTreeWithBuddy(ioManager, file, storageManager.getBufferCache(serviceCtx), - typeTraits, cmpFactories, bloomFilterFalsePositiveRate, + storageManager.getFileMapProvider(serviceCtx), typeTraits, cmpFactories, bloomFilterFalsePositiveRate, mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx), opTrackerProvider.getOperationTracker(serviceCtx), ioSchedulerProvider.getIoScheduler(serviceCtx), ioOpCallbackFactory.createIoOpCallback(), bloomFilterKeyFields, durable, metadataPageManagerFactory); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java index f6e311c..6416f6b 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java @@ -70,9 +70,9 @@ public class LSMBTreeLocalResource extends LsmResource { IIOManager ioManager = serviceCtx.getIoManager(); FileReference file = ioManager.resolve(path); List vbcs = vbcProvider.getVirtualBufferCaches(serviceCtx, file); - return LSMBTreeUtil.createLSMTree(ioManager, vbcs, file, storageManager.getBufferCache(serviceCtx), typeTraits, - cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, - mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx), + return LSMBTreeUtil.createLSMTree(ioManager, vbcs, file, storageManager.getBufferCache(serviceCtx), + storageManager.getFileMapProvider(serviceCtx), typeTraits, cmpFactories, bloomFilterKeyFields, + bloomFilterFalsePositiveRate, mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx), opTrackerProvider.getOperationTracker(serviceCtx), ioSchedulerProvider.getIoScheduler(serviceCtx), ioOpCallbackFactory.createIoOpCallback(), isPrimary, filterTypeTraits, filterCmpFactories, btreeFields, filterFields, durable, metadataPageManagerFactory); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java index c06edfd..cdde692 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java @@ -62,6 +62,7 @@ import org.apache.hyracks.storage.common.IIndexCursor; import org.apache.hyracks.storage.common.IModificationOperationCallback; import org.apache.hyracks.storage.common.ISearchOperationCallback; import org.apache.hyracks.storage.common.ISearchPredicate; +import org.apache.hyracks.storage.common.file.IFileMapProvider; /** * This is an lsm b-tree that does not have memory component and is modified @@ -92,12 +93,13 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex { ITreeIndexFrameFactory insertLeafFrameFactory, ITreeIndexFrameFactory deleteLeafFrameFactory, ILSMIndexFileManager fileManager, TreeIndexFactory diskBTreeFactory, TreeIndexFactory bulkLoadBTreeFactory, BloomFilterFactory bloomFilterFactory, - double bloomFilterFalsePositiveRate, IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy, - ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, + double bloomFilterFalsePositiveRate, IFileMapProvider diskFileMapProvider, int fieldCount, + IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, + ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, TreeIndexFactory transactionBTreeFactory, boolean durable) { super(ioManager, insertLeafFrameFactory, deleteLeafFrameFactory, fileManager, diskBTreeFactory, - bulkLoadBTreeFactory, bloomFilterFactory, bloomFilterFalsePositiveRate, cmpFactories, mergePolicy, - opTracker, ioScheduler, ioOpCallback, false, durable); + bulkLoadBTreeFactory, bloomFilterFactory, bloomFilterFalsePositiveRate, diskFileMapProvider, fieldCount, + cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, false, durable); this.transactionComponentFactory = new LSMBTreeDiskComponentFactory(transactionBTreeFactory, bloomFilterFactory, null); this.secondDiskComponents = new LinkedList<>(); @@ -299,13 +301,21 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex { BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(ioOpCallback); cb.afterFinalize(LSMOperationType.FLUSH, null); } - for (ILSMDiskComponent c : diskComponents) { - deactivateDiskComponent(c); + for (ILSMComponent c : diskComponents) { + LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c; + BTree btree = component.getBTree(); + BloomFilter bloomFilter = component.getBloomFilter(); + btree.deactivateCloseHandle(); + bloomFilter.deactivate(); } - for (ILSMDiskComponent c : secondDiskComponents) { - // Only deactivate non shared components + for (ILSMComponent c : secondDiskComponents) { + // Only deactivate non shared components (So components are not de-activated twice) if (!diskComponents.contains(c)) { - deactivateDiskComponent(c); + LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c; + BTree btree = component.getBTree(); + BloomFilter bloomFilter = component.getBloomFilter(); + btree.deactivateCloseHandle(); + bloomFilter.deactivate(); } } isActive = false; @@ -347,13 +357,17 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex { if (isActive) { throw new HyracksDataException("Failed to destroy the index since it is activated."); } - for (ILSMDiskComponent c : diskComponents) { - destroyDiskComponent(c); + for (ILSMComponent c : diskComponents) { + LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c; + component.getBTree().destroy(); + component.getBloomFilter().destroy(); // Remove from second list to avoid destroying twice secondDiskComponents.remove(c); } - for (ILSMDiskComponent c : secondDiskComponents) { - destroyDiskComponent(c); + for (ILSMComponent c : secondDiskComponents) { + LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c; + component.getBTree().destroy(); + component.getBloomFilter().destroy(); } diskComponents.clear(); secondDiskComponents.clear(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java index 9959583..1c99b17 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java @@ -72,6 +72,7 @@ import org.apache.hyracks.storage.common.ISearchOperationCallback; import org.apache.hyracks.storage.common.ISearchPredicate; import org.apache.hyracks.storage.common.MultiComparator; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.storage.common.file.IFileMapProvider; public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeIndex, ITwoPCIndex { @@ -99,12 +100,12 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, TreeIndexFactory bulkLoadBTreeFactory, TreeIndexFactory copyBtreeFactory, TreeIndexFactory buddyBtreeFactory, BloomFilterFactory bloomFilterFactory, - double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, - ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, + IFileMapProvider diskFileMapProvider, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, + ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, IBinaryComparatorFactory[] btreeCmpFactories, IBinaryComparatorFactory[] buddyBtreeCmpFactories, int[] buddyBTreeFields, boolean durable) { - super(ioManager, diskBufferCache, fileManager, bloomFilterFalsePositiveRate, mergePolicy, opTracker, - ioScheduler, ioOpCallback, durable); + super(ioManager, diskBufferCache, fileManager, diskFileMapProvider, bloomFilterFalsePositiveRate, mergePolicy, + opTracker, ioScheduler, ioOpCallback, durable); this.btreeCmpFactories = btreeCmpFactories; this.buddyBtreeCmpFactories = buddyBtreeCmpFactories; this.buddyBTreeFields = buddyBTreeFields; @@ -120,7 +121,12 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd @Override public void create() throws HyracksDataException { - super.create(); + if (isActive) { + throw new HyracksDataException("Failed to create the index since it is activated."); + } + fileManager.deleteDirs(); + fileManager.createDirs(); + diskComponents.clear(); secondDiskComponents.clear(); } @@ -767,12 +773,14 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd BTree btree = component.getBTree(); BTree buddyBtree = component.getBuddyBTree(); BloomFilter bloomFilter = component.getBloomFilter(); - btree.deactivate(); - buddyBtree.deactivate(); + btree.deactivateCloseHandle(); + buddyBtree.deactivateCloseHandle(); bloomFilter.deactivate(); - btree.purge(); - buddyBtree.purge(); - bloomFilter.purge(); + } + + @Override + protected void destroyMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException { + //do nothing since external index never use memory components } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java index a5c59d6..e830b3e 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java @@ -76,6 +76,7 @@ import org.apache.hyracks.storage.common.ISearchOperationCallback; import org.apache.hyracks.storage.common.ISearchPredicate; import org.apache.hyracks.storage.common.MultiComparator; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.storage.common.file.IFileMapProvider; public class LSMBTree extends AbstractLSMIndex implements ITreeIndex { @@ -102,11 +103,11 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex { TreeIndexFactory diskBTreeFactory, TreeIndexFactory bulkLoadBTreeFactory, BloomFilterFactory bloomFilterFactory, IComponentFilterHelper filterHelper, ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager, - double bloomFilterFalsePositiveRate, int fieldCount, IBinaryComparatorFactory[] cmpFactories, - ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, - ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, int[] btreeFields, int[] filterFields, - boolean durable) throws HyracksDataException { - super(ioManager, virtualBufferCaches, diskBTreeFactory.getBufferCache(), fileManager, + double bloomFilterFalsePositiveRate, IFileMapProvider diskFileMapProvider, int fieldCount, + IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, + ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, + int[] btreeFields, int[] filterFields, boolean durable) throws HyracksDataException { + super(ioManager, virtualBufferCaches, diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory, filterManager, filterFields, durable, filterHelper, btreeFields); this.insertLeafFrameFactory = insertLeafFrameFactory; @@ -115,7 +116,8 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex { int i = 0; for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) { LSMBTreeMemoryComponent mutableComponent = new LSMBTreeMemoryComponent( - new BTree(virtualBufferCache, new VirtualFreePageManager(virtualBufferCache), interiorFrameFactory, + new BTree(virtualBufferCache, virtualBufferCache.getFileMapProvider(), + new VirtualFreePageManager(virtualBufferCache), interiorFrameFactory, insertLeafFrameFactory, cmpFactories, fieldCount, ioManager.resolveAbsolutePath(fileManager.getBaseDir() + "_virtual_" + i)), virtualBufferCache, i == 0 ? true : false, @@ -135,18 +137,19 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex { ITreeIndexFrameFactory deleteLeafFrameFactory, ILSMIndexFileManager fileManager, TreeIndexFactory diskBTreeFactory, TreeIndexFactory bulkLoadBTreeFactory, BloomFilterFactory bloomFilterFactory, double bloomFilterFalsePositiveRate, - IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, - ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, - boolean durable) { - super(ioManager, diskBTreeFactory.getBufferCache(), fileManager, bloomFilterFalsePositiveRate, mergePolicy, - opTracker, ioScheduler, ioOpCallback, durable); + IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] cmpFactories, + ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, + ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, boolean durable) { + super(ioManager, diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider, + bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, durable); this.insertLeafFrameFactory = insertLeafFrameFactory; this.deleteLeafFrameFactory = deleteLeafFrameFactory; this.cmpFactories = cmpFactories; - this.needKeyDupCheck = needKeyDupCheck; - this.hasBloomFilter = true; componentFactory = new LSMBTreeDiskComponentFactory(diskBTreeFactory, bloomFilterFactory, null); bulkLoadComponentFactory = new LSMBTreeDiskComponentFactory(bulkLoadBTreeFactory, bloomFilterFactory, null); + this.needKeyDupCheck = needKeyDupCheck; + //TODO remove BloomFilter from external dataset's secondary LSMBTree index + this.hasBloomFilter = true; } public boolean hasBloomFilter() { @@ -180,6 +183,12 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex { } @Override + protected void destroyMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException { + LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c; + mutableComponent.getBTree().destroy(); + } + + @Override protected void clearDiskComponent(ILSMDiskComponent c) throws HyracksDataException { LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c; if (hasBloomFilter) { @@ -578,11 +587,9 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex { @Override protected void deactivateDiskComponent(ILSMDiskComponent c) throws HyracksDataException { LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c; - component.getBTree().deactivate(); - component.getBTree().purge(); + component.getBTree().deactivateCloseHandle(); if (hasBloomFilter) { component.getBloomFilter().deactivate(); - component.getBloomFilter().purge(); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java index 432a2b8..194eb3a 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java @@ -37,15 +37,17 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndex; import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory; +import org.apache.hyracks.storage.common.file.IFileMapProvider; public class LSMBTreeFileManager extends AbstractLSMIndexFileManager { + public static final String BTREE_STRING = "b"; private final TreeIndexFactory btreeFactory; private final boolean hasBloomFilter; - public LSMBTreeFileManager(IIOManager ioManager, FileReference file, + public LSMBTreeFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file, TreeIndexFactory btreeFactory, boolean hasBloomFilter) { - super(ioManager, file, null); + super(ioManager, fileMapProvider, file, null); this.btreeFactory = btreeFactory; this.hasBloomFilter = hasBloomFilter; } @@ -53,28 +55,28 @@ public class LSMBTreeFileManager extends AbstractLSMIndexFileManager { @Override public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException { String ts = getCurrentTimestamp(); - String baseName = baseDir + ts + DELIMITER + ts; + String baseName = baseDir + ts + SPLIT_STRING + ts; // Begin timestamp and end timestamp are identical since it is a flush - return new LSMComponentFileReferences(createFlushFile(baseName + DELIMITER + BTREE_SUFFIX), null, - hasBloomFilter ? createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX) : null); + return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING), null, + hasBloomFilter ? createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING) : null); } @Override public LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName) throws HyracksDataException { - String[] firstTimestampRange = firstFileName.split(DELIMITER); - String[] lastTimestampRange = lastFileName.split(DELIMITER); + String[] firstTimestampRange = firstFileName.split(SPLIT_STRING); + String[] lastTimestampRange = lastFileName.split(SPLIT_STRING); - String baseName = baseDir + firstTimestampRange[0] + DELIMITER + lastTimestampRange[1]; + String baseName = baseDir + firstTimestampRange[0] + SPLIT_STRING + lastTimestampRange[1]; // Get the range of timestamps by taking the earliest and the latest timestamps - return new LSMComponentFileReferences(createMergeFile(baseName + DELIMITER + BTREE_SUFFIX), null, - hasBloomFilter ? createMergeFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX) : null); + return new LSMComponentFileReferences(createMergeFile(baseName + SPLIT_STRING + BTREE_STRING), null, + hasBloomFilter ? createMergeFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING) : null); } private static FilenameFilter btreeFilter = new FilenameFilter() { @Override public boolean accept(File dir, String name) { - return !name.startsWith(".") && name.endsWith(BTREE_SUFFIX); + return !name.startsWith(".") && name.endsWith(BTREE_STRING); } }; @@ -93,7 +95,7 @@ public class LSMBTreeFileManager extends AbstractLSMIndexFileManager { cleanupAndGetValidFilesInternal(getCompoundFilter(transactionFilter, btreeFilter), btreeFactory, allBTreeFiles); HashSet btreeFilesSet = new HashSet<>(); for (ComparableFileName cmpFileName : allBTreeFiles) { - int index = cmpFileName.fileName.lastIndexOf(DELIMITER); + int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING); btreeFilesSet.add(cmpFileName.fileName.substring(0, index)); } @@ -198,19 +200,19 @@ public class LSMBTreeFileManager extends AbstractLSMIndexFileManager { public LSMComponentFileReferences getNewTransactionFileReference() throws IOException { String ts = getCurrentTimestamp(); // Create transaction lock file - Files.createFile(Paths.get(baseDir + TXN_PREFIX + ts)); + Files.createFile(Paths.get(baseDir + TRANSACTION_PREFIX + ts)); - String baseName = baseDir + ts + DELIMITER + ts; + String baseName = baseDir + ts + SPLIT_STRING + ts; // Begin timestamp and end timestamp are identical since it is a transaction - return new LSMComponentFileReferences(createFlushFile(baseName + DELIMITER + BTREE_SUFFIX), null, - createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); + return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING), null, + createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING)); } @Override public LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException { FilenameFilter transactionFilter; File dir = new File(baseDir); - String[] files = dir.list(txnFileNameFilter); + String[] files = dir.list(transactionFileNameFilter); if (files.length == 0) { return null; } @@ -234,9 +236,9 @@ public class LSMBTreeFileManager extends AbstractLSMIndexFileManager { File bloomFilterFile = null; for (String fileName : files) { - if (fileName.endsWith(BTREE_SUFFIX)) { + if (fileName.endsWith(BTREE_STRING)) { bTreeFile = new File(dir.getPath() + File.separator + fileName); - } else if (fileName.endsWith(BLOOM_FILTER_SUFFIX)) { + } else if (fileName.endsWith(BLOOM_FILTER_STRING)) { bloomFilterFile = new File(dir.getPath() + File.separator + fileName); } else { throw new HyracksDataException("unrecognized file found = " + fileName); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java index c866a45..0173c06 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java @@ -36,8 +36,11 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndex; import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory; +import org.apache.hyracks.storage.common.file.IFileMapProvider; public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager { + public static final String BUDDY_BTREE_STRING = "buddy"; + public static final String BTREE_STRING = "b"; private final TreeIndexFactory btreeFactory; private final TreeIndexFactory buddyBtreeFactory; @@ -45,21 +48,21 @@ public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager { private static FilenameFilter btreeFilter = new FilenameFilter() { @Override public boolean accept(File dir, String name) { - return !name.startsWith(".") && name.endsWith(BTREE_SUFFIX); + return !name.startsWith(".") && name.endsWith(BTREE_STRING); } }; private static FilenameFilter buddyBtreeFilter = new FilenameFilter() { @Override public boolean accept(File dir, String name) { - return !name.startsWith(".") && name.endsWith(DELETE_TREE_SUFFIX); + return !name.startsWith(".") && name.endsWith(BUDDY_BTREE_STRING); } }; - public LSMBTreeWithBuddyFileManager(IIOManager ioManager, FileReference file, + public LSMBTreeWithBuddyFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file, TreeIndexFactory btreeFactory, TreeIndexFactory buddyBtreeFactory) { - super(ioManager, file, null); + super(ioManager, fileMapProvider, file, null); this.buddyBtreeFactory = buddyBtreeFactory; this.btreeFactory = btreeFactory; } @@ -67,25 +70,25 @@ public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager { @Override public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException { String ts = getCurrentTimestamp(); - String baseName = baseDir + ts + DELIMITER + ts; + String baseName = baseDir + ts + SPLIT_STRING + ts; // Begin timestamp and end timestamp are identical since it is a flush - return new LSMComponentFileReferences(createFlushFile(baseName + DELIMITER + BTREE_SUFFIX), - createFlushFile(baseName + DELIMITER + DELETE_TREE_SUFFIX), - createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); + return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING), + createFlushFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING), + createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING)); } @Override public LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName) throws HyracksDataException { - String[] firstTimestampRange = firstFileName.split(DELIMITER); - String[] lastTimestampRange = lastFileName.split(DELIMITER); + String[] firstTimestampRange = firstFileName.split(SPLIT_STRING); + String[] lastTimestampRange = lastFileName.split(SPLIT_STRING); - String baseName = baseDir + firstTimestampRange[0] + DELIMITER + lastTimestampRange[1]; + String baseName = baseDir + firstTimestampRange[0] + SPLIT_STRING + lastTimestampRange[1]; // Get the range of timestamps by taking the earliest and the latest // timestamps - return new LSMComponentFileReferences(createMergeFile(baseName + DELIMITER + BTREE_SUFFIX), - createMergeFile(baseName + DELIMITER + DELETE_TREE_SUFFIX), - createMergeFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); + return new LSMComponentFileReferences(createMergeFile(baseName + SPLIT_STRING + BTREE_STRING), + createMergeFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING), + createMergeFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING)); } @Override @@ -103,7 +106,7 @@ public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager { allBTreeFiles); HashSet btreeFilesSet = new HashSet<>(); for (ComparableFileName cmpFileName : allBTreeFiles) { - int index = cmpFileName.fileName.lastIndexOf(DELIMITER); + int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING); btreeFilesSet.add(cmpFileName.fileName.substring(0, index)); } validateFiles(btreeFilesSet, allBuddyBTreeFiles, getCompoundFilter(buddyBtreeFilter, transactionFilefilter), @@ -203,19 +206,19 @@ public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager { public LSMComponentFileReferences getNewTransactionFileReference() throws IOException { String ts = getCurrentTimestamp(); // Create transaction lock file - Files.createFile(Paths.get(baseDir + TXN_PREFIX + ts)); + Files.createFile(Paths.get(baseDir + TRANSACTION_PREFIX + ts)); - String baseName = baseDir + ts + DELIMITER + ts; - return new LSMComponentFileReferences(createFlushFile(baseName + DELIMITER + BTREE_SUFFIX), - createFlushFile(baseName + DELIMITER + DELETE_TREE_SUFFIX), - createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); + String baseName = baseDir + ts + SPLIT_STRING + ts; + return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING), + createFlushFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING), + createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING)); } @Override public LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException { FilenameFilter transactionFilter; File dir = new File(baseDir); - String[] files = dir.list(txnFileNameFilter); + String[] files = dir.list(transactionFileNameFilter); if (files.length == 0) { return null; } @@ -240,11 +243,11 @@ public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager { File buddyBTreeFile = null; File bloomFilterFile = null; for (String fileName : files) { - if (fileName.endsWith(BTREE_SUFFIX)) { + if (fileName.endsWith(BTREE_STRING)) { bTreeFile = new File(dir.getPath() + File.separator + fileName); - } else if (fileName.endsWith(DELETE_TREE_SUFFIX)) { + } else if (fileName.endsWith(BUDDY_BTREE_STRING)) { buddyBTreeFile = new File(dir.getPath() + File.separator + fileName); - } else if (fileName.endsWith(BLOOM_FILTER_SUFFIX)) { + } else if (fileName.endsWith(BLOOM_FILTER_STRING)) { bloomFilterFile = new File(dir.getPath() + File.separator + fileName); } else { throw new HyracksDataException("unrecognized file found = " + fileName); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java index 792b872..259f25f 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java @@ -52,6 +52,7 @@ import org.apache.hyracks.storage.am.lsm.common.impls.ComponentFilterHelper; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager; import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.storage.common.file.IFileMapProvider; public class LSMBTreeUtil { @@ -59,12 +60,13 @@ public class LSMBTreeUtil { } public static LSMBTree createLSMTree(IIOManager ioManager, List virtualBufferCaches, - FileReference file, IBufferCache diskBufferCache, ITypeTraits[] typeTraits, - IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate, - ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, - ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, ITypeTraits[] filterTypeTraits, - IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields, int[] filterFields, boolean durable, - IMetadataPageManagerFactory freePageManagerFactory) throws HyracksDataException { + FileReference file, IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, + ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields, + double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, + ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, + ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields, + int[] filterFields, boolean durable, IMetadataPageManagerFactory freePageManagerFactory) + throws HyracksDataException { LSMBTreeTupleWriterFactory insertTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, false); LSMBTreeTupleWriterFactory deleteTupleWriterFactory = @@ -80,14 +82,15 @@ public class LSMBTreeUtil { ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory); ITreeIndexFrameFactory bulkLoadLeafFrameFactory = new BTreeNSMLeafFrameFactory(bulkLoadTupleWriterFactory); - TreeIndexFactory diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory, - interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length); + TreeIndexFactory diskBTreeFactory = + new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, + interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length); TreeIndexFactory bulkLoadBTreeFactory = - new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory, interiorFrameFactory, - bulkLoadLeafFrameFactory, cmpFactories, typeTraits.length); + new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, + interiorFrameFactory, bulkLoadLeafFrameFactory, cmpFactories, typeTraits.length); - BloomFilterFactory bloomFilterFactory = - needKeyDupCheck ? new BloomFilterFactory(diskBufferCache, bloomFilterKeyFields) : null; + BloomFilterFactory bloomFilterFactory = needKeyDupCheck + ? new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields) : null; ComponentFilterHelper filterHelper = null; LSMComponentFilterFrameFactory filterFrameFactory = null; @@ -101,20 +104,20 @@ public class LSMBTreeUtil { //Primary LSMBTree index has a BloomFilter. ILSMIndexFileManager fileNameManager = - new LSMBTreeFileManager(ioManager, file, diskBTreeFactory, needKeyDupCheck); + new LSMBTreeFileManager(ioManager, diskFileMapProvider, file, diskBTreeFactory, needKeyDupCheck); return new LSMBTree(ioManager, virtualBufferCaches, interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory, fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory, - filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, typeTraits.length, - cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, needKeyDupCheck, btreeFields, - filterFields, durable); + filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, diskFileMapProvider, + typeTraits.length, cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, needKeyDupCheck, + btreeFields, filterFields, durable); } public static ExternalBTree createExternalBTree(IIOManager ioManager, FileReference file, - IBufferCache diskBufferCache, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories, - int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, - ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, - boolean durable, IMetadataPageManagerFactory freePageManagerFactory) { + IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits, + IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate, + ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, + ILSMIOOperationCallback ioOpCallback, boolean durable, IMetadataPageManagerFactory freePageManagerFactory) { LSMBTreeTupleWriterFactory insertTupleWriterFactory = new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, false); LSMBTreeTupleWriterFactory deleteTupleWriterFactory = @@ -133,31 +136,35 @@ public class LSMBTreeUtil { ITreeIndexFrameFactory transactionLeafFrameFactory = new BTreeNSMLeafFrameFactory(transactionTupleWriterFactory); - TreeIndexFactory diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory, - interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length); - TreeIndexFactory bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, + TreeIndexFactory diskBTreeFactory = + new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, + interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length); + TreeIndexFactory bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, interiorFrameFactory, insertLeafFrameFactory, cmpFactories, typeTraits.length); - BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, bloomFilterKeyFields); + BloomFilterFactory bloomFilterFactory = + new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields); // This is the component factory for transactions TreeIndexFactory transactionBTreeFactory = - new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory, interiorFrameFactory, - transactionLeafFrameFactory, cmpFactories, typeTraits.length); + new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, + interiorFrameFactory, transactionLeafFrameFactory, cmpFactories, typeTraits.length); //TODO remove BloomFilter from external dataset's secondary LSMBTree index - ILSMIndexFileManager fileNameManager = new LSMBTreeFileManager(ioManager, file, diskBTreeFactory, true); + ILSMIndexFileManager fileNameManager = + new LSMBTreeFileManager(ioManager, diskFileMapProvider, file, diskBTreeFactory, true); // the disk only index uses an empty ArrayList for virtual buffer caches - return new ExternalBTree(ioManager, interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory, - fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory, - bloomFilterFalsePositiveRate, cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, - transactionBTreeFactory, durable); + ExternalBTree lsmTree = new ExternalBTree(ioManager, interiorFrameFactory, insertLeafFrameFactory, + deleteLeafFrameFactory, fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory, + bloomFilterFalsePositiveRate, diskFileMapProvider, typeTraits.length, cmpFactories, mergePolicy, + opTracker, ioScheduler, ioOpCallback, transactionBTreeFactory, durable); + return lsmTree; } public static ExternalBTreeWithBuddy createExternalBTreeWithBuddy(IIOManager ioManager, FileReference file, - IBufferCache diskBufferCache, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories, - double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, - ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] buddyBTreeFields, - boolean durable, IMetadataPageManagerFactory freePageManagerFactory) { + IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits, + IBinaryComparatorFactory[] cmpFactories, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, + ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, + int[] buddyBTreeFields, boolean durable, IMetadataPageManagerFactory freePageManagerFactory) { ITypeTraits[] buddyBtreeTypeTraits = new ITypeTraits[buddyBTreeFields.length]; IBinaryComparatorFactory[] buddyBtreeCmpFactories = new IBinaryComparatorFactory[buddyBTreeFields.length]; for (int i = 0; i < buddyBtreeTypeTraits.length; i++) { @@ -177,30 +184,32 @@ public class LSMBTreeUtil { ITreeIndexFrameFactory insertLeafFrameFactory = new BTreeNSMLeafFrameFactory(insertTupleWriterFactory); ITreeIndexFrameFactory copyTupleLeafFrameFactory = new BTreeNSMLeafFrameFactory(copyTupleWriterFactory); ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory); - TreeIndexFactory diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory, - interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length); + TreeIndexFactory diskBTreeFactory = + new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, + interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length); - TreeIndexFactory bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, + TreeIndexFactory bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory, interiorFrameFactory, insertLeafFrameFactory, cmpFactories, typeTraits.length); int[] bloomFilterKeyFields = new int[buddyBtreeCmpFactories.length]; for (int i = 0; i < buddyBtreeCmpFactories.length; i++) { bloomFilterKeyFields[i] = i; } - BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, bloomFilterKeyFields); + BloomFilterFactory bloomFilterFactory = + new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields); // buddy b-tree factory - TreeIndexFactory diskBuddyBTreeFactory = - new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory, buddyBtreeInteriorFrameFactory, - buddyBtreeLeafFrameFactory, buddyBtreeCmpFactories, buddyBtreeTypeTraits.length); + TreeIndexFactory diskBuddyBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, + diskFileMapProvider, freePageManagerFactory, buddyBtreeInteriorFrameFactory, buddyBtreeLeafFrameFactory, + buddyBtreeCmpFactories, buddyBtreeTypeTraits.length); - ILSMIndexFileManager fileNameManager = - new LSMBTreeWithBuddyFileManager(ioManager, file, diskBTreeFactory, diskBuddyBTreeFactory); + ILSMIndexFileManager fileNameManager = new LSMBTreeWithBuddyFileManager(ioManager, diskFileMapProvider, file, + diskBTreeFactory, diskBuddyBTreeFactory); // the disk only index uses an empty ArrayList for virtual buffer caches ExternalBTreeWithBuddy lsmTree = new ExternalBTreeWithBuddy(ioManager, interiorFrameFactory, insertLeafFrameFactory, buddyBtreeLeafFrameFactory, diskBufferCache, fileNameManager, - bulkLoadBTreeFactory, diskBTreeFactory, diskBuddyBTreeFactory, bloomFilterFactory, + bulkLoadBTreeFactory, diskBTreeFactory, diskBuddyBTreeFactory, bloomFilterFactory, diskFileMapProvider, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, cmpFactories, buddyBtreeCmpFactories, buddyBTreeFields, durable); return lsmTree; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml index 37e1c49..bc1e146 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml @@ -82,9 +82,8 @@ commons-lang3 - org.apache.hyracks - hyracks-util - ${project.version} + commons-io + commons-io \ No newline at end of file http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java index 18d9a67..54d64af 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java @@ -34,28 +34,28 @@ import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences * merge). */ public interface ILSMIndexFileManager { - void createDirs() throws HyracksDataException; + public void createDirs(); - void deleteDirs() throws HyracksDataException; + public void deleteDirs() throws HyracksDataException; - LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException; + public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException; - LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName) + public LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName) throws HyracksDataException; - String getBaseDir(); + public String getBaseDir(); // Deletes invalid files, and returns list of valid files from baseDir. // The returned valid files are correctly sorted (based on the recency of data). - List cleanupAndGetValidFiles() throws HyracksDataException; + public List cleanupAndGetValidFiles() throws HyracksDataException; - Comparator getFileNameComparator(); + public Comparator getFileNameComparator(); /** * @return delete existing transaction disk component file reference * @throws HyracksDataException */ - void deleteTransactionFiles() throws HyracksDataException; + public void deleteTransactionFiles() throws HyracksDataException; /** * Rename files of a transaction removing the transaction prefix and return the component file reference in order to be committed @@ -63,18 +63,18 @@ public interface ILSMIndexFileManager { * @return the renamed component file references * @throws HyracksDataException */ - LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException; + public LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException; /** * Recover transaction files without returning them * * @throws HyracksDataException */ - void recoverTransaction() throws HyracksDataException; + public void recoverTransaction() throws HyracksDataException; /** * @return a reference to the transaction disk component file reference * @throws IOException */ - LSMComponentFileReferences getNewTransactionFileReference() throws IOException; + public LSMComponentFileReferences getNewTransactionFileReference() throws IOException; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java index 72d5423..090ca68 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java @@ -23,6 +23,6 @@ import org.apache.hyracks.api.io.IIOManager; public class AbortRecoverLSMIndexFileManager extends AbstractLSMIndexFileManager { public AbortRecoverLSMIndexFileManager(IIOManager ioManager, FileReference file) { - super(ioManager, file, null); + super(ioManager, null, file, null); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java index c5bf2ca..2be2184 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java @@ -63,6 +63,7 @@ import org.apache.hyracks.storage.common.IIndexCursor; import org.apache.hyracks.storage.common.IModificationOperationCallback; import org.apache.hyracks.storage.common.ISearchOperationCallback; import org.apache.hyracks.storage.common.buffercache.IBufferCache; +import org.apache.hyracks.storage.common.file.IFileMapProvider; public abstract class AbstractLSMIndex implements ILSMIndex { protected final ILSMHarness lsmHarness; @@ -74,9 +75,11 @@ public abstract class AbstractLSMIndex implements ILSMIndex { protected final List memoryComponents; protected final List virtualBufferCaches; protected AtomicInteger currentMutableComponentId; + // On-disk components. protected final IBufferCache diskBufferCache; protected final ILSMIndexFileManager fileManager; + protected final IFileMapProvider diskFileMapProvider; // components with lower indexes are newer than components with higher index protected final List diskComponents; protected final List inactiveDiskComponents; @@ -92,14 +95,16 @@ public abstract class AbstractLSMIndex implements ILSMIndex { protected boolean memoryComponentsAllocated = false; public AbstractLSMIndex(IIOManager ioManager, List virtualBufferCaches, - IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, double bloomFilterFalsePositiveRate, - ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, - ILSMIOOperationCallback ioOpCallback, ILSMComponentFilterFrameFactory filterFrameFactory, - LSMComponentFilterManager filterManager, int[] filterFields, boolean durable, - IComponentFilterHelper filterHelper, int[] treeFields) { + IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider, + double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, + ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, + ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager, + int[] filterFields, boolean durable, IComponentFilterHelper filterHelper, + int[] treeFields) { this.ioManager = ioManager; this.virtualBufferCaches = virtualBufferCaches; this.diskBufferCache = diskBufferCache; + this.diskFileMapProvider = diskFileMapProvider; this.fileManager = fileManager; this.bloomFilterFalsePositiveRate = bloomFilterFalsePositiveRate; this.ioScheduler = ioScheduler; @@ -125,10 +130,12 @@ public abstract class AbstractLSMIndex implements ILSMIndex { // The constructor used by external indexes public AbstractLSMIndex(IIOManager ioManager, IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, - double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, - ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean durable) { + IFileMapProvider diskFileMapProvider, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, + ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, + boolean durable) { this.ioManager = ioManager; this.diskBufferCache = diskBufferCache; + this.diskFileMapProvider = diskFileMapProvider; this.fileManager = fileManager; this.bloomFilterFalsePositiveRate = bloomFilterFalsePositiveRate; this.ioScheduler = ioScheduler; @@ -155,6 +162,7 @@ public abstract class AbstractLSMIndex implements ILSMIndex { if (isActive) { throw HyracksDataException.create(ErrorCode.CANNOT_CREATE_ACTIVE_INDEX); } + fileManager.deleteDirs(); fileManager.createDirs(); diskComponents.clear(); } @@ -230,9 +238,16 @@ public abstract class AbstractLSMIndex implements ILSMIndex { throw HyracksDataException.create(ErrorCode.CANNOT_DESTROY_ACTIVE_INDEX); } destroyDiskComponents(); + destroyMemoryComponents(); fileManager.deleteDirs(); } + protected void destroyMemoryComponents() throws HyracksDataException { + for (ILSMMemoryComponent c : memoryComponents) { + destroyMemoryComponent(c); + } + } + protected void destroyDiskComponents() throws HyracksDataException { List immutableComponents = diskComponents; for (ILSMDiskComponent c : immutableComponents) { @@ -638,6 +653,8 @@ public abstract class AbstractLSMIndex implements ILSMIndex { protected abstract void destroyDiskComponent(ILSMDiskComponent c) throws HyracksDataException; + protected abstract void destroyMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException; + protected abstract void clearDiskComponent(ILSMDiskComponent c) throws HyracksDataException; protected abstract void clearMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException;