asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From buyin...@apache.org
Subject [6/7] asterixdb git commit: Revert "ASTERIXDB-1945 [STO] Cleanup Buffer Cache API"
Date Mon, 19 Jun 2017 22:46:43 GMT
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/build/IndexBuilder.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/build/IndexBuilder.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/build/IndexBuilder.java
index 9f648c5..f1ae6b0 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/build/IndexBuilder.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/build/IndexBuilder.java
@@ -19,13 +19,10 @@
 package org.apache.hyracks.storage.am.common.build;
 
 import java.io.IOException;
-import java.util.logging.Level;
-import java.util.logging.Logger;
 
 import org.apache.hyracks.api.application.INCServiceContext;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.api.util.IoUtil;
 import org.apache.hyracks.storage.am.common.api.IIndexBuilder;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
 import org.apache.hyracks.storage.common.IIndex;
@@ -38,7 +35,6 @@ import org.apache.hyracks.storage.common.LocalResource;
 import org.apache.hyracks.storage.common.file.IResourceIdFactory;
 
 public class IndexBuilder implements IIndexBuilder {
-    private static final Logger LOGGER = Logger.getLogger(IndexBuilder.class.getName());
 
     protected final INCServiceContext ctx;
     protected final IStorageManager storageManager;
@@ -85,22 +81,8 @@ public class IndexBuilder implements IIndexBuilder {
                 //The reason for this is to handle many cases such as:
                 //1. Crash while delete index is running (we don't do global cleanup on restart)
                 //2. Node leaves and then join with old data
-                LOGGER.log(Level.WARNING,
-                        "Removing existing index on index create for the index: " + resourceRef.getRelativePath());
                 lcManager.unregister(resourceRef.getRelativePath());
-                index.destroy();
             } else {
-                if (resourceRef.getFile().exists()) {
-                    // Index is not registered but the index file exists
-                    // This is another big problem that we need to disallow soon
-                    // We can only disallow this if we have a global cleanup after crash
-                    // on reboot
-                    LOGGER.log(Level.WARNING,
-                            "Deleting " + resourceRef.getRelativePath()
-                                    + " on index create. The index is not registered"
-                                    + " but the file exists in the filesystem");
-                    IoUtil.delete(resourceRef);
-                }
                 index = resource.createInstance(ctx);
             }
             index.create();

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java
index cc3bfe7..c00cecb 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java
@@ -37,6 +37,7 @@ import org.apache.hyracks.storage.am.common.util.TreeIndexStatsGatherer;
 import org.apache.hyracks.storage.common.IStorageManager;
 import org.apache.hyracks.storage.common.LocalResource;
 import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
 
 public class TreeIndexStatsOperatorNodePushable extends AbstractUnaryOutputSourceOperatorNodePushable {
     private final IHyracksTaskContext ctx;
@@ -68,11 +69,14 @@ public class TreeIndexStatsOperatorNodePushable extends AbstractUnaryOutputSourc
         try {
             writer.open();
             IBufferCache bufferCache = storageManager.getBufferCache(ctx.getJobletContext().getServiceContext());
+            IFileMapProvider fileMapProvider =
+                    storageManager.getFileMapProvider(ctx.getJobletContext().getServiceContext());
             LocalResource resource = treeIndexHelper.getResource();
             IIOManager ioManager = ctx.getIoManager();
             FileReference fileRef = ioManager.resolve(resource.getPath());
+            int indexFileId = fileMapProvider.lookupFileId(fileRef);
             TreeIndexStatsGatherer statsGatherer = new TreeIndexStatsGatherer(bufferCache, treeIndex.getPageManager(),
-                    fileRef, treeIndex.getRootPageId());
+                    indexFileId, treeIndex.getRootPageId());
             TreeIndexStats stats = statsGatherer.gatherStats(treeIndex.getLeafFrameFactory().createFrame(),
                     treeIndex.getInteriorFrameFactory().createFrame(),
                     treeIndex.getPageManager().createMetadataFrame());

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
index 8ffd0eb..163a3d1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
@@ -40,6 +40,7 @@ import org.apache.hyracks.storage.common.buffercache.IBufferCache;
 import org.apache.hyracks.storage.common.buffercache.ICachedPage;
 import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue;
 import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
 
 public abstract class AbstractTreeIndex implements ITreeIndex {
 
@@ -48,6 +49,7 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
     protected int rootPage = 1;
 
     protected final IBufferCache bufferCache;
+    protected final IFileMapProvider fileMapProvider;
     protected final IPageManager freePageManager;
 
     protected final ITreeIndexFrameFactory interiorFrameFactory;
@@ -57,16 +59,21 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
     protected final int fieldCount;
 
     protected FileReference file;
-    private int fileId = -1;
+    protected int fileId = -1;
 
     protected boolean isActive = false;
+    //hasEverBeenActivated is to stop the throwing of an exception of deactivating an index that
+    //was never activated or failed to activate in try/finally blocks, as there's no way to know if
+    //an index is activated or not from the outside.
+    protected boolean hasEverBeenActivated = false;
 
     protected int bulkloadLeafStart = 0;
 
-    public AbstractTreeIndex(IBufferCache bufferCache, IPageManager freePageManager,
+    public AbstractTreeIndex(IBufferCache bufferCache, IFileMapProvider fileMapProvider, IPageManager freePageManager,
             ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory,
             IBinaryComparatorFactory[] cmpFactories, int fieldCount, FileReference file) {
         this.bufferCache = bufferCache;
+        this.fileMapProvider = fileMapProvider;
         this.freePageManager = freePageManager;
         this.interiorFrameFactory = interiorFrameFactory;
         this.leafFrameFactory = leafFrameFactory;
@@ -80,8 +87,9 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
         if (isActive) {
             throw HyracksDataException.create(ErrorCode.CANNOT_CREATE_ACTIVE_INDEX);
         }
-        fileId = bufferCache.createFile(file);
-        bufferCache.openFile(fileId);
+        synchronized (fileMapProvider) {
+            fileId = createAndOpen(bufferCache, fileMapProvider, file);
+        }
         freePageManager.open(fileId);
         freePageManager.init(interiorFrameFactory, leafFrameFactory);
         setRootPage();
@@ -89,6 +97,27 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
         bufferCache.closeFile(fileId);
     }
 
+    public static int createAndOpen(IBufferCache bufferCache, IFileMapProvider fileMapProvider, FileReference file)
+            throws HyracksDataException {
+        int fileId;
+        boolean fileIsMapped = fileMapProvider.isMapped(file);
+        if (!fileIsMapped) {
+            bufferCache.createFile(file);
+        }
+        fileId = fileMapProvider.lookupFileId(file);
+        try {
+            // Also creates the file if it doesn't exist yet.
+            bufferCache.openFile(fileId);
+        } catch (HyracksDataException e) {
+            // Revert state of buffer cache since file failed to open.
+            if (!fileIsMapped) {
+                bufferCache.deleteFile(fileId, false);
+            }
+            throw e;
+        }
+        return fileId;
+    }
+
     private void setRootPage() throws HyracksDataException {
         rootPage = freePageManager.getRootPageId();
         bulkloadLeafStart = freePageManager.getBulkLoadLeaf();
@@ -99,35 +128,49 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
         if (isActive) {
             throw HyracksDataException.create(ErrorCode.CANNOT_ACTIVATE_ACTIVE_INDEX);
         }
-        if (fileId >= 0) {
-            bufferCache.openFile(fileId);
-        } else {
-            fileId = bufferCache.openFile(file);
+        boolean fileIsMapped = false;
+        synchronized (fileMapProvider) {
+            fileIsMapped = fileMapProvider.isMapped(file);
+            if (!fileIsMapped) {
+                bufferCache.createFile(file);
+            }
+            fileId = fileMapProvider.lookupFileId(file);
+            try {
+                // Also creates the file if it doesn't exist yet.
+                bufferCache.openFile(fileId);
+            } catch (HyracksDataException e) {
+                // Revert state of buffer cache since file failed to open.
+                if (!fileIsMapped) {
+                    bufferCache.deleteFile(fileId, false);
+                }
+                throw e;
+            }
         }
         freePageManager.open(fileId);
         setRootPage();
         // TODO: Should probably have some way to check that the tree is physically consistent
         // or that the file we just opened actually is a tree
         isActive = true;
+        hasEverBeenActivated = true;
     }
 
     @Override
     public synchronized void deactivate() throws HyracksDataException {
-        if (!isActive) {
+        if (!isActive && hasEverBeenActivated) {
             throw HyracksDataException.create(ErrorCode.CANNOT_DEACTIVATE_INACTIVE_INDEX);
         }
-        freePageManager.close();
-        bufferCache.closeFile(fileId);
+        if (isActive) {
+            freePageManager.close();
+            bufferCache.closeFile(fileId);
+        }
+
         isActive = false;
     }
 
-    public void purge() throws HyracksDataException {
-        if (isActive) {
-            throw HyracksDataException.create(ErrorCode.CANNOT_PURGE_ACTIVE_INDEX);
-        }
+    public synchronized void deactivateCloseHandle() throws HyracksDataException {
+        deactivate();
         bufferCache.purgeHandle(fileId);
-        // after purging, the fileId has no mapping and no meaning
-        fileId = -1;
+
     }
 
     @Override
@@ -135,7 +178,13 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
         if (isActive) {
             throw HyracksDataException.create(ErrorCode.CANNOT_DESTROY_ACTIVE_INDEX);
         }
-        bufferCache.deleteFile(file);
+
+        if (fileId == -1) {
+            return;
+        }
+        bufferCache.deleteFile(fileId, false);
+        file.delete();
+        fileId = -1;
     }
 
     @Override
@@ -235,7 +284,7 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
             queue = bufferCache.createFIFOQueue();
 
             if (!isEmptyTree(leafFrame)) {
-                throw HyracksDataException.create(ErrorCode.CANNOT_BULK_LOAD_NON_EMPTY_TREE);
+                throw new HyracksDataException("Cannot bulk-load a non-empty tree.");
             }
 
             this.cmp = MultiComparator.create(cmpFactories);
@@ -263,6 +312,9 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
             pagesToWrite = new ArrayList<>();
         }
 
+        @Override
+        public abstract void add(ITupleReference tuple) throws HyracksDataException;
+
         protected void handleException() throws HyracksDataException {
             // Unlatch and unpin pages that weren't in the queue to avoid leaking memory.
             for (NodeFrontier nodeFrontier : nodeFrontiers) {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
index 959f27a..2b47238 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
@@ -22,7 +22,7 @@ import java.util.ArrayList;
 import java.util.Random;
 
 import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.util.MathUtil;
 import org.apache.hyracks.storage.am.common.api.IMetadataPageManager;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrame;
@@ -30,25 +30,25 @@ import org.apache.hyracks.storage.common.arraylist.IntArrayList;
 import org.apache.hyracks.storage.common.buffercache.IBufferCache;
 import org.apache.hyracks.storage.common.buffercache.ICachedPage;
 import org.apache.hyracks.storage.common.file.BufferedFileHandle;
-import org.apache.hyracks.util.MathUtil;
 
 public class TreeIndexBufferCacheWarmup {
     private final IBufferCache bufferCache;
     private final IMetadataPageManager freePageManager;
-    private final FileReference fileRef;
+    private final int fileId;
     private final ArrayList<IntArrayList> pagesByLevel = new ArrayList<>();
     private final Random rnd = new Random();
 
-    public TreeIndexBufferCacheWarmup(IBufferCache bufferCache, IMetadataPageManager freePageManager,
-            FileReference fileRef) {
+    public TreeIndexBufferCacheWarmup(IBufferCache bufferCache,
+            IMetadataPageManager freePageManager, int fileId) {
         this.bufferCache = bufferCache;
         this.freePageManager = freePageManager;
-        this.fileRef = fileRef;
+        this.fileId = fileId;
     }
 
-    public void warmup(ITreeIndexFrame frame, ITreeIndexMetadataFrame metaFrame, int[] warmupTreeLevels,
+    public void warmup(ITreeIndexFrame frame,
+            ITreeIndexMetadataFrame metaFrame, int[] warmupTreeLevels,
             int[] warmupRepeats) throws HyracksDataException {
-        int fileId = bufferCache.openFile(fileRef);
+        bufferCache.openFile(fileId);
 
         // scan entire file to determine pages in each level
         int maxPageId = freePageManager.getMaxPageId(metaFrame);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
index 5410282..7e14b4a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
@@ -19,7 +19,6 @@
 package org.apache.hyracks.storage.am.common.util;
 
 import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
 import org.apache.hyracks.storage.am.common.api.IPageManager;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexMetadataFrame;
@@ -32,21 +31,25 @@ public class TreeIndexStatsGatherer {
     private final TreeIndexStats treeIndexStats = new TreeIndexStats();
     private final IBufferCache bufferCache;
     private final IPageManager freePageManager;
-    private final FileReference fileRef;
+    private final int fileId;
     private final int rootPage;
 
-    public TreeIndexStatsGatherer(IBufferCache bufferCache, IPageManager freePageManager, FileReference fileRef,
-            int rootPage) {
+    public TreeIndexStatsGatherer(IBufferCache bufferCache,
+            IPageManager freePageManager, int fileId, int rootPage) {
         this.bufferCache = bufferCache;
         this.freePageManager = freePageManager;
-        this.fileRef = fileRef;
+        this.fileId = fileId;
         this.rootPage = rootPage;
     }
 
-    public TreeIndexStats gatherStats(ITreeIndexFrame leafFrame, ITreeIndexFrame interiorFrame,
-            ITreeIndexMetadataFrame metaFrame) throws HyracksDataException {
-        int fileId = bufferCache.openFile(fileRef);
+    public TreeIndexStats gatherStats(ITreeIndexFrame leafFrame,
+            ITreeIndexFrame interiorFrame, ITreeIndexMetadataFrame metaFrame)
+            throws HyracksDataException {
+
+        bufferCache.openFile(fileId);
+
         treeIndexStats.begin();
+
         int maxPageId = freePageManager.getMaxPageId(metaFrame);
         for (int pageId = 0; pageId <= maxPageId; pageId++) {
             ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResource.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResource.java
index f1e9cf1..6860b97 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResource.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeLocalResource.java
@@ -57,9 +57,9 @@ public class ExternalBTreeLocalResource extends LSMBTreeLocalResource {
     public ILSMIndex createInstance(INCServiceContext serviceCtx) throws HyracksDataException {
         IIOManager ioManager = serviceCtx.getIoManager();
         FileReference file = ioManager.resolve(path);
-        return LSMBTreeUtil.createExternalBTree(ioManager, file, storageManager.getBufferCache(serviceCtx), typeTraits,
-                cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate,
-                mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx),
+        return LSMBTreeUtil.createExternalBTree(ioManager, file, storageManager.getBufferCache(serviceCtx),
+                storageManager.getFileMapProvider(serviceCtx), typeTraits, cmpFactories, bloomFilterKeyFields,
+                bloomFilterFalsePositiveRate, mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx),
                 opTrackerProvider.getOperationTracker(serviceCtx), ioSchedulerProvider.getIoScheduler(serviceCtx),
                 ioOpCallbackFactory.createIoOpCallback(), durable, metadataPageManagerFactory);
     }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResource.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResource.java
index 317a479..3819066 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResource.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/ExternalBTreeWithBuddyLocalResource.java
@@ -61,7 +61,7 @@ public class ExternalBTreeWithBuddyLocalResource extends LSMBTreeLocalResource {
         IIOManager ioManager = serviceCtx.getIoManager();
         FileReference file = ioManager.resolve(path);
         return LSMBTreeUtil.createExternalBTreeWithBuddy(ioManager, file, storageManager.getBufferCache(serviceCtx),
-                typeTraits, cmpFactories, bloomFilterFalsePositiveRate,
+                storageManager.getFileMapProvider(serviceCtx), typeTraits, cmpFactories, bloomFilterFalsePositiveRate,
                 mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx),
                 opTrackerProvider.getOperationTracker(serviceCtx), ioSchedulerProvider.getIoScheduler(serviceCtx),
                 ioOpCallbackFactory.createIoOpCallback(), bloomFilterKeyFields, durable, metadataPageManagerFactory);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java
index f6e311c..6416f6b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/dataflow/LSMBTreeLocalResource.java
@@ -70,9 +70,9 @@ public class LSMBTreeLocalResource extends LsmResource {
         IIOManager ioManager = serviceCtx.getIoManager();
         FileReference file = ioManager.resolve(path);
         List<IVirtualBufferCache> vbcs = vbcProvider.getVirtualBufferCaches(serviceCtx, file);
-        return LSMBTreeUtil.createLSMTree(ioManager, vbcs, file, storageManager.getBufferCache(serviceCtx), typeTraits,
-                cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate,
-                mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx),
+        return LSMBTreeUtil.createLSMTree(ioManager, vbcs, file, storageManager.getBufferCache(serviceCtx),
+                storageManager.getFileMapProvider(serviceCtx), typeTraits, cmpFactories, bloomFilterKeyFields,
+                bloomFilterFalsePositiveRate, mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx),
                 opTrackerProvider.getOperationTracker(serviceCtx), ioSchedulerProvider.getIoScheduler(serviceCtx),
                 ioOpCallbackFactory.createIoOpCallback(), isPrimary, filterTypeTraits, filterCmpFactories, btreeFields,
                 filterFields, durable, metadataPageManagerFactory);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
index c06edfd..cdde692 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
@@ -62,6 +62,7 @@ import org.apache.hyracks.storage.common.IIndexCursor;
 import org.apache.hyracks.storage.common.IModificationOperationCallback;
 import org.apache.hyracks.storage.common.ISearchOperationCallback;
 import org.apache.hyracks.storage.common.ISearchPredicate;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
 
 /**
  * This is an lsm b-tree that does not have memory component and is modified
@@ -92,12 +93,13 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
             ITreeIndexFrameFactory insertLeafFrameFactory, ITreeIndexFrameFactory deleteLeafFrameFactory,
             ILSMIndexFileManager fileManager, TreeIndexFactory<BTree> diskBTreeFactory,
             TreeIndexFactory<BTree> bulkLoadBTreeFactory, BloomFilterFactory bloomFilterFactory,
-            double bloomFilterFalsePositiveRate, IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy,
-            ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+            double bloomFilterFalsePositiveRate, IFileMapProvider diskFileMapProvider, int fieldCount,
+            IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+            ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
             TreeIndexFactory<BTree> transactionBTreeFactory, boolean durable) {
         super(ioManager, insertLeafFrameFactory, deleteLeafFrameFactory, fileManager, diskBTreeFactory,
-                bulkLoadBTreeFactory, bloomFilterFactory, bloomFilterFalsePositiveRate, cmpFactories, mergePolicy,
-                opTracker, ioScheduler, ioOpCallback, false, durable);
+                bulkLoadBTreeFactory, bloomFilterFactory, bloomFilterFalsePositiveRate, diskFileMapProvider, fieldCount,
+                cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, false, durable);
         this.transactionComponentFactory =
                 new LSMBTreeDiskComponentFactory(transactionBTreeFactory, bloomFilterFactory, null);
         this.secondDiskComponents = new LinkedList<>();
@@ -299,13 +301,21 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
             BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(ioOpCallback);
             cb.afterFinalize(LSMOperationType.FLUSH, null);
         }
-        for (ILSMDiskComponent c : diskComponents) {
-            deactivateDiskComponent(c);
+        for (ILSMComponent c : diskComponents) {
+            LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
+            BTree btree = component.getBTree();
+            BloomFilter bloomFilter = component.getBloomFilter();
+            btree.deactivateCloseHandle();
+            bloomFilter.deactivate();
         }
-        for (ILSMDiskComponent c : secondDiskComponents) {
-            // Only deactivate non shared components
+        for (ILSMComponent c : secondDiskComponents) {
+            // Only deactivate non shared components (So components are not de-activated twice)
             if (!diskComponents.contains(c)) {
-                deactivateDiskComponent(c);
+                LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
+                BTree btree = component.getBTree();
+                BloomFilter bloomFilter = component.getBloomFilter();
+                btree.deactivateCloseHandle();
+                bloomFilter.deactivate();
             }
         }
         isActive = false;
@@ -347,13 +357,17 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
         if (isActive) {
             throw new HyracksDataException("Failed to destroy the index since it is activated.");
         }
-        for (ILSMDiskComponent c : diskComponents) {
-            destroyDiskComponent(c);
+        for (ILSMComponent c : diskComponents) {
+            LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
+            component.getBTree().destroy();
+            component.getBloomFilter().destroy();
             // Remove from second list to avoid destroying twice
             secondDiskComponents.remove(c);
         }
-        for (ILSMDiskComponent c : secondDiskComponents) {
-            destroyDiskComponent(c);
+        for (ILSMComponent c : secondDiskComponents) {
+            LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
+            component.getBTree().destroy();
+            component.getBloomFilter().destroy();
         }
         diskComponents.clear();
         secondDiskComponents.clear();

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
index 9959583..1c99b17 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
@@ -72,6 +72,7 @@ import org.apache.hyracks.storage.common.ISearchOperationCallback;
 import org.apache.hyracks.storage.common.ISearchPredicate;
 import org.apache.hyracks.storage.common.MultiComparator;
 import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
 
 public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeIndex, ITwoPCIndex {
 
@@ -99,12 +100,12 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
             IBufferCache diskBufferCache, ILSMIndexFileManager fileManager,
             TreeIndexFactory<BTree> bulkLoadBTreeFactory, TreeIndexFactory<BTree> copyBtreeFactory,
             TreeIndexFactory<BTree> buddyBtreeFactory, BloomFilterFactory bloomFilterFactory,
-            double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
-            ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+            IFileMapProvider diskFileMapProvider, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
+            ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
             IBinaryComparatorFactory[] btreeCmpFactories, IBinaryComparatorFactory[] buddyBtreeCmpFactories,
             int[] buddyBTreeFields, boolean durable) {
-        super(ioManager, diskBufferCache, fileManager, bloomFilterFalsePositiveRate, mergePolicy, opTracker,
-                ioScheduler, ioOpCallback, durable);
+        super(ioManager, diskBufferCache, fileManager, diskFileMapProvider, bloomFilterFalsePositiveRate, mergePolicy,
+                opTracker, ioScheduler, ioOpCallback, durable);
         this.btreeCmpFactories = btreeCmpFactories;
         this.buddyBtreeCmpFactories = buddyBtreeCmpFactories;
         this.buddyBTreeFields = buddyBTreeFields;
@@ -120,7 +121,12 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
 
     @Override
     public void create() throws HyracksDataException {
-        super.create();
+        if (isActive) {
+            throw new HyracksDataException("Failed to create the index since it is activated.");
+        }
+        fileManager.deleteDirs();
+        fileManager.createDirs();
+        diskComponents.clear();
         secondDiskComponents.clear();
     }
 
@@ -767,12 +773,14 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
         BTree btree = component.getBTree();
         BTree buddyBtree = component.getBuddyBTree();
         BloomFilter bloomFilter = component.getBloomFilter();
-        btree.deactivate();
-        buddyBtree.deactivate();
+        btree.deactivateCloseHandle();
+        buddyBtree.deactivateCloseHandle();
         bloomFilter.deactivate();
-        btree.purge();
-        buddyBtree.purge();
-        bloomFilter.purge();
+    }
+
+    @Override
+    protected void destroyMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+        //do nothing since external index never use memory components
     }
 
     @Override

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index a5c59d6..e830b3e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -76,6 +76,7 @@ import org.apache.hyracks.storage.common.ISearchOperationCallback;
 import org.apache.hyracks.storage.common.ISearchPredicate;
 import org.apache.hyracks.storage.common.MultiComparator;
 import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
 
 public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
 
@@ -102,11 +103,11 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
             TreeIndexFactory<BTree> diskBTreeFactory, TreeIndexFactory<BTree> bulkLoadBTreeFactory,
             BloomFilterFactory bloomFilterFactory, IComponentFilterHelper filterHelper,
             ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
-            double bloomFilterFalsePositiveRate, int fieldCount, IBinaryComparatorFactory[] cmpFactories,
-            ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
-            ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, int[] btreeFields, int[] filterFields,
-            boolean durable) throws HyracksDataException {
-        super(ioManager, virtualBufferCaches, diskBTreeFactory.getBufferCache(), fileManager,
+            double bloomFilterFalsePositiveRate, IFileMapProvider diskFileMapProvider, int fieldCount,
+            IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+            ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck,
+            int[] btreeFields, int[] filterFields, boolean durable) throws HyracksDataException {
+        super(ioManager, virtualBufferCaches, diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider,
                 bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory,
                 filterManager, filterFields, durable, filterHelper, btreeFields);
         this.insertLeafFrameFactory = insertLeafFrameFactory;
@@ -115,7 +116,8 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
         int i = 0;
         for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) {
             LSMBTreeMemoryComponent mutableComponent = new LSMBTreeMemoryComponent(
-                    new BTree(virtualBufferCache, new VirtualFreePageManager(virtualBufferCache), interiorFrameFactory,
+                    new BTree(virtualBufferCache, virtualBufferCache.getFileMapProvider(),
+                            new VirtualFreePageManager(virtualBufferCache), interiorFrameFactory,
                             insertLeafFrameFactory, cmpFactories, fieldCount,
                             ioManager.resolveAbsolutePath(fileManager.getBaseDir() + "_virtual_" + i)),
                     virtualBufferCache, i == 0 ? true : false,
@@ -135,18 +137,19 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
             ITreeIndexFrameFactory deleteLeafFrameFactory, ILSMIndexFileManager fileManager,
             TreeIndexFactory<BTree> diskBTreeFactory, TreeIndexFactory<BTree> bulkLoadBTreeFactory,
             BloomFilterFactory bloomFilterFactory, double bloomFilterFalsePositiveRate,
-            IBinaryComparatorFactory[] cmpFactories, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
-            ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck,
-            boolean durable) {
-        super(ioManager, diskBTreeFactory.getBufferCache(), fileManager, bloomFilterFalsePositiveRate, mergePolicy,
-                opTracker, ioScheduler, ioOpCallback, durable);
+            IFileMapProvider diskFileMapProvider, int fieldCount, IBinaryComparatorFactory[] cmpFactories,
+            ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
+            ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, boolean durable) {
+        super(ioManager, diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider,
+                bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, durable);
         this.insertLeafFrameFactory = insertLeafFrameFactory;
         this.deleteLeafFrameFactory = deleteLeafFrameFactory;
         this.cmpFactories = cmpFactories;
-        this.needKeyDupCheck = needKeyDupCheck;
-        this.hasBloomFilter = true;
         componentFactory = new LSMBTreeDiskComponentFactory(diskBTreeFactory, bloomFilterFactory, null);
         bulkLoadComponentFactory = new LSMBTreeDiskComponentFactory(bulkLoadBTreeFactory, bloomFilterFactory, null);
+        this.needKeyDupCheck = needKeyDupCheck;
+        //TODO remove BloomFilter from external dataset's secondary LSMBTree index
+        this.hasBloomFilter = true;
     }
 
     public boolean hasBloomFilter() {
@@ -180,6 +183,12 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
     }
 
     @Override
+    protected void destroyMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+        LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
+        mutableComponent.getBTree().destroy();
+    }
+
+    @Override
     protected void clearDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
         LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
         if (hasBloomFilter) {
@@ -578,11 +587,9 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
     @Override
     protected void deactivateDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
         LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
-        component.getBTree().deactivate();
-        component.getBTree().purge();
+        component.getBTree().deactivateCloseHandle();
         if (hasBloomFilter) {
             component.getBloomFilter().deactivate();
-            component.getBloomFilter().purge();
         }
     }
 

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
index 432a2b8..194eb3a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java
@@ -37,15 +37,17 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndex;
 import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
 import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
 import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
 
 public class LSMBTreeFileManager extends AbstractLSMIndexFileManager {
+    public static final String BTREE_STRING = "b";
 
     private final TreeIndexFactory<? extends ITreeIndex> btreeFactory;
     private final boolean hasBloomFilter;
 
-    public LSMBTreeFileManager(IIOManager ioManager, FileReference file,
+    public LSMBTreeFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file,
             TreeIndexFactory<? extends ITreeIndex> btreeFactory, boolean hasBloomFilter) {
-        super(ioManager, file, null);
+        super(ioManager, fileMapProvider, file, null);
         this.btreeFactory = btreeFactory;
         this.hasBloomFilter = hasBloomFilter;
     }
@@ -53,28 +55,28 @@ public class LSMBTreeFileManager extends AbstractLSMIndexFileManager {
     @Override
     public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException {
         String ts = getCurrentTimestamp();
-        String baseName = baseDir + ts + DELIMITER + ts;
+        String baseName = baseDir + ts + SPLIT_STRING + ts;
         // Begin timestamp and end timestamp are identical since it is a flush
-        return new LSMComponentFileReferences(createFlushFile(baseName + DELIMITER + BTREE_SUFFIX), null,
-                hasBloomFilter ? createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX) : null);
+        return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING), null,
+                hasBloomFilter ? createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING) : null);
     }
 
     @Override
     public LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName)
             throws HyracksDataException {
-        String[] firstTimestampRange = firstFileName.split(DELIMITER);
-        String[] lastTimestampRange = lastFileName.split(DELIMITER);
+        String[] firstTimestampRange = firstFileName.split(SPLIT_STRING);
+        String[] lastTimestampRange = lastFileName.split(SPLIT_STRING);
 
-        String baseName = baseDir + firstTimestampRange[0] + DELIMITER + lastTimestampRange[1];
+        String baseName = baseDir + firstTimestampRange[0] + SPLIT_STRING + lastTimestampRange[1];
         // Get the range of timestamps by taking the earliest and the latest timestamps
-        return new LSMComponentFileReferences(createMergeFile(baseName + DELIMITER + BTREE_SUFFIX), null,
-                hasBloomFilter ? createMergeFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX) : null);
+        return new LSMComponentFileReferences(createMergeFile(baseName + SPLIT_STRING + BTREE_STRING), null,
+                hasBloomFilter ? createMergeFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING) : null);
     }
 
     private static FilenameFilter btreeFilter = new FilenameFilter() {
         @Override
         public boolean accept(File dir, String name) {
-            return !name.startsWith(".") && name.endsWith(BTREE_SUFFIX);
+            return !name.startsWith(".") && name.endsWith(BTREE_STRING);
         }
     };
 
@@ -93,7 +95,7 @@ public class LSMBTreeFileManager extends AbstractLSMIndexFileManager {
         cleanupAndGetValidFilesInternal(getCompoundFilter(transactionFilter, btreeFilter), btreeFactory, allBTreeFiles);
         HashSet<String> btreeFilesSet = new HashSet<>();
         for (ComparableFileName cmpFileName : allBTreeFiles) {
-            int index = cmpFileName.fileName.lastIndexOf(DELIMITER);
+            int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
             btreeFilesSet.add(cmpFileName.fileName.substring(0, index));
         }
 
@@ -198,19 +200,19 @@ public class LSMBTreeFileManager extends AbstractLSMIndexFileManager {
     public LSMComponentFileReferences getNewTransactionFileReference() throws IOException {
         String ts = getCurrentTimestamp();
         // Create transaction lock file
-        Files.createFile(Paths.get(baseDir + TXN_PREFIX + ts));
+        Files.createFile(Paths.get(baseDir + TRANSACTION_PREFIX + ts));
 
-        String baseName = baseDir + ts + DELIMITER + ts;
+        String baseName = baseDir + ts + SPLIT_STRING + ts;
         // Begin timestamp and end timestamp are identical since it is a transaction
-        return new LSMComponentFileReferences(createFlushFile(baseName + DELIMITER + BTREE_SUFFIX), null,
-                createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
+        return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING), null,
+                createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING));
     }
 
     @Override
     public LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException {
         FilenameFilter transactionFilter;
         File dir = new File(baseDir);
-        String[] files = dir.list(txnFileNameFilter);
+        String[] files = dir.list(transactionFileNameFilter);
         if (files.length == 0) {
             return null;
         }
@@ -234,9 +236,9 @@ public class LSMBTreeFileManager extends AbstractLSMIndexFileManager {
         File bloomFilterFile = null;
 
         for (String fileName : files) {
-            if (fileName.endsWith(BTREE_SUFFIX)) {
+            if (fileName.endsWith(BTREE_STRING)) {
                 bTreeFile = new File(dir.getPath() + File.separator + fileName);
-            } else if (fileName.endsWith(BLOOM_FILTER_SUFFIX)) {
+            } else if (fileName.endsWith(BLOOM_FILTER_STRING)) {
                 bloomFilterFile = new File(dir.getPath() + File.separator + fileName);
             } else {
                 throw new HyracksDataException("unrecognized file found = " + fileName);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
index c866a45..0173c06 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java
@@ -36,8 +36,11 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndex;
 import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
 import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
 import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
 
 public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager {
+    public static final String BUDDY_BTREE_STRING = "buddy";
+    public static final String BTREE_STRING = "b";
 
     private final TreeIndexFactory<? extends ITreeIndex> btreeFactory;
     private final TreeIndexFactory<? extends ITreeIndex> buddyBtreeFactory;
@@ -45,21 +48,21 @@ public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager {
     private static FilenameFilter btreeFilter = new FilenameFilter() {
         @Override
         public boolean accept(File dir, String name) {
-            return !name.startsWith(".") && name.endsWith(BTREE_SUFFIX);
+            return !name.startsWith(".") && name.endsWith(BTREE_STRING);
         }
     };
 
     private static FilenameFilter buddyBtreeFilter = new FilenameFilter() {
         @Override
         public boolean accept(File dir, String name) {
-            return !name.startsWith(".") && name.endsWith(DELETE_TREE_SUFFIX);
+            return !name.startsWith(".") && name.endsWith(BUDDY_BTREE_STRING);
         }
     };
 
-    public LSMBTreeWithBuddyFileManager(IIOManager ioManager, FileReference file,
+    public LSMBTreeWithBuddyFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file,
             TreeIndexFactory<? extends ITreeIndex> btreeFactory,
             TreeIndexFactory<? extends ITreeIndex> buddyBtreeFactory) {
-        super(ioManager, file, null);
+        super(ioManager, fileMapProvider, file, null);
         this.buddyBtreeFactory = buddyBtreeFactory;
         this.btreeFactory = btreeFactory;
     }
@@ -67,25 +70,25 @@ public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager {
     @Override
     public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException {
         String ts = getCurrentTimestamp();
-        String baseName = baseDir + ts + DELIMITER + ts;
+        String baseName = baseDir + ts + SPLIT_STRING + ts;
         // Begin timestamp and end timestamp are identical since it is a flush
-        return new LSMComponentFileReferences(createFlushFile(baseName + DELIMITER + BTREE_SUFFIX),
-                createFlushFile(baseName + DELIMITER + DELETE_TREE_SUFFIX),
-                createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
+        return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING),
+                createFlushFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING),
+                createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING));
     }
 
     @Override
     public LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName)
             throws HyracksDataException {
-        String[] firstTimestampRange = firstFileName.split(DELIMITER);
-        String[] lastTimestampRange = lastFileName.split(DELIMITER);
+        String[] firstTimestampRange = firstFileName.split(SPLIT_STRING);
+        String[] lastTimestampRange = lastFileName.split(SPLIT_STRING);
 
-        String baseName = baseDir + firstTimestampRange[0] + DELIMITER + lastTimestampRange[1];
+        String baseName = baseDir + firstTimestampRange[0] + SPLIT_STRING + lastTimestampRange[1];
         // Get the range of timestamps by taking the earliest and the latest
         // timestamps
-        return new LSMComponentFileReferences(createMergeFile(baseName + DELIMITER + BTREE_SUFFIX),
-                createMergeFile(baseName + DELIMITER + DELETE_TREE_SUFFIX),
-                createMergeFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
+        return new LSMComponentFileReferences(createMergeFile(baseName + SPLIT_STRING + BTREE_STRING),
+                createMergeFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING),
+                createMergeFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING));
     }
 
     @Override
@@ -103,7 +106,7 @@ public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager {
                 allBTreeFiles);
         HashSet<String> btreeFilesSet = new HashSet<>();
         for (ComparableFileName cmpFileName : allBTreeFiles) {
-            int index = cmpFileName.fileName.lastIndexOf(DELIMITER);
+            int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING);
             btreeFilesSet.add(cmpFileName.fileName.substring(0, index));
         }
         validateFiles(btreeFilesSet, allBuddyBTreeFiles, getCompoundFilter(buddyBtreeFilter, transactionFilefilter),
@@ -203,19 +206,19 @@ public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager {
     public LSMComponentFileReferences getNewTransactionFileReference() throws IOException {
         String ts = getCurrentTimestamp();
         // Create transaction lock file
-        Files.createFile(Paths.get(baseDir + TXN_PREFIX + ts));
+        Files.createFile(Paths.get(baseDir + TRANSACTION_PREFIX + ts));
 
-        String baseName = baseDir + ts + DELIMITER + ts;
-        return new LSMComponentFileReferences(createFlushFile(baseName + DELIMITER + BTREE_SUFFIX),
-                createFlushFile(baseName + DELIMITER + DELETE_TREE_SUFFIX),
-                createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX));
+        String baseName = baseDir + ts + SPLIT_STRING + ts;
+        return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + BTREE_STRING),
+                createFlushFile(baseName + SPLIT_STRING + BUDDY_BTREE_STRING),
+                createFlushFile(baseName + SPLIT_STRING + BLOOM_FILTER_STRING));
     }
 
     @Override
     public LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException {
         FilenameFilter transactionFilter;
         File dir = new File(baseDir);
-        String[] files = dir.list(txnFileNameFilter);
+        String[] files = dir.list(transactionFileNameFilter);
         if (files.length == 0) {
             return null;
         }
@@ -240,11 +243,11 @@ public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager {
         File buddyBTreeFile = null;
         File bloomFilterFile = null;
         for (String fileName : files) {
-            if (fileName.endsWith(BTREE_SUFFIX)) {
+            if (fileName.endsWith(BTREE_STRING)) {
                 bTreeFile = new File(dir.getPath() + File.separator + fileName);
-            } else if (fileName.endsWith(DELETE_TREE_SUFFIX)) {
+            } else if (fileName.endsWith(BUDDY_BTREE_STRING)) {
                 buddyBTreeFile = new File(dir.getPath() + File.separator + fileName);
-            } else if (fileName.endsWith(BLOOM_FILTER_SUFFIX)) {
+            } else if (fileName.endsWith(BLOOM_FILTER_STRING)) {
                 bloomFilterFile = new File(dir.getPath() + File.separator + fileName);
             } else {
                 throw new HyracksDataException("unrecognized file found = " + fileName);

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java
index 792b872..259f25f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/utils/LSMBTreeUtil.java
@@ -52,6 +52,7 @@ import org.apache.hyracks.storage.am.lsm.common.impls.ComponentFilterHelper;
 import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager;
 import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
 import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
 
 public class LSMBTreeUtil {
 
@@ -59,12 +60,13 @@ public class LSMBTreeUtil {
     }
 
     public static LSMBTree createLSMTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
-            FileReference file, IBufferCache diskBufferCache, ITypeTraits[] typeTraits,
-            IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate,
-            ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
-            ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck, ITypeTraits[] filterTypeTraits,
-            IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields, int[] filterFields, boolean durable,
-            IMetadataPageManagerFactory freePageManagerFactory) throws HyracksDataException {
+            FileReference file, IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider,
+            ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields,
+            double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+            ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean needKeyDupCheck,
+            ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] btreeFields,
+            int[] filterFields, boolean durable, IMetadataPageManagerFactory freePageManagerFactory)
+            throws HyracksDataException {
         LSMBTreeTupleWriterFactory insertTupleWriterFactory =
                 new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, false);
         LSMBTreeTupleWriterFactory deleteTupleWriterFactory =
@@ -80,14 +82,15 @@ public class LSMBTreeUtil {
         ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
         ITreeIndexFrameFactory bulkLoadLeafFrameFactory = new BTreeNSMLeafFrameFactory(bulkLoadTupleWriterFactory);
 
-        TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory,
-                interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length);
+        TreeIndexFactory<BTree> diskBTreeFactory =
+                new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory,
+                        interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length);
         TreeIndexFactory<BTree> bulkLoadBTreeFactory =
-                new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory, interiorFrameFactory,
-                        bulkLoadLeafFrameFactory, cmpFactories, typeTraits.length);
+                new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory,
+                        interiorFrameFactory, bulkLoadLeafFrameFactory, cmpFactories, typeTraits.length);
 
-        BloomFilterFactory bloomFilterFactory =
-                needKeyDupCheck ? new BloomFilterFactory(diskBufferCache, bloomFilterKeyFields) : null;
+        BloomFilterFactory bloomFilterFactory = needKeyDupCheck
+                ? new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields) : null;
 
         ComponentFilterHelper filterHelper = null;
         LSMComponentFilterFrameFactory filterFrameFactory = null;
@@ -101,20 +104,20 @@ public class LSMBTreeUtil {
 
         //Primary LSMBTree index has a BloomFilter.
         ILSMIndexFileManager fileNameManager =
-                new LSMBTreeFileManager(ioManager, file, diskBTreeFactory, needKeyDupCheck);
+                new LSMBTreeFileManager(ioManager, diskFileMapProvider, file, diskBTreeFactory, needKeyDupCheck);
 
         return new LSMBTree(ioManager, virtualBufferCaches, interiorFrameFactory, insertLeafFrameFactory,
                 deleteLeafFrameFactory, fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory,
-                filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, typeTraits.length,
-                cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, needKeyDupCheck, btreeFields,
-                filterFields, durable);
+                filterHelper, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, diskFileMapProvider,
+                typeTraits.length, cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback, needKeyDupCheck,
+                btreeFields, filterFields, durable);
     }
 
     public static ExternalBTree createExternalBTree(IIOManager ioManager, FileReference file,
-            IBufferCache diskBufferCache, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
-            int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
-            ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
-            boolean durable, IMetadataPageManagerFactory freePageManagerFactory) {
+            IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
+            IBinaryComparatorFactory[] cmpFactories, int[] bloomFilterKeyFields, double bloomFilterFalsePositiveRate,
+            ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
+            ILSMIOOperationCallback ioOpCallback, boolean durable, IMetadataPageManagerFactory freePageManagerFactory) {
         LSMBTreeTupleWriterFactory insertTupleWriterFactory =
                 new LSMBTreeTupleWriterFactory(typeTraits, cmpFactories.length, false);
         LSMBTreeTupleWriterFactory deleteTupleWriterFactory =
@@ -133,31 +136,35 @@ public class LSMBTreeUtil {
         ITreeIndexFrameFactory transactionLeafFrameFactory =
                 new BTreeNSMLeafFrameFactory(transactionTupleWriterFactory);
 
-        TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory,
-                interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length);
-        TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache,
+        TreeIndexFactory<BTree> diskBTreeFactory =
+                new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory,
+                        interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length);
+        TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
                 freePageManagerFactory, interiorFrameFactory, insertLeafFrameFactory, cmpFactories, typeTraits.length);
 
-        BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, bloomFilterKeyFields);
+        BloomFilterFactory bloomFilterFactory =
+                new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields);
 
         // This is the component factory for transactions
         TreeIndexFactory<BTree> transactionBTreeFactory =
-                new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory, interiorFrameFactory,
-                        transactionLeafFrameFactory, cmpFactories, typeTraits.length);
+                new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory,
+                        interiorFrameFactory, transactionLeafFrameFactory, cmpFactories, typeTraits.length);
         //TODO remove BloomFilter from external dataset's secondary LSMBTree index
-        ILSMIndexFileManager fileNameManager = new LSMBTreeFileManager(ioManager, file, diskBTreeFactory, true);
+        ILSMIndexFileManager fileNameManager =
+                new LSMBTreeFileManager(ioManager, diskFileMapProvider, file, diskBTreeFactory, true);
         // the disk only index uses an empty ArrayList for virtual buffer caches
-        return new ExternalBTree(ioManager, interiorFrameFactory, insertLeafFrameFactory, deleteLeafFrameFactory,
-                fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory,
-                bloomFilterFalsePositiveRate, cmpFactories, mergePolicy, opTracker, ioScheduler, ioOpCallback,
-                transactionBTreeFactory, durable);
+        ExternalBTree lsmTree = new ExternalBTree(ioManager, interiorFrameFactory, insertLeafFrameFactory,
+                deleteLeafFrameFactory, fileNameManager, diskBTreeFactory, bulkLoadBTreeFactory, bloomFilterFactory,
+                bloomFilterFalsePositiveRate, diskFileMapProvider, typeTraits.length, cmpFactories, mergePolicy,
+                opTracker, ioScheduler, ioOpCallback, transactionBTreeFactory, durable);
+        return lsmTree;
     }
 
     public static ExternalBTreeWithBuddy createExternalBTreeWithBuddy(IIOManager ioManager, FileReference file,
-            IBufferCache diskBufferCache, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
-            double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
-            ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] buddyBTreeFields,
-            boolean durable, IMetadataPageManagerFactory freePageManagerFactory) {
+            IBufferCache diskBufferCache, IFileMapProvider diskFileMapProvider, ITypeTraits[] typeTraits,
+            IBinaryComparatorFactory[] cmpFactories, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
+            ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+            int[] buddyBTreeFields, boolean durable, IMetadataPageManagerFactory freePageManagerFactory) {
         ITypeTraits[] buddyBtreeTypeTraits = new ITypeTraits[buddyBTreeFields.length];
         IBinaryComparatorFactory[] buddyBtreeCmpFactories = new IBinaryComparatorFactory[buddyBTreeFields.length];
         for (int i = 0; i < buddyBtreeTypeTraits.length; i++) {
@@ -177,30 +184,32 @@ public class LSMBTreeUtil {
         ITreeIndexFrameFactory insertLeafFrameFactory = new BTreeNSMLeafFrameFactory(insertTupleWriterFactory);
         ITreeIndexFrameFactory copyTupleLeafFrameFactory = new BTreeNSMLeafFrameFactory(copyTupleWriterFactory);
         ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
-        TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory,
-                interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length);
+        TreeIndexFactory<BTree> diskBTreeFactory =
+                new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider, freePageManagerFactory,
+                        interiorFrameFactory, copyTupleLeafFrameFactory, cmpFactories, typeTraits.length);
 
-        TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache,
+        TreeIndexFactory<BTree> bulkLoadBTreeFactory = new BTreeFactory(ioManager, diskBufferCache, diskFileMapProvider,
                 freePageManagerFactory, interiorFrameFactory, insertLeafFrameFactory, cmpFactories, typeTraits.length);
 
         int[] bloomFilterKeyFields = new int[buddyBtreeCmpFactories.length];
         for (int i = 0; i < buddyBtreeCmpFactories.length; i++) {
             bloomFilterKeyFields[i] = i;
         }
-        BloomFilterFactory bloomFilterFactory = new BloomFilterFactory(diskBufferCache, bloomFilterKeyFields);
+        BloomFilterFactory bloomFilterFactory =
+                new BloomFilterFactory(diskBufferCache, diskFileMapProvider, bloomFilterKeyFields);
 
         // buddy b-tree factory
-        TreeIndexFactory<BTree> diskBuddyBTreeFactory =
-                new BTreeFactory(ioManager, diskBufferCache, freePageManagerFactory, buddyBtreeInteriorFrameFactory,
-                        buddyBtreeLeafFrameFactory, buddyBtreeCmpFactories, buddyBtreeTypeTraits.length);
+        TreeIndexFactory<BTree> diskBuddyBTreeFactory = new BTreeFactory(ioManager, diskBufferCache,
+                diskFileMapProvider, freePageManagerFactory, buddyBtreeInteriorFrameFactory, buddyBtreeLeafFrameFactory,
+                buddyBtreeCmpFactories, buddyBtreeTypeTraits.length);
 
-        ILSMIndexFileManager fileNameManager =
-                new LSMBTreeWithBuddyFileManager(ioManager, file, diskBTreeFactory, diskBuddyBTreeFactory);
+        ILSMIndexFileManager fileNameManager = new LSMBTreeWithBuddyFileManager(ioManager, diskFileMapProvider, file,
+                diskBTreeFactory, diskBuddyBTreeFactory);
 
         // the disk only index uses an empty ArrayList for virtual buffer caches
         ExternalBTreeWithBuddy lsmTree = new ExternalBTreeWithBuddy(ioManager, interiorFrameFactory,
                 insertLeafFrameFactory, buddyBtreeLeafFrameFactory, diskBufferCache, fileNameManager,
-                bulkLoadBTreeFactory, diskBTreeFactory, diskBuddyBTreeFactory, bloomFilterFactory,
+                bulkLoadBTreeFactory, diskBTreeFactory, diskBuddyBTreeFactory, bloomFilterFactory, diskFileMapProvider,
                 bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, cmpFactories,
                 buddyBtreeCmpFactories, buddyBTreeFields, durable);
         return lsmTree;

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml
index 37e1c49..bc1e146 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/pom.xml
@@ -82,9 +82,8 @@
       <artifactId>commons-lang3</artifactId>
     </dependency>
     <dependency>
-      <groupId>org.apache.hyracks</groupId>
-      <artifactId>hyracks-util</artifactId>
-      <version>${project.version}</version>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
     </dependency>
   </dependencies>
 </project>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java
index 18d9a67..54d64af 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java
@@ -34,28 +34,28 @@ import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences
  * merge).
  */
 public interface ILSMIndexFileManager {
-    void createDirs() throws HyracksDataException;
+    public void createDirs();
 
-    void deleteDirs() throws HyracksDataException;
+    public void deleteDirs() throws HyracksDataException;
 
-    LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException;
+    public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException;
 
-    LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName)
+    public LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName)
             throws HyracksDataException;
 
-    String getBaseDir();
+    public String getBaseDir();
 
     // Deletes invalid files, and returns list of valid files from baseDir.
     // The returned valid files are correctly sorted (based on the recency of data).
-    List<LSMComponentFileReferences> cleanupAndGetValidFiles() throws HyracksDataException;
+    public List<LSMComponentFileReferences> cleanupAndGetValidFiles() throws HyracksDataException;
 
-    Comparator<String> getFileNameComparator();
+    public Comparator<String> getFileNameComparator();
 
     /**
      * @return delete existing transaction disk component file reference
      * @throws HyracksDataException
      */
-    void deleteTransactionFiles() throws HyracksDataException;
+    public void deleteTransactionFiles() throws HyracksDataException;
 
     /**
      * Rename files of a transaction removing the transaction prefix and return the component file reference in order to be committed
@@ -63,18 +63,18 @@ public interface ILSMIndexFileManager {
      * @return the renamed component file references
      * @throws HyracksDataException
      */
-    LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException;
+    public LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException;
 
     /**
      * Recover transaction files without returning them
      *
      * @throws HyracksDataException
      */
-    void recoverTransaction() throws HyracksDataException;
+    public void recoverTransaction() throws HyracksDataException;
 
     /**
      * @return a reference to the transaction disk component file reference
      * @throws IOException
      */
-    LSMComponentFileReferences getNewTransactionFileReference() throws IOException;
+    public LSMComponentFileReferences getNewTransactionFileReference() throws IOException;
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java
index 72d5423..090ca68 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java
@@ -23,6 +23,6 @@ import org.apache.hyracks.api.io.IIOManager;
 
 public class AbortRecoverLSMIndexFileManager extends AbstractLSMIndexFileManager {
     public AbortRecoverLSMIndexFileManager(IIOManager ioManager, FileReference file) {
-        super(ioManager, file, null);
+        super(ioManager, null, file, null);
     }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/d9000469/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index c5bf2ca..2be2184 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -63,6 +63,7 @@ import org.apache.hyracks.storage.common.IIndexCursor;
 import org.apache.hyracks.storage.common.IModificationOperationCallback;
 import org.apache.hyracks.storage.common.ISearchOperationCallback;
 import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
 
 public abstract class AbstractLSMIndex implements ILSMIndex {
     protected final ILSMHarness lsmHarness;
@@ -74,9 +75,11 @@ public abstract class AbstractLSMIndex implements ILSMIndex {
     protected final List<ILSMMemoryComponent> memoryComponents;
     protected final List<IVirtualBufferCache> virtualBufferCaches;
     protected AtomicInteger currentMutableComponentId;
+
     // On-disk components.
     protected final IBufferCache diskBufferCache;
     protected final ILSMIndexFileManager fileManager;
+    protected final IFileMapProvider diskFileMapProvider;
     // components with lower indexes are newer than components with higher index
     protected final List<ILSMDiskComponent> diskComponents;
     protected final List<ILSMDiskComponent> inactiveDiskComponents;
@@ -92,14 +95,16 @@ public abstract class AbstractLSMIndex implements ILSMIndex {
     protected boolean memoryComponentsAllocated = false;
 
     public AbstractLSMIndex(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
-            IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, double bloomFilterFalsePositiveRate,
-            ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler,
-            ILSMIOOperationCallback ioOpCallback, ILSMComponentFilterFrameFactory filterFrameFactory,
-            LSMComponentFilterManager filterManager, int[] filterFields, boolean durable,
-            IComponentFilterHelper filterHelper, int[] treeFields) {
+            IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider,
+            double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+            ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+            ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager,
+            int[] filterFields, boolean durable, IComponentFilterHelper filterHelper,
+            int[] treeFields) {
         this.ioManager = ioManager;
         this.virtualBufferCaches = virtualBufferCaches;
         this.diskBufferCache = diskBufferCache;
+        this.diskFileMapProvider = diskFileMapProvider;
         this.fileManager = fileManager;
         this.bloomFilterFalsePositiveRate = bloomFilterFalsePositiveRate;
         this.ioScheduler = ioScheduler;
@@ -125,10 +130,12 @@ public abstract class AbstractLSMIndex implements ILSMIndex {
 
     // The constructor used by external indexes
     public AbstractLSMIndex(IIOManager ioManager, IBufferCache diskBufferCache, ILSMIndexFileManager fileManager,
-            double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
-            ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean durable) {
+            IFileMapProvider diskFileMapProvider, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy,
+            ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
+            boolean durable) {
         this.ioManager = ioManager;
         this.diskBufferCache = diskBufferCache;
+        this.diskFileMapProvider = diskFileMapProvider;
         this.fileManager = fileManager;
         this.bloomFilterFalsePositiveRate = bloomFilterFalsePositiveRate;
         this.ioScheduler = ioScheduler;
@@ -155,6 +162,7 @@ public abstract class AbstractLSMIndex implements ILSMIndex {
         if (isActive) {
             throw HyracksDataException.create(ErrorCode.CANNOT_CREATE_ACTIVE_INDEX);
         }
+        fileManager.deleteDirs();
         fileManager.createDirs();
         diskComponents.clear();
     }
@@ -230,9 +238,16 @@ public abstract class AbstractLSMIndex implements ILSMIndex {
             throw HyracksDataException.create(ErrorCode.CANNOT_DESTROY_ACTIVE_INDEX);
         }
         destroyDiskComponents();
+        destroyMemoryComponents();
         fileManager.deleteDirs();
     }
 
+    protected void destroyMemoryComponents() throws HyracksDataException {
+        for (ILSMMemoryComponent c : memoryComponents) {
+            destroyMemoryComponent(c);
+        }
+    }
+
     protected void destroyDiskComponents() throws HyracksDataException {
         List<ILSMDiskComponent> immutableComponents = diskComponents;
         for (ILSMDiskComponent c : immutableComponents) {
@@ -638,6 +653,8 @@ public abstract class AbstractLSMIndex implements ILSMIndex {
 
     protected abstract void destroyDiskComponent(ILSMDiskComponent c) throws HyracksDataException;
 
+    protected abstract void destroyMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException;
+
     protected abstract void clearDiskComponent(ILSMDiskComponent c) throws HyracksDataException;
 
     protected abstract void clearMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException;


Mime
View raw message