asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ima...@apache.org
Subject [4/5] incubator-asterixdb-hyracks git commit: Make LSM bulkload append-only and write-once.
Date Mon, 23 Nov 2015 17:07:26 GMT
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/LinkedMetaDataPageManager.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/LinkedMetaDataPageManager.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/LinkedMetaDataPageManager.java
new file mode 100644
index 0000000..096908c
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/LinkedMetaDataPageManager.java
@@ -0,0 +1,458 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.common.freepage;
+
+import java.util.logging.Logger;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import org.apache.hyracks.storage.common.buffercache.BufferCache;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue;
+import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+
+public class LinkedMetaDataPageManager implements IMetaDataPageManager {
+
+    private static final byte META_PAGE_LEVEL_INDICATOR = -1;
+    private static final byte FREE_PAGE_LEVEL_INDICATOR = -2;
+    public static final int NO_FILTER_IN_PLACE = -1;
+    public static final int NO_FILTER_APPEND_ONLY = -2;
+    private final IBufferCache bufferCache;
+    private int headPage = -1;
+    private int fileId = -1;
+    private final ITreeIndexMetaDataFrameFactory metaDataFrameFactory;
+    private boolean appendOnly = false;
+    ICachedPage confiscatedMetaNode;
+    ICachedPage filterPage;
+    private static Logger LOGGER = Logger
+            .getLogger(LinkedMetaDataPageManager.class.getName());
+
+    public LinkedMetaDataPageManager(IBufferCache bufferCache, ITreeIndexMetaDataFrameFactory metaDataFrameFactory) {
+        this.bufferCache = bufferCache;
+        this.metaDataFrameFactory = metaDataFrameFactory;
+    }
+
+    @Override
+    public void addFreePage(ITreeIndexMetaDataFrame metaFrame, int freePage) throws HyracksDataException {
+
+        ICachedPage metaNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, getFirstMetadataPage()), false);
+        metaNode.acquireWriteLatch();
+
+        try {
+            metaFrame.setPage(metaNode);
+
+            if (metaFrame.hasSpace()) {
+                metaFrame.addFreePage(freePage);
+            } else {
+                // allocate a new page in the chain of meta pages
+                int newPage = metaFrame.getFreePage();
+                if (newPage < 0) {
+                    throw new Exception("Inconsistent Meta Page State. It has no space, but it also has no entries.");
+                }
+
+                ICachedPage newNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, newPage), false);
+                newNode.acquireWriteLatch();
+
+                try {
+                    int metaMaxPage = metaFrame.getMaxPage();
+
+                    // copy metaDataPage to newNode
+                    System.arraycopy(metaNode.getBuffer().array(), 0, newNode.getBuffer().array(), 0, metaNode
+                            .getBuffer().capacity());
+
+                    metaFrame.initBuffer(META_PAGE_LEVEL_INDICATOR);
+                    metaFrame.setNextPage(newPage);
+                    metaFrame.setMaxPage(metaMaxPage);
+                    metaFrame.addFreePage(freePage);
+                } finally {
+                    newNode.releaseWriteLatch(true);
+                    bufferCache.unpin(newNode);
+                }
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        } finally {
+            metaNode.releaseWriteLatch(true);
+            bufferCache.unpin(metaNode);
+        }
+    }
+
+    @Override
+    public int getFreePage(ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException {
+        ICachedPage metaNode;
+        if (!appendOnly) {
+            metaNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, getFirstMetadataPage()), false);
+        } else {
+            metaNode = confiscatedMetaNode;
+        }
+
+        metaNode.acquireWriteLatch();
+
+        int freePage = IBufferCache.INVALID_PAGEID;
+        try {
+            metaFrame.setPage(metaNode);
+            freePage = metaFrame.getFreePage();
+            if (freePage < 0) { // no free page entry on this page
+                int nextPage = metaFrame.getNextPage();
+                if (nextPage > 0) { // sibling may have free pages
+                    ICachedPage nextNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, nextPage), false);
+
+                    nextNode.acquireWriteLatch();
+                    // we copy over the free space entries of nextpage into the
+                    // first meta page (metaDataPage)
+                    // we need to link the first page properly to the next page
+                    // of nextpage
+                    try {
+                        // remember entries that remain unchanged
+                        int maxPage = metaFrame.getMaxPage();
+
+                        // copy entire page (including sibling pointer, free
+                        // page entries, and all other info)
+                        // after this copy nextPage is considered a free page
+                        System.arraycopy(nextNode.getBuffer().array(), 0, metaNode.getBuffer().array(), 0, nextNode
+                                .getBuffer().capacity());
+
+                        // reset unchanged entry
+                        metaFrame.setMaxPage(maxPage);
+
+                        freePage = metaFrame.getFreePage();
+                        // sibling also has no free pages, this "should" not
+                        // happen, but we deal with it anyway just to be safe
+                        if (freePage < 0) {
+                            freePage = nextPage;
+                        } else {
+                            metaFrame.addFreePage(nextPage);
+                        }
+                    } finally {
+                        nextNode.releaseWriteLatch(true);
+                        bufferCache.unpin(nextNode);
+                    }
+                } else {
+                    freePage = metaFrame.getMaxPage();
+                    freePage++;
+                    metaFrame.setMaxPage(freePage);
+                }
+            }
+        } finally {
+            if (!appendOnly) {
+                metaNode.releaseWriteLatch(true);
+                bufferCache.unpin(metaNode);
+            } else {
+                metaNode.releaseWriteLatch(false);
+            }
+        }
+
+        return freePage;
+    }
+
+    @Override
+    public int getMaxPage(ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException {
+        ICachedPage metaNode;
+        if (!appendOnly || (appendOnly && confiscatedMetaNode == null)) {
+            int mdPage = getFirstMetadataPage();
+            if( mdPage <0 ){
+                return IBufferCache.INVALID_PAGEID;
+            }
+            metaNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, mdPage), false);
+        } else {
+            metaNode = confiscatedMetaNode;
+        }
+        metaNode.acquireReadLatch();
+        int maxPage = -1;
+        try {
+            metaFrame.setPage(metaNode);
+            maxPage = metaFrame.getMaxPage();
+        } finally {
+            metaNode.releaseReadLatch();
+            if (!appendOnly || (appendOnly && confiscatedMetaNode == null)) {
+                bufferCache.unpin(metaNode);
+            }
+        }
+        return maxPage;
+    }
+
+    @Override
+    public void setFilterPageId(int filterPageId) throws HyracksDataException {
+        ICachedPage metaNode;
+        if (!appendOnly) {
+            int mdPage = getFirstMetadataPage();
+            if( mdPage <0 ){
+                return;
+            }
+            metaNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, mdPage), false);
+        } else {
+            metaNode = confiscatedMetaNode;
+        }
+        ITreeIndexMetaDataFrame metaFrame = metaDataFrameFactory.createFrame();
+        metaNode.acquireWriteLatch();
+        try {
+            metaFrame.setPage(metaNode);
+            metaFrame.setLSMComponentFilterPageId(filterPageId);
+        } finally {
+            if (!appendOnly) {
+                metaNode.releaseWriteLatch(true);
+                bufferCache.unpin(metaNode);
+            } else {
+                metaNode.releaseWriteLatch(false);
+            }
+        }
+    }
+
+    @Override
+    public int getFilterPageId() throws HyracksDataException {
+        ICachedPage metaNode;
+        int filterPageId = NO_FILTER_IN_PLACE;
+        if (!appendOnly || (appendOnly && confiscatedMetaNode == null)) {
+            metaNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, getFirstMetadataPage()), false);
+        } else {
+            metaNode = confiscatedMetaNode;
+        }
+        ITreeIndexMetaDataFrame metaFrame = metaDataFrameFactory.createFrame();
+        metaNode.acquireReadLatch();
+        try {
+            metaFrame.setPage(metaNode);
+            filterPageId = metaFrame.getLSMComponentFilterPageId();
+            if(appendOnly && filterPageId == -1){
+                //hint to filter manager that we are in append-only mode
+                filterPageId = NO_FILTER_APPEND_ONLY;
+            }
+        } finally {
+            metaNode.releaseReadLatch();
+            if (!appendOnly || (appendOnly && confiscatedMetaNode == null)) {
+                bufferCache.unpin(metaNode);
+            }
+        }
+        return filterPageId;
+    }
+
+    @Override
+    public void init(ITreeIndexMetaDataFrame metaFrame, int currentMaxPage) throws HyracksDataException {
+        // initialize meta data page
+        int metaPage = getFirstMetadataPage();
+        if(metaPage == IBufferCache.INVALID_PAGEID){
+            throw new HyracksDataException("No valid metadata found in this file.");
+        }
+        ICachedPage metaNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, getFirstMetadataPage()), true);
+
+        metaNode.acquireWriteLatch();
+        try {
+            metaFrame.setPage(metaNode);
+            metaFrame.initBuffer(META_PAGE_LEVEL_INDICATOR);
+            metaFrame.setMaxPage(currentMaxPage);
+        } finally {
+            metaNode.releaseWriteLatch(true);
+            bufferCache.unpin(metaNode);
+        }
+    }
+
+    @Override
+    public void init(ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException {
+        if (confiscatedMetaNode != null) { // don't init twice
+            return;
+        }
+        ICachedPage metaNode = bufferCache.confiscatePage(BufferCache.INVALID_DPID);
+        try {
+            metaFrame.setPage(metaNode);
+            metaFrame.initBuffer(META_PAGE_LEVEL_INDICATOR);
+            metaFrame.setMaxPage(0);
+        } finally {
+            confiscatedMetaNode = metaNode;
+            appendOnly = true;
+        }
+    }
+
+    @Override
+    public ITreeIndexMetaDataFrameFactory getMetaDataFrameFactory() {
+        return metaDataFrameFactory;
+    }
+
+    @Override
+    public byte getFreePageLevelIndicator() {
+        return FREE_PAGE_LEVEL_INDICATOR;
+    }
+
+    @Override
+    public byte getMetaPageLevelIndicator() {
+        return META_PAGE_LEVEL_INDICATOR;
+    }
+
+    @Override
+    public boolean isFreePage(ITreeIndexMetaDataFrame metaFrame) {
+        return metaFrame.getLevel() == FREE_PAGE_LEVEL_INDICATOR;
+    }
+
+    @Override
+    public boolean isMetaPage(ITreeIndexMetaDataFrame metaFrame) {
+        return (metaFrame.getLevel() == META_PAGE_LEVEL_INDICATOR);
+    }
+
+    @Override
+    public void open(int fileId) {
+        this.fileId = fileId;
+    }
+
+    @Override
+    public void close() throws HyracksDataException {
+        if (appendOnly && fileId >= 0 && confiscatedMetaNode != null) {
+            IFIFOPageQueue queue = bufferCache.createFIFOQueue();
+            writeFilterPage(queue);
+            ITreeIndexMetaDataFrame metaFrame = metaDataFrameFactory.createFrame();
+            metaFrame.setPage(confiscatedMetaNode);
+            metaFrame.setValid(true);
+            int finalMetaPage = getMaxPage(metaFrame) + 1;
+            bufferCache.setPageDiskId(confiscatedMetaNode, BufferedFileHandle.getDiskPageId(fileId,finalMetaPage));
+            queue.put(confiscatedMetaNode);
+            bufferCache.finishQueue();
+            confiscatedMetaNode = null;
+        }
+    }
+
+    private void writeFilterPage(IFIFOPageQueue queue) throws HyracksDataException {
+        if(filterPage != null) {
+            ITreeIndexMetaDataFrame metaFrame = metaDataFrameFactory.createFrame();
+            metaFrame.setPage(confiscatedMetaNode);
+            int finalFilterPage = getFreePage(metaFrame);
+            bufferCache.setPageDiskId(filterPage, BufferedFileHandle.getDiskPageId(fileId, finalFilterPage));
+            queue.put(filterPage);
+        }
+    }
+
+    /**
+     * For storage on append-only media (such as HDFS), the meta data page has to be written last.
+     * However, some implementations still write the meta data to the front. To deal with this as well
+     * as to provide downward compatibility, this method tries to find the meta data page first in the
+     * last and then in the first page of the file.
+     *
+     * @return The Id of the page holding the meta data
+     * @throws HyracksDataException
+     */
+    @Override
+    public int getFirstMetadataPage() throws HyracksDataException {
+        if (headPage != IBufferCache.INVALID_PAGEID)
+            return headPage;
+
+        ITreeIndexMetaDataFrame metaFrame = metaDataFrameFactory.createFrame();
+
+        int pages = bufferCache.getNumPagesOfFile(fileId);
+        //if there are no pages in the file yet, we're just initializing
+        if(pages == 0){
+            return 0;
+        }
+        //look at the front (modify in-place index)
+        int page = 0;
+        ICachedPage metaNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, page), false);
+        try {
+            metaNode.acquireReadLatch();
+            metaFrame.setPage(metaNode);
+
+            if (isMetaPage(metaFrame)) {
+                headPage = page;
+                return headPage;
+            }
+        } finally {
+            metaNode.releaseReadLatch();
+            bufferCache.unpin(metaNode);
+        }
+        //otherwise, look at the back. (append-only index)
+        page = pages-1 >0 ? pages -1 : 0;
+        metaNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, page), false);
+        try {
+            metaNode.acquireReadLatch();
+            metaFrame.setPage(metaNode);
+
+            if (isMetaPage(metaFrame)) {
+                headPage = page;
+                return headPage;
+            }
+        } finally {
+            metaNode.releaseReadLatch();
+            bufferCache.unpin(metaNode);
+        }
+        //if we find nothing, this isn't a tree (or isn't one yet).
+        if(pages>0){
+            return IBufferCache.INVALID_PAGEID;
+        }
+        else{
+            return 0;
+        }
+    }
+
+    @Override
+    public long getLSN() throws HyracksDataException {
+        ICachedPage metaNode;
+        if (!appendOnly || (appendOnly && confiscatedMetaNode == null)) {
+            metaNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, getFirstMetadataPage()), false);
+        } else {
+            metaNode = confiscatedMetaNode;
+        }
+        ITreeIndexMetaDataFrame metaFrame = metaDataFrameFactory.createFrame();
+        metaNode.acquireReadLatch();
+        try {
+            metaFrame.setPage(metaNode);
+            return metaFrame.getLSN();
+        } finally {
+            metaNode.releaseReadLatch();
+            if (!appendOnly || (appendOnly && confiscatedMetaNode == null)) {
+                bufferCache.unpin(metaNode);
+            }
+        }
+    }
+
+    @Override
+    public void setLSN(long lsn) throws HyracksDataException {
+        ICachedPage metaNode;
+        if (!appendOnly) {
+            metaNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, getFirstMetadataPage()), false);
+        } else {
+            metaNode = confiscatedMetaNode;
+        }
+        ITreeIndexMetaDataFrame metaFrame = metaDataFrameFactory.createFrame();
+        metaNode.acquireWriteLatch();
+        try {
+            metaFrame.setPage(metaNode);
+            metaFrame.setLSN(lsn);
+        } finally {
+            if (!appendOnly) {
+                metaNode.releaseWriteLatch(true);
+                bufferCache.unpin(metaNode);
+            } else {
+                metaNode.releaseWriteLatch(false);
+            }
+        }
+    }
+
+    @Override
+    public void setFilterPage(ICachedPage filterPage) {
+        this.filterPage = filterPage;
+    }
+
+    @Override
+    public ICachedPage getFilterPage() {
+        return this.filterPage;
+    }
+
+    @Override
+    public boolean appendOnlyMode() {
+        return appendOnly;
+    }
+}
+

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
index e7b535f..015404d 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/impls/AbstractTreeIndex.java
@@ -25,29 +25,22 @@ import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.io.FileReference;
 import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.storage.am.common.api.IFreePageManager;
-import org.apache.hyracks.storage.am.common.api.IIndexBulkLoader;
-import org.apache.hyracks.storage.am.common.api.ITreeIndex;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexAccessor;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
-import org.apache.hyracks.storage.am.common.api.IndexException;
-import org.apache.hyracks.storage.am.common.api.TreeIndexException;
+import org.apache.hyracks.storage.am.common.api.*;
 import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
 import org.apache.hyracks.storage.common.buffercache.IBufferCache;
 import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue;
 import org.apache.hyracks.storage.common.file.BufferedFileHandle;
 import org.apache.hyracks.storage.common.file.IFileMapProvider;
 
 public abstract class AbstractTreeIndex implements ITreeIndex {
 
-    protected final static int rootPage = 1;
+    public static final int MINIMAL_TREE_PAGE_COUNT = 2;
+    protected int rootPage = 1;
 
     protected final IBufferCache bufferCache;
     protected final IFileMapProvider fileMapProvider;
-    protected final IFreePageManager freePageManager;
+    protected final IMetaDataPageManager freePageManager;
 
     protected final ITreeIndexFrameFactory interiorFrameFactory;
     protected final ITreeIndexFrameFactory leafFrameFactory;
@@ -58,10 +51,17 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
     protected FileReference file;
     protected int fileId = -1;
 
-    private boolean isActivated = false;
+    protected boolean isActive = false;
+    //hasEverBeenActivated is to stop the throwing of an exception of deactivating an index that
+    //was never activated or failed to activate in try/finally blocks, as there's no way to know if
+    //an index is activated or not from the outside.
+    protected boolean hasEverBeenActivated = false;
+    protected boolean appendOnly = false;
+
+    protected int bulkloadLeafStart = 0;
 
     public AbstractTreeIndex(IBufferCache bufferCache, IFileMapProvider fileMapProvider,
-            IFreePageManager freePageManager, ITreeIndexFrameFactory interiorFrameFactory,
+            IMetaDataPageManager freePageManager, ITreeIndexFrameFactory interiorFrameFactory,
             ITreeIndexFrameFactory leafFrameFactory, IBinaryComparatorFactory[] cmpFactories, int fieldCount,
             FileReference file) {
         this.bufferCache = bufferCache;
@@ -75,7 +75,11 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
     }
 
     public synchronized void create() throws HyracksDataException {
-        if (isActivated) {
+        create(false);
+    }
+
+    private synchronized void create(boolean appendOnly) throws HyracksDataException {
+        if (isActive) {
             throw new HyracksDataException("Failed to create the index since it is activated.");
         }
 
@@ -99,8 +103,14 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
         }
 
         freePageManager.open(fileId);
-        initEmptyTree();
-        freePageManager.close();
+        setRootAndMetadataPages(appendOnly);
+        if (!appendOnly) {
+            initEmptyTree();
+            freePageManager.close();
+        } else {
+            this.appendOnly = true;
+            initCachedMetadataPage();
+        }
         bufferCache.closeFile(fileId);
     }
 
@@ -108,7 +118,6 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
         ITreeIndexFrame frame = leafFrameFactory.createFrame();
         ITreeIndexMetaDataFrame metaFrame = freePageManager.getMetaDataFrameFactory().createFrame();
         freePageManager.init(metaFrame, rootPage);
-
         ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), true);
         rootNode.acquireWriteLatch();
         try {
@@ -120,8 +129,28 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
         }
     }
 
+    private void setRootAndMetadataPages(boolean appendOnly) throws HyracksDataException{
+        if (!appendOnly) {
+            // regular or empty tree
+            rootPage = 1;
+            bulkloadLeafStart = 2;
+        } else {
+            // bulkload-only tree (used e.g. for HDFS). -1 is meta page, -2 is root page
+            int numPages = bufferCache.getNumPagesOfFile(fileId);
+            //the root page is the last page before the metadata page
+            rootPage = numPages > MINIMAL_TREE_PAGE_COUNT ? numPages - MINIMAL_TREE_PAGE_COUNT : 0;
+            //leaves start from the very beginning of the file.
+            bulkloadLeafStart = 0;
+        }
+    }
+
+    private void initCachedMetadataPage() throws HyracksDataException {
+        ITreeIndexMetaDataFrame metaFrame = freePageManager.getMetaDataFrameFactory().createFrame();
+        freePageManager.init(metaFrame);
+    }
+
     public synchronized void activate() throws HyracksDataException {
-        if (isActivated) {
+        if (isActive) {
             throw new HyracksDataException("Failed to activate the index since it is already activated.");
         }
 
@@ -144,26 +173,44 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
             }
         }
         freePageManager.open(fileId);
+        int mdPageLoc = freePageManager.getFirstMetadataPage();
+        ITreeIndexMetaDataFrame metaFrame = freePageManager.getMetaDataFrameFactory().createFrame();
+        int numPages = freePageManager.getMaxPage(metaFrame);
+        if(mdPageLoc > 1 || (mdPageLoc == 1 && numPages <= MINIMAL_TREE_PAGE_COUNT -1  )){ //md page doesn't count itself
+            appendOnly = true;
+        }
+        else{
+            appendOnly = false;
+        }
+        setRootAndMetadataPages(appendOnly);
 
         // TODO: Should probably have some way to check that the tree is physically consistent
         // or that the file we just opened actually is a tree
 
-        isActivated = true;
+        isActive = true;
+        hasEverBeenActivated = true;
     }
 
     public synchronized void deactivate() throws HyracksDataException {
-        if (!isActivated) {
+        if (!isActive && hasEverBeenActivated) {
             throw new HyracksDataException("Failed to deactivate the index since it is already deactivated.");
         }
+        if (isActive) {
+            freePageManager.close();
+            bufferCache.closeFile(fileId);
+        }
 
-        bufferCache.closeFile(fileId);
-        freePageManager.close();
+        isActive = false;
+    }
+
+    public synchronized void deactivateCloseHandle() throws HyracksDataException {
+        deactivate();
+        bufferCache.purgeHandle(fileId);
 
-        isActivated = false;
     }
 
     public synchronized void destroy() throws HyracksDataException {
-        if (isActivated) {
+        if (isActive) {
             throw new HyracksDataException("Failed to destroy the index since it is activated.");
         }
 
@@ -176,13 +223,19 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
     }
 
     public synchronized void clear() throws HyracksDataException {
-        if (!isActivated) {
+        if (!isActive) {
             throw new HyracksDataException("Failed to clear the index since it is not activated.");
         }
         initEmptyTree();
     }
 
     public boolean isEmptyTree(ITreeIndexFrame frame) throws HyracksDataException {
+        if (rootPage == -1) {
+            return true;
+        }
+        if(freePageManager.appendOnlyMode() && bufferCache.getNumPagesOfFile(fileId) <= MINIMAL_TREE_PAGE_COUNT){
+            return true;
+        }
         ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), false);
         rootNode.acquireReadLatch();
         try {
@@ -198,6 +251,8 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
         }
     }
 
+
+
     public byte getTreeHeight(ITreeIndexFrame frame) throws HyracksDataException {
         ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), false);
         rootNode.acquireReadLatch();
@@ -234,7 +289,7 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
         return cmpFactories;
     }
 
-    public IFreePageManager getFreePageManager() {
+    public IMetaDataPageManager getMetaManager() {
         return freePageManager;
     }
 
@@ -256,13 +311,28 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
         protected final ITreeIndexTupleWriter tupleWriter;
         protected ITreeIndexFrame leafFrame;
         protected ITreeIndexFrame interiorFrame;
-        private boolean releasedLatches;
+        // Immutable bulk loaders write their root page at page -2, as needed e.g. by append-only file systems such as HDFS. 
+        // Since loading this tree relies on the root page actually being at that point, no further inserts into that tree are allowed.
+        // Currently, this is not enforced.
+        protected boolean releasedLatches;
+        public boolean appendOnly = false;
+        protected final IFIFOPageQueue queue;
+
+        public AbstractTreeIndexBulkLoader(float fillFactor, boolean appendOnly) throws TreeIndexException,
+                HyracksDataException {
+            //Initialize the tree 
+            if (appendOnly) {
+                create(appendOnly);
+                this.appendOnly = appendOnly;
+                activate();
+            }
 
-        public AbstractTreeIndexBulkLoader(float fillFactor) throws TreeIndexException, HyracksDataException {
             leafFrame = leafFrameFactory.createFrame();
             interiorFrame = interiorFrameFactory.createFrame();
             metaFrame = freePageManager.getMetaDataFrameFactory().createFrame();
 
+            queue = bufferCache.createFIFOQueue();
+
             if (!isEmptyTree(leafFrame)) {
                 throw new TreeIndexException("Cannot bulk-load a non-empty tree.");
             }
@@ -276,8 +346,8 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
 
             NodeFrontier leafFrontier = new NodeFrontier(leafFrame.createTupleReference());
             leafFrontier.pageId = freePageManager.getFreePage(metaFrame);
-            leafFrontier.page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, leafFrontier.pageId), true);
-            leafFrontier.page.acquireWriteLatch();
+            leafFrontier.page = bufferCache.confiscatePage(BufferedFileHandle
+                    .getDiskPageId(fileId, leafFrontier.pageId));
 
             interiorFrame.setPage(leafFrontier.page);
             interiorFrame.initBuffer((byte) 0);
@@ -294,48 +364,50 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
         public abstract void add(ITupleReference tuple) throws IndexException, HyracksDataException;
 
         protected void handleException() throws HyracksDataException {
-            // Unlatch and unpin pages.
+            // Unlatch and unpin pages that weren't in the queue to avoid leaking memory.
             for (NodeFrontier nodeFrontier : nodeFrontiers) {
-                nodeFrontier.page.releaseWriteLatch(true);
-                bufferCache.unpin(nodeFrontier.page);
+                ICachedPage frontierPage = nodeFrontier.page;
+                if (frontierPage.confiscated()) {
+                    bufferCache.returnPage(frontierPage,false);
+                }
             }
             releasedLatches = true;
         }
 
         @Override
         public void end() throws HyracksDataException {
-            // copy the root generated from the bulk-load to *the* root page location
-            ICachedPage newRoot = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), true);
-            newRoot.acquireWriteLatch();
-            NodeFrontier lastNodeFrontier = nodeFrontiers.get(nodeFrontiers.size() - 1);
-            try {
-                System.arraycopy(lastNodeFrontier.page.getBuffer().array(), 0, newRoot.getBuffer().array(), 0,
-                        lastNodeFrontier.page.getBuffer().capacity());
-            } finally {
-                newRoot.releaseWriteLatch(true);
-                bufferCache.unpin(newRoot);
-
-                // register old root as a free page
-                freePageManager.addFreePage(metaFrame, lastNodeFrontier.pageId);
-
-                if (!releasedLatches) {
-                    for (int i = 0; i < nodeFrontiers.size(); i++) {
-                        try {
-                            nodeFrontiers.get(i).page.releaseWriteLatch(true);
-                        } catch (Exception e) {
-                            //ignore illegal monitor state exception
-                        }
-                        bufferCache.unpin(nodeFrontiers.get(i).page);
-                    }
+            //move the root page to the first data page if necessary
+            bufferCache.finishQueue();
+            if (!appendOnly) {
+                ICachedPage newRoot = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), true);
+                newRoot.acquireWriteLatch();
+                //root will be the highest frontier
+                NodeFrontier lastNodeFrontier = nodeFrontiers.get(nodeFrontiers.size() - 1);
+                ICachedPage oldRoot = bufferCache.pin(
+                        BufferedFileHandle.getDiskPageId(fileId, lastNodeFrontier.pageId), false);
+                oldRoot.acquireReadLatch();
+                lastNodeFrontier.page = oldRoot;
+                try {
+                    System.arraycopy(lastNodeFrontier.page.getBuffer().array(), 0, newRoot.getBuffer().array(), 0,
+                            lastNodeFrontier.page.getBuffer().capacity());
+                } finally {
+                    newRoot.releaseWriteLatch(true);
+                    bufferCache.flushDirtyPage(newRoot);
+                    bufferCache.unpin(newRoot);
+                    oldRoot.releaseReadLatch();
+                    bufferCache.unpin(oldRoot);
+
+                    // register old root as a free page
+                    freePageManager.addFreePage(metaFrame, lastNodeFrontier.pageId);
+
                 }
             }
         }
 
         protected void addLevel() throws HyracksDataException {
             NodeFrontier frontier = new NodeFrontier(tupleWriter.createTupleReference());
-            frontier.pageId = freePageManager.getFreePage(metaFrame);
-            frontier.page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, frontier.pageId), true);
-            frontier.page.acquireWriteLatch();
+            frontier.page = bufferCache.confiscatePage(IBufferCache.INVALID_DPID);
+            frontier.pageId = -1;
             frontier.lastTuple.setFieldCount(cmp.getKeyFieldCount());
             interiorFrame.setPage(frontier.page);
             interiorFrame.initBuffer((byte) nodeFrontiers.size());
@@ -349,6 +421,7 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
         public void setLeafFrame(ITreeIndexFrame leafFrame) {
             this.leafFrame = leafFrame;
         }
+
     }
 
     public class TreeIndexInsertBulkLoader implements IIndexBulkLoader {
@@ -373,6 +446,11 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
             // do nothing
         }
 
+        @Override
+        public void abort() {
+
+        }
+
     }
 
     @Override
@@ -383,7 +461,7 @@ public abstract class AbstractTreeIndex implements ITreeIndex {
     public IBinaryComparatorFactory[] getCmpFactories() {
         return cmpFactories;
     }
-    
+
     @Override
     public boolean hasMemoryComponents() {
         return true;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
index 14c1ad9..398a3f3 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
@@ -22,7 +22,7 @@ import java.util.ArrayList;
 import java.util.Random;
 
 import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.common.api.IFreePageManager;
+import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
 import org.apache.hyracks.storage.am.common.ophelpers.IntArrayList;
@@ -32,13 +32,13 @@ import org.apache.hyracks.storage.common.file.BufferedFileHandle;
 
 public class TreeIndexBufferCacheWarmup {
 	private final IBufferCache bufferCache;
-	private final IFreePageManager freePageManager;
+	private final IMetaDataPageManager freePageManager;
 	private final int fileId;
 	private final ArrayList<IntArrayList> pagesByLevel = new ArrayList<IntArrayList>();
 	private final Random rnd = new Random();
 
 	public TreeIndexBufferCacheWarmup(IBufferCache bufferCache,
-			IFreePageManager freePageManager, int fileId) {
+			IMetaDataPageManager freePageManager, int fileId) {
 		this.bufferCache = bufferCache;
 		this.freePageManager = freePageManager;
 		this.fileId = fileId;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStats.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStats.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStats.java
index ad1d888..8251c6f 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStats.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStats.java
@@ -20,7 +20,7 @@ package org.apache.hyracks.storage.am.common.util;
 
 import java.text.DecimalFormat;
 
-import org.apache.hyracks.storage.am.common.api.IFreePageManager;
+import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
 
@@ -57,7 +57,7 @@ public class TreeIndexStats {
 	}
 
 	public void add(ITreeIndexMetaDataFrame metaFrame,
-			IFreePageManager freePageManager) {
+			IMetaDataPageManager freePageManager) {
 		if (freePageManager.isFreePage(metaFrame)) {
 			freePages++;
 		} else if (freePageManager.isMetaPage(metaFrame)) {

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
index a12df46..3c15b23 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
@@ -19,7 +19,7 @@
 package org.apache.hyracks.storage.am.common.util;
 
 import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.common.api.IFreePageManager;
+import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
 import org.apache.hyracks.storage.common.buffercache.IBufferCache;
@@ -30,12 +30,12 @@ public class TreeIndexStatsGatherer {
 
 	private final TreeIndexStats treeIndexStats = new TreeIndexStats();
 	private final IBufferCache bufferCache;
-	private final IFreePageManager freePageManager;
+	private final IMetaDataPageManager freePageManager;
 	private final int fileId;
 	private final int rootPage;
 
 	public TreeIndexStatsGatherer(IBufferCache bufferCache,
-			IFreePageManager freePageManager, int fileId, int rootPage) {
+			IMetaDataPageManager freePageManager, int fileId, int rootPage) {
 		this.bufferCache = bufferCache;
 		this.freePageManager = freePageManager;
 		this.fileId = fileId;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
index 3ade4fa..7861730 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java
@@ -32,19 +32,8 @@ import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
 import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterSpecification;
 import org.apache.hyracks.storage.am.btree.impls.BTree;
 import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeBulkLoader;
-import org.apache.hyracks.storage.am.common.api.IFreePageManager;
-import org.apache.hyracks.storage.am.common.api.IIndexBulkLoader;
-import org.apache.hyracks.storage.am.common.api.IIndexCursor;
-import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
-import org.apache.hyracks.storage.am.common.api.IModificationOperationCallback;
-import org.apache.hyracks.storage.am.common.api.ISearchOperationCallback;
-import org.apache.hyracks.storage.am.common.api.ISearchPredicate;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
-import org.apache.hyracks.storage.am.common.api.ITwoPCIndexBulkLoader;
-import org.apache.hyracks.storage.am.common.api.IndexException;
-import org.apache.hyracks.storage.am.common.api.TreeIndexException;
+import org.apache.hyracks.storage.am.common.api.*;
+import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
 import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
 import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
 import org.apache.hyracks.storage.am.lsm.btree.tuples.LSMBTreeRefrencingTupleWriterFactory;
@@ -510,7 +499,7 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
             frameTupleWriterFactory = ((LSMBTreeDiskComponent) component).getBTree().getLeafFrameFactory()
                     .getTupleWriterFactory();
             bulkLoader = (BTreeBulkLoader) ((LSMBTreeDiskComponent) component).getBTree().createBulkLoader(fillFactor,
-                    verifyInput, numElementsHint, false);
+                    verifyInput, numElementsHint, false, true);
 
             int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElementsHint);
             BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement,
@@ -652,7 +641,7 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex {
     }
 
     @Override
-    public IFreePageManager getFreePageManager() {
+    public IMetaDataPageManager getMetaManager() {
         return null;
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
index bf0040a..4b5e5c8 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
@@ -35,19 +35,7 @@ import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterSpecification;
 import org.apache.hyracks.storage.am.btree.impls.BTree;
 import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeBulkLoader;
 import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
-import org.apache.hyracks.storage.am.common.api.IFreePageManager;
-import org.apache.hyracks.storage.am.common.api.IIndexBulkLoader;
-import org.apache.hyracks.storage.am.common.api.IIndexCursor;
-import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
-import org.apache.hyracks.storage.am.common.api.IModificationOperationCallback;
-import org.apache.hyracks.storage.am.common.api.ISearchOperationCallback;
-import org.apache.hyracks.storage.am.common.api.ISearchPredicate;
-import org.apache.hyracks.storage.am.common.api.ITreeIndex;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import org.apache.hyracks.storage.am.common.api.ITwoPCIndexBulkLoader;
-import org.apache.hyracks.storage.am.common.api.IndexException;
-import org.apache.hyracks.storage.am.common.api.TreeIndexException;
+import org.apache.hyracks.storage.am.common.api.*;
 import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
 import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
 import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
@@ -299,7 +287,8 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
     public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint,
             boolean checkIfEmptyIndex) throws TreeIndexException {
         try {
-            return new LSMTwoPCBTreeWithBuddyBulkLoader(fillLevel, verifyInput, 0, checkIfEmptyIndex, false);
+            return new LSMTwoPCBTreeWithBuddyBulkLoader(fillLevel, verifyInput, 0, checkIfEmptyIndex,
+                    false);
         } catch (HyracksDataException e) {
             throw new TreeIndexException(e);
         }
@@ -503,14 +492,8 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
     public void markAsValid(ILSMComponent lsmComponent) throws HyracksDataException {
         LSMBTreeWithBuddyDiskComponent component = (LSMBTreeWithBuddyDiskComponent) lsmComponent;
         // Flush the bloom filter first.
-        int fileId = component.getBloomFilter().getFileId();
-        IBufferCache bufferCache = component.getBTree().getBufferCache();
-        int startPage = 0;
-        int maxPage = component.getBloomFilter().getNumPages();
-        forceFlushDirtyPages(bufferCache, fileId, startPage, maxPage);
-        forceFlushDirtyPages(component.getBTree());
+        markAsValidInternal(component.getBTree().getBufferCache(),component.getBloomFilter());
         markAsValidInternal(component.getBTree());
-        forceFlushDirtyPages(component.getBuddyBTree());
         markAsValidInternal(component.getBuddyBTree());
     }
 
@@ -590,7 +573,7 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
     }
 
     @Override
-    public IFreePageManager getFreePageManager() {
+    public IMetaDataPageManager getMetaManager() {
         // This method should never be called for disk only indexes
         return null;
     }
@@ -625,13 +608,9 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
                 .createLSMComponentInstance(new LSMComponentFileReferences(insertFileRef, deleteFileRef,
                         bloomFilterFileRef));
         if (createComponent) {
-            component.getBTree().create();
-            component.getBuddyBTree().create();
             component.getBloomFilter().create();
         }
 
-        component.getBTree().activate();
-        component.getBuddyBTree().activate();
         component.getBloomFilter().activate();
         return component;
     }
@@ -696,9 +675,9 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
 
             // Create the three loaders
             btreeBulkLoader = (BTreeBulkLoader) ((LSMBTreeWithBuddyDiskComponent) component).getBTree()
-                    .createBulkLoader(fillFactor, verifyInput, numElementsHint, false);
+                    .createBulkLoader(fillFactor, verifyInput, numElementsHint, false,true);
             buddyBtreeBulkLoader = (BTreeBulkLoader) ((LSMBTreeWithBuddyDiskComponent) component).getBuddyBTree()
-                    .createBulkLoader(fillFactor, verifyInput, numElementsHint, false);
+                    .createBulkLoader(fillFactor, verifyInput, numElementsHint, false, true);
             int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElementsHint);
             BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement,
                     bloomFilterFalsePositiveRate);
@@ -886,7 +865,7 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
     public boolean isPrimaryIndex() {
         return false;
     }
-    
+
     @Override
     public Set<String> getLSMComponentPhysicalFiles(ILSMComponent lsmComponent) {
         Set<String> files = new HashSet<String>();
@@ -900,7 +879,18 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd
     }
 
     @Override
+    public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
+            boolean checkIfEmptyIndex, boolean appendOnly) throws IndexException {
+        if(!appendOnly){
+            throw new IndexException("LSM Indices do not support in-place inserts");
+        }
+        else{
+            return createBulkLoader(fillFactor, verifyInput,numElementsHint,checkIfEmptyIndex);
+        }
+   }
+
+    @Override
     public void allocateMemoryComponents() throws HyracksDataException {
         //do nothing since external index never use memory components
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index 822b320..04f76d5 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -39,19 +39,8 @@ import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeAccessor;
 import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeBulkLoader;
 import org.apache.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
 import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
-import org.apache.hyracks.storage.am.common.api.IFreePageManager;
-import org.apache.hyracks.storage.am.common.api.IIndexAccessor;
-import org.apache.hyracks.storage.am.common.api.IIndexBulkLoader;
-import org.apache.hyracks.storage.am.common.api.IIndexCursor;
-import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
-import org.apache.hyracks.storage.am.common.api.IModificationOperationCallback;
-import org.apache.hyracks.storage.am.common.api.ISearchOperationCallback;
-import org.apache.hyracks.storage.am.common.api.ISearchPredicate;
-import org.apache.hyracks.storage.am.common.api.ITreeIndex;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import org.apache.hyracks.storage.am.common.api.IndexException;
-import org.apache.hyracks.storage.am.common.api.TreeIndexException;
+import org.apache.hyracks.storage.am.common.api.*;
+import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
 import org.apache.hyracks.storage.am.common.exceptions.TreeIndexDuplicateKeyException;
 import org.apache.hyracks.storage.am.common.impls.AbstractSearchPredicate;
 import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
@@ -73,7 +62,7 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
 import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
-import org.apache.hyracks.storage.am.lsm.common.freepage.VirtualFreePageManager;
+import org.apache.hyracks.storage.am.lsm.common.freepage.VirtualMetaDataPageManager;
 import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
 import org.apache.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
 import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
@@ -119,7 +108,7 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
         int i = 0;
         for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) {
             LSMBTreeMemoryComponent mutableComponent = new LSMBTreeMemoryComponent(new BTree(virtualBufferCache,
-                    virtualBufferCache.getFileMapProvider(), new VirtualFreePageManager(
+                    virtualBufferCache.getFileMapProvider(), new VirtualMetaDataPageManager(
                             virtualBufferCache.getNumPages()), interiorFrameFactory, insertLeafFrameFactory,
                     cmpFactories, fieldCount, new FileReference(new File(fileManager.getBaseDir() + "_virtual_" + i))),
                     virtualBufferCache, i == 0 ? true : false, filterFactory == null ? null
@@ -213,7 +202,7 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
             LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
             BTree btree = component.getBTree();
             BloomFilter bloomFilter = component.getBloomFilter();
-            btree.deactivate();
+            btree.deactivateCloseHandle();
             bloomFilter.deactivate();
         }
         deallocateMemoryComponents();
@@ -449,7 +438,7 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
 
         LSMBTreeDiskComponent component = createDiskComponent(componentFactory, flushOp.getBTreeFlushTarget(),
                 flushOp.getBloomFilterFlushTarget(), true);
-        IIndexBulkLoader bulkLoader = component.getBTree().createBulkLoader(1.0f, false, numElements, false);
+        IIndexBulkLoader bulkLoader = component.getBTree().createBulkLoader(1.0f, false, numElements, false, true);
         IIndexBulkLoader builder = component.getBloomFilter().createBuilder(numElements,
                 bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements());
 
@@ -465,15 +454,18 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
             scanCursor.close();
             builder.end();
         }
-        bulkLoader.end();
 
         if (component.getLSMComponentFilter() != null) {
             List<ITupleReference> filterTuples = new ArrayList<ITupleReference>();
             filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
             filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
             filterManager.updateFilterInfo(component.getLSMComponentFilter(), filterTuples);
-            filterManager.writeFilterInfo(component.getLSMComponentFilter(), component.getBTree());
+            filterManager.writeFilterInfo(component.getLSMComponentFilter(), component.getBTree()
+            );
         }
+
+        bulkLoader.end();
+
         return component;
     }
 
@@ -522,7 +514,8 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
         LSMBTreeDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getBTreeMergeTarget(),
                 mergeOp.getBloomFilterMergeTarget(), true);
 
-        IIndexBulkLoader bulkLoader = mergedComponent.getBTree().createBulkLoader(1.0f, false, numElements, false);
+        IIndexBulkLoader bulkLoader = mergedComponent.getBTree()
+                .createBulkLoader(1.0f, false, numElements, false, true);
         IIndexBulkLoader builder = mergedComponent.getBloomFilter().createBuilder(numElements,
                 bloomFilterSpec.getNumHashes(), bloomFilterSpec.getNumBucketsPerElements());
         try {
@@ -536,8 +529,6 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
             cursor.close();
             builder.end();
         }
-        bulkLoader.end();
-
         if (mergedComponent.getLSMComponentFilter() != null) {
             List<ITupleReference> filterTuples = new ArrayList<ITupleReference>();
             for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
@@ -545,9 +536,12 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
                 filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
             }
             filterManager.updateFilterInfo(mergedComponent.getLSMComponentFilter(), filterTuples);
-            filterManager.writeFilterInfo(mergedComponent.getLSMComponentFilter(), mergedComponent.getBTree());
+            filterManager.writeFilterInfo(mergedComponent.getLSMComponentFilter(), mergedComponent.getBTree()
+            );
         }
 
+        bulkLoader.end();
+
         return mergedComponent;
     }
 
@@ -557,14 +551,12 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
         // Create new BTree instance.
         LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) factory
                 .createLSMComponentInstance(new LSMComponentFileReferences(btreeFileRef, null, bloomFilterFileRef));
-        if (createComponent) {
-            component.getBTree().create();
-            component.getBloomFilter().create();
-        }
         // BTree will be closed during cleanup of merge().
-        component.getBTree().activate();
+        if (!createComponent) {
+            component.getBTree().activate();
+        }
         component.getBloomFilter().activate();
-        if (component.getLSMComponentFilter() != null) {
+        if (component.getLSMComponentFilter() != null && !createComponent) {
             filterManager.readFilterInfo(component.getLSMComponentFilter(), component.getBTree());
         }
         return component;
@@ -591,13 +583,7 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
         // The order of forcing the dirty page to be flushed is critical. The
         // bloom filter must be always done first.
         LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) lsmComponent;
-        // Flush the bloom filter first.
-        int fileId = component.getBloomFilter().getFileId();
-        IBufferCache bufferCache = component.getBTree().getBufferCache();
-        int startPage = 0;
-        int maxPage = component.getBloomFilter().getNumPages();
-        forceFlushDirtyPages(bufferCache, fileId, startPage, maxPage);
-        forceFlushDirtyPages(component.getBTree());
+        markAsValidInternal(component.getBTree().getBufferCache(),component.getBloomFilter());
         markAsValidInternal(component.getBTree());
     }
 
@@ -623,7 +609,7 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
                 throw new TreeIndexException(e);
             }
             bulkLoader = (BTreeBulkLoader) ((LSMBTreeDiskComponent) component).getBTree().createBulkLoader(fillFactor,
-                    verifyInput, numElementsHint, false);
+                    verifyInput, numElementsHint, false, true);
 
             int maxBucketsPerElement = BloomCalculations.maxBucketsPerElement(numElementsHint);
             BloomFilterSpecification bloomFilterSpec = BloomCalculations.computeBloomSpec(maxBucketsPerElement,
@@ -672,10 +658,9 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
         protected void cleanupArtifacts() throws HyracksDataException, IndexException {
             if (!cleanedUpArtifacts) {
                 cleanedUpArtifacts = true;
-                // We make sure to end the bloom filter load to release latches.
                 if (!endedBloomFilterLoad) {
-                    builder.end();
-                    endedBloomFilterLoad = true;
+                        builder.abort();
+	                    endedBloomFilterLoad = true;
                 }
                 ((LSMBTreeDiskComponent) component).getBTree().deactivate();
                 ((LSMBTreeDiskComponent) component).getBTree().destroy();
@@ -691,13 +676,14 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
                     builder.end();
                     endedBloomFilterLoad = true;
                 }
-                bulkLoader.end();
 
                 if (component.getLSMComponentFilter() != null) {
                     filterManager.writeFilterInfo(component.getLSMComponentFilter(),
                             ((LSMBTreeDiskComponent) component).getBTree());
                 }
 
+                bulkLoader.end();
+
                 if (isEmptyComponent) {
                     cleanupArtifacts();
                 } else {
@@ -705,6 +691,18 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
                 }
             }
         }
+
+        @Override
+        public void abort() throws HyracksDataException {
+            if(bulkLoader != null){
+                bulkLoader.abort();
+            }
+
+            if(builder != null){
+                builder.abort();
+            }
+
+        }
     }
 
     public LSMBTreeOpContext createOpContext(IModificationOperationCallback modificationCallback,
@@ -768,10 +766,10 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
     }
 
     @Override
-    public IFreePageManager getFreePageManager() {
+    public IMetaDataPageManager getMetaManager() {
         LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) memoryComponents
                 .get(currentMutableComponentId.get());
-        return mutableComponent.getBTree().getFreePageManager();
+        return mutableComponent.getBTree().getMetaManager();
     }
 
     @Override
@@ -820,6 +818,16 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex {
     }
 
     @Override
+    public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
+            boolean checkIfEmptyIndex, boolean appendOnly) throws IndexException {
+        try {
+            return new LSMBTreeBulkLoader(fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex);
+        } catch (HyracksDataException e) {
+            throw new TreeIndexException(e);
+        }
+    }
+
+    @Override
     public Set<String> getLSMComponentPhysicalFiles(ILSMComponent lsmComponent) {
         Set<String> files = new HashSet<String>();
         LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) lsmComponent;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
index e8e54ff..c175336 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/util/LSMBTreeUtils.java
@@ -28,11 +28,11 @@ import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory;
 import org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
 import org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
 import org.apache.hyracks.storage.am.btree.impls.BTree;
-import org.apache.hyracks.storage.am.common.api.IFreePageManagerFactory;
+import org.apache.hyracks.storage.am.common.api.IMetadataManagerFactory;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
 import org.apache.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
-import org.apache.hyracks.storage.am.common.freepage.LinkedListFreePageManagerFactory;
+import org.apache.hyracks.storage.am.common.freepage.LinkedListMetadataManagerFactory;
 import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
 import org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTree;
 import org.apache.hyracks.storage.am.lsm.btree.impls.ExternalBTreeWithBuddy;
@@ -74,7 +74,7 @@ public class LSMBTreeUtils {
         ITreeIndexFrameFactory deleteLeafFrameFactory = new BTreeNSMLeafFrameFactory(deleteTupleWriterFactory);
         ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
         ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
-        IFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(diskBufferCache,
+        IMetadataManagerFactory freePageManagerFactory = new LinkedListMetadataManagerFactory(diskBufferCache,
                 metaFrameFactory);
 
         TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,
@@ -123,7 +123,7 @@ public class LSMBTreeUtils {
         ITreeIndexFrameFactory deleteLeafFrameFactory = new BTreeNSMLeafFrameFactory(deleteTupleWriterFactory);
         ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
         ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
-        IFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(diskBufferCache,
+        IMetadataManagerFactory freePageManagerFactory = new LinkedListMetadataManagerFactory(diskBufferCache,
                 metaFrameFactory);
         // This is the tuple writer that can do both inserts and deletes
         LSMBTreeRefrencingTupleWriterFactory referencingTupleWriterFactory = new LSMBTreeRefrencingTupleWriterFactory(
@@ -181,7 +181,7 @@ public class LSMBTreeUtils {
         ITreeIndexFrameFactory copyTupleLeafFrameFactory = new BTreeNSMLeafFrameFactory(copyTupleWriterFactory);
         ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(insertTupleWriterFactory);
         ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
-        IFreePageManagerFactory freePageManagerFactory = new LinkedListFreePageManagerFactory(diskBufferCache,
+        IMetadataManagerFactory freePageManagerFactory = new LinkedListMetadataManagerFactory(diskBufferCache,
                 metaFrameFactory);
 
         TreeIndexFactory<BTree> diskBTreeFactory = new BTreeFactory(diskBufferCache, diskFileMapProvider,

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterManager.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterManager.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterManager.java
index 1a64865..20598ca 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterManager.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMComponentFilterManager.java
@@ -22,7 +22,9 @@ import java.util.List;
 
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.api.IIndexBulkLoader;
 import org.apache.hyracks.storage.am.common.api.ITreeIndex;
+import org.apache.hyracks.storage.am.common.impls.AbstractTreeIndex.AbstractTreeIndexBulkLoader;
 
 public interface ILSMComponentFilterManager {
 
@@ -31,6 +33,7 @@ public interface ILSMComponentFilterManager {
 
     public boolean readFilterInfo(ILSMComponentFilter filter, ITreeIndex treeIndex) throws HyracksDataException;
 
-    public void writeFilterInfo(ILSMComponentFilter filter, ITreeIndex treeIndex) throws HyracksDataException;
+    public void writeFilterInfo(ILSMComponentFilter filter, ITreeIndex treeIndex)
+            throws HyracksDataException;
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexInternal.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexInternal.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexInternal.java
index 08be21d..40d32a0 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexInternal.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexInternal.java
@@ -63,7 +63,7 @@ public interface ILSMIndexInternal extends ILSMIndex {
 
     /**
      * Populates the context's component holder with a snapshot of the components involved in the operation.
-     *
+     * 
      * @param ctx
      *            - the operation's context
      * @throws HyracksDataException
@@ -75,8 +75,8 @@ public interface ILSMIndexInternal extends ILSMIndex {
     public void addInactiveDiskComponent(ILSMComponent diskComponent);
 
     /**
-     * Persistent the LSM component
-     *
+     * Persist the LSM component
+     * 
      * @param lsmComponent
      *            , the component to be persistent
      * @throws HyracksDataException

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualFreePageManager.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualFreePageManager.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualFreePageManager.java
deleted file mode 100644
index d62bc84..0000000
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualFreePageManager.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.lsm.common.freepage;
-
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.common.api.IVirtualFreePageManager;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
-
-public class VirtualFreePageManager implements IVirtualFreePageManager {
-    protected final int capacity;
-    protected final AtomicInteger currentPageId = new AtomicInteger();
-
-    public VirtualFreePageManager(int capacity) {
-        // We start the currentPageId from 1, because the BTree uses
-        // the first page as metadata page, and the second page as root page.
-        // (when returning free pages we first increment, then get)
-        currentPageId.set(1);
-        this.capacity = capacity;
-    }
-
-    @Override
-    public int getFreePage(ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException {
-        // The very first call returns page id 2 because the BTree uses
-        // the first page as metadata page, and the second page as root page.
-        return currentPageId.incrementAndGet();
-    }
-
-    @Override
-    public int getMaxPage(ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException {
-        return currentPageId.get();
-    }
-
-    @Override
-    public void init(ITreeIndexMetaDataFrame metaFrame, int currentMaxPage) throws HyracksDataException {
-        currentPageId.set(1);
-    }
-
-    @Override
-    public ITreeIndexMetaDataFrameFactory getMetaDataFrameFactory() {
-        return NullMetadataFrameFactory.INSTANCE;
-    }
-
-    public int getCapacity() {
-        return capacity - 2;
-    }
-
-    public void reset() {
-        currentPageId.set(1);
-    }
-
-    @Override
-    public void addFreePage(ITreeIndexMetaDataFrame metaFrame, int freePage) throws HyracksDataException {
-    }
-
-    @Override
-    public byte getMetaPageLevelIndicator() {
-        return 0;
-    }
-
-    @Override
-    public byte getFreePageLevelIndicator() {
-        return 0;
-    }
-
-    @Override
-    public boolean isMetaPage(ITreeIndexMetaDataFrame metaFrame) {
-        return false;
-    }
-
-    @Override
-    public boolean isFreePage(ITreeIndexMetaDataFrame metaFrame) {
-        return false;
-    }
-
-    @Override
-    public int getFirstMetadataPage() {
-        // Method doesn't make sense for this free page manager.
-        return -1;
-    }
-
-    @Override
-    public void open(int fileId) {
-        // Method doesn't make sense for this free page manager.
-    }
-
-    @Override
-    public void close() {
-        // Method doesn't make sense for this free page manager.
-    }
-
-    private static class NullMetadataFrameFactory implements ITreeIndexMetaDataFrameFactory {
-        private static final NullMetadataFrameFactory INSTANCE = new NullMetadataFrameFactory();
-
-        @Override
-        public ITreeIndexMetaDataFrame createFrame() {
-            return null;
-        }
-
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualMetaDataPageManager.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualMetaDataPageManager.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualMetaDataPageManager.java
new file mode 100644
index 0000000..08ac2f8
--- /dev/null
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualMetaDataPageManager.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.common.freepage;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.common.api.IVirtualMetaDataPageManager;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+
+public class VirtualMetaDataPageManager implements IVirtualMetaDataPageManager {
+    protected final int capacity;
+    protected final AtomicInteger currentPageId = new AtomicInteger();
+
+    public VirtualMetaDataPageManager(int capacity) {
+        // We start the currentPageId from 1, because the BTree uses
+        // the first page as metadata page, and the second page as root page.
+        // (when returning free pages we first increment, then get)
+        currentPageId.set(1);
+        this.capacity = capacity;
+    }
+
+    @Override
+    public int getFreePage(ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException {
+        // The very first call returns page id 2 because the BTree uses
+        // the first page as metadata page, and the second page as root page.
+        return currentPageId.incrementAndGet();
+    }
+
+    @Override
+    public int getMaxPage(ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException {
+        return currentPageId.get();
+    }
+
+    @Override
+    public void init(ITreeIndexMetaDataFrame metaFrame, int currentMaxPage) throws HyracksDataException {
+        currentPageId.set(1);
+    }
+
+    @Override
+    public ITreeIndexMetaDataFrameFactory getMetaDataFrameFactory() {
+        return NullMetadataFrameFactory.INSTANCE;
+    }
+
+    public int getCapacity() {
+        return capacity - 2;
+    }
+
+    public void reset() {
+        currentPageId.set(1);
+    }
+
+    @Override
+    public void addFreePage(ITreeIndexMetaDataFrame metaFrame, int freePage) throws HyracksDataException {
+    }
+
+    @Override
+    public byte getMetaPageLevelIndicator() {
+        return 0;
+    }
+
+    @Override
+    public byte getFreePageLevelIndicator() {
+        return 0;
+    }
+
+    @Override
+    public boolean isMetaPage(ITreeIndexMetaDataFrame metaFrame) {
+        return false;
+    }
+
+    @Override
+    public boolean isFreePage(ITreeIndexMetaDataFrame metaFrame) {
+        return false;
+    }
+
+    @Override
+    public int getFirstMetadataPage() {
+        //MD page in a virtual context is always 0, because it is by nature an in-place modification tree
+        return 0;
+    }
+
+    @Override
+    public void open(int fileId) {
+        // Method doesn't make sense for this free page manager.
+    }
+
+    @Override
+    public void close() {
+        // Method doesn't make sense for this free page manager.
+    }
+
+    private static class NullMetadataFrameFactory implements ITreeIndexMetaDataFrameFactory {
+        private static final NullMetadataFrameFactory INSTANCE = new NullMetadataFrameFactory();
+
+        @Override
+        public ITreeIndexMetaDataFrame createFrame() {
+            return null;
+        }
+
+    }
+
+    @Override
+    public void init(ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException {
+        // Method doesn't make sense for this free page manager.
+    }
+
+    @Override
+    public int getFilterPageId() throws HyracksDataException {
+        // Method doesn't make sense for this free page manager.
+        return 0;
+    }
+
+    @Override
+    public void setFilterPageId(int filterPageId) throws HyracksDataException {
+        // Method doesn't make sense for this free page manager.
+    }
+
+    @Override
+    public long getLSN() throws HyracksDataException {
+        // Method doesn't make sense for this free page manager.
+        return -1;
+    }
+
+    @Override
+    public void setLSN(long lsn) throws HyracksDataException {
+        // Method doesn't make sense for this free page manager.
+    }
+
+    public void setFilterPage(ICachedPage page) {
+        // Method doesn't make sense for this free page manager.
+    }
+
+    public ICachedPage getFilterPage(){
+        return null;
+    }
+
+    @Override
+    public boolean appendOnlyMode() {
+        return false;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index adfadaa..441dda1 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -31,6 +31,7 @@ import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.replication.IReplicationJob.ReplicationExecutionType;
 import org.apache.hyracks.api.replication.IReplicationJob.ReplicationOperation;
+import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
 import org.apache.hyracks.storage.am.common.api.ITreeIndex;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
@@ -134,67 +135,12 @@ public abstract class AbstractLSMIndex implements ILSMIndexInternal {
         filterFields = null;
     }
 
-    protected void forceFlushDirtyPages(ITreeIndex treeIndex) throws HyracksDataException {
-        int fileId = treeIndex.getFileId();
-        IBufferCache bufferCache = treeIndex.getBufferCache();
-        // Flush all dirty pages of the tree.
-        // By default, metadata and data are flushed asynchronously in the buffercache.
-        // This means that the flush issues writes to the OS, but the data may still lie in filesystem buffers.
-        ITreeIndexMetaDataFrame metadataFrame = treeIndex.getFreePageManager().getMetaDataFrameFactory().createFrame();
-        int startPage = 0;
-        int maxPage = treeIndex.getFreePageManager().getMaxPage(metadataFrame);
-        forceFlushDirtyPages(bufferCache, fileId, startPage, maxPage);
-    }
-
-    protected void forceFlushDirtyPages(IBufferCache bufferCache, int fileId, int startPageId, int endPageId)
-            throws HyracksDataException {
-        for (int i = startPageId; i <= endPageId; i++) {
-            ICachedPage page = bufferCache.tryPin(BufferedFileHandle.getDiskPageId(fileId, i));
-            // If tryPin returns null, it means the page is not cached, and therefore cannot be dirty.
-            if (page == null) {
-                continue;
-            }
-            try {
-                bufferCache.flushDirtyPage(page);
-            } finally {
-                bufferCache.unpin(page);
-            }
-        }
-        // Forces all pages of given file to disk. This guarantees the data makes it to disk.
-        // If the index is not durable, then the flush is not necessary.
-        if (durable) {
-            bufferCache.force(fileId, true);
-        }
-    }
-
     protected void markAsValidInternal(ITreeIndex treeIndex) throws HyracksDataException {
         int fileId = treeIndex.getFileId();
         IBufferCache bufferCache = treeIndex.getBufferCache();
-        ITreeIndexMetaDataFrame metadataFrame = treeIndex.getFreePageManager().getMetaDataFrameFactory().createFrame();
-        // Mark the component as a valid component by flushing the metadata page to disk
-        int metadataPageId = treeIndex.getFreePageManager().getFirstMetadataPage();
-        ICachedPage metadataPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, metadataPageId), false);
-        metadataPage.acquireWriteLatch();
-        try {
-            metadataFrame.setPage(metadataPage);
-            metadataFrame.setValid(true);
-        } finally {
-            metadataPage.releaseWriteLatch(true);
-            bufferCache.unpin(metadataPage);
-        }
-
+        treeIndex.getMetaManager().close();
         // WARNING: flushing the metadata page should be done after releasing the write latch; otherwise, the page
         // won't be flushed to disk because it won't be dirty until the write latch has been released.
-        metadataPage = bufferCache.tryPin(BufferedFileHandle.getDiskPageId(fileId, metadataPageId));
-        if (metadataPage != null) {
-            try {
-                // Flush the single modified page to disk.
-                bufferCache.flushDirtyPage(metadataPage);
-            } finally {
-                bufferCache.unpin(metadataPage);
-            }
-        }
-
         // Force modified metadata page to disk.
         // If the index is not durable, then the flush is not necessary.
         if (durable) {
@@ -202,6 +148,12 @@ public abstract class AbstractLSMIndex implements ILSMIndexInternal {
         }
     }
 
+    protected void markAsValidInternal(IBufferCache bufferCache, BloomFilter filter) throws HyracksDataException {
+        if(durable){
+            bufferCache.force(filter.getFileId(),true);
+        }
+    }
+
     @Override
     public void addComponent(ILSMComponent c) throws HyracksDataException {
         diskComponents.add(0, c);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
index 3ad1396..c5fccdc 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java
@@ -82,8 +82,11 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage
         IBufferCache bufferCache = treeIndex.getBufferCache();
         treeIndex.activate();
         try {
-            int metadataPage = treeIndex.getFreePageManager().getFirstMetadataPage();
-            ITreeIndexMetaDataFrame metadataFrame = treeIndex.getFreePageManager().getMetaDataFrameFactory()
+            int metadataPage = treeIndex.getMetaManager().getFirstMetadataPage();
+            if(metadataPage <0 ){
+                return false;
+            }
+            ITreeIndexMetaDataFrame metadataFrame = treeIndex.getMetaManager().getMetaDataFrameFactory()
                     .createFrame();
             ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(treeIndex.getFileId(), metadataPage),
                     false);



Mime
View raw message