asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ima...@apache.org
Subject [5/5] incubator-asterixdb-hyracks git commit: Make LSM bulkload append-only and write-once.
Date Mon, 23 Nov 2015 17:07:27 GMT
Make LSM bulkload append-only and write-once.

Allows for usage of LSM indexes with underlying storage that is append-only.
This also results in a small improvement for LSM component bulk load speed.

- Tree metadata (filters, etc) now lie at the back of the tree file in
  append-only mode.
-- Note that you should *not* ever give the append-only flag on bulk-load,
   if the tree is ever to be modified in place.
- Append-only operations bypass the buffer cache for writes, but utilize
  the buffer cache for memory allocation and reads.
- Addresses ASTERIXDB-1059

Change-Id: I80fb891b5310252143854a336b591bf3f8cd4ba7
Reviewed-on: https://asterix-gerrit.ics.uci.edu/255
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Young-Seok Kim <kisskys@gmail.com>
Reviewed-by: Murtadha Hubail <hubailmor@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/commit/1a659da1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/tree/1a659da1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/diff/1a659da1

Branch: refs/heads/master
Commit: 1a659da1f05faa630c35c10fa0f3af3c2053ed40
Parents: baec39a
Author: Ian Maxon <imaxon@apache.org>
Authored: Thu Nov 19 16:07:22 2015 -0800
Committer: Ian Maxon <imaxon@apache.org>
Committed: Mon Nov 23 09:00:34 2015 -0800

----------------------------------------------------------------------
 .../org/apache/hyracks/api/io/IIOManager.java   |   4 +-
 .../control/cc/ClusterControllerService.java    |   5 +-
 .../control/nc/NodeControllerService.java       |   4 +
 .../apache/hyracks/control/nc/io/IOManager.java |   7 +-
 .../am/bloomfilter/impls/BloomFilter.java       | 104 +++--
 .../frames/BTreeFieldPrefixNSMLeafFrame.java    |   2 +-
 .../am/btree/frames/BTreeNSMInteriorFrame.java  |   2 +-
 .../am/btree/frames/BTreeNSMLeafFrame.java      |   2 +-
 .../hyracks/storage/am/btree/impls/BTree.java   | 151 +++---
 .../storage/am/btree/util/BTreeUtils.java       |   9 +-
 .../storage/am/common/api/IFreePageManager.java |  53 ---
 .../am/common/api/IFreePageManagerFactory.java  |  23 -
 .../hyracks/storage/am/common/api/IIndex.java   |   6 +-
 .../storage/am/common/api/IIndexBulkLoader.java |  61 +--
 .../am/common/api/IMetaDataPageManager.java     | 133 ++++++
 .../am/common/api/IMetadataManagerFactory.java  |  25 +
 .../storage/am/common/api/ITreeIndex.java       |   2 +-
 .../storage/am/common/api/ITreeIndexFrame.java  |   2 +-
 .../am/common/api/IVirtualFreePageManager.java  |  25 -
 .../common/api/IVirtualMetaDataPageManager.java |  25 +
 .../TreeIndexStatsOperatorNodePushable.java     |   8 +-
 .../am/common/frames/AbstractSlotManager.java   |   4 +-
 .../am/common/frames/LIFOMetaDataFrame.java     |   2 +-
 .../freepage/LinkedListFreePageManager.java     | 234 ----------
 .../LinkedListFreePageManagerFactory.java       |  41 --
 .../LinkedListMetadataManagerFactory.java       |  42 ++
 .../freepage/LinkedMetaDataPageManager.java     | 458 ++++++++++++++++++
 .../am/common/impls/AbstractTreeIndex.java      | 200 +++++---
 .../common/util/TreeIndexBufferCacheWarmup.java |   6 +-
 .../storage/am/common/util/TreeIndexStats.java  |   4 +-
 .../am/common/util/TreeIndexStatsGatherer.java  |   6 +-
 .../am/lsm/btree/impls/ExternalBTree.java       |  19 +-
 .../lsm/btree/impls/ExternalBTreeWithBuddy.java |  50 +-
 .../storage/am/lsm/btree/impls/LSMBTree.java    |  94 ++--
 .../am/lsm/btree/util/LSMBTreeUtils.java        |  10 +-
 .../common/api/ILSMComponentFilterManager.java  |   5 +-
 .../am/lsm/common/api/ILSMIndexInternal.java    |   6 +-
 .../common/freepage/VirtualFreePageManager.java | 120 -----
 .../freepage/VirtualMetaDataPageManager.java    | 162 +++++++
 .../am/lsm/common/impls/AbstractLSMIndex.java   |  64 +--
 .../impls/AbstractLSMIndexFileManager.java      |   7 +-
 .../am/lsm/common/impls/BTreeFactory.java       |  13 +-
 .../am/lsm/common/impls/IndexFactory.java       |   6 +-
 .../common/impls/LSMComponentFilterManager.java |  81 ++--
 .../impls/MultitenantVirtualBufferCache.java    |  64 ++-
 .../am/lsm/common/impls/TreeIndexFactory.java   |   4 +-
 .../am/lsm/common/impls/VirtualBufferCache.java |  73 ++-
 .../invertedindex/impls/LSMInvertedIndex.java   | 172 +++----
 .../impls/PartitionedLSMInvertedIndex.java      |   4 +-
 .../inmemory/InMemoryInvertedIndex.java         |  17 +-
 .../PartitionedInMemoryInvertedIndex.java       |  10 +-
 .../ondisk/OnDiskInvertedIndex.java             |  72 ++-
 .../invertedindex/util/InvertedIndexUtils.java  |  10 +-
 .../am/lsm/rtree/impls/AbstractLSMRTree.java    |  36 +-
 .../am/lsm/rtree/impls/ExternalRTree.java       |   8 +-
 .../storage/am/lsm/rtree/impls/LSMRTree.java    | 141 +++---
 .../impls/LSMRTreeWithAntiMatterTuples.java     |  39 +-
 .../am/lsm/rtree/impls/RTreeFactory.java        |  13 +-
 .../am/lsm/rtree/utils/LSMRTreeUtils.java       |   8 +-
 .../am/rtree/frames/RStarTreePolicy.java        |   4 +-
 .../storage/am/rtree/frames/RTreeNSMFrame.java  |  20 +-
 .../am/rtree/frames/RTreeNSMInteriorFrame.java  |  17 +-
 .../am/rtree/frames/RTreeNSMLeafFrame.java      |   2 +-
 .../storage/am/rtree/frames/RTreePolicy.java    |   4 +-
 .../hyracks/storage/am/rtree/impls/RTree.java   | 182 +++++---
 .../am/rtree/impls/RTreeSearchCursor.java       |   2 +
 .../storage/am/rtree/util/RTreeUtils.java       |   6 +-
 .../buffercache/AsyncFIFOPageQueueManager.java  | 163 +++++++
 .../storage/common/buffercache/BufferCache.java | 468 ++++++++++++++++---
 .../storage/common/buffercache/CachedPage.java  |  75 ++-
 .../ClockPageReplacementStrategy.java           |  64 ++-
 .../common/buffercache/DebugBufferCache.java    |  53 ++-
 .../common/buffercache/FIFOLocalWriter.java     |  54 +++
 .../common/buffercache/IBufferCache.java        |  31 +-
 .../storage/common/buffercache/ICachedPage.java |   6 +
 .../common/buffercache/ICachedPageInternal.java |   3 +
 .../common/buffercache/IFIFOPageQueue.java      |  22 +
 .../common/buffercache/IFIFOPageWriter.java     |  25 +
 .../buffercache/IPageReplacementStrategy.java   |   2 +
 .../storage/common/buffercache/IQueueInfo.java  |  33 ++
 .../storage/common/buffercache/QueueInfo.java   |  41 ++
 .../am/btree/OrderedIndexExamplesTest.java      |   1 +
 .../storage/am/btree/OrderedIndexTestUtils.java |   4 +-
 .../storage/am/common/TreeIndexTestUtils.java   |  11 +-
 .../storage/am/btree/BTreeSearchCursorTest.java |  15 +-
 .../storage/am/btree/BTreeStatsTest.java        |  12 +-
 .../storage/am/btree/BTreeUpdateSearchTest.java |  12 +-
 .../LSMBTreeSearchOperationCallbackTest.java    |   2 +-
 .../am/lsm/btree/perf/InMemoryBTreeRunner.java  |   6 +-
 .../lsm/common/VirtualFreePageManagerTest.java  |   6 +-
 .../common/AbstractInvertedIndexDeleteTest.java |  21 +-
 .../common/AbstractInvertedIndexLoadTest.java   |   9 +-
 .../common/AbstractInvertedIndexSearchTest.java |  13 +-
 .../util/LSMInvertedIndexTestContext.java       |   6 +-
 .../util/LSMInvertedIndexTestUtils.java         |   4 +-
 .../storage/am/rtree/RTreeSearchCursorTest.java |  13 +-
 .../hyracks/storage/common/BufferCacheTest.java |  96 ----
 hyracks/pom.xml                                 |   1 -
 98 files changed, 2940 insertions(+), 1527 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
index a0cca95..b5c7aa0 100644
--- a/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
+++ b/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
@@ -55,5 +55,7 @@ public interface IIOManager {
 
     public void setExecutor(Executor executor);
 
+    public long getSize(IFileHandle fileHandle);
+
     public void deleteWorkspaceFiles();
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java
index e311903..1c27376 100644
--- a/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java
+++ b/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerService.java
@@ -637,11 +637,10 @@ public class ClusterControllerService extends AbstractRemoteService {
         deploymentRunMap.remove(deploymentKey);
     }
 
-    public synchronized void setShutdownRun(ShutdownRun sRun) {
+    public synchronized void setShutdownRun(ShutdownRun sRun){
         shutdownCallback = sRun;
     }
-
-    public synchronized ShutdownRun getShutdownRun() {
+    public synchronized ShutdownRun getShutdownRun(){
         return shutdownCallback;
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
index ab0f16b..b96abf8 100644
--- a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
+++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
@@ -38,6 +38,7 @@ import java.util.TimerTask;
 import java.util.concurrent.Executor;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -333,6 +334,9 @@ public class NodeControllerService extends AbstractRemoteService {
         if (!shuttedDown) {
             LOGGER.log(Level.INFO, "Stopping NodeControllerService");
             executor.shutdownNow();
+            if(!executor.awaitTermination(10, TimeUnit.SECONDS)){
+                LOGGER.log(Level.SEVERE, "Some jobs failed to exit, continuing shutdown abnormally");
+            }
             partitionManager.close();
             datasetPartitionManager.close();
             heartbeatTask.cancel();

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
index f673a07..6881f73 100644
--- a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
+++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
@@ -260,6 +260,11 @@ public class IOManager implements IIOManager {
     }
 
     @Override
+    public long getSize(IFileHandle fileHandle) {
+        return ((FileHandle) fileHandle).getFileReference().getFile().length();
+    }
+
+    @Override
     public void deleteWorkspaceFiles() {
         for (IODeviceHandle ioDevice : workAreaIODevices) {
             File workspaceFolder = new File(ioDevice.getPath(), ioDevice.getWorkAreaPath());
@@ -277,4 +282,4 @@ public class IOManager implements IIOManager {
             return name.endsWith(WORKSPACE_FILE_SUFFIX);
         }
     };
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-bloomfilter/src/main/java/org/apache/hyracks/storage/am/bloomfilter/impls/BloomFilter.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-bloomfilter/src/main/java/org/apache/hyracks/storage/am/bloomfilter/impls/BloomFilter.java b/hyracks/hyracks-storage-am-bloomfilter/src/main/java/org/apache/hyracks/storage/am/bloomfilter/impls/BloomFilter.java
index b62e483..ed65902 100644
--- a/hyracks/hyracks-storage-am-bloomfilter/src/main/java/org/apache/hyracks/storage/am/bloomfilter/impls/BloomFilter.java
+++ b/hyracks/hyracks-storage-am-bloomfilter/src/main/java/org/apache/hyracks/storage/am/bloomfilter/impls/BloomFilter.java
@@ -28,6 +28,7 @@ import org.apache.hyracks.storage.am.common.api.IIndexBulkLoader;
 import org.apache.hyracks.storage.am.common.api.IndexException;
 import org.apache.hyracks.storage.common.buffercache.IBufferCache;
 import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue;
 import org.apache.hyracks.storage.common.file.BufferedFileHandle;
 import org.apache.hyracks.storage.common.file.IFileMapProvider;
 
@@ -73,7 +74,7 @@ public class BloomFilter {
 
     public int getNumPages() throws HyracksDataException {
         if (!isActivated) {
-            throw new HyracksDataException("The bloom filter is not activated.");
+            activate();
         }
         return numPages;
     }
@@ -141,17 +142,6 @@ public class BloomFilter {
             throw new HyracksDataException("Failed to create the bloom filter since it is activated.");
         }
         prepareFile();
-        ICachedPage metaPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, METADATA_PAGE_ID), true);
-        metaPage.acquireWriteLatch();
-        try {
-            metaPage.getBuffer().putInt(NUM_PAGES_OFFSET, 0);
-            metaPage.getBuffer().putInt(NUM_HASHES_USED_OFFSET, 0);
-            metaPage.getBuffer().putLong(NUM_ELEMENTS_OFFSET, 0L);
-            metaPage.getBuffer().putLong(NUM_BITS_OFFSET, 0L);
-        } finally {
-            metaPage.releaseWriteLatch(true);
-            bufferCache.unpin(metaPage);
-        }
         bufferCache.closeFile(fileId);
     }
 
@@ -166,6 +156,13 @@ public class BloomFilter {
     }
 
     private void readBloomFilterMetaData() throws HyracksDataException {
+        if (bufferCache.getNumPagesOfFile(fileId) == 0) {
+            numPages = 0;
+            numHashes = 0;
+            numElements = 0;
+            numBits = 0;
+            return;
+        }
         ICachedPage metaPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, METADATA_PAGE_ID), false);
         metaPage.acquireReadLatch();
         try {
@@ -211,12 +208,15 @@ public class BloomFilter {
         private final int numHashes;
         private final long numBits;
         private final int numPages;
+        private IFIFOPageQueue queue;
+        private ICachedPage[] pages;
+        private ICachedPage metaDataPage = null;
 
         public BloomFilterBuilder(long numElements, int numHashes, int numBitsPerElement) throws HyracksDataException {
             if (!isActivated) {
                 throw new HyracksDataException("Failed to create the bloom filter builder since it is not activated.");
             }
-
+            queue = bufferCache.createFIFOQueue();
             this.numElements = numElements;
             this.numHashes = numHashes;
             numBits = this.numElements * numBitsPerElement;
@@ -225,18 +225,12 @@ public class BloomFilter {
                 throw new HyracksDataException("Cannot create a bloom filter with his huge number of pages.");
             }
             numPages = (int) tmp;
-            persistBloomFilterMetaData();
-            readBloomFilterMetaData();
+            pages = new ICachedPage[numPages];
             int currentPageId = 1;
             while (currentPageId <= numPages) {
-                ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), true);
-                page.acquireWriteLatch();
-                try {
-                    initPage(page.getBuffer().array());
-                } finally {
-                    page.releaseWriteLatch(true);
-                    bufferCache.unpin(page);
-                }
+                ICachedPage page = bufferCache.confiscatePage(BufferedFileHandle.getDiskPageId(fileId, currentPageId));
+                initPage(page.getBuffer().array());
+                pages[currentPageId - 1] = page;
                 ++currentPageId;
             }
         }
@@ -254,18 +248,14 @@ public class BloomFilter {
             }
         }
 
-        private void persistBloomFilterMetaData() throws HyracksDataException {
-            ICachedPage metaPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, METADATA_PAGE_ID), false);
-            metaPage.acquireWriteLatch();
-            try {
-                metaPage.getBuffer().putInt(NUM_PAGES_OFFSET, numPages);
-                metaPage.getBuffer().putInt(NUM_HASHES_USED_OFFSET, numHashes);
-                metaPage.getBuffer().putLong(NUM_ELEMENTS_OFFSET, numElements);
-                metaPage.getBuffer().putLong(NUM_BITS_OFFSET, numBits);
-            } finally {
-                metaPage.releaseWriteLatch(true);
-                bufferCache.unpin(metaPage);
+        private void allocateAndInitMetaDataPage() throws HyracksDataException {
+            if (metaDataPage == null) {
+                metaDataPage = bufferCache.confiscatePage(BufferedFileHandle.getDiskPageId(fileId, METADATA_PAGE_ID));
             }
+            metaDataPage.getBuffer().putInt(NUM_PAGES_OFFSET, numPages);
+            metaDataPage.getBuffer().putInt(NUM_HASHES_USED_OFFSET, numHashes);
+            metaDataPage.getBuffer().putLong(NUM_ELEMENTS_OFFSET, numElements);
+            metaDataPage.getBuffer().putLong(NUM_BITS_OFFSET, numBits);
         }
 
         @Override
@@ -277,28 +267,42 @@ public class BloomFilter {
             MurmurHash128Bit.hash3_x64_128(tuple, keyFields, SEED, hashes);
             for (int i = 0; i < numHashes; ++i) {
                 long hash = Math.abs((hashes[0] + i * hashes[1]) % numBits);
+                ICachedPage page = pages[((int) (hash / numBitsPerPage))];
+                ByteBuffer buffer = page.getBuffer();
+                int byteIndex = (int) (hash % numBitsPerPage) >> 3; // divide by 8
+                byte b = buffer.get(byteIndex);
+                int bitIndex = (int) (hash % numBitsPerPage) & 0x07; // mod 8
+                b = (byte) (b | (1 << bitIndex));
 
-                // we increment the page id by one, since the metadata page id of the filter is 0.
-                ICachedPage page = bufferCache.pin(
-                        BufferedFileHandle.getDiskPageId(fileId, (int) (hash / numBitsPerPage) + 1), false);
-                page.acquireWriteLatch();
-                try {
-                    ByteBuffer buffer = page.getBuffer();
-                    int byteIndex = (int) (hash % numBitsPerPage) >> 3; // divide by 8
-                    byte b = buffer.array()[byteIndex];
-                    int bitIndex = (int) (hash % numBitsPerPage) & 0x07; // mod 8
-                    b = (byte) (b | (1 << bitIndex));
-                    buffer.array()[byteIndex] = b;
-                } finally {
-                    page.releaseWriteLatch(true);
-                    bufferCache.unpin(page);
-                }
+                buffer.put(byteIndex, b);
             }
         }
 
         @Override
         public void end() throws HyracksDataException, IndexException {
+            allocateAndInitMetaDataPage();
+            queue.put(metaDataPage);
+            for (ICachedPage p : pages) {
+                queue.put(p);
+            }
+            bufferCache.finishQueue();
+            BloomFilter.this.numBits = numBits;
+            BloomFilter.this.numHashes = numHashes;
+            BloomFilter.this.numElements = numElements;
+            BloomFilter.this.numPages = numPages;
+        }
+
+        @Override
+        public void abort() throws HyracksDataException {
+            for (ICachedPage p : pages) {
+                if (p != null) {
+                    bufferCache.returnPage(p, false);
+                }
+            }
+            if (metaDataPage != null ){
+                bufferCache.returnPage(metaDataPage,false);
+            }
         }
 
     }
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java b/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java
index f4fd394..e908c1b 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java
@@ -91,7 +91,7 @@ public class BTreeFieldPrefixNSMLeafFrame implements IBTreeLeafFrame {
     }
 
     @Override
-    public int getBytesRequriedToWriteTuple(ITupleReference tuple) {
+    public int getBytesRequiredToWriteTuple(ITupleReference tuple) {
         return tupleWriter.bytesRequired(tuple) + slotManager.getSlotSize();
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java b/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
index ab5e55f..5b2bdfc 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
@@ -59,7 +59,7 @@ public class BTreeNSMInteriorFrame extends TreeIndexNSMFrame implements IBTreeIn
     }
 
     @Override
-    public int getBytesRequriedToWriteTuple(ITupleReference tuple) {
+    public int getBytesRequiredToWriteTuple(ITupleReference tuple) {
         return tupleWriter.bytesRequired(tuple) + childPtrSize + slotManager.getSlotSize();
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java b/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java
index c96a2cf..5172a92 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java
@@ -50,7 +50,7 @@ public class BTreeNSMLeafFrame extends TreeIndexNSMFrame implements IBTreeLeafFr
     }
 
     @Override
-    public int getBytesRequriedToWriteTuple(ITupleReference tuple) {
+    public int getBytesRequiredToWriteTuple(ITupleReference tuple) {
         return tupleWriter.bytesRequired(tuple) + slotManager.getSlotSize();
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java b/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
index 3d02065..8c192a1 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/impls/BTree.java
@@ -31,7 +31,6 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.io.FileReference;
 import org.apache.hyracks.data.std.primitive.IntegerPointable;
 import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
 import org.apache.hyracks.dataflow.common.util.TupleUtils;
 import org.apache.hyracks.storage.am.btree.api.IBTreeFrame;
 import org.apache.hyracks.storage.am.btree.api.IBTreeInteriorFrame;
@@ -41,22 +40,8 @@ import org.apache.hyracks.storage.am.btree.exceptions.BTreeException;
 import org.apache.hyracks.storage.am.btree.exceptions.BTreeNotUpdateableException;
 import org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrame;
 import org.apache.hyracks.storage.am.btree.impls.BTreeOpContext.PageValidationInfo;
-import org.apache.hyracks.storage.am.common.api.IFreePageManager;
-import org.apache.hyracks.storage.am.common.api.IIndexAccessor;
-import org.apache.hyracks.storage.am.common.api.IIndexBulkLoader;
-import org.apache.hyracks.storage.am.common.api.IIndexCursor;
-import org.apache.hyracks.storage.am.common.api.IModificationOperationCallback;
-import org.apache.hyracks.storage.am.common.api.ISearchOperationCallback;
-import org.apache.hyracks.storage.am.common.api.ISearchPredicate;
-import org.apache.hyracks.storage.am.common.api.ISplitKey;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexAccessor;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
-import org.apache.hyracks.storage.am.common.api.IndexException;
-import org.apache.hyracks.storage.am.common.api.TreeIndexException;
-import org.apache.hyracks.storage.am.common.api.UnsortedInputException;
+import org.apache.hyracks.storage.am.common.api.*;
+import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
 import org.apache.hyracks.storage.am.common.exceptions.TreeIndexDuplicateKeyException;
 import org.apache.hyracks.storage.am.common.exceptions.TreeIndexNonExistentKeyException;
 import org.apache.hyracks.storage.am.common.frames.FrameOpSpaceStatus;
@@ -66,6 +51,7 @@ import org.apache.hyracks.storage.am.common.impls.NodeFrontier;
 import org.apache.hyracks.storage.am.common.impls.TreeIndexDiskOrderScanCursor;
 import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
 import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
+import org.apache.hyracks.storage.common.buffercache.BufferCache;
 import org.apache.hyracks.storage.common.buffercache.IBufferCache;
 import org.apache.hyracks.storage.common.buffercache.ICachedPage;
 import org.apache.hyracks.storage.common.file.BufferedFileHandle;
@@ -83,7 +69,7 @@ public class BTree extends AbstractTreeIndex {
     private final ReadWriteLock treeLatch;
     private final int maxTupleSize;
 
-    public BTree(IBufferCache bufferCache, IFileMapProvider fileMapProvider, IFreePageManager freePageManager,
+    public BTree(IBufferCache bufferCache, IFileMapProvider fileMapProvider, IMetaDataPageManager freePageManager,
             ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory,
             IBinaryComparatorFactory[] cmpFactories, int fieldCount, FileReference file) {
         super(bufferCache, fileMapProvider, freePageManager, interiorFrameFactory, leafFrameFactory, cmpFactories,
@@ -100,8 +86,8 @@ public class BTree extends AbstractTreeIndex {
         TreeIndexDiskOrderScanCursor cursor = (TreeIndexDiskOrderScanCursor) icursor;
         ctx.reset();
         RangePredicate diskOrderScanPred = new RangePredicate(null, null, true, true, ctx.cmp, ctx.cmp);
-        int currentPageId = rootPage;
         int maxPageId = freePageManager.getMaxPage(ctx.metaFrame);
+        int currentPageId = bulkloadLeafStart;
         ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), false);
         page.acquireReadLatch();
         try {
@@ -128,7 +114,9 @@ public class BTree extends AbstractTreeIndex {
                 NoOpOperationCallback.INSTANCE);
         PageValidationInfo pvi = accessor.ctx.createPageValidationInfo(null);
         accessor.ctx.validationInfos.addFirst(pvi);
-        validate(accessor.ctx, rootPage);
+        if (isActive) {
+            validate(accessor.ctx, rootPage);
+        }
     }
 
     private void validate(BTreeOpContext ctx, int pageId) throws HyracksDataException {
@@ -316,8 +304,8 @@ public class BTree extends AbstractTreeIndex {
     }
 
     private void insert(ITupleReference tuple, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
-        int tupleSize = Math.max(ctx.leafFrame.getBytesRequriedToWriteTuple(tuple),
-                ctx.interiorFrame.getBytesRequriedToWriteTuple(tuple));
+        int tupleSize = Math.max(ctx.leafFrame.getBytesRequiredToWriteTuple(tuple),
+                ctx.interiorFrame.getBytesRequiredToWriteTuple(tuple));
         if (tupleSize > maxTupleSize) {
             throw new TreeIndexException("Space required for record (" + tupleSize
                     + ") larger than maximum acceptable size (" + maxTupleSize + ")");
@@ -327,8 +315,8 @@ public class BTree extends AbstractTreeIndex {
     }
 
     private void upsert(ITupleReference tuple, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
-        int tupleSize = Math.max(ctx.leafFrame.getBytesRequriedToWriteTuple(tuple),
-                ctx.interiorFrame.getBytesRequriedToWriteTuple(tuple));
+        int tupleSize = Math.max(ctx.leafFrame.getBytesRequiredToWriteTuple(tuple),
+                ctx.interiorFrame.getBytesRequiredToWriteTuple(tuple));
         if (tupleSize > maxTupleSize) {
             throw new TreeIndexException("Space required for record (" + tupleSize
                     + ") larger than maximum acceptable size (" + maxTupleSize + ")");
@@ -344,8 +332,8 @@ public class BTree extends AbstractTreeIndex {
         if (fieldCount == ctx.cmp.getKeyFieldCount()) {
             throw new BTreeNotUpdateableException("Cannot perform updates when the entire tuple forms the key.");
         }
-        int tupleSize = Math.max(ctx.leafFrame.getBytesRequriedToWriteTuple(tuple),
-                ctx.interiorFrame.getBytesRequriedToWriteTuple(tuple));
+        int tupleSize = Math.max(ctx.leafFrame.getBytesRequiredToWriteTuple(tuple),
+                ctx.interiorFrame.getBytesRequiredToWriteTuple(tuple));
         if (tupleSize > maxTupleSize) {
             throw new TreeIndexException("Space required for record (" + tupleSize
                     + ") larger than maximum acceptable size (" + maxTupleSize + ")");
@@ -756,7 +744,7 @@ public class BTree extends AbstractTreeIndex {
                     bufferCache.unpin(node);
                 }
                 if (restartOp) {
-                    // Wait for the SMO to finish before restarting.
+                    // Wait for the SMO to persistFrontiers before restarting.
                     treeLatch.readLock().lock();
                     treeLatch.readLock().unlock();
                     ctx.pageLsns.removeLast();
@@ -942,8 +930,13 @@ public class BTree extends AbstractTreeIndex {
     @Override
     public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
             boolean checkIfEmptyIndex) throws TreeIndexException {
+        return createBulkLoader(fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, false);
+    }
+
+    public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
+            boolean checkIfEmptyIndex, boolean appendOnly) throws TreeIndexException {
         try {
-            return new BTreeBulkLoader(fillFactor, verifyInput);
+            return new BTreeBulkLoader(fillFactor, verifyInput, appendOnly);
         } catch (HyracksDataException e) {
             throw new TreeIndexException(e);
         }
@@ -952,19 +945,22 @@ public class BTree extends AbstractTreeIndex {
     public class BTreeBulkLoader extends AbstractTreeIndex.AbstractTreeIndexBulkLoader {
         protected final ISplitKey splitKey;
         protected final boolean verifyInput;
+        protected List<ICachedPage> pagesToWrite;
 
-        public BTreeBulkLoader(float fillFactor, boolean verifyInput) throws TreeIndexException, HyracksDataException {
-            super(fillFactor);
+        public BTreeBulkLoader(float fillFactor, boolean verifyInput, boolean appendOnly) throws TreeIndexException,
+                HyracksDataException {
+            super(fillFactor, appendOnly);
             this.verifyInput = verifyInput;
             splitKey = new BTreeSplitKey(leafFrame.getTupleWriter().createTupleReference());
             splitKey.getTuple().setFieldCount(cmp.getKeyFieldCount());
+            pagesToWrite = new ArrayList<ICachedPage>();
         }
 
         @Override
         public void add(ITupleReference tuple) throws IndexException, HyracksDataException {
             try {
-                int tupleSize = Math.max(leafFrame.getBytesRequriedToWriteTuple(tuple),
-                        interiorFrame.getBytesRequriedToWriteTuple(tuple));
+                int tupleSize = Math.max(leafFrame.getBytesRequiredToWriteTuple(tuple),
+                        interiorFrame.getBytesRequiredToWriteTuple(tuple));
                 if (tupleSize > maxTupleSize) {
                     throw new TreeIndexException("Space required for record (" + tupleSize
                             + ") larger than maximum acceptable size (" + maxTupleSize + ")");
@@ -980,7 +976,7 @@ public class BTree extends AbstractTreeIndex {
                     leafFrame.compress();
                     spaceUsed = leafFrame.getBuffer().capacity() - leafFrame.getTotalFreeSpace();
                 }
-
+                //full, allocate new page
                 if (spaceUsed + spaceNeeded > leafMaxBytes) {
                     leafFrontier.lastTuple.resetByTupleIndex(leafFrame, leafFrame.getTupleCount() - 1);
                     if (verifyInput) {
@@ -992,18 +988,20 @@ public class BTree extends AbstractTreeIndex {
                             .getBuffer().array(), 0);
                     splitKey.getTuple().resetByTupleOffset(splitKey.getBuffer(), 0);
                     splitKey.setLeftPage(leafFrontier.pageId);
+
+                    pagesToWrite.clear();
+                    propagateBulk(1, pagesToWrite);
                     leafFrontier.pageId = freePageManager.getFreePage(metaFrame);
 
                     ((IBTreeLeafFrame) leafFrame).setNextLeaf(leafFrontier.pageId);
-                    leafFrontier.page.releaseWriteLatch(true);
-                    bufferCache.unpin(leafFrontier.page);
+                    queue.put(leafFrontier.page);
+                    for (ICachedPage c : pagesToWrite) {
+                        queue.put(c);
+                    }
 
                     splitKey.setRightPage(leafFrontier.pageId);
-                    propagateBulk(1);
-
-                    leafFrontier.page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, leafFrontier.pageId),
-                            true);
-                    leafFrontier.page.acquireWriteLatch();
+                    leafFrontier.page = bufferCache.confiscatePage(BufferedFileHandle.getDiskPageId(fileId,
+                            leafFrontier.pageId));
                     leafFrame.setPage(leafFrontier.page);
                     leafFrame.initBuffer((byte) 0);
                 } else {
@@ -1039,7 +1037,7 @@ public class BTree extends AbstractTreeIndex {
             }
         }
 
-        protected void propagateBulk(int level) throws HyracksDataException {
+        protected void propagateBulk(int level, List<ICachedPage> pagesToWrite) throws HyracksDataException {
             if (splitKey.getBuffer() == null)
                 return;
 
@@ -1063,25 +1061,73 @@ public class BTree extends AbstractTreeIndex {
                 tupleWriter.writeTupleFields(frontier.lastTuple, 0, cmp.getKeyFieldCount(), splitKey.getBuffer()
                         .array(), 0);
                 splitKey.getTuple().resetByTupleOffset(splitKey.getBuffer(), 0);
-                splitKey.setLeftPage(frontier.pageId);
 
                 ((IBTreeInteriorFrame) interiorFrame).deleteGreatest();
+                int finalPageId = freePageManager.getFreePage(metaFrame);
+                bufferCache.setPageDiskId(frontier.page, BufferedFileHandle.getDiskPageId(fileId, finalPageId));
+                pagesToWrite.add(frontier.page);
+                splitKey.setLeftPage(finalPageId);
 
-                frontier.page.releaseWriteLatch(true);
-                bufferCache.unpin(frontier.page);
-                frontier.pageId = freePageManager.getFreePage(metaFrame);
-
-                splitKey.setRightPage(frontier.pageId);
-                propagateBulk(level + 1);
-
-                frontier.page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, frontier.pageId), true);
-                frontier.page.acquireWriteLatch();
+                propagateBulk(level + 1, pagesToWrite);
+                frontier.page = bufferCache.confiscatePage(BufferCache.INVALID_DPID);
                 interiorFrame.setPage(frontier.page);
                 interiorFrame.initBuffer((byte) level);
             }
             ((IBTreeInteriorFrame) interiorFrame).insertSorted(tuple);
         }
 
+        private void persistFrontiers(int level, int rightPage) throws HyracksDataException {
+            if (level >= nodeFrontiers.size()) {
+                //at root
+                if (appendOnly) {
+                    rootPage = nodeFrontiers.get(level - 1).pageId;
+                }
+                releasedLatches = true;
+                return;
+            }
+            if (level < 1) {
+                ICachedPage lastLeaf = nodeFrontiers.get(level).page;
+                int lastLeafPage = nodeFrontiers.get(level).pageId;
+                setPageDpid(lastLeaf,nodeFrontiers.get(level).pageId);
+                queue.put(lastLeaf);
+                nodeFrontiers.get(level).page = null;
+                persistFrontiers(level + 1, lastLeafPage);
+                return;
+            }
+            NodeFrontier frontier = nodeFrontiers.get(level);
+            interiorFrame.setPage(frontier.page);
+            //just finalize = the layer right above the leaves has correct righthand pointers already
+            if (rightPage < 0) {
+                throw new HyracksDataException("Error in index creation. Internal node appears to have no rightmost guide");
+            }
+            ((IBTreeInteriorFrame) interiorFrame).setRightmostChildPageId(rightPage);
+            int finalPageId = freePageManager.getFreePage(metaFrame);
+            setPageDpid(frontier.page, finalPageId);
+            queue.put(frontier.page);
+            frontier.pageId = finalPageId;
+
+            persistFrontiers(level + 1, finalPageId);
+        }
+
+        @Override
+        protected void handleException() throws HyracksDataException {
+            super.handleException();
+        }
+
+        @Override
+        public void end() throws HyracksDataException {
+            persistFrontiers(0, -1);
+            super.end();
+        }
+
+        @Override
+        public void abort() throws HyracksDataException {
+            super.handleException();
+        }
+
+        private void setPageDpid(ICachedPage page, int pageId){
+            bufferCache.setPageDiskId(page, BufferedFileHandle.getDiskPageId(fileId,pageId));
+        }
     }
 
     @SuppressWarnings("rawtypes")
@@ -1109,7 +1155,8 @@ public class BTree extends AbstractTreeIndex {
             tuple.resetByTupleIndex(interiorFrame, i);
             // Print child pointer.
             int numFields = tuple.getFieldCount();
-            int childPageId = IntegerPointable.getInteger(tuple.getFieldData(numFields - 1), tuple.getFieldStart(numFields - 1) + tuple.getFieldLength(numFields - 1));
+            int childPageId = IntegerPointable.getInteger(tuple.getFieldData(numFields - 1),
+                    tuple.getFieldStart(numFields - 1) + tuple.getFieldLength(numFields - 1));
             strBuilder.append("(" + childPageId + ") ");
             String tupleString = TupleUtils.printTuple(tuple, fieldSerdes);
             strBuilder.append(tupleString + " | ");

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java b/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java
index 5aab7ae..7c2abb1 100644
--- a/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java
+++ b/hyracks/hyracks-storage-am-btree/src/main/java/org/apache/hyracks/storage/am/btree/util/BTreeUtils.java
@@ -29,12 +29,12 @@ import org.apache.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
 import org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
 import org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
 import org.apache.hyracks.storage.am.btree.impls.BTree;
-import org.apache.hyracks.storage.am.common.api.IFreePageManager;
+import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
 import org.apache.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
-import org.apache.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
+import org.apache.hyracks.storage.am.common.freepage.LinkedMetaDataPageManager;
 import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
 import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
 import org.apache.hyracks.storage.common.buffercache.IBufferCache;
@@ -48,13 +48,14 @@ public class BTreeUtils {
         ITreeIndexFrameFactory leafFrameFactory = getLeafFrameFactory(tupleWriterFactory, leafType);
         ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
         ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
-        IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
+        IMetaDataPageManager freePageManager;
+        freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
         BTree btree = new BTree(bufferCache, fileMapProvider, freePageManager, interiorFrameFactory, leafFrameFactory,
                 cmpFactories, typeTraits.length, file);
         return btree;
     }
 
-    public static BTree createBTree(IBufferCache bufferCache, IFreePageManager freePageManager,
+    public static BTree createBTree(IBufferCache bufferCache, IMetaDataPageManager freePageManager,
             IFileMapProvider fileMapProvider, ITypeTraits[] typeTraits, IBinaryComparatorFactory[] cmpFactories,
             BTreeLeafFrameType leafType, FileReference file) throws BTreeException {
         TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IFreePageManager.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IFreePageManager.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IFreePageManager.java
deleted file mode 100644
index 57bcd6c..0000000
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IFreePageManager.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.storage.am.common.api;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public interface IFreePageManager {
-	public void open(int fileId);
-	
-	public void close();
-	
-	public int getFreePage(ITreeIndexMetaDataFrame metaFrame)
-			throws HyracksDataException;
-
-	public void addFreePage(ITreeIndexMetaDataFrame metaFrame, int freePage)
-			throws HyracksDataException;
-
-	public int getMaxPage(ITreeIndexMetaDataFrame metaFrame)
-			throws HyracksDataException;
-
-	public void init(ITreeIndexMetaDataFrame metaFrame, int currentMaxPage)
-			throws HyracksDataException;
-
-	public ITreeIndexMetaDataFrameFactory getMetaDataFrameFactory();
-
-	// required to return negative values
-	public byte getMetaPageLevelIndicator();
-
-	public byte getFreePageLevelIndicator();
-
-	// determined by examining level indicator
-	public boolean isMetaPage(ITreeIndexMetaDataFrame metaFrame);
-
-	public boolean isFreePage(ITreeIndexMetaDataFrame metaFrame);
-	
-	public int getFirstMetadataPage();		
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IFreePageManagerFactory.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IFreePageManagerFactory.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IFreePageManagerFactory.java
deleted file mode 100644
index 3dbe0ab..0000000
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IFreePageManagerFactory.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.storage.am.common.api;
-
-public interface IFreePageManagerFactory {
-    public IFreePageManager createFreePageManager();
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IIndex.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IIndex.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IIndex.java
index 9fe9f06..e670390 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IIndex.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IIndex.java
@@ -126,9 +126,11 @@ public interface IIndex {
      */
     public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
             boolean checkIfEmptyIndex) throws IndexException;
-    
+
+    public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
+            boolean checkIfEmptyIndex, boolean appendOnly) throws IndexException;
+
     /**
-     * 
      * @return true if the index needs memory components
      */
     public boolean hasMemoryComponents();

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IIndexBulkLoader.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IIndexBulkLoader.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IIndexBulkLoader.java
index d390154..b837022 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IIndexBulkLoader.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IIndexBulkLoader.java
@@ -16,31 +16,36 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.hyracks.storage.am.common.api;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-
-public interface IIndexBulkLoader {
-    /**
-     * Append a tuple to the index in the context of a bulk load.
-     * 
-     * @param tuple
-     *            Tuple to be inserted.
-     * @throws IndexException
-     *             If the input stream is invalid for bulk loading (e.g., is not sorted).
-     * @throws HyracksDataException
-     *             If the BufferCache throws while un/pinning or un/latching.
-     */
-    public void add(ITupleReference tuple) throws IndexException, HyracksDataException;
-
-    /**
-     * Finalize the bulk loading operation in the given context.
-     * 
-     * @throws IndexException
-     * @throws HyracksDataException
-     *             If the BufferCache throws while un/pinning or un/latching.
-     */
-    public void end() throws IndexException, HyracksDataException;
-
-}
+package org.apache.hyracks.storage.am.common.api;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+
+public interface IIndexBulkLoader {
+    /**
+     * Append a tuple to the index in the context of a bulk load.
+     * 
+     * @param tuple
+     *            Tuple to be inserted.
+     * @throws IndexException
+     *             If the input stream is invalid for bulk loading (e.g., is not sorted).
+     * @throws HyracksDataException
+     *             If the BufferCache throws while un/pinning or un/latching.
+     */
+    public void add(ITupleReference tuple) throws IndexException, HyracksDataException;
+
+    /**
+     * Finalize the bulk loading operation in the given context.
+     * 
+     * @throws IndexException
+     * @throws HyracksDataException
+     *             If the BufferCache throws while un/pinning or un/latching.
+     */
+    public void end() throws IndexException, HyracksDataException;
+
+    /**
+     * Release all resources held by this bulkloader, with no guarantee of
+     * persisted content.
+     */
+    void abort() throws HyracksDataException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IMetaDataPageManager.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IMetaDataPageManager.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IMetaDataPageManager.java
new file mode 100644
index 0000000..432eb59
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IMetaDataPageManager.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.common.api;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+
+public interface IMetaDataPageManager {
+    /**
+     * This is the class through which one interfaces with index metadata.
+     * The index metadata contains information such as the LSN of the index, free page information,
+     * and filter page locations.
+     */
+    /**
+     * Open an index file's metadata
+     * @param fileId The file which to open the metadata of
+     */
+    public void open(int fileId);
+
+    /**
+     * Close an index file's metadata.
+     * @throws HyracksDataException
+     */
+
+    public void close() throws HyracksDataException;
+
+    /**
+     * Get the locaiton of a free page to use for index operations
+     * @param metaFrame A metadata frame to use to wrap the raw page
+     * @return A page location, or -1 if no free page could be found or allocated
+     * @throws HyracksDataException
+     */
+
+    public int getFreePage(ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException;
+
+    /**
+     * Add a page back to the pool of free pages within an index file
+     * @param metaFrame A metadata frame to use to wrap the raw page
+     * @param freePage The page which to return to the free space
+     * @throws HyracksDataException
+     */
+
+    public void addFreePage(ITreeIndexMetaDataFrame metaFrame, int freePage) throws HyracksDataException;
+
+    /**
+     * Gets the highest page offset according to the metadata
+     * @param metaFrame A metadata frame to use to wrap the raw page
+     * @return The locaiton of the highest offset page
+     * @throws HyracksDataException
+     */
+
+    public int getMaxPage(ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException;
+
+    /**
+     * Initializes the index metadata
+     * @param metaFrame A metadata farme to use to wrap the raw page
+     * @param currentMaxPage The highest page offset to consider valid
+     * @throws HyracksDataException
+     */
+
+    public void init(ITreeIndexMetaDataFrame metaFrame, int currentMaxPage) throws HyracksDataException;
+
+    public ITreeIndexMetaDataFrameFactory getMetaDataFrameFactory();
+
+    // required to return negative values
+    public byte getMetaPageLevelIndicator();
+
+    public byte getFreePageLevelIndicator();
+
+    // determined by examining level indicator
+
+    public boolean isMetaPage(ITreeIndexMetaDataFrame metaFrame);
+
+    public boolean isFreePage(ITreeIndexMetaDataFrame metaFrame);
+
+    /**
+     * Determines where the metadata page is located in an index file
+     * @return The locaiton of the metadata page, or -1 if the file appears to be corrupt
+     * @throws HyracksDataException
+     */
+
+    public int getFirstMetadataPage() throws HyracksDataException;
+
+    /**
+     * Initializes the metadata manager on an open index file
+     * @param metaFrame A metadata frame used to wrap the raw page
+     * @throws HyracksDataException
+     */
+
+    void init(ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException;
+
+    /**
+     * Locate the filter page in an index file
+     * @return The offset of the filter page if it exists, or less than zero if no filter page exists yet
+     * @throws HyracksDataException
+     */
+
+    int getFilterPageId() throws HyracksDataException;
+
+    void setFilterPageId(int filterPageId) throws HyracksDataException;
+
+    long getLSN() throws HyracksDataException;
+
+    void setLSN(long lsn) throws HyracksDataException;
+
+    /**
+     * Set the cached page to manage for filter data
+     * @param page The page to manage
+     */
+
+    void setFilterPage(ICachedPage page);
+
+    ICachedPage getFilterPage();
+
+    boolean appendOnlyMode();
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IMetadataManagerFactory.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IMetadataManagerFactory.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IMetadataManagerFactory.java
new file mode 100644
index 0000000..e9dd674
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IMetadataManagerFactory.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.common.api;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public interface IMetadataManagerFactory {
+    public IMetaDataPageManager createFreePageManager() throws HyracksDataException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndex.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndex.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndex.java
index d3a7c6d..a29f329 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndex.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndex.java
@@ -41,7 +41,7 @@ public interface ITreeIndex extends IIndex {
     /**
      * @return The index's free page manager.
      */
-    public IFreePageManager getFreePageManager();
+    public IMetaDataPageManager getMetaManager();
 
     /**
      * @return The number of fields tuples of this index have.

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java
index 18fe4c0..9ac09a3 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/ITreeIndexFrame.java
@@ -65,7 +65,7 @@ public interface ITreeIndexFrame {
 
     public int getMaxTupleSize(int pageSize);
 
-    public int getBytesRequriedToWriteTuple(ITupleReference tuple);
+    public int getBytesRequiredToWriteTuple(ITupleReference tuple);
 
     // for debugging
     public String printHeader();

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IVirtualFreePageManager.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IVirtualFreePageManager.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IVirtualFreePageManager.java
deleted file mode 100644
index 4247e64..0000000
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IVirtualFreePageManager.java
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.storage.am.common.api;
-
-public interface IVirtualFreePageManager extends IFreePageManager {
-    public int getCapacity();
-
-    public void reset();
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IVirtualMetaDataPageManager.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IVirtualMetaDataPageManager.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IVirtualMetaDataPageManager.java
new file mode 100644
index 0000000..0073f59
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IVirtualMetaDataPageManager.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.common.api;
+
+public interface IVirtualMetaDataPageManager extends IMetaDataPageManager {
+    public int getCapacity();
+
+    public void reset();
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java
index fff3d57..48e65bb 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java
@@ -68,11 +68,11 @@ public class TreeIndexStatsOperatorNodePushable extends AbstractUnaryOutputSourc
             IBufferCache bufferCache = opDesc.getStorageManager().getBufferCache(ctx);
             IFileMapProvider fileMapProvider = opDesc.getStorageManager().getFileMapProvider(ctx);
             int indexFileId = fileMapProvider.lookupFileId(treeIndexHelper.getFileReference());
-            statsGatherer = new TreeIndexStatsGatherer(bufferCache, treeIndex.getFreePageManager(), indexFileId,
+            statsGatherer = new TreeIndexStatsGatherer(bufferCache, treeIndex.getMetaManager(), indexFileId,
                     treeIndex.getRootPageId());
-            TreeIndexStats stats = statsGatherer.gatherStats(treeIndex.getLeafFrameFactory().createFrame(),
-                    treeIndex.getInteriorFrameFactory().createFrame(),
-                    treeIndex.getFreePageManager().getMetaDataFrameFactory().createFrame());
+            TreeIndexStats stats = statsGatherer.gatherStats(treeIndex.getLeafFrameFactory().createFrame(), treeIndex
+                    .getInteriorFrameFactory().createFrame(), treeIndex.getMetaManager().getMetaDataFrameFactory()
+                    .createFrame());
             // Write the stats output as a single string field.
             FrameTupleAppender appender = new FrameTupleAppender(new VSizeFrame(ctx));
             ArrayTupleBuilder tb = new ArrayTupleBuilder(1);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/AbstractSlotManager.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/AbstractSlotManager.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/AbstractSlotManager.java
index 667f76c..7d61659 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/AbstractSlotManager.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/AbstractSlotManager.java
@@ -24,8 +24,8 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
 
 public abstract class AbstractSlotManager implements ISlotManager {
 	
-	protected final int GREATEST_KEY_INDICATOR = -1;
-    protected final int ERROR_INDICATOR = -2;
+    public static final int GREATEST_KEY_INDICATOR = -1;
+    public static final int ERROR_INDICATOR = -2;
 	
 	protected static final int slotSize = 4;
 	protected ITreeIndexFrame frame;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/LIFOMetaDataFrame.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/LIFOMetaDataFrame.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/LIFOMetaDataFrame.java
index 2d1ab42..16fdecd 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/LIFOMetaDataFrame.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/frames/LIFOMetaDataFrame.java
@@ -112,7 +112,7 @@ public class LIFOMetaDataFrame implements ITreeIndexMetaDataFrame {
     public void initBuffer(byte level) {
         buf.putInt(tupleCountOff, 0);
         buf.putInt(freeSpaceOff, lsnOff + 8);
-        //buf.putInt(maxPageOff, -1);
+        buf.putInt(maxPageOff, 0);
         buf.put(levelOff, level);
         buf.putInt(nextPageOff, -1);
         buf.putInt(additionalFilteringPageOff, -1);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/LinkedListFreePageManager.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/LinkedListFreePageManager.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/LinkedListFreePageManager.java
deleted file mode 100644
index 55ddbb7..0000000
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/LinkedListFreePageManager.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.storage.am.common.freepage;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.common.api.IFreePageManager;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
-import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.buffercache.ICachedPage;
-import org.apache.hyracks.storage.common.file.BufferedFileHandle;
-
-public class LinkedListFreePageManager implements IFreePageManager {
-
-	private static final byte META_PAGE_LEVEL_INDICATOR = -1;
-	private static final byte FREE_PAGE_LEVEL_INDICATOR = -2;
-	private final IBufferCache bufferCache;
-	private final int headPage;	
-	private int fileId = -1;
-	private final ITreeIndexMetaDataFrameFactory metaDataFrameFactory;
-
-	public LinkedListFreePageManager(IBufferCache bufferCache,
-			int headPage, ITreeIndexMetaDataFrameFactory metaDataFrameFactory) {
-		this.bufferCache = bufferCache;
-		this.headPage = headPage;
-		this.metaDataFrameFactory = metaDataFrameFactory;
-	}
-
-	@Override
-	public void addFreePage(ITreeIndexMetaDataFrame metaFrame, int freePage)
-			throws HyracksDataException {
-
-		ICachedPage metaNode = bufferCache.pin(
-				BufferedFileHandle.getDiskPageId(fileId, headPage), false);
-		metaNode.acquireWriteLatch();
-
-		try {
-			metaFrame.setPage(metaNode);
-
-			if (metaFrame.hasSpace()) {
-				metaFrame.addFreePage(freePage);
-			} else {
-				// allocate a new page in the chain of meta pages
-				int newPage = metaFrame.getFreePage();
-				if (newPage < 0) {
-					throw new Exception(
-							"Inconsistent Meta Page State. It has no space, but it also has no entries.");
-				}
-
-				ICachedPage newNode = bufferCache.pin(
-						BufferedFileHandle.getDiskPageId(fileId, newPage),
-						false);
-				newNode.acquireWriteLatch();
-
-				try {
-					int metaMaxPage = metaFrame.getMaxPage();
-
-					// copy metaDataPage to newNode
-					System.arraycopy(metaNode.getBuffer().array(), 0, newNode
-							.getBuffer().array(), 0, metaNode.getBuffer()
-							.capacity());
-
-					metaFrame.initBuffer(META_PAGE_LEVEL_INDICATOR);
-					metaFrame.setNextPage(newPage);
-					metaFrame.setMaxPage(metaMaxPage);
-					metaFrame.addFreePage(freePage);
-				} finally {
-					newNode.releaseWriteLatch(true);
-					bufferCache.unpin(newNode);
-				}
-			}
-		} catch (Exception e) {
-			e.printStackTrace();
-		} finally {
-			metaNode.releaseWriteLatch(true);
-			bufferCache.unpin(metaNode);
-		}
-	}
-
-	@Override
-	public int getFreePage(ITreeIndexMetaDataFrame metaFrame)
-			throws HyracksDataException {
-		ICachedPage metaNode = bufferCache.pin(
-				BufferedFileHandle.getDiskPageId(fileId, headPage), false);
-
-		metaNode.acquireWriteLatch();
-
-		int freePage = -1;
-		try {
-			metaFrame.setPage(metaNode);
-			freePage = metaFrame.getFreePage();
-			if (freePage < 0) { // no free page entry on this page
-				int nextPage = metaFrame.getNextPage();
-				if (nextPage > 0) { // sibling may have free pages
-					ICachedPage nextNode = bufferCache.pin(
-							BufferedFileHandle.getDiskPageId(fileId, nextPage),
-							false);
-
-					nextNode.acquireWriteLatch();
-					// we copy over the free space entries of nextpage into the
-					// first meta page (metaDataPage)
-					// we need to link the first page properly to the next page
-					// of nextpage
-					try {
-						// remember entries that remain unchanged
-						int maxPage = metaFrame.getMaxPage();
-
-						// copy entire page (including sibling pointer, free
-						// page entries, and all other info)
-						// after this copy nextPage is considered a free page
-						System.arraycopy(nextNode.getBuffer().array(), 0,
-								metaNode.getBuffer().array(), 0, nextNode
-										.getBuffer().capacity());
-
-						// reset unchanged entry
-						metaFrame.setMaxPage(maxPage);
-
-						freePage = metaFrame.getFreePage();
-						// sibling also has no free pages, this "should" not
-						// happen, but we deal with it anyway just to be safe
-						if (freePage < 0) {
-							freePage = nextPage;
-						} else {
-							metaFrame.addFreePage(nextPage);
-						}
-					} finally {
-						nextNode.releaseWriteLatch(true);
-						bufferCache.unpin(nextNode);
-					}
-				} else {
-					freePage = metaFrame.getMaxPage();
-					freePage++;
-					metaFrame.setMaxPage(freePage);
-				}
-			}
-		} finally {
-			metaNode.releaseWriteLatch(true);
-			bufferCache.unpin(metaNode);
-		}
-
-		return freePage;
-	}
-
-	@Override
-	public int getMaxPage(ITreeIndexMetaDataFrame metaFrame)
-			throws HyracksDataException {
-		ICachedPage metaNode = bufferCache.pin(
-				BufferedFileHandle.getDiskPageId(fileId, headPage), false);
-		metaNode.acquireWriteLatch();
-		int maxPage = -1;
-		try {
-			metaFrame.setPage(metaNode);
-			maxPage = metaFrame.getMaxPage();
-		} finally {
-			metaNode.releaseWriteLatch(true);
-			bufferCache.unpin(metaNode);
-		}
-		return maxPage;
-	}
-
-	@Override
-	public void init(ITreeIndexMetaDataFrame metaFrame, int currentMaxPage)
-			throws HyracksDataException {
-		// initialize meta data page
-		ICachedPage metaNode = bufferCache.pin(
-				BufferedFileHandle.getDiskPageId(fileId, headPage), true);
-
-		metaNode.acquireWriteLatch();
-		try {
-			metaFrame.setPage(metaNode);
-			metaFrame.initBuffer(META_PAGE_LEVEL_INDICATOR);
-			metaFrame.setMaxPage(currentMaxPage);
-		} finally {
-			metaNode.releaseWriteLatch(true);
-			bufferCache.unpin(metaNode);
-		}
-	}
-
-	@Override
-	public ITreeIndexMetaDataFrameFactory getMetaDataFrameFactory() {
-		return metaDataFrameFactory;
-	}
-
-	@Override
-	public byte getFreePageLevelIndicator() {
-		return FREE_PAGE_LEVEL_INDICATOR;
-	}
-
-	@Override
-	public byte getMetaPageLevelIndicator() {
-		return META_PAGE_LEVEL_INDICATOR;
-	}
-
-	@Override
-	public boolean isFreePage(ITreeIndexMetaDataFrame metaFrame) {
-		return metaFrame.getLevel() == FREE_PAGE_LEVEL_INDICATOR;
-	}
-
-	@Override
-	public boolean isMetaPage(ITreeIndexMetaDataFrame metaFrame) {
-		return metaFrame.getLevel() == META_PAGE_LEVEL_INDICATOR;
-	}
-
-    @Override
-    public int getFirstMetadataPage() {
-        return headPage;
-    }
-
-	@Override
-	public void open(int fileId) {
-		this.fileId = fileId;
-	}
-
-	@Override
-	public void close() {
-		fileId = -1;
-	}
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/LinkedListFreePageManagerFactory.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/LinkedListFreePageManagerFactory.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/LinkedListFreePageManagerFactory.java
deleted file mode 100644
index 16b5c02..0000000
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/LinkedListFreePageManagerFactory.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.common.freepage;
-
-import org.apache.hyracks.storage.am.common.api.IFreePageManager;
-import org.apache.hyracks.storage.am.common.api.IFreePageManagerFactory;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
-import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-
-public class LinkedListFreePageManagerFactory implements IFreePageManagerFactory {
-
-    private final ITreeIndexMetaDataFrameFactory metaDataFrameFactory;
-    private final IBufferCache bufferCache;
-
-    public LinkedListFreePageManagerFactory(IBufferCache bufferCache,
-            ITreeIndexMetaDataFrameFactory metaDataFrameFactory) {
-        this.metaDataFrameFactory = metaDataFrameFactory;
-        this.bufferCache = bufferCache;
-    }
-
-    public IFreePageManager createFreePageManager() {
-        return new LinkedListFreePageManager(bufferCache, 0, metaDataFrameFactory);
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/LinkedListMetadataManagerFactory.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/LinkedListMetadataManagerFactory.java b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/LinkedListMetadataManagerFactory.java
new file mode 100644
index 0000000..a911413
--- /dev/null
+++ b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/LinkedListMetadataManagerFactory.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.common.freepage;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
+import org.apache.hyracks.storage.am.common.api.IMetadataManagerFactory;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+
+public class LinkedListMetadataManagerFactory implements IMetadataManagerFactory {
+
+    private final ITreeIndexMetaDataFrameFactory metaDataFrameFactory;
+    private final IBufferCache bufferCache;
+
+    public LinkedListMetadataManagerFactory(IBufferCache bufferCache,
+                                            ITreeIndexMetaDataFrameFactory metaDataFrameFactory) {
+        this.metaDataFrameFactory = metaDataFrameFactory;
+        this.bufferCache = bufferCache;
+    }
+
+    public IMetaDataPageManager createFreePageManager() throws HyracksDataException {
+        return new LinkedMetaDataPageManager(bufferCache, metaDataFrameFactory);
+    }
+}


Mime
View raw message