asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ima...@apache.org
Subject [1/5] incubator-asterixdb-hyracks git commit: Make LSM bulkload append-only and write-once.
Date Mon, 23 Nov 2015 17:07:23 GMT
Repository: incubator-asterixdb-hyracks
Updated Branches:
  refs/heads/master baec39a7e -> 1a659da1f


http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java
index 0c0a618..305b577 100644
--- a/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java
+++ b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java
@@ -21,24 +21,37 @@ package org.apache.hyracks.storage.common.buffercache;
 import java.nio.ByteBuffer;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 /**
  * @author yingyib
  */
-class CachedPage implements ICachedPageInternal {
+public class CachedPage implements ICachedPageInternal {
     final int cpid;
     final ByteBuffer buffer;
-    final AtomicInteger pinCount;
+    public final AtomicInteger pinCount;
     final AtomicBoolean dirty;
-    final ReadWriteLock latch;
+    final ReentrantReadWriteLock latch;
     private final Object replacementStrategyObject;
     private final IPageReplacementStrategy pageReplacementStrategy;
-    volatile long dpid;
+    volatile long dpid; // disk page id (composed of file id and page id)
     CachedPage next;
     volatile boolean valid;
-    volatile boolean virtual;
+    final AtomicBoolean confiscated;
+    private IQueueInfo queueInfo;
+
+    //Constructor for making dummy entry for FIFO queue
+    public CachedPage(){
+        this.cpid = -1;
+        this.buffer = null;
+        this.pageReplacementStrategy = null;
+        this.dirty = new AtomicBoolean(false);
+        this.confiscated = new AtomicBoolean(true);
+        pinCount = null;
+        queueInfo = null;
+        replacementStrategyObject = null;
+        latch =null;
+    }
 
     public CachedPage(int cpid, ByteBuffer buffer, IPageReplacementStrategy pageReplacementStrategy) {
         this.cpid = cpid;
@@ -50,14 +63,17 @@ class CachedPage implements ICachedPageInternal {
         replacementStrategyObject = pageReplacementStrategy.createPerPageStrategyObject(cpid);
         dpid = -1;
         valid = false;
-        virtual = false;
+        confiscated = new AtomicBoolean(false);
+        queueInfo = null;
     }
 
     public void reset(long dpid) {
         this.dpid = dpid;
         dirty.set(false);
         valid = false;
+        confiscated.set(false);
         pageReplacementStrategy.notifyCachePageReset(this);
+        queueInfo = null;
     }
 
     public void invalidate() {
@@ -76,10 +92,11 @@ class CachedPage implements ICachedPageInternal {
 
     @Override
     public boolean pinIfGoodVictim() {
-        if (virtual)
+        if (confiscated.get())
             return false; //i am not a good victim because i cant flush!
-        else
+        else {
             return pinCount.compareAndSet(0, 1);
+        }
     }
 
     @Override
@@ -104,11 +121,43 @@ class CachedPage implements ICachedPageInternal {
 
     @Override
     public void releaseWriteLatch(boolean markDirty) {
-        if (markDirty) {
-            if (dirty.compareAndSet(false, true)) {
-                pinCount.incrementAndGet();
+        try {
+            if (markDirty) {
+                if (dirty.compareAndSet(false, true)) {
+                    pinCount.incrementAndGet();
+                }
             }
+        } finally {
+            latch.writeLock().unlock();
         }
-        latch.writeLock().unlock();
     }
+
+    @Override
+    public boolean confiscated() {
+        return confiscated.get();
+    }
+
+    @Override
+    public IQueueInfo getQueueInfo() {
+        return queueInfo;
+    }
+
+    @Override
+    public void setQueueInfo(IQueueInfo queueInfo) {
+        this.queueInfo = queueInfo;
+    }
+
+    @Override
+    public long getDiskPageId() {
+        return dpid;
+    }
+
+    CachedPage getNext() {
+        return next;
+    }
+
+    void setNext(CachedPage next) {
+        this.next = next;
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java
index 6050b22..6a82b97 100644
--- a/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java
+++ b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java
@@ -25,9 +25,10 @@ public class ClockPageReplacementStrategy implements IPageReplacementStrategy {
     private static final int MAX_UNSUCCESSFUL_CYCLE_COUNT = 3;
 
     private IBufferCacheInternal bufferCache;
-    private int clockPtr;
+    private AtomicInteger clockPtr;
     private ICacheMemoryAllocator allocator;
-    private AtomicInteger numPages = new AtomicInteger(0);
+    private AtomicInteger numPages;
+    private AtomicInteger cpIdCounter;
     private final int pageSize;
     private final int maxAllowedNumPages;
 
@@ -35,7 +36,9 @@ public class ClockPageReplacementStrategy implements IPageReplacementStrategy {
         this.allocator = allocator;
         this.pageSize = pageSize;
         this.maxAllowedNumPages = maxAllowedNumPages;
-        clockPtr = 0;
+        this.clockPtr = new AtomicInteger(0);
+        this.numPages = new AtomicInteger(0);
+        this.cpIdCounter = new AtomicInteger(0);
     }
 
     @Override
@@ -61,9 +64,7 @@ public class ClockPageReplacementStrategy implements IPageReplacementStrategy {
     @Override
     public ICachedPageInternal findVictim() {
         ICachedPageInternal cachedPage = null;
-        int pageCount = getNumPages();
-        // pageCount is a lower-bound of numPages.
-        if (pageCount >= maxAllowedNumPages) {
+        if (numPages.get() >= maxAllowedNumPages) {
             cachedPage = findVictimByEviction();
         } else {
             cachedPage = allocatePage();
@@ -72,10 +73,12 @@ public class ClockPageReplacementStrategy implements IPageReplacementStrategy {
     }
 
     private ICachedPageInternal findVictimByEviction() {
-        int startClockPtr = clockPtr;
+        //check if we're starved from confiscation
+        assert (maxAllowedNumPages > 0);
+        int startClockPtr = clockPtr.get();
         int cycleCount = 0;
         do {
-            ICachedPageInternal cPage = bufferCache.getPage(clockPtr);
+            ICachedPageInternal cPage = bufferCache.getPage(clockPtr.get());
 
             /*
              * We do two things here:
@@ -88,36 +91,26 @@ public class ClockPageReplacementStrategy implements IPageReplacementStrategy {
             AtomicBoolean accessedFlag = getPerPageObject(cPage);
             if (!accessedFlag.compareAndSet(true, false)) {
                 if (cPage.pinIfGoodVictim()) {
-                    return cPage;
+                        return cPage;
                 }
             }
-            /**
-             * The clockPtr may miss the last added pages in this round.
-             */
-            clockPtr = (clockPtr + 1) % getNumPages();
-            if (clockPtr == startClockPtr) {
+            advanceClock();
+            if (clockPtr.get() == startClockPtr) {
                 ++cycleCount;
             }
         } while (cycleCount < MAX_UNSUCCESSFUL_CYCLE_COUNT);
         return null;
     }
 
-    /**
-     * The number returned here could only be smaller or equal to the actual number
-     * of pages, because numPages is monotonically incremented.
-     */
     @Override
     public int getNumPages() {
         return numPages.get();
     }
 
     private ICachedPageInternal allocatePage() {
-        CachedPage cPage = null;
-        synchronized (this) {
-            cPage = new CachedPage(numPages.get(), allocator.allocate(pageSize, 1)[0], this);
-            bufferCache.addPage(cPage);
-            numPages.incrementAndGet();
-        }
+        CachedPage cPage = new CachedPage(cpIdCounter.getAndIncrement(), allocator.allocate(pageSize, 1)[0], this);
+        bufferCache.addPage(cPage);
+        numPages.incrementAndGet();
         AtomicBoolean accessedFlag = getPerPageObject(cPage);
         if (!accessedFlag.compareAndSet(true, false)) {
             if (cPage.pinIfGoodVictim()) {
@@ -127,6 +120,21 @@ public class ClockPageReplacementStrategy implements IPageReplacementStrategy {
         return null;
     }
 
+    //derived from RoundRobinAllocationPolicy in Apache directmemory
+    private int advanceClock(){
+        boolean clockInDial = false;
+        int newClockPtr = 0;
+        do
+        {
+            int currClockPtr = clockPtr.get();
+            newClockPtr = ( currClockPtr + 1 ) % numPages.get();
+            clockInDial = clockPtr.compareAndSet( currClockPtr, newClockPtr );
+        }
+        while ( !clockInDial );
+        return newClockPtr;
+
+    }
+
     private AtomicBoolean getPerPageObject(ICachedPageInternal cPage) {
         return (AtomicBoolean) cPage.getReplacementStrategyObject();
     }
@@ -140,4 +148,10 @@ public class ClockPageReplacementStrategy implements IPageReplacementStrategy {
     public int getMaxAllowedNumPages() {
         return maxAllowedNumPages;
     }
-}
\ No newline at end of file
+
+    @Override
+    public void adviseWontNeed(ICachedPageInternal cPage) {
+        //make the page appear as if it wasn't accessed even if it was
+        getPerPageObject(cPage).set(false);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java
index 10ce65a..9f7960f 100644
--- a/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java
+++ b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java
@@ -182,15 +182,51 @@ public class DebugBufferCache implements IBufferCache {
     }
 
     @Override
-    public ICachedPage pinVirtual(long vpid) throws HyracksDataException {
-        pinCount.addAndGet(1);
-        return bufferCache.pinVirtual(vpid);
+    public int getNumPagesOfFile(int fileId) throws HyracksDataException {
+        return bufferCache.getNumPagesOfFile(fileId);
     }
 
     @Override
-    public ICachedPage unpinVirtual(long vpid, long dpid) throws HyracksDataException {
-        unpinCount.addAndGet(1);
-        return bufferCache.unpinVirtual(vpid, dpid);
+    public void adviseWontNeed(ICachedPage page) {
+        bufferCache.adviseWontNeed(page);
+    }
+
+    @Override
+    public ICachedPage confiscatePage(long dpid) throws HyracksDataException {
+        return bufferCache.confiscatePage(dpid);
+    }
+
+    @Override
+    public void returnPage(ICachedPage page) {
+        bufferCache.returnPage(page);
+    }
+
+    @Override
+    public IFIFOPageQueue createFIFOQueue() {
+        return bufferCache.createFIFOQueue();
+    }
+
+    @Override
+    public void finishQueue() {
+        bufferCache.finishQueue();
+    }
+
+    @Override
+    public void copyPage(ICachedPage src, ICachedPage dst) {
+        // TODO Auto-generated method stub
+        
+    }
+
+    @Override
+    public void setPageDiskId(ICachedPage page, long dpid) {
+        // TODO Auto-generated method stub
+        
+    }
+
+    @Override
+    public void returnPage(ICachedPage page, boolean reinsert) {
+        // TODO Auto-generated method stub
+        
     }
 
     @Override
@@ -207,4 +243,9 @@ public class DebugBufferCache implements IBufferCache {
     public IIOReplicationManager getIOReplicationManager() {
         return null;
     }
+
+    @Override
+    public void purgeHandle(int fileId) throws HyracksDataException {
+        bufferCache.purgeHandle(fileId);
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/FIFOLocalWriter.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/FIFOLocalWriter.java b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/FIFOLocalWriter.java
new file mode 100644
index 0000000..2f66b15
--- /dev/null
+++ b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/FIFOLocalWriter.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hyracks.storage.common.buffercache;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+
+public class FIFOLocalWriter implements IFIFOPageWriter {
+    private static FIFOLocalWriter instance;
+    private static boolean DEBUG = false;
+
+    public static FIFOLocalWriter instance() {
+        if(instance == null) {
+            instance = new FIFOLocalWriter();
+        }
+        return instance;
+    }
+
+    @Override
+    public void write(ICachedPage page, IBufferCache ibufferCache) throws HyracksDataException {
+        BufferCache bufferCache = (BufferCache)ibufferCache;
+        CachedPage cPage = (CachedPage)page;
+        BufferedFileHandle fInfo = bufferCache.getFileInfo(cPage);
+        if (fInfo.fileHasBeenDeleted()) {
+            return;
+        }
+        cPage.buffer.position(0);
+        cPage.buffer.limit(bufferCache.getPageSize());
+        bufferCache.ioManager.syncWrite(fInfo.getFileHandle(), (long) BufferedFileHandle.getPageId(cPage.dpid) * bufferCache.getPageSize(),
+                cPage.buffer);
+        bufferCache.returnPage(cPage);
+        if(DEBUG) System.out.println("[FIFO] Return page: " + cPage.cpid + "," + cPage.dpid);
+    }
+    
+    @Override
+    public void sync(int fileId, IBufferCache ibufferCache) throws HyracksDataException {
+        BufferCache bufferCache = (BufferCache)ibufferCache;
+        BufferedFileHandle fInfo = bufferCache.getFileInfo(fileId);
+        bufferCache.ioManager.sync(fInfo.getFileHandle(), true);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
index dc4ee3b..20e268d 100644
--- a/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
+++ b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
@@ -19,12 +19,17 @@
 package org.apache.hyracks.storage.common.buffercache;
 
 import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.IFileHandle;
 import org.apache.hyracks.api.io.FileReference;
 import org.apache.hyracks.api.replication.IIOReplicationManager;
 
 public interface IBufferCache {
-    public void createFile(FileReference fileRef) throws HyracksDataException;
 
+    public static final long INVALID_DPID = -1l;
+    public static final int INVALID_PAGEID = -1;
+
+    public void createFile(FileReference fileRef) throws HyracksDataException;
+    
     public int createMemFile() throws HyracksDataException;
 
     public void openFile(int fileId) throws HyracksDataException;
@@ -39,26 +44,42 @@ public interface IBufferCache {
 
     public ICachedPage pin(long dpid, boolean newPage) throws HyracksDataException;
 
-    public ICachedPage pinVirtual(long vpid) throws HyracksDataException;
-
-    public ICachedPage unpinVirtual(long vpid, long dpid) throws HyracksDataException;
-
     public void unpin(ICachedPage page) throws HyracksDataException;
 
     public void flushDirtyPage(ICachedPage page) throws HyracksDataException;
 
+    public void adviseWontNeed(ICachedPage page);
+
+    public ICachedPage confiscatePage(long dpid) throws HyracksDataException;
+
+    public void returnPage(ICachedPage page);
+
+    public void returnPage(ICachedPage page, boolean reinsert);
+
     public void force(int fileId, boolean metadata) throws HyracksDataException;
 
     public int getPageSize();
 
     public int getNumPages();
 
+    public int getNumPagesOfFile(int fileId) throws HyracksDataException;
+
     public int getFileReferenceCount(int fileId);
 
     public void close() throws HyracksDataException;
 
+    public IFIFOPageQueue createFIFOQueue();
+
+    public void finishQueue();
+
+    void copyPage(ICachedPage src, ICachedPage dst);
+
+    void setPageDiskId(ICachedPage page, long dpid);
+
     public boolean isReplicationEnabled();
 
     public IIOReplicationManager getIOReplicationManager();
 
+    void purgeHandle(int fileId) throws HyracksDataException;
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPage.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPage.java b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPage.java
index c8c3ab1..0f77730 100644
--- a/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPage.java
+++ b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPage.java
@@ -30,4 +30,10 @@ public interface ICachedPage {
     public void acquireWriteLatch();
 
     public void releaseWriteLatch(boolean markDirty);
+
+    public boolean confiscated();
+
+    public IQueueInfo getQueueInfo();
+
+    public void setQueueInfo(IQueueInfo queueInfo);
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPageInternal.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPageInternal.java b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPageInternal.java
index 908af18..0193ed8 100644
--- a/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPageInternal.java
+++ b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICachedPageInternal.java
@@ -21,7 +21,10 @@ package org.apache.hyracks.storage.common.buffercache;
 public interface ICachedPageInternal extends ICachedPage {
     public int getCachedPageId();
 
+    public long getDiskPageId();
+
     public Object getReplacementStrategyObject();
 
     public boolean pinIfGoodVictim();
+
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IFIFOPageQueue.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IFIFOPageQueue.java b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IFIFOPageQueue.java
new file mode 100644
index 0000000..0fe5767
--- /dev/null
+++ b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IFIFOPageQueue.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hyracks.storage.common.buffercache;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public interface IFIFOPageQueue {
+    public void put(ICachedPage page) throws HyracksDataException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IFIFOPageWriter.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IFIFOPageWriter.java b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IFIFOPageWriter.java
new file mode 100644
index 0000000..2281a44
--- /dev/null
+++ b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IFIFOPageWriter.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hyracks.storage.common.buffercache;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+
+public interface IFIFOPageWriter {
+    public void write(ICachedPage page, IBufferCache bufferCache) throws HyracksDataException;
+
+    void sync(int fileId, IBufferCache ibufferCache) throws HyracksDataException;
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IPageReplacementStrategy.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IPageReplacementStrategy.java b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IPageReplacementStrategy.java
index 2ecf6b3..c39d3ed 100644
--- a/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IPageReplacementStrategy.java
+++ b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IPageReplacementStrategy.java
@@ -27,6 +27,8 @@ public interface IPageReplacementStrategy {
 
     public void notifyCachePageAccess(ICachedPageInternal cPage);
 
+    public void adviseWontNeed(ICachedPageInternal cPage);
+
     public ICachedPageInternal findVictim();
 
     public int getNumPages();

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IQueueInfo.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IQueueInfo.java b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IQueueInfo.java
new file mode 100644
index 0000000..f214639
--- /dev/null
+++ b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IQueueInfo.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hyracks.storage.common.buffercache;
+
+public interface IQueueInfo {
+
+    /**
+     * Returns whether this queued page should notify all waiters (i.e. it is a low water mark)
+     * @return
+     */
+
+    boolean hasWaiters();
+
+    /**
+     * Whether or not this cached page is a poisoned page.
+     * @return
+     */
+
+    boolean isPoison();
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/QueueInfo.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/QueueInfo.java b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/QueueInfo.java
new file mode 100644
index 0000000..bc69bc8
--- /dev/null
+++ b/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/QueueInfo.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.common.buffercache;
+
+public class QueueInfo implements IQueueInfo{
+
+    private final boolean poison;
+    private final boolean waiters;
+
+    public QueueInfo(boolean waiters, boolean poison){
+        this.waiters = waiters;
+        this.poison = poison;
+    }
+
+    @Override
+    public boolean hasWaiters(){
+        return waiters;
+
+    }
+
+    @Override
+    public boolean isPoison(){
+        return poison;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexExamplesTest.java b/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
index b8f2166..8c01105 100644
--- a/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
+++ b/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
@@ -692,6 +692,7 @@ public abstract class OrderedIndexExamplesTest {
 
         int ins = 1000;
         for (int i = 1; i < ins; i++) {
+
             ITreeIndex treeIndex = createTreeIndex(typeTraits, cmpFactories, bloomFilterKeyFields, null, null, null,
                     null);
             treeIndex.create();

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexTestUtils.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexTestUtils.java b/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexTestUtils.java
index e1b71d2..40b9f05 100644
--- a/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexTestUtils.java
+++ b/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexTestUtils.java
@@ -63,8 +63,8 @@ public class OrderedIndexTestUtils extends TreeIndexTestUtils {
             DataInput dataIn = new DataInputStream(inStream);
             Object actualObj = fieldSerdes[i].deserialize(dataIn);
             if (!actualObj.equals(expected.getField(i))) {
-                fail("Actual and expected fields do not match on field " + i + ".\nExpected: " + expected.getField(i)
-                        + "\nActual  : " + actualObj);
+                 fail("Actual and expected fields do not match on field " + i + ".\nExpected: " + expected.getField(i)
+                         + "\nActual  : " + actualObj);
             }
         }
     }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/TreeIndexTestUtils.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/TreeIndexTestUtils.java b/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/TreeIndexTestUtils.java
index bf03aea..464291a 100644
--- a/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/TreeIndexTestUtils.java
+++ b/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/common/TreeIndexTestUtils.java
@@ -138,7 +138,12 @@ public abstract class TreeIndexTestUtils {
                             + ctx.getCheckTuples().size() + "\nActual  : " + actualCount);
                 }
             } finally {
-                diskOrderCursor.close();
+                try {
+                    diskOrderCursor.close();
+                }
+                catch(Exception ex){
+                    LOGGER.log(Level.WARNING,"Error during scan cursor close",ex);
+                }
             }
         } catch (UnsupportedOperationException e) {
             // Ignore exception because some indexes, e.g. the LSMTrees, don't
@@ -251,9 +256,9 @@ public abstract class TreeIndexTestUtils {
         int c = 1;
         for (CheckTuple checkTuple : checkTuples) {
             if (LOGGER.isLoggable(Level.INFO)) {
-                if (c % (numTuples / 10) == 0) {
+                //if (c % (numTuples / 10) == 0) {
                     LOGGER.info("Bulk Loading Tuple " + c + "/" + numTuples);
-                }
+                //}
             }
             createTupleFromCheckTuple(checkTuple, tupleBuilder, tuple, ctx.getFieldSerdes());
             bulkLoader.add(tuple);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchCursorTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchCursorTest.java b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchCursorTest.java
index 6e009e8..15d63f6 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchCursorTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeSearchCursorTest.java
@@ -28,6 +28,8 @@ import java.util.Random;
 import java.util.TreeSet;
 import java.util.logging.Level;
 
+import org.apache.hyracks.storage.am.common.api.*;
+import org.apache.hyracks.storage.am.common.freepage.LinkedMetaDataPageManager;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
@@ -53,13 +55,8 @@ import org.apache.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
 import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
 import org.apache.hyracks.storage.am.btree.util.AbstractBTreeTest;
 import org.apache.hyracks.storage.am.common.TestOperationCallback;
-import org.apache.hyracks.storage.am.common.api.IFreePageManager;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexAccessor;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
 import org.apache.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
-import org.apache.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
 import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
 import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
 import org.apache.hyracks.storage.common.buffercache.IBufferCache;
@@ -97,7 +94,7 @@ public class BTreeSearchCursorTest extends AbstractBTreeTest {
         IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
         IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
 
-        IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
+        IMetaDataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
 
         BTree btree = new BTree(bufferCache, harness.getFileMapProvider(), freePageManager, interiorFrameFactory,
                 leafFrameFactory, cmpFactories, fieldCount, harness.getFileReference());
@@ -174,7 +171,7 @@ public class BTreeSearchCursorTest extends AbstractBTreeTest {
         IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
         IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
 
-        IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
+        IMetaDataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
 
         BTree btree = new BTree(bufferCache, harness.getFileMapProvider(), freePageManager, interiorFrameFactory,
                 leafFrameFactory, cmpFactories, fieldCount, harness.getFileReference());
@@ -248,7 +245,7 @@ public class BTreeSearchCursorTest extends AbstractBTreeTest {
         IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
         IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
 
-        IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
+        IMetaDataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
 
         BTree btree = new BTree(bufferCache, harness.getFileMapProvider(), freePageManager, interiorFrameFactory,
                 leafFrameFactory, cmpFactories, fieldCount, harness.getFileReference());

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeStatsTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeStatsTest.java b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeStatsTest.java
index 54505dd..4049185 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeStatsTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeStatsTest.java
@@ -22,6 +22,7 @@ import java.io.DataOutput;
 import java.util.Random;
 import java.util.logging.Level;
 
+import org.apache.hyracks.storage.am.common.api.*;
 import org.junit.Test;
 
 import org.apache.hyracks.api.comm.IFrame;
@@ -46,14 +47,9 @@ import org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
 import org.apache.hyracks.storage.am.btree.impls.BTree;
 import org.apache.hyracks.storage.am.btree.util.AbstractBTreeTest;
 import org.apache.hyracks.storage.am.common.TestOperationCallback;
-import org.apache.hyracks.storage.am.common.api.IFreePageManager;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexAccessor;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
-import org.apache.hyracks.storage.am.common.api.TreeIndexException;
+import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
 import org.apache.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
-import org.apache.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
+import org.apache.hyracks.storage.am.common.freepage.LinkedMetaDataPageManager;
 import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
 import org.apache.hyracks.storage.am.common.util.TreeIndexBufferCacheWarmup;
 import org.apache.hyracks.storage.am.common.util.TreeIndexStats;
@@ -99,7 +95,7 @@ public class BTreeStatsTest extends AbstractBTreeTest {
         IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
         ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
 
-        IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
+        IMetaDataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
 
         BTree btree = new BTree(bufferCache, fmp, freePageManager, interiorFrameFactory, leafFrameFactory,
                 cmpFactories, fieldCount, harness.getFileReference());

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpdateSearchTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpdateSearchTest.java b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpdateSearchTest.java
index 4915751..40fc0ef 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpdateSearchTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/BTreeUpdateSearchTest.java
@@ -21,6 +21,8 @@ package org.apache.hyracks.storage.am.btree;
 import java.util.Random;
 import java.util.logging.Level;
 
+import org.apache.hyracks.storage.am.common.api.*;
+import org.apache.hyracks.storage.am.common.freepage.LinkedMetaDataPageManager;
 import org.junit.Test;
 
 import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
@@ -41,14 +43,8 @@ import org.apache.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
 import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
 import org.apache.hyracks.storage.am.btree.util.AbstractBTreeTest;
 import org.apache.hyracks.storage.am.common.TestOperationCallback;
-import org.apache.hyracks.storage.am.common.api.IFreePageManager;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexAccessor;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
-import org.apache.hyracks.storage.am.common.api.TreeIndexException;
+import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
 import org.apache.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
-import org.apache.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
 import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
 import org.apache.hyracks.storage.common.buffercache.IBufferCache;
 
@@ -81,7 +77,7 @@ public class BTreeUpdateSearchTest extends AbstractBTreeTest {
 
         IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
 
-        IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
+        IMetaDataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
         BTree btree = new BTree(bufferCache, harness.getFileMapProvider(), freePageManager, interiorFrameFactory,
                 leafFrameFactory, cmpFactories, fieldCount, harness.getFileReference());
         btree.create();

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java
index 00d637a..6712d1c 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/LSMBTreeSearchOperationCallbackTest.java
@@ -279,7 +279,7 @@ public class LSMBTreeSearchOperationCallbackTest extends AbstractSearchOperation
                 throw new IllegalArgumentException("Invalid range: [" + begin + ", " + end + "]");
             }
 
-            IIndexBulkLoader bulkloader = index.createBulkLoader(1.0f, false, end - begin, true);
+            IIndexBulkLoader bulkloader = index.createBulkLoader(1.0f, false, end - begin, true, true);
             for (int i = begin; i <= end; i++) {
                 TupleUtils.createIntegerTuple(builder, tuple, i);
                 bulkloader.add(tuple);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/InMemoryBTreeRunner.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/InMemoryBTreeRunner.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/InMemoryBTreeRunner.java
index 62bc7d0..bcb11d3 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/InMemoryBTreeRunner.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/perf/InMemoryBTreeRunner.java
@@ -31,7 +31,7 @@ import org.apache.hyracks.storage.am.btree.exceptions.BTreeException;
 import org.apache.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
 import org.apache.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
 import org.apache.hyracks.storage.am.btree.impls.BTree;
-import org.apache.hyracks.storage.am.common.api.IFreePageManager;
+import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexAccessor;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
 import org.apache.hyracks.storage.am.common.api.TreeIndexException;
@@ -39,7 +39,7 @@ import org.apache.hyracks.storage.am.common.datagen.DataGenThread;
 import org.apache.hyracks.storage.am.common.datagen.TupleBatch;
 import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
 import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
-import org.apache.hyracks.storage.am.lsm.common.freepage.VirtualFreePageManager;
+import org.apache.hyracks.storage.am.lsm.common.freepage.VirtualMetaDataPageManager;
 import org.apache.hyracks.storage.am.lsm.common.impls.VirtualBufferCache;
 import org.apache.hyracks.storage.common.buffercache.HeapBufferAllocator;
 import org.apache.hyracks.storage.common.buffercache.IBufferCache;
@@ -71,7 +71,7 @@ public class InMemoryBTreeRunner extends Thread implements IExperimentRunner {
         TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
         ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
         ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
-        IFreePageManager freePageManager = new VirtualFreePageManager(bufferCache.getNumPages());
+        IMetaDataPageManager freePageManager = new VirtualMetaDataPageManager(bufferCache.getNumPages());
         btree = new BTree(bufferCache, new TransientFileMapManager(), freePageManager, interiorFrameFactory,
                 leafFrameFactory, cmpFactories, typeTraits.length, file);
     }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/VirtualFreePageManagerTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/VirtualFreePageManagerTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/VirtualFreePageManagerTest.java
index 634589d..86f75d4 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/VirtualFreePageManagerTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/VirtualFreePageManagerTest.java
@@ -21,16 +21,16 @@ package org.apache.hyracks.storage.am.lsm.common;
 
 import static org.junit.Assert.assertEquals;
 
+import org.apache.hyracks.storage.am.lsm.common.freepage.VirtualMetaDataPageManager;
 import org.junit.Test;
 
 import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.lsm.common.freepage.VirtualFreePageManager;
 
 public class VirtualFreePageManagerTest {
 
     private final int NUM_PAGES = 100;
 
-    private void testInMemoryFreePageManager(VirtualFreePageManager virtualFreePageManager) throws HyracksDataException {
+    private void testInMemoryFreePageManager(VirtualMetaDataPageManager virtualFreePageManager) throws HyracksDataException {
         // The first two pages are reserved for the BTree's metadata page and
         // root page.
         // The "actual" capacity is therefore numPages - 2.
@@ -53,7 +53,7 @@ public class VirtualFreePageManagerTest {
 
     @Test
     public void test01() throws HyracksDataException {
-        VirtualFreePageManager virtualFreePageManager = new VirtualFreePageManager(NUM_PAGES);
+        VirtualMetaDataPageManager virtualFreePageManager = new VirtualMetaDataPageManager(NUM_PAGES);
         testInMemoryFreePageManager(virtualFreePageManager);
         // We expect exactly the same behavior after a reset().
         virtualFreePageManager.reset();

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexDeleteTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexDeleteTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexDeleteTest.java
index 4dbfe6a..e59f85c 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexDeleteTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexDeleteTest.java
@@ -45,15 +45,22 @@ public abstract class AbstractInvertedIndexDeleteTest extends AbstractInvertedIn
     protected void runTest(LSMInvertedIndexTestContext testCtx, TupleGenerator tupleGen) throws IOException,
             IndexException {
         IIndex invIndex = testCtx.getIndex();
-        invIndex.create();
-        invIndex.activate();
+
+        if ((invIndexType != InvertedIndexType.LSM) && (invIndexType != InvertedIndexType.PARTITIONED_LSM) || !bulkLoad) {
+            invIndex.create();
+            invIndex.activate();
+        }
 
         for (int i = 0; i < numInsertRounds; i++) {
             // Start generating documents ids from 0 again.
             tupleGen.reset();
-
             if (bulkLoad) {
-                LSMInvertedIndexTestUtils.bulkLoadInvIndex(testCtx, tupleGen, NUM_DOCS_TO_INSERT);
+                if ((invIndexType != InvertedIndexType.LSM) && (invIndexType != InvertedIndexType.PARTITIONED_LSM)) {
+                    LSMInvertedIndexTestUtils.bulkLoadInvIndex(testCtx, tupleGen, NUM_DOCS_TO_INSERT, false);
+                } else {
+
+                    LSMInvertedIndexTestUtils.bulkLoadInvIndex(testCtx, tupleGen, NUM_DOCS_TO_INSERT, true);
+                }
             } else {
                 LSMInvertedIndexTestUtils.insertIntoInvIndex(testCtx, tupleGen, NUM_DOCS_TO_INSERT);
             }
@@ -74,7 +81,8 @@ public abstract class AbstractInvertedIndexDeleteTest extends AbstractInvertedIn
 
     @Test
     public void wordTokensInvIndexTest() throws IOException, IndexException {
-        LSMInvertedIndexTestContext testCtx = LSMInvertedIndexTestUtils.createWordInvIndexTestContext(harness, invIndexType);
+        LSMInvertedIndexTestContext testCtx = LSMInvertedIndexTestUtils.createWordInvIndexTestContext(harness,
+                invIndexType);
         TupleGenerator tupleGen = LSMInvertedIndexTestUtils.createStringDocumentTupleGen(harness.getRandom());
         runTest(testCtx, tupleGen);
     }
@@ -89,7 +97,8 @@ public abstract class AbstractInvertedIndexDeleteTest extends AbstractInvertedIn
 
     @Test
     public void ngramTokensInvIndexTest() throws IOException, IndexException {
-        LSMInvertedIndexTestContext testCtx = LSMInvertedIndexTestUtils.createNGramInvIndexTestContext(harness, invIndexType);
+        LSMInvertedIndexTestContext testCtx = LSMInvertedIndexTestUtils.createNGramInvIndexTestContext(harness,
+                invIndexType);
         TupleGenerator tupleGen = LSMInvertedIndexTestUtils.createPersonNamesTupleGen(harness.getRandom());
         runTest(testCtx, tupleGen);
     }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexLoadTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexLoadTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexLoadTest.java
index 4467394..78b6d0e 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexLoadTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexLoadTest.java
@@ -42,11 +42,12 @@ public abstract class AbstractInvertedIndexLoadTest extends AbstractInvertedInde
     protected void runTest(LSMInvertedIndexTestContext testCtx, TupleGenerator tupleGen) throws IOException,
             IndexException {
         IIndex invIndex = testCtx.getIndex();
-        invIndex.create();
-        invIndex.activate();
-
+        if(invIndexType != InvertedIndexType.PARTITIONED_ONDISK && invIndexType != InvertedIndexType.ONDISK) {
+            invIndex.create();
+            invIndex.activate();
+        }
         if (bulkLoad) {
-            LSMInvertedIndexTestUtils.bulkLoadInvIndex(testCtx, tupleGen, NUM_DOCS_TO_INSERT);
+                LSMInvertedIndexTestUtils.bulkLoadInvIndex(testCtx, tupleGen, NUM_DOCS_TO_INSERT, true);
         } else {
             LSMInvertedIndexTestUtils.insertIntoInvIndex(testCtx, tupleGen, NUM_DOCS_TO_INSERT);
         }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexSearchTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexSearchTest.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexSearchTest.java
index 2c239c5..3a188fe 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexSearchTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/common/AbstractInvertedIndexSearchTest.java
@@ -54,11 +54,16 @@ public abstract class AbstractInvertedIndexSearchTest extends AbstractInvertedIn
     protected void runTest(LSMInvertedIndexTestContext testCtx, TupleGenerator tupleGen,
             List<IInvertedIndexSearchModifier> searchModifiers) throws IOException, IndexException {
         IIndex invIndex = testCtx.getIndex();
-        invIndex.create();
-        invIndex.activate();
-
+        if ((invIndexType != InvertedIndexType.LSM) && (invIndexType != InvertedIndexType.PARTITIONED_LSM) || !bulkLoad) {
+            invIndex.create();
+            invIndex.activate();
+        }
         if (bulkLoad) {
-            LSMInvertedIndexTestUtils.bulkLoadInvIndex(testCtx, tupleGen, NUM_DOCS_TO_INSERT);
+            if ((invIndexType != InvertedIndexType.LSM) && (invIndexType != InvertedIndexType.PARTITIONED_LSM)) {
+                LSMInvertedIndexTestUtils.bulkLoadInvIndex(testCtx, tupleGen, NUM_DOCS_TO_INSERT, false);
+            } else {
+                LSMInvertedIndexTestUtils.bulkLoadInvIndex(testCtx, tupleGen, NUM_DOCS_TO_INSERT, true);
+            }
         } else {
             LSMInvertedIndexTestUtils.insertIntoInvIndex(testCtx, tupleGen, NUM_DOCS_TO_INSERT);
         }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
index 7a9db2f..061e231 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestContext.java
@@ -40,7 +40,7 @@ import org.apache.hyracks.storage.am.btree.OrderedIndexTestContext;
 import org.apache.hyracks.storage.am.common.CheckTuple;
 import org.apache.hyracks.storage.am.common.api.IIndex;
 import org.apache.hyracks.storage.am.common.api.IndexException;
-import org.apache.hyracks.storage.am.lsm.common.freepage.VirtualFreePageManager;
+import org.apache.hyracks.storage.am.lsm.common.freepage.VirtualMetaDataPageManager;
 import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex;
 import org.apache.hyracks.storage.am.lsm.invertedindex.common.LSMInvertedIndexTestHarness;
 import org.apache.hyracks.storage.am.lsm.invertedindex.exceptions.InvertedIndexException;
@@ -129,14 +129,14 @@ public class LSMInvertedIndexTestContext extends OrderedIndexTestContext {
         switch (invIndexType) {
             case INMEMORY: {
                 invIndex = InvertedIndexUtils.createInMemoryBTreeInvertedindex(harness.getVirtualBufferCaches().get(0),
-                        new VirtualFreePageManager(harness.getVirtualBufferCaches().get(0).getNumPages()),
+                        new VirtualMetaDataPageManager(harness.getVirtualBufferCaches().get(0).getNumPages()),
                         invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory,
                         new FileReference(new File(harness.getOnDiskDir())));
                 break;
             }
             case PARTITIONED_INMEMORY: {
                 invIndex = InvertedIndexUtils.createPartitionedInMemoryBTreeInvertedindex(harness
-                        .getVirtualBufferCaches().get(0), new VirtualFreePageManager(harness.getVirtualBufferCaches()
+                        .getVirtualBufferCaches().get(0), new VirtualMetaDataPageManager(harness.getVirtualBufferCaches()
                         .get(0).getNumPages()), invListTypeTraits, invListCmpFactories, tokenTypeTraits,
                         tokenCmpFactories, tokenizerFactory, new FileReference(new File(harness.getOnDiskDir())));
                 break;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestUtils.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestUtils.java b/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestUtils.java
index fd94870..cc0400a 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestUtils.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestUtils.java
@@ -199,7 +199,7 @@ public class LSMInvertedIndexTestUtils {
         return testCtx;
     }
 
-    public static void bulkLoadInvIndex(LSMInvertedIndexTestContext testCtx, TupleGenerator tupleGen, int numDocs)
+    public static void bulkLoadInvIndex(LSMInvertedIndexTestContext testCtx, TupleGenerator tupleGen, int numDocs, boolean appendOnly)
             throws IndexException, IOException {
         SortedSet<CheckTuple> tmpMemIndex = new TreeSet<CheckTuple>();
         // First generate the expected index by inserting the documents one-by-one.
@@ -210,7 +210,7 @@ public class LSMInvertedIndexTestUtils {
         ISerializerDeserializer[] fieldSerdes = testCtx.getFieldSerdes();
 
         // Use the expected index to bulk-load the actual index.
-        IIndexBulkLoader bulkLoader = testCtx.getIndex().createBulkLoader(1.0f, false, numDocs, true);
+        IIndexBulkLoader bulkLoader = testCtx.getIndex().createBulkLoader(1.0f, false, numDocs, true, appendOnly);
         ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(testCtx.getFieldSerdes().length);
         ArrayTupleReference tuple = new ArrayTupleReference();
         Iterator<CheckTuple> checkTupleIter = tmpMemIndex.iterator();

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeSearchCursorTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeSearchCursorTest.java b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeSearchCursorTest.java
index c444d68..b613724 100644
--- a/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeSearchCursorTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/org/apache/hyracks/storage/am/rtree/RTreeSearchCursorTest.java
@@ -23,6 +23,8 @@ import java.util.ArrayList;
 import java.util.Random;
 import java.util.logging.Level;
 
+import org.apache.hyracks.storage.am.common.api.*;
+import org.apache.hyracks.storage.am.common.freepage.LinkedMetaDataPageManager;
 import org.junit.Before;
 import org.junit.Test;
 
@@ -36,15 +38,8 @@ import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
 import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference;
 import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
 import org.apache.hyracks.dataflow.common.util.TupleUtils;
-import org.apache.hyracks.storage.am.common.api.IFreePageManager;
-import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexAccessor;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
-import org.apache.hyracks.storage.am.common.api.TreeIndexException;
+import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
 import org.apache.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
-import org.apache.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
 import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
 import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
 import org.apache.hyracks.storage.am.common.util.HashMultiSet;
@@ -119,7 +114,7 @@ public class RTreeSearchCursorTest extends AbstractRTreeTest {
 
         IRTreeInteriorFrame interiorFrame = (IRTreeInteriorFrame) interiorFrameFactory.createFrame();
         IRTreeLeafFrame leafFrame = (IRTreeLeafFrame) leafFrameFactory.createFrame();
-        IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, 0, metaFrameFactory);
+        IMetaDataPageManager freePageManager = new LinkedMetaDataPageManager(bufferCache, metaFrameFactory);
 
         RTree rtree = new RTree(bufferCache, harness.getFileMapProvider(), freePageManager, interiorFrameFactory,
                 leafFrameFactory, cmpFactories, fieldCount, harness.getFileReference());

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheTest.java b/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheTest.java
index 1f365df..2c584d5 100644
--- a/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheTest.java
+++ b/hyracks/hyracks-tests/hyracks-storage-common-test/src/test/java/org/apache/hyracks/storage/common/BufferCacheTest.java
@@ -319,102 +319,6 @@ public class BufferCacheTest {
         bufferCache.close();
     }
 
-    @Test
-    public void virtualPageTest() throws HyracksDataException {
-        TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
-        IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
-        IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
-
-        List<Integer> fileIds = new ArrayList<Integer>();
-        Map<Integer, ArrayList<Integer>> pageContents = new HashMap<Integer, ArrayList<Integer>>();
-        ArrayList<Integer> memVals;
-        int num = 10;
-        int testPageId = 0;
-        int lastRealPage = 0;
-        String fileName = getFileName();
-        FileReference file = new FileReference(new File(fileName));
-        bufferCache.createFile(file);
-        int memFileId = bufferCache.createMemFile();
-        int fileId = fmp.lookupFileId(file);
-        bufferCache.openFile(fileId);
-        fileIds.add(fileId);
-
-        // try and write a few somethings into an on-disk paged file
-        ICachedPage page = null;
-        for (; lastRealPage < 10; lastRealPage++) {
-            page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, lastRealPage), true);
-            page.acquireWriteLatch();
-            try {
-                ArrayList<Integer> values = new ArrayList<Integer>();
-                for (int j = 0; j < num; j++) {
-                    int x = Math.abs(rnd.nextInt());
-                    page.getBuffer().putInt(j * 4, x);
-                    values.add(x);
-                }
-                pageContents.put(lastRealPage, values);
-            } finally {
-                page.releaseWriteLatch(true);
-                bufferCache.unpin(page);
-            }
-        }
-        //now try the same thing, but for a virtual page
-        page = bufferCache.pinVirtual(BufferedFileHandle.getDiskPageId(memFileId, testPageId));
-        page.acquireWriteLatch();
-        try {
-            ArrayList<Integer> values = new ArrayList<Integer>();
-            for (int j = 0; j < num; j++) {
-                int x = Math.abs(rnd.nextInt());
-                page.getBuffer().putInt(j * 4, x);
-                values.add(x);
-            }
-            memVals = values;
-        } finally {
-            page.releaseWriteLatch(true);
-            //no unpin here.
-        }
-        //write some more stuff...
-        for (; lastRealPage < 20; lastRealPage++) {
-            page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, lastRealPage), true);
-            page.acquireWriteLatch();
-            try {
-                ArrayList<Integer> values = new ArrayList<Integer>();
-                for (int j = 0; j < num; j++) {
-                    int x = Math.abs(rnd.nextInt());
-                    page.getBuffer().putInt(j * 4, x);
-                    values.add(x);
-                }
-                pageContents.put(lastRealPage, values);
-            } finally {
-                page.releaseWriteLatch(true);
-                bufferCache.unpin(page);
-            }
-        }
-        //now try putting the virtual page after the other pages
-        ICachedPage realPage = bufferCache.unpinVirtual(BufferedFileHandle.getDiskPageId(memFileId, testPageId),
-                BufferedFileHandle.getDiskPageId(fileId, lastRealPage));
-        bufferCache.unpin(realPage);
-        pageContents.put(lastRealPage, memVals);
-
-        bufferCache.closeFile(fileId);
-
-        //now try reading it back!
-        bufferCache.openFile(fileId);
-        for (int i : pageContents.keySet()) {
-            page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, i), false);
-            page.acquireReadLatch();
-            try {
-                ArrayList<Integer> values = pageContents.get(i);
-                for (int j = 0; j < values.size(); j++) {
-                    Assert.assertEquals(values.get(j).intValue(), page.getBuffer().getInt(j * 4));
-                }
-            } finally {
-                page.releaseReadLatch();
-                bufferCache.unpin(page);
-            }
-        }
-        bufferCache.closeFile(fileId);
-    }
-
     @AfterClass
     public static void cleanup() throws Exception {
         for (String s : openedFiles) {

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/1a659da1/hyracks/pom.xml
----------------------------------------------------------------------
diff --git a/hyracks/pom.xml b/hyracks/pom.xml
index 61e06e4..7af92c7 100644
--- a/hyracks/pom.xml
+++ b/hyracks/pom.xml
@@ -103,7 +103,6 @@
     <module>hyracks-client</module>
     <module>hyracks-dataflow-common</module>
     <module>hyracks-dataflow-std</module>
-    <module>hyracks-dataflow-hadoop</module>
     <module>hyracks-control</module>
     <module>hyracks-net</module>
     <module>hyracks-data</module>


Mime
View raw message