asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mb...@apache.org
Subject [01/12] incubator-asterixdb git commit: ASTERIXDB-1436: Big Object Support For Storage
Date Fri, 13 May 2016 02:40:51 GMT
Repository: incubator-asterixdb
Updated Branches:
  refs/heads/master a98d0de2d -> 1defc92ae


http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
index 3c15b23..7a9e588 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
@@ -53,7 +53,7 @@ public class TreeIndexStatsGatherer {
 		int maxPageId = freePageManager.getMaxPage(metaFrame);
 		for (int pageId = 0; pageId <= maxPageId; pageId++) {
 			ICachedPage page = bufferCache.pin(
-					BufferedFileHandle.getDiskPageId(fileId, pageId), false);
+					BufferedFileHandle.getDiskPageId(fileId, pageId), false, leafFrame.getLargePageHelper());
 			page.acquireReadLatch();
 			try {
 				metaFrame.setPage(page);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualMetaDataPageManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualMetaDataPageManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualMetaDataPageManager.java
index cd28f0f..c83cbf2 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualMetaDataPageManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/freepage/VirtualMetaDataPageManager.java
@@ -22,10 +22,10 @@ package org.apache.hyracks.storage.am.lsm.common.freepage;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.am.common.api.IVirtualMetaDataPageManager;
 import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import org.apache.hyracks.storage.am.common.api.IVirtualMetaDataPageManager;
 import org.apache.hyracks.storage.common.buffercache.ICachedPage;
 
 public class VirtualMetaDataPageManager implements IVirtualMetaDataPageManager {
@@ -48,6 +48,11 @@ public class VirtualMetaDataPageManager implements IVirtualMetaDataPageManager {
     }
 
     @Override
+    public int getFreePageBlock(ITreeIndexMetaDataFrame metaFrame, int count) throws HyracksDataException {
+        return currentPageId.getAndUpdate(operand -> operand + count) + 1;
+    }
+
+    @Override
     public int getMaxPage(ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException {
         return currentPageId.get();
     }
@@ -75,6 +80,11 @@ public class VirtualMetaDataPageManager implements IVirtualMetaDataPageManager {
     }
 
     @Override
+    public void addFreePageBlock(ITreeIndexMetaDataFrame metaFrame, int startingPage, int count)
+            throws HyracksDataException {
+    }
+
+    @Override
     public byte getMetaPageLevelIndicator() {
         return 0;
     }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java
index 122db2c..c4fce24 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java
@@ -25,6 +25,7 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.io.FileReference;
 import org.apache.hyracks.api.replication.IIOReplicationManager;
 import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
+import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
 import org.apache.hyracks.storage.common.buffercache.ICachedPage;
 import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue;
 import org.apache.hyracks.storage.common.file.IFileMapManager;
@@ -71,6 +72,11 @@ public class MultitenantVirtualBufferCache implements IVirtualBufferCache {
     }
 
     @Override
+    public ICachedPage pin(long dpid, boolean newPage, ILargePageHelper helper) throws HyracksDataException {
+        return vbc.pin(dpid, newPage, helper);
+    }
+
+    @Override
     public void unpin(ICachedPage page) throws HyracksDataException {
         vbc.unpin(page);
     }
@@ -157,6 +163,11 @@ public class MultitenantVirtualBufferCache implements IVirtualBufferCache {
     }
 
     @Override
+    public ICachedPage confiscateLargePage(long dpid, int multiplier) throws HyracksDataException {
+        return vbc.confiscateLargePage(dpid, multiplier);
+    }
+
+    @Override
     public void returnPage(ICachedPage page) {
         vbc.returnPage(page);
     }
@@ -205,4 +216,9 @@ public class MultitenantVirtualBufferCache implements IVirtualBufferCache {
     public void purgeHandle(int fileId) throws HyracksDataException {
 
     }
+
+    @Override
+    public void resizePage(ICachedPage page, int multiple) {
+        vbc.resizePage(page, multiple);
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
index 6b8cf35..e62f824 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
@@ -21,6 +21,7 @@ package org.apache.hyracks.storage.am.lsm.common.impls;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -34,6 +35,7 @@ import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
 import org.apache.hyracks.storage.common.buffercache.ICacheMemoryAllocator;
 import org.apache.hyracks.storage.common.buffercache.ICachedPage;
 import org.apache.hyracks.storage.common.buffercache.IFIFOPageQueue;
+import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
 import org.apache.hyracks.storage.common.buffercache.IQueueInfo;
 import org.apache.hyracks.storage.common.file.BufferedFileHandle;
 import org.apache.hyracks.storage.common.file.IFileMapManager;
@@ -53,6 +55,7 @@ public class VirtualBufferCache implements IVirtualBufferCache {
     private final ArrayList<VirtualPage> pages;
 
     private volatile int nextFree;
+    private final AtomicInteger largePages;
 
     private boolean open;
 
@@ -65,6 +68,7 @@ public class VirtualBufferCache implements IVirtualBufferCache {
         buckets = new CacheBucket[this.numPages];
         pages = new ArrayList<VirtualPage>();
         nextFree = 0;
+        largePages = new AtomicInteger(0);
         open = false;
     }
 
@@ -164,6 +168,11 @@ public class VirtualBufferCache implements IVirtualBufferCache {
     }
 
     @Override
+    public ICachedPage pin(long dpid, boolean newPage, ILargePageHelper helper) throws HyracksDataException {
+        return pin(dpid, newPage);
+    }
+
+    @Override
     public ICachedPage pin(long dpid, boolean newPage) throws HyracksDataException {
         VirtualPage page = null;
         int hash = hash(dpid);
@@ -200,7 +209,7 @@ public class VirtualBufferCache implements IVirtualBufferCache {
     }
 
     private VirtualPage getOrAllocPage(long dpid) {
-        VirtualPage page = null;
+        VirtualPage page;
         synchronized (pages) {
             if (nextFree >= pages.size()) {
                 page = new VirtualPage(allocator.allocate(pageSize, 1)[0]);
@@ -215,6 +224,38 @@ public class VirtualBufferCache implements IVirtualBufferCache {
     }
 
     @Override
+    public void resizePage(ICachedPage cPage, int multiplier) {
+        ByteBuffer oldBuffer = cPage.getBuffer();
+        int origMultiplier = oldBuffer.capacity() / pageSize;
+        if (origMultiplier == multiplier) {
+            // no-op
+            return;
+        }
+        if (origMultiplier == 1) {
+            synchronized (pages) {
+                pages.remove(cPage);
+                nextFree--;
+            }
+        }
+        ByteBuffer newBuffer = allocator.allocate(pageSize * multiplier, 1)[0];
+        oldBuffer.position(0);
+        if (multiplier < origMultiplier) {
+            oldBuffer.limit(newBuffer.capacity());
+        }
+        newBuffer.put(oldBuffer);
+        if (origMultiplier == 1) {
+            largePages.getAndAdd(multiplier);
+        } else if (multiplier == 1) {
+            largePages.getAndAdd(-origMultiplier);
+            pages.add(0, (VirtualPage) cPage);
+            nextFree++;
+        } else {
+            largePages.getAndAdd(multiplier - origMultiplier);
+        }
+        ((VirtualPage)cPage).buffer = newBuffer;
+    }
+
+    @Override
     public void unpin(ICachedPage page) throws HyracksDataException {
     }
 
@@ -243,19 +284,18 @@ public class VirtualBufferCache implements IVirtualBufferCache {
         }
         pages.trimToSize();
         pages.ensureCapacity(numPages + OVERFLOW_PADDING);
-        ByteBuffer[] buffers = allocator.ensureAvailabilityThenAllocate(pageSize, numPages);
+        allocator.reserveAllocation(pageSize, numPages);
         for (int i = 0; i < numPages; i++) {
-            pages.add(new VirtualPage(buffers[i]));
             buckets[i] = new CacheBucket();
         }
         nextFree = 0;
+        largePages.set(0);
         open = true;
     }
 
     @Override
     public void reset() {
         for (int i = 0; i < numPages; i++) {
-            pages.get(i).reset();
             buckets[i].cachedPage = null;
         }
         int excess = pages.size() - numPages;
@@ -265,6 +305,7 @@ public class VirtualBufferCache implements IVirtualBufferCache {
             }
         }
         nextFree = 0;
+        largePages.set(0);
     }
 
     @Override
@@ -285,6 +326,7 @@ public class VirtualBufferCache implements IVirtualBufferCache {
         sb.append(String.format("Page size = %d\n", pageSize));
         sb.append(String.format("Capacity = %d\n", numPages));
         sb.append(String.format("Allocated pages = %d\n", pages.size()));
+        sb.append(String.format("Allocated large pages = %d\n", largePages.get()));
         sb.append(String.format("Next free page = %d\n", nextFree));
         return sb.toString();
     }
@@ -296,7 +338,7 @@ public class VirtualBufferCache implements IVirtualBufferCache {
 
     @Override
     public boolean isFull() {
-        return nextFree >= numPages;
+        return (nextFree  + largePages.get()) >= numPages;
     }
 
     private static class CacheBucket {
@@ -309,7 +351,7 @@ public class VirtualBufferCache implements IVirtualBufferCache {
     }
 
     private class VirtualPage implements ICachedPage {
-        final ByteBuffer buffer;
+        ByteBuffer buffer;
         final ReadWriteLock latch;
         volatile long dpid;
         VirtualPage next;
@@ -410,6 +452,11 @@ public class VirtualBufferCache implements IVirtualBufferCache {
     }
 
     @Override
+    public ICachedPage confiscateLargePage(long dpid, int multiplier) throws HyracksDataException {
+        throw new UnsupportedOperationException("Virtual buffer caches don't have FIFO writers");
+    }
+
+    @Override
     public void copyPage(ICachedPage src, ICachedPage dst) {
         throw new UnsupportedOperationException("Virtual buffer caches don't have FIFO writers");
     }
@@ -443,4 +490,5 @@ public class VirtualBufferCache implements IVirtualBufferCache {
     public void purgeHandle(int fileId) throws HyracksDataException {
 
     }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java
index 40986f4..56827a5 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java
@@ -21,9 +21,11 @@ package org.apache.hyracks.storage.am.rtree.frames;
 
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.apache.hyracks.storage.am.common.api.IMetaDataPageManager;
 import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProvider;
 import org.apache.hyracks.storage.am.common.api.ISplitKey;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
+import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
 import org.apache.hyracks.storage.am.common.frames.TreeIndexNSMFrame;
@@ -31,10 +33,11 @@ import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
 import org.apache.hyracks.storage.am.rtree.api.IRTreeFrame;
 import org.apache.hyracks.storage.am.rtree.api.IRTreePolicy;
 import org.apache.hyracks.storage.am.rtree.impls.UnorderedSlotManager;
+import org.apache.hyracks.storage.common.buffercache.IBufferCache;
 
 public abstract class RTreeNSMFrame extends TreeIndexNSMFrame implements IRTreeFrame {
-    protected static final int pageNsnOff = smFlagOff + 1;
-    protected static final int rightPageOff = pageNsnOff + 8;
+    protected static final int pageNsnOff = flagOff + 1; // 22
+    protected static final int rightPageOff = pageNsnOff + 8; // 30
 
     protected ITreeIndexTupleReference[] mbrTuples;
     protected ITreeIndexTupleReference cmpFrameTuple;
@@ -46,8 +49,8 @@ public abstract class RTreeNSMFrame extends TreeIndexNSMFrame implements IRTreeF
     protected final boolean isPointMBR;
 
     public RTreeNSMFrame(ITreeIndexTupleWriter tupleWriter, IPrimitiveValueProvider[] keyValueProviders,
-            RTreePolicyType rtreePolicyType, boolean isPointMBR) {
-        super(tupleWriter, new UnorderedSlotManager());
+                         RTreePolicyType rtreePolicyType, boolean isPointMBR) {
+        super(tupleWriter, new UnorderedSlotManager(), null);
         this.mbrTuples = new ITreeIndexTupleReference[keyValueProviders.length];
         for (int i = 0; i < keyValueProviders.length; i++) {
             this.mbrTuples[i] = tupleWriter.createTupleReference();
@@ -98,12 +101,6 @@ public abstract class RTreeNSMFrame extends TreeIndexNSMFrame implements IRTreeF
     }
 
     @Override
-    protected void resetSpaceParams() {
-        buf.putInt(freeSpaceOff, rightPageOff + 4);
-        buf.putInt(totalFreeSpaceOff, buf.capacity() - (rightPageOff + 4));
-    }
-
-    @Override
     public int getRightPage() {
         return buf.getInt(rightPageOff);
     }
@@ -118,7 +115,8 @@ public abstract class RTreeNSMFrame extends TreeIndexNSMFrame implements IRTreeF
     }
 
     @Override
-    public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey)
+    public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey,
+                      IMetaDataPageManager freePageManager, ITreeIndexMetaDataFrame metaFrame, IBufferCache bufferCache)
             throws HyracksDataException {
         rtreePolicy.split(this, buf, rightFrame, slotManager, frameTuple, tuple, splitKey);
     }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrame.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrame.java
index c83b924..a368c9a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrame.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrame.java
@@ -27,7 +27,6 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
 import org.apache.hyracks.data.std.primitive.IntegerPointable;
 import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
-import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
 import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProvider;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleReference;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
@@ -47,7 +46,7 @@ public class RTreeNSMInteriorFrame extends RTreeNSMFrame implements IRTreeInteri
     private final int keyFieldCount;
 
     public RTreeNSMInteriorFrame(ITreeIndexTupleWriter tupleWriter, IPrimitiveValueProvider[] keyValueProviders,
-            RTreePolicyType rtreePolicyType, boolean isPointMBR) {
+                                 RTreePolicyType rtreePolicyType, boolean isPointMBR) {
         super(tupleWriter, keyValueProviders, rtreePolicyType, isPointMBR);
         keyFieldCount = keyValueProviders.length;
         frameTuple.setFieldCount(keyFieldCount);
@@ -174,7 +173,7 @@ public class RTreeNSMInteriorFrame extends RTreeNSMFrame implements IRTreeInteri
     }
 
     @Override
-    public FrameOpSpaceStatus hasSpaceInsert(ITupleReference tuple) {
+    public FrameOpSpaceStatus hasSpaceInsert(ITupleReference tuple) throws HyracksDataException {
         int bytesRequired = tupleWriter.bytesRequired(tuple) + childPtrSize;
         if (bytesRequired + slotManager.getSlotSize() <= buf.capacity() - buf.getInt(freeSpaceOff)
                 - (buf.getInt(tupleCountOff) * slotManager.getSlotSize()))

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrameFactory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrameFactory.java
index d4fd2af..02177f4 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrameFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrameFactory.java
@@ -24,6 +24,7 @@ import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
 import org.apache.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
+import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
 
 public class RTreeNSMInteriorFrameFactory implements ITreeIndexFrameFactory {
 
@@ -51,12 +52,17 @@ public class RTreeNSMInteriorFrameFactory implements ITreeIndexFrameFactory {
         for (int i = 0; i < keyValueProviders.length; i++) {
             keyValueProviders[i] = keyValueProviderFactories[i].createPrimitiveValueProvider();
         }
-        return new RTreeNSMInteriorFrame(tupleWriterFactory.createTupleWriter(), keyValueProviders, rtreePolicyType,
-                isPointMBR);
+        return new RTreeNSMInteriorFrame(tupleWriterFactory.createTupleWriter(), keyValueProviders,
+                rtreePolicyType, isPointMBR);
     }
 
     @Override
     public ITreeIndexTupleWriterFactory getTupleWriterFactory() {
         return tupleWriterFactory;
     }
+
+    @Override
+    public ILargePageHelper getLargePageHelper() {
+        return null;
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrame.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrame.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrame.java
index 79b7f6d..ed8f14c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrame.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrame.java
@@ -30,7 +30,7 @@ import org.apache.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
 public class RTreeNSMLeafFrame extends RTreeNSMFrame implements IRTreeLeafFrame {
 
     public RTreeNSMLeafFrame(ITreeIndexTupleWriter tupleWriter, IPrimitiveValueProvider[] keyValueProviders,
-            RTreePolicyType rtreePolicyType, boolean isPointMBR) {
+                             RTreePolicyType rtreePolicyType, boolean isPointMBR) {
         super(tupleWriter, keyValueProviders, rtreePolicyType, isPointMBR);
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrameFactory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrameFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrameFactory.java
index b45136a..6373c7e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrameFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrameFactory.java
@@ -24,6 +24,7 @@ import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
 import org.apache.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
+import org.apache.hyracks.storage.common.buffercache.ILargePageHelper;
 
 public class RTreeNSMLeafFrameFactory implements ITreeIndexFrameFactory {
 
@@ -51,12 +52,17 @@ public class RTreeNSMLeafFrameFactory implements ITreeIndexFrameFactory {
         for (int i = 0; i < keyValueProviders.length; i++) {
             keyValueProviders[i] = keyValueProviderFactories[i].createPrimitiveValueProvider();
         }
-        return new RTreeNSMLeafFrame(tupleWriterFactory.createTupleWriter(), keyValueProviders, rtreePolicyType,
-                isPointMBR);
+        return new RTreeNSMLeafFrame(tupleWriterFactory.createTupleWriter(), keyValueProviders,
+                rtreePolicyType, isPointMBR);
     }
 
     @Override
     public ITreeIndexTupleWriterFactory getTupleWriterFactory() {
         return tupleWriterFactory;
     }
+
+    @Override
+    public ILargePageHelper getLargePageHelper() {
+        return null;
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
index 342d941..84180aa 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
@@ -412,7 +412,8 @@ public class RTree extends AbstractTreeIndex {
                         rightFrame.setPage(rightNode);
                         rightFrame.initBuffer((byte) ctx.interiorFrame.getLevel());
                         rightFrame.setRightPage(ctx.interiorFrame.getRightPage());
-                        ctx.interiorFrame.split(rightFrame, tuple, ctx.splitKey);
+                        ctx.interiorFrame.split(rightFrame, tuple, ctx.splitKey, freePageManager, ctx.metaFrame,
+                                bufferCache);
                         ctx.interiorFrame.setRightPage(rightPageId);
                     } else {
                         rightFrame = (IRTreeFrame) leafFrameFactory.createFrame();
@@ -420,7 +421,8 @@ public class RTree extends AbstractTreeIndex {
                         rightFrame.initBuffer((byte) 0);
                         rightFrame.setRightPage(ctx.interiorFrame.getRightPage());
                         ctx.modificationCallback.found(null, tuple);
-                        ctx.leafFrame.split(rightFrame, tuple, ctx.splitKey);
+                        ctx.leafFrame.split(rightFrame, tuple, ctx.splitKey, freePageManager, ctx.metaFrame,
+                                bufferCache);
                         ctx.leafFrame.setRightPage(rightPageId);
                     }
                     succeeded = true;
@@ -494,6 +496,10 @@ public class RTree extends AbstractTreeIndex {
                 }
                 break;
             }
+
+            default: {
+                throw new IllegalStateException("NYI: " + spaceStatus);
+            }
         }
     }
 
@@ -1043,8 +1049,8 @@ public class RTree extends AbstractTreeIndex {
 
             NodeFrontier frontier = nodeFrontiers.get(level);
             interiorFrame.setPage(frontier.page);
-            //see if we have space for two tuples. this works around a  tricky boundary condition with sequential bulk load where
-            //finalization can possibly lead to a split
+            //see if we have space for two tuples. this works around a  tricky boundary condition with sequential bulk
+            // load where finalization can possibly lead to a split
             //TODO: accomplish this without wasting 1 tuple
             int sizeOfTwoTuples = 2 * (mbrTuple.getTupleSize() + RTreeNSMInteriorFrame.childPtrSize);
             FrameOpSpaceStatus spaceForTwoTuples = (((RTreeNSMInteriorFrame) interiorFrame)

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/AsyncFIFOPageQueueManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/AsyncFIFOPageQueueManager.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/AsyncFIFOPageQueueManager.java
index 4b23bb1..1bbe6db 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/AsyncFIFOPageQueueManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/AsyncFIFOPageQueueManager.java
@@ -133,30 +133,31 @@ public class AsyncFIFOPageQueueManager implements Runnable {
 
     @Override
     public void run() {
-        if(DEBUG) System.out.println("[FIFO] Writer started");
+        if (DEBUG) System.out.println("[FIFO] Writer started");
         boolean die = false;
         while (!die) {
+            ICachedPage entry;
             try {
-                ICachedPage entry = queue.take();
-                if(entry.getQueueInfo() != null && entry.getQueueInfo().hasWaiters()){
-                    synchronized(entry) {
-                        if(entry.getQueueInfo().isPoison()) { die = true; }
-                        entry.notifyAll();
-                        continue;
-                    }
+                entry = queue.take();
+            } catch(InterruptedException e) {
+                break;
+            }
+            if (entry.getQueueInfo() != null && entry.getQueueInfo().hasWaiters()){
+                synchronized(entry) {
+                    if(entry.getQueueInfo().isPoison()) { die = true; }
+                    entry.notifyAll();
+                    continue;
                 }
+            }
 
-                if(DEBUG) System.out.println("[FIFO] Write " + BufferedFileHandle.getFileId(((CachedPage)entry).dpid)+","
-                        + BufferedFileHandle.getPageId(((CachedPage)entry).dpid));
+            if (DEBUG) System.out.println("[FIFO] Write " + BufferedFileHandle.getFileId(((CachedPage)entry).dpid)+","
+                    + BufferedFileHandle.getPageId(((CachedPage)entry).dpid));
 
-                try {
-                    pageQueue.getWriter().write(entry, bufferCache);
-                } catch (HyracksDataException e) {
-                    //TODO: What do we do, if we could not write the page?
-                    e.printStackTrace();
-                }
-            } catch(InterruptedException e) {
-                continue;
+            try {
+                pageQueue.getWriter().write(entry, bufferCache);
+            } catch (HyracksDataException e) {
+                //TODO: What do we do, if we could not write the page?
+                e.printStackTrace();
             }
         }
     }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
index 6020d7b..2b6dd73 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
@@ -20,13 +20,22 @@ package org.apache.hyracks.storage.common.buffercache;
 
 import java.io.IOException;
 import java.io.OutputStream;
-import java.util.*;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Executor;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Supplier;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -65,7 +74,7 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent {
     private ConcurrentHashMap<CachedPage, StackTraceElement[]> pinnedPageOwner;
     //!DEBUG
     private IIOReplicationManager ioReplicationManager;
-    public List<ICachedPageInternal> cachedPages = new ArrayList<ICachedPageInternal>();
+    private List<ICachedPageInternal> cachedPages = new ArrayList<>();
 
     private boolean closed;
 
@@ -166,6 +175,11 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent {
 
     @Override
     public ICachedPage pin(long dpid, boolean newPage) throws HyracksDataException {
+        return pin(dpid, newPage, null);
+    }
+
+    @Override
+    public ICachedPage pin(long dpid, boolean newPage, ILargePageHelper helper) throws HyracksDataException {
         // Calling the pinSanityCheck should be used only for debugging, since
         // the synchronized block over the fileInfoMap is a hot spot.
         if (DEBUG) {
@@ -183,7 +197,7 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent {
                             }
                         }
                     }
-                }finally{
+                } finally{
                     confiscateLock.unlock();
                 }
             }
@@ -191,7 +205,7 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent {
             // disk.
             synchronized (cPage) {
                 if (!cPage.valid) {
-                    read(cPage);
+                    read(cPage, helper);
                     cPage.valid = true;
                 }
             }
@@ -202,11 +216,10 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent {
         if(DEBUG){
             pinnedPageOwner.put((CachedPage) cPage, Thread.currentThread().getStackTrace());
         }
+        cPage.setLargePageHelper(helper);
         return cPage;
     }
 
-
-
     private boolean isVirtual(long vpid) throws HyracksDataException {
         CachedPage virtPage = findPage(vpid, true);
         return virtPage.confiscated.get();
@@ -544,11 +557,26 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent {
         return false;
     }
 
-    private void read(CachedPage cPage) throws HyracksDataException {
+    private void read(CachedPage cPage, ILargePageHelper helper) throws HyracksDataException {
         BufferedFileHandle fInfo = getFileInfo(cPage);
         cPage.buffer.clear();
         ioManager.syncRead(fInfo.getFileHandle(), (long) BufferedFileHandle.getPageId(cPage.dpid) * pageSize,
                 cPage.buffer);
+        if (helper != null) {
+            int totalPages = helper.getSupplementalBlockNumPages(cPage) + 1;
+            if (totalPages > 1) {
+                resizePage(cPage, totalPages);
+                cPage.buffer.position(pageSize);
+                cPage.buffer.limit(totalPages * pageSize);
+                ioManager.syncRead(fInfo.getFileHandle(), (long) helper.getSupplementalBlockPageId(cPage) * pageSize,
+                        cPage.buffer);
+            }
+        }
+    }
+
+    @Override
+    public void resizePage(ICachedPage cPage, int totalPages) {
+        pageReplacementStrategy.resizePage((ICachedPageInternal) cPage, totalPages);
     }
 
     BufferedFileHandle getFileInfo(CachedPage cPage) throws HyracksDataException {
@@ -567,13 +595,25 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent {
 
     void write(CachedPage cPage) throws HyracksDataException {
         BufferedFileHandle fInfo = getFileInfo(cPage);
-        //synchronize on fInfo to prevent the file handle from being deleted until the page is written.
+        // synchronize on fInfo to prevent the file handle from being deleted until the page is written.
         synchronized (fInfo) {
             if (!fInfo.fileHasBeenDeleted()) {
-                cPage.buffer.position(0);
-                cPage.buffer.limit(pageSize);
+                ByteBuffer buf = cPage.buffer.duplicate();
+                buf.position(0);
+                buf.limit(pageSize);
                 ioManager.syncWrite(fInfo.getFileHandle(), (long) BufferedFileHandle.getPageId(cPage.dpid) * pageSize,
-                        cPage.buffer);
+                        buf);
+                if (cPage.largePageHelper != null) {
+                    int totalPages = cPage.largePageHelper.getSupplementalBlockNumPages(cPage) + 1;
+                    if (totalPages > 1) {
+                        buf.limit(totalPages * pageSize);
+                        long offset = (long) cPage.largePageHelper.getSupplementalBlockPageId(cPage) * pageSize;
+                        ioManager.syncWrite(fInfo.getFileHandle(), offset, buf);
+                    }
+                    assert buf.capacity() == (pageSize * totalPages);
+                } else {
+                    assert buf.capacity() == pageSize;
+                }
             }
         }
     }
@@ -583,12 +623,13 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent {
         if (closed) {
             throw new HyracksDataException("unpin called on a closed cache");
         }
-        if(DEBUG){
+        int pinCount = ((CachedPage) page).pinCount.decrementAndGet();
+        if (DEBUG && pinCount == 0) {
             pinnedPageOwner.remove(page);
         }
-        ((CachedPage) page).pinCount.decrementAndGet();
     }
 
+
     private int hash(long dpid) {
         int hashValue = (int) dpid ^ (Integer.reverse((int) (dpid >>> 32)) >>> 1);
         return hashValue % pageMap.length;
@@ -677,10 +718,13 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent {
                     int curPage = 0;
                     while (true) {
                         synchronized (cachedPages) {
-                            if (curPage >= pageReplacementStrategy.getNumPages()) {
+                            if (curPage >= cachedPages.size()) {
                                 break;
                             }
-                            cleanPage((CachedPage) cachedPages.get(curPage), false);
+                            CachedPage cPage = (CachedPage) cachedPages.get(curPage);
+                            if (cPage != null) {
+                                cleanPage(cPage, false);
+                            }
                         }
                         curPage++;
                     }
@@ -970,8 +1014,99 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent {
     @Override
     public void addPage(ICachedPageInternal page) {
         synchronized (cachedPages) {
-            cachedPages.add(page);
+            final int cpid = page.getCachedPageId();
+            if (cpid < cachedPages.size()) {
+                ICachedPageInternal old = cachedPages.set(cpid, page);
+                if (DEBUG) {
+                    assert old == null;
+                }
+            } else {
+                if (cpid > cachedPages.size()) {
+                    // 4 > 1 -> [exiting, null, null, null, new]
+                    cachedPages.addAll(Collections.nCopies(cpid - cachedPages.size(), null));
+                }
+                cachedPages.add(page);
+            }
+        }
+    }
+
+    @Override
+    public boolean removePage(ICachedPageInternal victimPage) {
+        CachedPage victim = (CachedPage)victimPage;
+        // Case 1 from findPage()
+        if (victim.dpid < 0) { // new page
+            if (!victim.pinCount.compareAndSet(0, 1)) {
+                return false;
+            }
+            // now that we have the pin, ensure the victim's dpid still is < 0, if it's not, decrement
+            // pin count and try again
+            if (victim.dpid >= 0) {
+                victim.pinCount.decrementAndGet();
+                return false;
+            }
+        } else {
+            // Case 2a/b
+            int pageHash = hash(victim.dpid);
+            CacheBucket bucket = pageMap[pageHash];
+            bucket.bucketLock.lock();
+            try {
+                if (!victim.pinCount.compareAndSet(0, 1)) {
+                    return false;
+                }
+                // now that we have the pin, ensure the victim's bucket hasn't changed, if it has, decrement
+                // pin count and try again
+                if (pageHash != hash(victim.dpid)) {
+                    victim.pinCount.decrementAndGet();
+                    return false;
+                }
+                // readjust the next pointers to remove this page from
+                // the pagemap
+                CachedPage curr = bucket.cachedPage;
+                CachedPage prev = null;
+                boolean found = false;
+                //traverse the bucket's linked list to find the victim.
+                while (curr != null) {
+                    if (curr == victim) { // we found where the victim
+                        // resides in the hash table
+                        // if this is the first page in the bucket
+                        if (prev == null) {
+                            if(DEBUG) {
+                                assert curr != curr.next;
+                            }
+                            bucket.cachedPage = bucket.cachedPage.next;
+                            found = true;
+                            break;
+                            // if it isn't we need to make the previous
+                            // node point to where it should
+                        } else {
+                            if(DEBUG) {
+                                assert curr.next != curr;
+                            }
+                            prev.next = curr.next;
+                            curr.next = null;
+                            if(DEBUG) {
+                                assert prev.next != prev;
+                            }
+                            found = true;
+                            break;
+                        }
+                    }
+                    // go to the next entry
+                    prev = curr;
+                    curr = curr.next;
+                }
+                assert found;
+            } finally {
+                bucket.bucketLock.unlock();
+            }
+        }
+        synchronized (cachedPages) {
+            ICachedPageInternal old = cachedPages.set(victim.cpid, null);
+            if (DEBUG) {
+                assert old == victim;
+            };
         }
+        return true;
     }
 
     @Override
@@ -1000,10 +1135,20 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent {
 
     @Override
     public ICachedPage confiscatePage(long dpid) throws HyracksDataException {
+        return confiscatePage(dpid, () -> pageReplacementStrategy.findVictim());
+    }
+
+    @Override
+    public ICachedPage confiscateLargePage(long dpid, int multiplier) throws HyracksDataException {
+        return confiscatePage(dpid, () -> pageReplacementStrategy.findVictim(multiplier));
+    }
+
+    private ICachedPage confiscatePage(long dpid, Supplier<ICachedPageInternal> victimSupplier)
+            throws HyracksDataException {
         while (true) {
             int startCleanedCount = cleanerThread.cleanedCount;
             ICachedPage returnPage = null;
-            CachedPage victim = (CachedPage) pageReplacementStrategy.findVictim();
+            CachedPage victim = (CachedPage) victimSupplier.get();
             if (victim != null) {
                 if(DEBUG) {
                     assert !victim.confiscated.get();
@@ -1079,6 +1224,7 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent {
             // if we found a page after all that, go ahead and finish
             if (returnPage != null) {
                 ((CachedPage) returnPage).confiscated.set(true);
+                ((CachedPage) returnPage).setLargePageHelper(null);
                 if (DEBUG) {
                     confiscateLock.lock();
                     try{
@@ -1167,6 +1313,7 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent {
             }
         }
         pageReplacementStrategy.adviseWontNeed(cPage);
+        cPage.largePageHelper = null;
     }
 
     @Override
@@ -1219,4 +1366,5 @@ public class BufferCache implements IBufferCacheInternal, ILifeCycleComponent {
                 }
         }
     }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java
index cda81ad..ddab742 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/CachedPage.java
@@ -28,7 +28,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
  */
 public class CachedPage implements ICachedPageInternal {
     final int cpid;
-    final ByteBuffer buffer;
+    ByteBuffer buffer;
     public final AtomicInteger pinCount;
     final AtomicBoolean dirty;
     final ReentrantReadWriteLock latch;
@@ -39,6 +39,7 @@ public class CachedPage implements ICachedPageInternal {
     volatile boolean valid;
     final AtomicBoolean confiscated;
     private IQueueInfo queueInfo;
+    ILargePageHelper largePageHelper;
 
     //Constructor for making dummy entry for FIFO queue
     public CachedPage(){
@@ -74,6 +75,7 @@ public class CachedPage implements ICachedPageInternal {
         confiscated.set(false);
         pageReplacementStrategy.notifyCachePageReset(this);
         queueInfo = null;
+        largePageHelper = null;
     }
 
     public void invalidate() {
@@ -160,4 +162,12 @@ public class CachedPage implements ICachedPageInternal {
         this.next = next;
     }
 
+    public ILargePageHelper getLargePageHelper() {
+        return largePageHelper;
+    }
+
+    public void setLargePageHelper(ILargePageHelper largePageHelper) {
+        this.largePageHelper = largePageHelper;
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java
index 5c89bb6..ea27757 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ClockPageReplacementStrategy.java
@@ -18,10 +18,15 @@
  */
 package org.apache.hyracks.storage.common.buffercache;
 
+import java.nio.ByteBuffer;
+import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 
 public class ClockPageReplacementStrategy implements IPageReplacementStrategy {
+    private static final Logger LOGGER = Logger.getLogger(ClockPageReplacementStrategy.class.getName());
     private static final int MAX_UNSUCCESSFUL_CYCLE_COUNT = 3;
 
     private IBufferCacheInternal bufferCache;
@@ -31,6 +36,7 @@ public class ClockPageReplacementStrategy implements IPageReplacementStrategy {
     private AtomicInteger cpIdCounter;
     private final int pageSize;
     private final int maxAllowedNumPages;
+    private final ConcurrentLinkedQueue<Integer> cpIdFreeList;
 
     public ClockPageReplacementStrategy(ICacheMemoryAllocator allocator, int pageSize, int maxAllowedNumPages) {
         this.allocator = allocator;
@@ -39,6 +45,7 @@ public class ClockPageReplacementStrategy implements IPageReplacementStrategy {
         this.clockPtr = new AtomicInteger(0);
         this.numPages = new AtomicInteger(0);
         this.cpIdCounter = new AtomicInteger(0);
+        cpIdFreeList = new ConcurrentLinkedQueue<>();
     }
 
     @Override
@@ -52,6 +59,11 @@ public class ClockPageReplacementStrategy implements IPageReplacementStrategy {
     }
 
     @Override
+    public IBufferCacheInternal getBufferCache() {
+        return bufferCache;
+    }
+
+    @Override
     public void notifyCachePageReset(ICachedPageInternal cPage) {
         getPerPageObject(cPage).set(false);
     }
@@ -63,13 +75,26 @@ public class ClockPageReplacementStrategy implements IPageReplacementStrategy {
 
     @Override
     public ICachedPageInternal findVictim() {
-        ICachedPageInternal cachedPage;
-        if (numPages.get() >= maxAllowedNumPages) {
-            cachedPage = findVictimByEviction();
-        } else {
-            cachedPage = allocatePage();
+        return findVictim(1);
+    }
+
+    @Override
+    public ICachedPageInternal findVictim(int multiplier) {
+        while (numPages.get() + (multiplier - 1) >= maxAllowedNumPages) {
+            // TODO: is dropping pages on the floor enough to adhere to memory budget?
+            ICachedPageInternal victim = findVictimByEviction();
+            if (victim == null) {
+                return null;
+            }
+            int multiple = victim.getBuffer().capacity() / pageSize;
+            if (multiple == multiplier) {
+                return victim;
+            } else if (bufferCache.removePage(victim)) {
+                cpIdFreeList.add(victim.getCachedPageId());
+                numPages.getAndAdd(-multiple);
+            }
         }
-        return cachedPage;
+        return allocatePage(multiplier);
     }
 
     private ICachedPageInternal findVictimByEviction() {
@@ -82,26 +107,32 @@ public class ClockPageReplacementStrategy implements IPageReplacementStrategy {
         boolean looped = false;
         while (true) {
             ICachedPageInternal cPage = bufferCache.getPage(clockPtr);
-
-            /*
-             * We do two things here:
-             * 1. If the page has been accessed, then we skip it -- The CAS would return
-             * false if the current value is false which makes the page a possible candidate
-             * for replacement.
-             * 2. We check with the buffer manager if it feels it's a good idea to use this
-             * page as a victim.
-             */
-            AtomicBoolean accessedFlag = getPerPageObject(cPage);
-            if (!accessedFlag.compareAndSet(true, false)) {
-                if (cPage.isGoodVictim()) {
-                    return cPage;
+            if (cPage != null) {
+                /*
+                 * We do two things here:
+                 * 1. If the page has been accessed, then we skip it -- The CAS would return
+                 * false if the current value is false which makes the page a possible candidate
+                 * for replacement.
+                 * 2. We check with the buffer manager if it feels it's a good idea to use this
+                 * page as a victim.
+                 */
+                AtomicBoolean accessedFlag = getPerPageObject(cPage);
+                if (!accessedFlag.compareAndSet(true, false)) {
+                    if (cPage.isGoodVictim()) {
+                        return cPage;
+                    }
                 }
             }
             if (clockPtr < lastClockPtr) {
                 looped = true;
             }
             if (looped && clockPtr >= startClockPtr) {
-                if (++cycleCount >= MAX_UNSUCCESSFUL_CYCLE_COUNT) {
+                cycleCount++;
+                if (LOGGER.isLoggable(Level.WARNING)) {
+                    LOGGER.warning("completed " + cycleCount + "/" + MAX_UNSUCCESSFUL_CYCLE_COUNT
+                            + " clock cycle(s) without finding victim");
+                }
+                if (cycleCount >= MAX_UNSUCCESSFUL_CYCLE_COUNT) {
                     return null;
                 }
                 looped = false;
@@ -116,17 +147,56 @@ public class ClockPageReplacementStrategy implements IPageReplacementStrategy {
         return numPages.get();
     }
 
-    private ICachedPageInternal allocatePage() {
-        CachedPage cPage = new CachedPage(cpIdCounter.getAndIncrement(), allocator.allocate(pageSize, 1)[0], this);
+    private ICachedPageInternal allocatePage(int multiplier) {
+        Integer cpId = cpIdFreeList.poll();
+        if (cpId == null) {
+            cpId = cpIdCounter.getAndIncrement();
+        }
+        CachedPage cPage = new CachedPage(cpId, allocator.allocate(pageSize * multiplier, 1)[0], this);
         bufferCache.addPage(cPage);
-        numPages.incrementAndGet();
-        AtomicBoolean accessedFlag = getPerPageObject(cPage);
-        if (!accessedFlag.compareAndSet(true, false)) {
-            if (cPage.isGoodVictim()) {
-                return cPage;
+        numPages.getAndAdd(multiplier);
+        return cPage;
+    }
+
+    @Override
+    public void resizePage(ICachedPageInternal cPage, int multiplier) {
+        ByteBuffer oldBuffer = cPage.getBuffer();
+        int origMultiplier = oldBuffer.capacity() / pageSize;
+        if (origMultiplier == multiplier) {
+            // no-op
+            return;
+        }
+        final int newSize = pageSize * multiplier;
+        oldBuffer.position(0);
+        if (multiplier < origMultiplier) {
+            oldBuffer.limit(newSize);
+        } else {
+            while (numPages.get() + (multiplier - origMultiplier) >= maxAllowedNumPages) {
+                ICachedPageInternal victim = findVictimByEviction();
+                if (victim != null) {
+                    int multiple = victim.getBuffer().capacity() / pageSize;
+                    if (bufferCache.removePage(victim)) {
+                        // TODO(mblow): is dropping pages on the floor enough to adhere to memory budget?
+                        cpIdFreeList.add(victim.getCachedPageId());
+                        numPages.getAndAdd(-multiple);
+                    }
+                } else {
+                    // TODO(mblow): what should we do with we need to resize and don't have the budget?
+                    // we don't have the budget to resize- proceed anyway, but log
+                    if (LOGGER.isLoggable(Level.WARNING)) {
+                        LOGGER.warning("Exceeding buffer cache budget of " + maxAllowedNumPages + " by "
+                                + (numPages.get() + (multiplier - origMultiplier) - maxAllowedNumPages)
+                                + " pages in order to satisfy large page read");
+                    }
+                    break;
+
+                }
             }
         }
-        return null;
+        ByteBuffer newBuffer = allocator.allocate(newSize, 1)[0];
+        newBuffer.put(oldBuffer);
+        numPages.getAndAdd(multiplier - origMultiplier);
+        ((CachedPage) cPage).buffer = newBuffer;
     }
 
     //derived from RoundRobinAllocationPolicy in Apache directmemory
@@ -134,13 +204,11 @@ public class ClockPageReplacementStrategy implements IPageReplacementStrategy {
 
         boolean clockInDial;
         int currClockPtr;
-        do
-        {
+        do {
             currClockPtr = clockPtr.get();
-            int newClockPtr = ( currClockPtr + 1 ) % numPages.get();
-            clockInDial = clockPtr.compareAndSet( currClockPtr, newClockPtr );
-        }
-        while ( !clockInDial );
+            int newClockPtr = (currClockPtr + 1) % cpIdCounter.get();
+            clockInDial = clockPtr.compareAndSet(currClockPtr, newClockPtr);
+        } while (!clockInDial);
         return currClockPtr;
 
     }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java
index 0c7f679..c78012b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java
@@ -88,6 +88,13 @@ public class DebugBufferCache implements IBufferCache {
     }
 
     @Override
+    public ICachedPage pin(long dpid, boolean newPage, ILargePageHelper helper) throws HyracksDataException {
+        ICachedPage page = bufferCache.pin(dpid, newPage, helper);
+        pinCount.addAndGet(1);
+        return page;
+    }
+
+    @Override
     public void unpin(ICachedPage page) throws HyracksDataException {
         bufferCache.unpin(page);
         unpinCount.addAndGet(1);
@@ -197,6 +204,11 @@ public class DebugBufferCache implements IBufferCache {
     }
 
     @Override
+    public ICachedPage confiscateLargePage(long dpid, int multiplier) throws HyracksDataException {
+        return bufferCache.confiscateLargePage(dpid, multiplier);
+    }
+
+    @Override
     public void returnPage(ICachedPage page) {
         bufferCache.returnPage(page);
     }
@@ -248,4 +260,9 @@ public class DebugBufferCache implements IBufferCache {
     public void purgeHandle(int fileId) throws HyracksDataException {
         bufferCache.purgeHandle(fileId);
     }
+
+    @Override
+    public void resizePage(ICachedPage page, int multiple) {
+        bufferCache.resizePage(page, multiple);
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DirectBufferAllocator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DirectBufferAllocator.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DirectBufferAllocator.java
index 59455ff..fc05c6b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DirectBufferAllocator.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DirectBufferAllocator.java
@@ -36,4 +36,8 @@ public class DirectBufferAllocator implements ICacheMemoryAllocator {
     public ByteBuffer[] ensureAvailabilityThenAllocate(int pageSize, int numPages) throws HyracksDataException {
         return allocate(pageSize, numPages);
     }
+
+    @Override
+    public void reserveAllocation(int pageSize, int numPages) throws HyracksDataException {
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/FIFOLocalWriter.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/FIFOLocalWriter.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/FIFOLocalWriter.java
index 78be314..6774ddd 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/FIFOLocalWriter.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/FIFOLocalWriter.java
@@ -16,7 +16,6 @@
 package org.apache.hyracks.storage.common.buffercache;
 
 import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.storage.common.file.BufferedFileHandle;
 
 public class FIFOLocalWriter implements IFIFOPageWriter {
     private static FIFOLocalWriter instance;

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/HeapBufferAllocator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/HeapBufferAllocator.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/HeapBufferAllocator.java
index 83850c0..d70ed39 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/HeapBufferAllocator.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/HeapBufferAllocator.java
@@ -36,4 +36,8 @@ public class HeapBufferAllocator implements ICacheMemoryAllocator {
     public ByteBuffer[] ensureAvailabilityThenAllocate(int pageSize, int numPages) throws HyracksDataException {
         return allocate(pageSize, numPages);
     }
+
+    @Override
+    public void reserveAllocation(int pageSize, int numPages) throws HyracksDataException {
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
index ffde5c4..feac69e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
@@ -19,7 +19,6 @@
 package org.apache.hyracks.storage.common.buffercache;
 
 import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.IFileHandle;
 import org.apache.hyracks.api.io.FileReference;
 import org.apache.hyracks.api.replication.IIOReplicationManager;
 
@@ -44,6 +43,8 @@ public interface IBufferCache {
 
     public ICachedPage pin(long dpid, boolean newPage) throws HyracksDataException;
 
+    public ICachedPage pin(long dpid, boolean newPage, ILargePageHelper helper) throws HyracksDataException;
+
     public void unpin(ICachedPage page) throws HyracksDataException;
 
     public void flushDirtyPage(ICachedPage page) throws HyracksDataException;
@@ -52,6 +53,8 @@ public interface IBufferCache {
 
     public ICachedPage confiscatePage(long dpid) throws HyracksDataException;
 
+    public ICachedPage confiscateLargePage(long dpid, int multiplier) throws HyracksDataException;
+
     public void returnPage(ICachedPage page);
 
     public void returnPage(ICachedPage page, boolean reinsert);
@@ -82,4 +85,5 @@ public interface IBufferCache {
 
     void purgeHandle(int fileId) throws HyracksDataException;
 
+    void resizePage(ICachedPage page, int multiple);
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCacheInternal.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCacheInternal.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCacheInternal.java
index ec05542..22ba71f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCacheInternal.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCacheInternal.java
@@ -24,4 +24,6 @@ public interface IBufferCacheInternal extends IBufferCache {
     public ICachedPageInternal getPage(int cpid);
 
     public void addPage(ICachedPageInternal page);
+
+    boolean removePage(ICachedPageInternal victim);
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICacheMemoryAllocator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICacheMemoryAllocator.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICacheMemoryAllocator.java
index e8eb6e3..829d276 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICacheMemoryAllocator.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ICacheMemoryAllocator.java
@@ -40,4 +40,15 @@ public interface ICacheMemoryAllocator {
      */
     public ByteBuffer[] ensureAvailabilityThenAllocate(int pageSize, int numPages) throws HyracksDataException;
 
+    /**
+     * Reserves the allocation with the ResourceMemoryManager (if exists). Otherwise, a HyracksDataException is thrown.
+     * Typically the memory will be subsequently allocated with call to allocate()
+     *
+     * @param pageSize
+     * @param numPages
+     * @return
+     * @throws HyracksDataException
+     */
+    public void reserveAllocation(int pageSize, int numPages) throws HyracksDataException;
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java
new file mode 100644
index 0000000..22bec97
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ILargePageHelper.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.common.buffercache;
+
+public interface ILargePageHelper {
+
+    int getSupplementalBlockNumPages(CachedPage cPage);
+    int getSupplementalBlockPageId(CachedPage cPage);
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IPageReplacementStrategy.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IPageReplacementStrategy.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IPageReplacementStrategy.java
index 661d363..1a59a65 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IPageReplacementStrategy.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IPageReplacementStrategy.java
@@ -23,6 +23,8 @@ public interface IPageReplacementStrategy {
 
     public void setBufferCache(IBufferCacheInternal bufferCache);
 
+    public IBufferCacheInternal getBufferCache();
+
     public void notifyCachePageReset(ICachedPageInternal cPage);
 
     public void notifyCachePageAccess(ICachedPageInternal cPage);
@@ -31,9 +33,13 @@ public interface IPageReplacementStrategy {
 
     public ICachedPageInternal findVictim();
 
+    public ICachedPageInternal findVictim(int multiplier);
+
     public int getNumPages();
 
     public int getPageSize();
 
     public int getMaxAllowedNumPages();
+
+    public void resizePage(ICachedPageInternal page, int multiplier);
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ResourceHeapBufferAllocator.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ResourceHeapBufferAllocator.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ResourceHeapBufferAllocator.java
index 934fe06..3b2e3cb 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ResourceHeapBufferAllocator.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/ResourceHeapBufferAllocator.java
@@ -44,7 +44,12 @@ public class ResourceHeapBufferAllocator implements ICacheMemoryAllocator {
 
     @Override
     public ByteBuffer[] ensureAvailabilityThenAllocate(int pageSize, int numPages) throws HyracksDataException {
-        memoryManager.allocateMemory(resourceName);
+        reserveAllocation(pageSize, numPages);
         return allocate(pageSize, numPages);
     }
+
+    @Override
+    public void reserveAllocation(int pageSize, int numPages) throws HyracksDataException {
+        memoryManager.allocateMemory(resourceName);
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/BufferedFileHandle.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/BufferedFileHandle.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/BufferedFileHandle.java
index aecbd17..177128e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/BufferedFileHandle.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/BufferedFileHandle.java
@@ -70,10 +70,10 @@ public class BufferedFileHandle {
     }
 
     public static int getFileId(long dpid) {
-        return (int) ((dpid >> 32) & 0xffffffff);
+        return (int) (dpid >> 32);
     }
 
     public static int getPageId(long dpid) {
-        return (int) (dpid & 0xffffffff);
+        return (int) dpid;
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexExamplesTest.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
index 2116c0b..0a9343e 100644
--- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/btree/OrderedIndexExamplesTest.java
@@ -19,15 +19,10 @@
 
 package org.apache.hyracks.storage.am.btree;
 
-import static org.junit.Assert.fail;
-
 import java.util.Random;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-import org.apache.hyracks.storage.common.buffercache.BufferCache;
-import org.junit.Test;
-
 import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
 import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
 import org.apache.hyracks.api.dataflow.value.ITypeTraits;
@@ -54,6 +49,9 @@ import org.apache.hyracks.storage.am.common.api.UnsortedInputException;
 import org.apache.hyracks.storage.am.common.exceptions.TreeIndexDuplicateKeyException;
 import org.apache.hyracks.storage.am.common.impls.TreeIndexDiskOrderScanCursor;
 import org.apache.hyracks.storage.am.common.ophelpers.MultiComparator;
+import org.junit.Test;
+
+import static org.junit.Assert.fail;
 
 @SuppressWarnings("rawtypes")
 public abstract class OrderedIndexExamplesTest {

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java
index d47b114..840acd5 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/FieldPrefixNSMTest.java
@@ -23,6 +23,7 @@ import java.io.DataOutput;
 import java.util.Random;
 import java.util.logging.Level;
 
+import org.apache.hyracks.storage.am.btree.frames.BTreeLargeFrameHelper;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -135,7 +136,8 @@ public class FieldPrefixNSMTest extends AbstractBTreeTest {
         int btreeFileId = fileMapProvider.lookupFileId(harness.getFileReference());
         bufferCache.openFile(btreeFileId);
         IHyracksTaskContext ctx = harness.getHyracksTaskContext();
-        ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(btreeFileId, 0), false);
+        ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(btreeFileId, 0), false,
+                BTreeLargeFrameHelper.INSTANCE);
         try {
 
             ITreeIndexTupleWriter tupleWriter = new TypeAwareTupleWriter(typeTraits);
@@ -206,7 +208,8 @@ public class FieldPrefixNSMTest extends AbstractBTreeTest {
                     }
                 }
 
-                ITupleReference tuple = createTuple(ctx, savedFields[i][0], savedFields[i][1], savedFields[i][2], false);
+                ITupleReference tuple = createTuple(ctx, savedFields[i][0], savedFields[i][1], savedFields[i][2],
+                        false);
                 try {
                     int tupleIndex = frame.findDeleteTupleIndex(tuple);
                     frame.delete(tuple, tupleIndex);

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageManagerTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageManagerTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageManagerTest.java
index 01863ca..cce2f82 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageManagerTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/org/apache/hyracks/storage/am/btree/StorageManagerTest.java
@@ -24,6 +24,7 @@ import java.util.List;
 import java.util.Random;
 import java.util.logging.Level;
 
+import org.apache.hyracks.storage.am.btree.frames.BTreeLargeFrameHelper;
 import org.junit.Test;
 
 import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -89,7 +90,8 @@ public class StorageManagerTest extends AbstractBTreeTest {
             }
 
             try {
-                ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
+                ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false,
+                        BTreeLargeFrameHelper.INSTANCE);
                 LatchType latch = null;
 
                 switch (fta) {

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/IntSerDeUtils.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/IntSerDeUtils.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/IntSerDeUtils.java
new file mode 100644
index 0000000..b5cb8b1
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/IntSerDeUtils.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.util;
+
+public class IntSerDeUtils {
+
+    public static int getInt(byte[] bytes, int offset) {
+        return ((bytes[offset] & 0xff) << 24) + ((bytes[offset + 1] & 0xff) << 16) + ((bytes[offset + 2] & 0xff) << 8)
+                + ((bytes[offset + 3] & 0xff) << 0);
+    }
+
+    /**
+     * put integer value into the array bytes at the offset offset
+     * @param bytes byte array to put data in
+     * @param offset offset from the beginning of the array to write the {@code value} in
+     * @param value value to write to {@code bytes[offset]}
+     */
+    public static void putInt(byte[] bytes, int offset, int value) {
+
+        bytes[offset++] = (byte) (value >> 24);
+        bytes[offset++] = (byte) (value >> 16);
+        bytes[offset++] = (byte) (value >> 8);
+        bytes[offset++] = (byte) (value);
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/1defc92a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ReflectionUtils.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ReflectionUtils.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ReflectionUtils.java
new file mode 100644
index 0000000..408d67f
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ReflectionUtils.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.util;
+
+public class ReflectionUtils {
+    public static <T> T createInstance(Class<? extends T> klass) {
+        T instance = null;
+        try {
+            instance = klass.newInstance();
+        } catch (InstantiationException e) {
+            throw new RuntimeException(e);
+        } catch (IllegalAccessException e) {
+            throw new RuntimeException(e);
+        }
+        return instance;
+    }
+}


Mime
View raw message