asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jianf...@apache.org
Subject [04/11] incubator-asterixdb-hyracks git commit: Implemented the memory-bounded HashGroupby and HashJoin for BigObject
Date Fri, 26 Feb 2016 05:54:01 GMT
http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/6abc63e2/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/buffermanager/VariableFramesMemoryManagerTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/buffermanager/VariableFramesMemoryManagerTest.java b/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/buffermanager/VariableFramesMemoryManagerTest.java
new file mode 100644
index 0000000..9d4a9a1
--- /dev/null
+++ b/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/buffermanager/VariableFramesMemoryManagerTest.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.dataflow.std.buffermanager;
+
+import static org.apache.hyracks.dataflow.std.buffermanager.Common.BUDGET;
+import static org.apache.hyracks.dataflow.std.buffermanager.Common.MIN_FRAME_SIZE;
+import static org.apache.hyracks.dataflow.std.buffermanager.Common.NUM_MIN_FRAME;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.hyracks.api.comm.FixedSizeFrame;
+import org.apache.hyracks.api.comm.FrameHelper;
+import org.apache.hyracks.api.comm.IFrame;
+import org.apache.hyracks.api.comm.IFrameTupleAppender;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+
+public class VariableFramesMemoryManagerTest {
+    VariableFrameMemoryManager framesMemoryManager;
+    FrameTupleAccessor fta;
+    Random random;
+    List<IFrame> frameList;
+
+    @Before
+    public void setUp() throws Exception {
+        VariableFramePool framePool = new VariableFramePool(Common.commonFrameManager, BUDGET);
+        FrameFreeSlotLastFit policy = new FrameFreeSlotLastFit(NUM_MIN_FRAME);
+        framesMemoryManager = new VariableFrameMemoryManager(framePool, policy);
+        RecordDescriptor recordDescriptor = new RecordDescriptor(new ISerializerDeserializer[] { null });
+        fta = new FrameTupleAccessor(recordDescriptor);
+        random = new Random(System.currentTimeMillis());
+        frameList = new ArrayList<>();
+    }
+
+    @Test
+    public void testNormalIncomingFrames() throws HyracksDataException {
+        HashMap<Integer, Integer> tupleSet = prepareTuples();
+        for (IFrame frame : frameList) {
+            assertTrue(framesMemoryManager.insertFrame(frame.getBuffer()) >= 0);
+        }
+        assertEquals(NUM_MIN_FRAME, framesMemoryManager.getNumFrames());
+        assertEveryTupleInFTAIsInFrameMemoryManager(tupleSet, framesMemoryManager);
+    }
+
+    @Test
+    public void testRandomTuplesAreAllStoredInBuffer() throws HyracksDataException {
+        Map<Integer, Integer> tupleSet = prepareRandomTuples();
+        for (IFrame frame : frameList) {
+            if (framesMemoryManager.insertFrame(frame.getBuffer()) < 0) {
+                fta.reset(frame.getBuffer());
+                for (int i = 0; i < fta.getTupleCount(); ++i) {
+                    int id = parseTuple(fta.getBuffer(),
+                            fta.getTupleStartOffset(i) + fta.getFieldStartOffset(i, 0) + fta.getFieldSlotsLength());
+                    tupleSet.remove(id);
+                    //                    System.out.println(
+                    //                            "can't appended id:" + id + ",frameSize:" + frame.getInitialFrameSize());
+                }
+            }
+        }
+        assertEveryTupleInFTAIsInFrameMemoryManager(tupleSet, framesMemoryManager);
+        framesMemoryManager.reset();
+    }
+
+    @Test
+    public void testResetShouldWork() throws HyracksDataException {
+        testNormalIncomingFrames();
+        framesMemoryManager.reset();
+        testRandomTuplesAreAllStoredInBuffer();
+        framesMemoryManager.reset();
+        testRandomTuplesAreAllStoredInBuffer();
+    }
+
+    @Test
+    public void testCloseShouldAlsoWork() throws HyracksDataException {
+        testRandomTuplesAreAllStoredInBuffer();
+        framesMemoryManager.close();
+        testRandomTuplesAreAllStoredInBuffer();
+        framesMemoryManager.close();
+        testRandomTuplesAreAllStoredInBuffer();
+    }
+
+    private HashMap<Integer, Integer> prepareRandomTuples() throws HyracksDataException {
+        frameList.clear();
+        HashMap<Integer, Integer> set = new HashMap<>(NUM_MIN_FRAME);
+        int[] fieldSlot = { 0 };
+        int id = 0;
+        int size = 0;
+        while (size < BUDGET) {
+            int tupleLength = random.nextInt(BUDGET / 3) + 4;
+            IFrame frame = new FixedSizeFrame(Common.commonFrameManager
+                    .allocateFrame(FrameHelper.calcAlignedFrameSizeToStore(1, tupleLength, MIN_FRAME_SIZE)));
+            IFrameTupleAppender appender = new FrameTupleAppender();
+            appender.reset(frame, true);
+            //            System.out.println("id:" + id + ",frameSize:" + frame.getInitialFrameSize() / MIN_FRAME_SIZE);
+            ByteBuffer buffer = ByteBuffer.allocate(tupleLength);
+            buffer.putInt(0, id);
+            assertTrue(appender.append(fieldSlot, buffer.array(), 0, buffer.capacity()));
+            set.put(id++, tupleLength);
+            size += frame.getFrameSize();
+            frameList.add(frame);
+        }
+        return set;
+    }
+
+    private HashMap<Integer, Integer> prepareTuples() throws HyracksDataException {
+        frameList.clear();
+        HashMap<Integer, Integer> set = new HashMap<>(NUM_MIN_FRAME);
+        for (int i = 0; i < NUM_MIN_FRAME; ++i) {
+            IFrame frame = new FixedSizeFrame(Common.commonFrameManager.allocateFrame(MIN_FRAME_SIZE));
+            IFrameTupleAppender appender = new FrameTupleAppender();
+            appender.reset(frame, true);
+
+            int[] fieldSlot = { 0 };
+            ByteBuffer buffer = ByteBuffer.allocate(MIN_FRAME_SIZE / 2);
+            buffer.putInt(0, i);
+            appender.append(fieldSlot, buffer.array(), 0, buffer.capacity());
+            set.put(i, buffer.capacity());
+            frameList.add(frame);
+        }
+        return set;
+    }
+
+    private void assertEveryTupleInFTAIsInFrameMemoryManager(Map<Integer, Integer> tupleSet,
+            VariableFrameMemoryManager framesMemoryManager) {
+        BufferInfo info = new BufferInfo(null, -1, -1);
+        for (int i = 0; i < framesMemoryManager.getNumFrames(); ++i) {
+            framesMemoryManager.getFrame(i, info);
+            fta.reset(info.getBuffer(), info.getStartOffset(), info.getLength());
+            for (int t = 0; t < fta.getTupleCount(); t++) {
+                int id = parseTuple(fta.getBuffer(), fta.getTupleStartOffset(t) + fta.getFieldSlotsLength() + fta
+                        .getFieldStartOffset(t, 0));
+                //                System.out.println("frameid:" + i + ",tuple:" + t + ",has id:" + id + ",length:" +
+                //                        (fta.getTupleEndOffset(t) - fta.getTupleStartOffset(t) - fta.getFieldSlotsLength()));
+                assertTrue(tupleSet.remove(id) == fta.getTupleEndOffset(t) - fta.getTupleStartOffset(t) - fta
+                        .getFieldSlotsLength());
+            }
+        }
+        assertTrue(tupleSet.isEmpty());
+    }
+
+    private int parseTuple(ByteBuffer buffer, int fieldStartOffset) {
+        return buffer.getInt(fieldStartOffset);
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/6abc63e2/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/buffermanager/VariableTupleMemoryManagerTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/buffermanager/VariableTupleMemoryManagerTest.java b/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/buffermanager/VariableTupleMemoryManagerTest.java
new file mode 100644
index 0000000..036cd41
--- /dev/null
+++ b/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/buffermanager/VariableTupleMemoryManagerTest.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.dataflow.std.buffermanager;
+
+import static org.junit.Assert.assertTrue;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.hyracks.api.comm.FixedSizeFrame;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import org.apache.hyracks.dataflow.common.util.IntSerDeUtils;
+import org.apache.hyracks.dataflow.std.structures.TuplePointer;
+import org.junit.Before;
+import org.junit.Test;
+
+public class VariableTupleMemoryManagerTest extends AbstractTupleMemoryManagerTest {
+    VariableDeletableTupleMemoryManager tupleMemoryManager;
+    final int EXTRA_BYTES_FOR_DELETABLE_FRAME = 4;
+
+    @Before
+    public void setup() {
+        VariableFramePool framePool = new VariableFramePool(Common.commonFrameManager, Common.BUDGET);
+        tupleMemoryManager = new VariableDeletableTupleMemoryManager(framePool, recordDescriptor);
+    }
+
+    @Test
+    public void testInsertTupleToMemoryManager() throws HyracksDataException {
+        int iTuplePerFrame = 3;
+        Map<Integer, Integer> mapPrepare = prepareFixedSizeTuples(iTuplePerFrame, EXTRA_BYTES_FOR_DELETABLE_FRAME, 0);
+        Map<TuplePointer, Integer> mapInserted = insertInFTAToBufferShouldAllSuccess();
+        assertEachTupleInFTAIsInBuffer(mapPrepare, mapInserted);
+    }
+
+    @Test
+    public void testReset() throws HyracksDataException {
+        testInsertVariableSizeTupleToMemoryManager();
+        tupleMemoryManager.reset();
+        testInsertTupleToMemoryManager();
+        tupleMemoryManager.reset();
+        testInsertVariableSizeTupleToMemoryManager();
+    }
+
+    @Test
+    public void testDeleteTupleInMemoryManager() throws HyracksDataException {
+        int iTuplePerFrame = 3;
+        Map<Integer, Integer> mapPrepare = prepareFixedSizeTuples(iTuplePerFrame, EXTRA_BYTES_FOR_DELETABLE_FRAME, 0);
+        Map<TuplePointer, Integer> mapInserted = insertInFTAToBufferShouldAllSuccess();
+        deleteRandomSelectedTuples(mapPrepare, mapInserted, 1);
+        assertEachTupleInFTAIsInBuffer(mapPrepare, mapInserted);
+    }
+
+    @Test
+    public void testReOrganizeSpace() throws HyracksDataException {
+        int iTuplePerFrame = 3;
+        Map<Integer, Integer> mapPrepare = prepareFixedSizeTuples(iTuplePerFrame, EXTRA_BYTES_FOR_DELETABLE_FRAME, 0);
+        Map<Integer, Integer> copyMap = new HashMap<>(mapPrepare);
+        Map<TuplePointer, Integer> mapInserted = insertInFTAToBufferShouldAllSuccess();
+        ByteBuffer buffer = deleteRandomSelectedTuples(mapPrepare, mapInserted, mapPrepare.size() / 2);
+        inFTA.reset(buffer);
+        //The deletable frame buffer will keep the deleted slot untouched, which will take more space.
+        // the reason is to not reuse the same TuplePointer outside.
+        Map<TuplePointer, Integer> mapInserted2 = insertInFTAToBufferMayNotAllSuccess();
+        assertTrue(mapInserted2.size() > 0);
+    }
+
+    @Test
+    public void testReOrganizeVariableSizeTuple() throws HyracksDataException {
+        Map<Integer, Integer> map = prepareVariableSizeTuples();
+        Map<TuplePointer, Integer> mapInserted = insertInFTAToBufferCouldFailForLargerTuples(map);
+        Map<Integer, Integer> copyMap = new HashMap<>(map);
+
+        ByteBuffer buffer = deleteRandomSelectedTuples(map, mapInserted, map.size() / 2);
+        inFTA.reset(buffer);
+
+        Map<TuplePointer, Integer> mapInserted2 = insertInFTAToBufferCouldFailForLargerTuples(copyMap);
+        Map<TuplePointer, Integer> mergedMap = new HashMap<>(mapInserted);
+        mergedMap.putAll(mapInserted2);
+
+        assertEachTupleInFTAIsInBuffer(copyMap, mergedMap);
+    }
+
+    @Test
+    public void testInsertVariableSizeTupleToMemoryManager() throws HyracksDataException {
+        Map<Integer, Integer> map = prepareVariableSizeTuples();
+        Map<TuplePointer, Integer> mapInserted = insertInFTAToBufferCouldFailForLargerTuples(map);
+        assertEachTupleInFTAIsInBuffer(map, mapInserted);
+    }
+
+    @Override
+    ITuplePointerAccessor getTupleAccessor() {
+        return tupleMemoryManager.createTupleAccessor();
+    }
+
+    private Map<TuplePointer, Integer> insertInFTAToBufferShouldAllSuccess() throws HyracksDataException {
+        Map<TuplePointer, Integer> tuplePointerIntegerMap = new HashMap<>();
+        for (int i = 0; i < inFTA.getTupleCount(); i++) {
+            TuplePointer tuplePointer = new TuplePointer();
+            assertTrue(tupleMemoryManager.insertTuple(inFTA, i, tuplePointer));
+            tuplePointerIntegerMap.put(tuplePointer,
+                    IntSerDeUtils.getInt(inFTA.getBuffer().array(), inFTA.getAbsoluteFieldStartOffset(i, 0)));
+        }
+        return tuplePointerIntegerMap;
+    }
+
+    private Map<TuplePointer, Integer> insertInFTAToBufferMayNotAllSuccess() throws HyracksDataException {
+        Map<TuplePointer, Integer> tuplePointerIntegerMap = new HashMap<>();
+        for (int i = 0; i < inFTA.getTupleCount(); i++) {
+            TuplePointer tuplePointer = new TuplePointer();
+            if (!tupleMemoryManager.insertTuple(inFTA, i, tuplePointer)) {
+                break;
+            }
+            tuplePointerIntegerMap.put(tuplePointer,
+                    IntSerDeUtils.getInt(inFTA.getBuffer().array(), inFTA.getAbsoluteFieldStartOffset(i, 0)));
+        }
+        return tuplePointerIntegerMap;
+    }
+
+    private Map<TuplePointer, Integer> insertInFTAToBufferCouldFailForLargerTuples(Map<Integer, Integer> map)
+            throws HyracksDataException {
+        Map<TuplePointer, Integer> tuplePointerIdMap = new HashMap<>();
+        int i = 0;
+        for (; i < inFTA.getTupleCount(); i++) {
+            TuplePointer tuplePointer = new TuplePointer();
+            if (!tupleMemoryManager.insertTuple(inFTA, i, tuplePointer)) {
+                break;
+            }
+            tuplePointerIdMap.put(tuplePointer,
+                    IntSerDeUtils.getInt(inFTA.getBuffer().array(), inFTA.getAbsoluteFieldStartOffset(i, 0)));
+        }
+        for (; i < inFTA.getTupleCount(); i++) {
+            map.remove(IntSerDeUtils.getInt(inFTA.getBuffer().array(), inFTA.getAbsoluteFieldStartOffset(i, 0)));
+        }
+        return tuplePointerIdMap;
+    }
+
+    private ByteBuffer deleteRandomSelectedTuples(Map<Integer, Integer> map, Map<TuplePointer, Integer> mapInserted,
+            int minNumOfRecordTobeDeleted)
+            throws HyracksDataException {
+        ByteBuffer buffer = ByteBuffer.allocate(Common.BUDGET);
+        FixedSizeFrame frame = new FixedSizeFrame(buffer);
+        FrameTupleAppender appender = new FrameTupleAppender();
+        appender.reset(frame, true);
+
+        assert (minNumOfRecordTobeDeleted < mapInserted.size());
+        int countDeleted = minNumOfRecordTobeDeleted + random.nextInt(mapInserted.size() - minNumOfRecordTobeDeleted);
+
+        ITuplePointerAccessor accessor = tupleMemoryManager.createTupleAccessor();
+        for (int i = 0; i < countDeleted; i++) {
+            Iterator<Map.Entry<TuplePointer, Integer>> iter = mapInserted.entrySet().iterator();
+            assert (iter.hasNext());
+            Map.Entry<TuplePointer, Integer> pair = iter.next();
+            accessor.reset(pair.getKey());
+            appender.append(accessor.getBuffer().array(), accessor.getTupleStartOffset(), accessor.getTupleLength());
+            map.remove(pair.getValue());
+            tupleMemoryManager.deleteTuple(pair.getKey());
+            iter.remove();
+        }
+        return buffer;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/6abc63e2/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/Common.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/Common.java b/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/Common.java
deleted file mode 100644
index a6f10a6..0000000
--- a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/Common.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.dataflow.std.sort.buffermanager;
-
-import org.apache.hyracks.control.nc.resources.memory.FrameManager;
-
-public class Common {
-    static int MIN_FRAME_SIZE = 256;
-    static int NUM_MIN_FRAME = 15;
-    static int BUDGET = NUM_MIN_FRAME * MIN_FRAME_SIZE;
-
-    static FrameManager commonFrameManager = new FrameManager(MIN_FRAME_SIZE);
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/6abc63e2/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/FrameFreeSlotBestFitUsingTreeMapTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/FrameFreeSlotBestFitUsingTreeMapTest.java b/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/FrameFreeSlotBestFitUsingTreeMapTest.java
deleted file mode 100644
index 58dd7e8..0000000
--- a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/FrameFreeSlotBestFitUsingTreeMapTest.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.dataflow.std.sort.buffermanager;
-
-import static org.junit.Assert.assertEquals;
-
-import org.junit.Before;
-import org.junit.Test;
-
-public class FrameFreeSlotBestFitUsingTreeMapTest {
-
-    static int size = 10;
-
-    FrameFreeSlotSmallestFit policy;
-
-    @Before
-    public void intial() {
-        policy = new FrameFreeSlotSmallestFit();
-    }
-
-    @Test
-    public void testAll() {
-
-        for (int i = 0; i < size; i++) {
-            policy.pushNewFrame(i, i);
-            assertEquals(i, policy.popBestFit(i));
-        }
-        assertEquals(-1, policy.popBestFit(0));
-
-        for (int i = 0; i < size; i++) {
-            policy.pushNewFrame(i, i);
-        }
-        for (int i = 0; i < size; i++) {
-            assertEquals(i, policy.popBestFit(i));
-        }
-
-    }
-
-    @Test
-    public void testReset(){
-        testAll();
-        policy.reset();
-        testAll();
-    }
-
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/6abc63e2/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/FrameFreeSlotBiggestFirstTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/FrameFreeSlotBiggestFirstTest.java b/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/FrameFreeSlotBiggestFirstTest.java
deleted file mode 100644
index 73eda47..0000000
--- a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/FrameFreeSlotBiggestFirstTest.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.dataflow.std.sort.buffermanager;
-
-import static junit.framework.Assert.assertEquals;
-
-import org.junit.Before;
-import org.junit.Test;
-
-public class FrameFreeSlotBiggestFirstTest {
-
-    static int size = 10;
-
-    FrameFreeSlotBiggestFirst policy;
-
-    @Before
-    public void intial() {
-        policy = new FrameFreeSlotBiggestFirst(size);
-    }
-
-    @Test
-    public void testAll() {
-
-        for (int i = 0; i < size; i++) {
-            policy.pushNewFrame(i, i);
-            assertEquals(i, policy.popBestFit(i));
-        }
-        assertEquals(-1, policy.popBestFit(0));
-
-        for (int i = 0; i < size; i++) {
-            policy.pushNewFrame(i, i);
-        }
-        for (int i = 0; i < size; i++) {
-            assertEquals(size - i - 1, policy.popBestFit(0));
-        }
-
-        for (int i = 0; i < size; i++) {
-            policy.pushNewFrame(i, i);
-        }
-        for (int i = 0; i < size / 2; i++) {
-            assertEquals(size - i - 1, policy.popBestFit(size / 2));
-        }
-        assertEquals(-1, policy.popBestFit(size / 2));
-        for (int i = 0; i < size / 2; i++) {
-            assertEquals(size / 2 - i - 1, policy.popBestFit(0));
-        }
-
-    }
-
-    @Test
-    public void testReset() {
-        testAll();
-        policy.reset();
-        testAll();
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/6abc63e2/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/FrameFreeSlotLastFitTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/FrameFreeSlotLastFitTest.java b/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/FrameFreeSlotLastFitTest.java
deleted file mode 100644
index 16338fd..0000000
--- a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/FrameFreeSlotLastFitTest.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.dataflow.std.sort.buffermanager;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import org.junit.Before;
-import org.junit.Test;
-
-public class FrameFreeSlotLastFitTest {
-
-    FrameFreeSlotLastFit zeroPolicy;
-    FrameFreeSlotLastFit unifiedPolicy;
-    FrameFreeSlotLastFit ascPolicy;
-    FrameFreeSlotLastFit dscPolicy;
-
-    static final int size = 10;
-    static final int medium = 5;
-
-    @Before
-    public void setUp() throws Exception {
-        zeroPolicy = new FrameFreeSlotLastFit(0);
-        unifiedPolicy = new FrameFreeSlotLastFit(size);
-        ascPolicy = new FrameFreeSlotLastFit(size);
-        dscPolicy = new FrameFreeSlotLastFit(size);
-    }
-
-    @Test
-    public void testPushAndPop() throws Exception {
-        for (int i = 0; i < size; i++) {
-            unifiedPolicy.pushNewFrame(i, medium);
-        }
-        for (int i = 0; i < size; i++) {
-            assertTrue(unifiedPolicy.popBestFit(medium) == size - i - 1);
-        }
-        assertTrue(unifiedPolicy.popBestFit(0) == -1);
-
-        for (int i = 0; i < size / 2; i++) {
-            ascPolicy.pushNewFrame(i, i);
-            assertEquals(ascPolicy.popBestFit(medium), -1);
-            dscPolicy.pushNewFrame(i, size - i - 1);
-            assertEquals(dscPolicy.popBestFit(medium), i);
-        }
-
-        for (int i = size / 2; i < size; i++) {
-            ascPolicy.pushNewFrame(i, i);
-            assertEquals(ascPolicy.popBestFit(medium), i);
-            dscPolicy.pushNewFrame(i, size - i - 1);
-            assertEquals(dscPolicy.popBestFit(medium), -1);
-        }
-
-        ascPolicy.reset();
-        for (int i = 0; i < size; i++) {
-            ascPolicy.pushNewFrame(size - i, size - i);
-        }
-
-        for (int i = 0; i < size; i++) {
-            assertEquals(size - i, ascPolicy.popBestFit(size - i));
-        }
-    }
-
-    @Test
-    public void testReset() throws Exception {
-        testPushAndPop();
-
-        zeroPolicy.reset();
-        unifiedPolicy.reset();
-        ascPolicy.reset();
-        dscPolicy.reset();
-        testPushAndPop();
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/6abc63e2/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/VariableFramePoolTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/VariableFramePoolTest.java b/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/VariableFramePoolTest.java
deleted file mode 100644
index e14c685..0000000
--- a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/VariableFramePoolTest.java
+++ /dev/null
@@ -1,220 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.dataflow.std.sort.buffermanager;
-
-import static org.apache.hyracks.dataflow.std.sort.buffermanager.Common.BUDGET;
-import static org.apache.hyracks.dataflow.std.sort.buffermanager.Common.MIN_FRAME_SIZE;
-import static org.apache.hyracks.dataflow.std.sort.buffermanager.Common.NUM_MIN_FRAME;
-import static org.apache.hyracks.dataflow.std.sort.buffermanager.Common.commonFrameManager;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public class VariableFramePoolTest {
-
-    VariableFramePool pool;
-    @Before
-    public void setUp() throws Exception {
-
-        pool = new VariableFramePool(commonFrameManager, BUDGET);
-    }
-
-    @Test
-    public void testGetMinFrameSize() throws Exception {
-        assertEquals(MIN_FRAME_SIZE, commonFrameManager.getInitialFrameSize());
-        assertEquals(MIN_FRAME_SIZE, pool.getMinFrameSize());
-    }
-
-    @Test
-    public void testGetMemoryBudgetBytes() throws Exception {
-        assertEquals(BUDGET, pool.getMemoryBudgetBytes());
-    }
-
-    @Test
-    public void testAllocateUniformFrameShouldSuccess() throws Exception {
-        testAllocateAllSpacesWithMinFrames();
-        testAllocateShouldFailAfterAllSpaceGetUsed();
-        pool.reset();
-        testAllocateAllSpacesWithMinFrames();
-        pool.close();
-    }
-
-    @Test
-    public void testResetShouldReuseExistingFrames() throws HyracksDataException {
-        Set<?> set1 = testAllocateAllSpacesWithMinFrames();
-        pool.reset();
-        Set<?> set2 = testAllocateAllSpacesWithMinFrames();
-        assertEquals(set1, set2);
-        pool.close();
-    }
-
-    @Test
-    public void testCloseShouldNotReuseExistingFrames() throws HyracksDataException {
-        Set<?> set1 = testAllocateAllSpacesWithMinFrames();
-        pool.close();
-        Set<?> set2 = testAllocateAllSpacesWithMinFrames();
-        assertFalse(set1.equals(set2));
-        pool.close();
-    }
-
-    @Test
-    public void testShouldReturnLargerFramesIfFitOneIsUsed() throws HyracksDataException {
-        Set<?> set = testAllocateVariableFrames();
-        pool.reset();
-        testShouldFindTheMatchFrames(set);
-        pool.reset();
-
-        // allocate seq: 1, 1, 2, 3, 4
-        ByteBuffer placeBuffer = pool.allocateFrame(MIN_FRAME_SIZE);
-        assertTrue(set.contains(new ByteBufferPtr(placeBuffer)));
-        for (int i = 1; i <= 4 ; i++) {
-            ByteBuffer buffer = pool.allocateFrame(i * MIN_FRAME_SIZE);
-            assertNotNull(buffer);
-            assertTrue(set.contains(new ByteBufferPtr(buffer)));
-        }
-        assertNull(pool.allocateFrame(MIN_FRAME_SIZE));
-        pool.close();
-    }
-
-    @Test
-    public void testShouldMergeIfNoLargerFrames() throws HyracksDataException {
-        Set<?> set = testAllocateAllSpacesWithMinFrames();
-        pool.reset();
-        int chunks = 5;
-        for (int i = 0; i < NUM_MIN_FRAME; i+= chunks) {
-            ByteBuffer buffer = pool.allocateFrame(chunks * MIN_FRAME_SIZE);
-            assertNotNull(buffer);
-            assertTrue(!set.contains(new ByteBufferPtr(buffer)));
-        }
-    }
-
-    @Test
-    public void testUseMiddleSizeFrameAndNeedToMergeSmallAndBigger() throws HyracksDataException {
-        Set<?> set = testAllocateVariableFrames();
-        pool.reset();
-        // allocate seq: 3, 6, 1;
-        ByteBuffer buffer = pool.allocateFrame(3 * MIN_FRAME_SIZE);
-        assertTrue(set.contains(new ByteBufferPtr(buffer)));
-        buffer = pool.allocateFrame(6 * MIN_FRAME_SIZE);
-        assertFalse(set.contains(new ByteBufferPtr(buffer)));
-        buffer = pool.allocateFrame(1 * MIN_FRAME_SIZE);
-        assertTrue(set.contains(new ByteBufferPtr(buffer)));
-        assertEquals(5 * MIN_FRAME_SIZE, buffer.capacity());
-        pool.reset();
-    }
-
-    private void testAllocateShouldFailAfterAllSpaceGetUsed() throws HyracksDataException {
-        for (int i = 0; i < NUM_MIN_FRAME; i++) {
-            assertNull(pool.allocateFrame(MIN_FRAME_SIZE));
-        }
-    }
-
-    private HashSet<ByteBufferPtr> testAllocateAllSpacesWithMinFrames() throws HyracksDataException {
-        HashSet<ByteBufferPtr> set = new HashSet<>();
-        for (int i = 0; i < NUM_MIN_FRAME; i++) {
-            ByteBuffer buffer = pool.allocateFrame(MIN_FRAME_SIZE);
-            assertNotNull(buffer);
-            assertTrue(!set.contains(new ByteBufferPtr(buffer)));
-            set.add(new ByteBufferPtr(buffer));
-        }
-        return set;
-    }
-
-    /**
-     * Pool will become 1,2,3,4,5
-     *
-     * @throws HyracksDataException
-     */
-    private Set<ByteBufferPtr> testAllocateVariableFrames() throws HyracksDataException {
-        int budget = BUDGET;
-        int allocate = 0;
-        int i = 1;
-        Set<ByteBufferPtr> set = new HashSet<>();
-        while (budget - allocate >= i * MIN_FRAME_SIZE) {
-            ByteBuffer buffer = pool.allocateFrame(i * MIN_FRAME_SIZE);
-            assertNotNull(buffer);
-            set.add(new ByteBufferPtr(buffer));
-            allocate += i++ * MIN_FRAME_SIZE;
-        }
-        return set;
-    }
-
-    private void testShouldFindTheMatchFrames(Set<?> set) throws HyracksDataException {
-        pool.reset();
-        List<Integer> list = Arrays.asList(1, 2, 3, 4, 5);
-
-        for (int i = 0; i < list.size(); i++) {
-            ByteBuffer buffer = pool.allocateFrame(list.get(i) * MIN_FRAME_SIZE);
-            assertNotNull(buffer);
-            assertTrue(set.contains(new ByteBufferPtr(buffer)));
-            assertEquals(list.get(i) * MIN_FRAME_SIZE, buffer.capacity());
-        }
-        pool.reset();
-        for (int i = list.size() - 1; i >= 0; i--) {
-            ByteBuffer buffer = pool.allocateFrame(list.get(i) * MIN_FRAME_SIZE);
-            assertNotNull(buffer);
-            assertTrue(set.contains(new ByteBufferPtr(buffer)));
-            assertEquals(list.get(i) * MIN_FRAME_SIZE, buffer.capacity());
-        }
-
-        Collections.shuffle(list);
-        pool.reset();
-        for (int i = 0; i < list.size(); i++) {
-            ByteBuffer buffer = pool.allocateFrame(list.get(i) * MIN_FRAME_SIZE);
-            assertNotNull(buffer);
-            assertTrue(set.contains(new ByteBufferPtr(buffer)));
-            assertEquals(list.get(i) * MIN_FRAME_SIZE, buffer.capacity());
-        }
-
-    }
-
-    public static class ByteBufferPtr {
-        ByteBuffer bytebuffer;
-
-        public ByteBufferPtr(ByteBuffer buffer) {
-            bytebuffer = buffer;
-        }
-
-        @Override
-        public int hashCode() {
-            return bytebuffer.hashCode();
-        }
-
-        @Override
-        public boolean equals(Object obj) {
-            return this.bytebuffer == ((ByteBufferPtr) obj).bytebuffer;
-        }
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/6abc63e2/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/VariableFramesMemoryManagerTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/VariableFramesMemoryManagerTest.java b/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/VariableFramesMemoryManagerTest.java
deleted file mode 100644
index 18f4f9b..0000000
--- a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/VariableFramesMemoryManagerTest.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.dataflow.std.sort.buffermanager;
-
-import static org.apache.hyracks.dataflow.std.sort.buffermanager.Common.BUDGET;
-import static org.apache.hyracks.dataflow.std.sort.buffermanager.Common.MIN_FRAME_SIZE;
-import static org.apache.hyracks.dataflow.std.sort.buffermanager.Common.NUM_MIN_FRAME;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.hyracks.api.comm.FixedSizeFrame;
-import org.apache.hyracks.api.comm.FrameHelper;
-import org.apache.hyracks.api.comm.IFrame;
-import org.apache.hyracks.api.comm.IFrameTupleAppender;
-import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-
-public class VariableFramesMemoryManagerTest {
-    VariableFrameMemoryManager framesMemoryManager;
-    FrameTupleAccessor fta;
-    Random random;
-    List<IFrame> frameList;
-
-    @Before
-    public void setUp() throws Exception {
-        VariableFramePool framePool = new VariableFramePool(Common.commonFrameManager, BUDGET);
-        FrameFreeSlotLastFit policy = new FrameFreeSlotLastFit(NUM_MIN_FRAME);
-        framesMemoryManager = new VariableFrameMemoryManager(framePool, policy);
-        RecordDescriptor recordDescriptor = new RecordDescriptor(new ISerializerDeserializer[] { null });
-        fta = new FrameTupleAccessor(recordDescriptor);
-        random = new Random(System.currentTimeMillis());
-        frameList = new ArrayList<>();
-    }
-
-    @Test
-    public void testNormalIncomingFrames() throws HyracksDataException {
-        HashMap<Integer, Integer> tupleSet = prepareTuples();
-        for (IFrame frame : frameList) {
-            assertTrue(framesMemoryManager.insertFrame(frame.getBuffer()) >=0);
-        }
-        assertEquals(NUM_MIN_FRAME, framesMemoryManager.getNumFrames());
-        assertEveryTupleInFTAIsInFrameMemoryManager(tupleSet, framesMemoryManager);
-    }
-
-    @Test
-    public void testRandomTuplesAreAllStoredInBuffer() throws HyracksDataException {
-        Map<Integer, Integer> tupleSet = prepareRandomTuples();
-        for (IFrame frame : frameList) {
-            if (framesMemoryManager.insertFrame(frame.getBuffer()) < 0) {
-                fta.reset(frame.getBuffer());
-                for (int i = 0; i < fta.getTupleCount(); ++i) {
-                    int id = parseTuple(fta.getBuffer(),
-                            fta.getTupleStartOffset(i) + fta.getFieldStartOffset(i, 0) + fta.getFieldSlotsLength());
-                    tupleSet.remove(id);
-                    //                    System.out.println(
-                    //                            "can't appended id:" + id + ",frameSize:" + frame.getInitialFrameSize());
-                }
-            }
-        }
-        assertEveryTupleInFTAIsInFrameMemoryManager(tupleSet, framesMemoryManager);
-        framesMemoryManager.reset();
-    }
-
-    @Test
-    public void testResetShouldWork() throws HyracksDataException {
-        testNormalIncomingFrames();
-        framesMemoryManager.reset();
-        testRandomTuplesAreAllStoredInBuffer();
-        framesMemoryManager.reset();
-        testRandomTuplesAreAllStoredInBuffer();
-    }
-
-    @Test
-    public void testCloseShouldAlsoWork() throws HyracksDataException {
-        testRandomTuplesAreAllStoredInBuffer();
-        framesMemoryManager.close();
-        testRandomTuplesAreAllStoredInBuffer();
-        framesMemoryManager.close();
-        testRandomTuplesAreAllStoredInBuffer();
-    }
-
-    private HashMap<Integer, Integer> prepareRandomTuples() throws HyracksDataException {
-        frameList.clear();
-        HashMap<Integer, Integer> set = new HashMap<>(NUM_MIN_FRAME);
-        int[] fieldSlot = { 0 };
-        int id = 0;
-        int size = 0;
-        while (size < BUDGET) {
-            int tupleLength = random.nextInt(BUDGET / 3) + 4;
-            IFrame frame = new FixedSizeFrame(Common.commonFrameManager
-                    .allocateFrame(FrameHelper.calcAlignedFrameSizeToStore(1, tupleLength, MIN_FRAME_SIZE)));
-            IFrameTupleAppender appender = new FrameTupleAppender();
-            appender.reset(frame, true);
-            //            System.out.println("id:" + id + ",frameSize:" + frame.getInitialFrameSize() / MIN_FRAME_SIZE);
-            ByteBuffer buffer = ByteBuffer.allocate(tupleLength);
-            buffer.putInt(0, id);
-            assertTrue(appender.append(fieldSlot, buffer.array(), 0, buffer.capacity()));
-            set.put(id++, tupleLength);
-            size += frame.getFrameSize();
-            frameList.add(frame);
-        }
-        return set;
-    }
-
-    private HashMap<Integer, Integer> prepareTuples() throws HyracksDataException {
-        frameList.clear();
-        HashMap<Integer, Integer> set = new HashMap<>(NUM_MIN_FRAME);
-        for (int i = 0; i < NUM_MIN_FRAME; ++i) {
-            IFrame frame = new FixedSizeFrame(Common.commonFrameManager.allocateFrame(MIN_FRAME_SIZE));
-            IFrameTupleAppender appender = new FrameTupleAppender();
-            appender.reset(frame, true);
-
-            int[] fieldSlot = { 0 };
-            ByteBuffer buffer = ByteBuffer.allocate(MIN_FRAME_SIZE / 2);
-            buffer.putInt(0, i);
-            appender.append(fieldSlot, buffer.array(), 0, buffer.capacity());
-            set.put(i, buffer.capacity());
-            frameList.add(frame);
-        }
-        return set;
-    }
-
-    private void assertEveryTupleInFTAIsInFrameMemoryManager(Map<Integer, Integer> tupleSet,
-            VariableFrameMemoryManager framesMemoryManager) {
-        for (int i = 0; i < framesMemoryManager.getNumFrames(); ++i) {
-            fta.reset(framesMemoryManager.getFrame(i), framesMemoryManager.getFrameStartOffset(i),
-                    framesMemoryManager.getFrameSize(i));
-            for (int t = 0; t < fta.getTupleCount(); t++) {
-                int id = parseTuple(fta.getBuffer(), fta.getTupleStartOffset(t) + fta.getFieldSlotsLength() + fta
-                        .getFieldStartOffset(t, 0));
-                //                System.out.println("frameid:" + i + ",tuple:" + t + ",has id:" + id + ",length:" +
-                //                        (fta.getTupleEndOffset(t) - fta.getTupleStartOffset(t) - fta.getFieldSlotsLength()));
-                assertTrue(tupleSet.remove(id) == fta.getTupleEndOffset(t) - fta.getTupleStartOffset(t) - fta
-                        .getFieldSlotsLength());
-            }
-        }
-        assertTrue(tupleSet.isEmpty());
-    }
-
-    private int parseTuple(ByteBuffer buffer, int fieldStartOffset) {
-        return buffer.getInt(fieldStartOffset);
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/6abc63e2/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/VariableTupleMemoryManagerTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/VariableTupleMemoryManagerTest.java b/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/VariableTupleMemoryManagerTest.java
deleted file mode 100644
index d14e15b..0000000
--- a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/sort/buffermanager/VariableTupleMemoryManagerTest.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.dataflow.std.sort.buffermanager;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Random;
-
-import org.junit.Before;
-import org.junit.Test;
-
-import org.apache.hyracks.api.comm.FixedSizeFrame;
-import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import org.apache.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
-import org.apache.hyracks.dataflow.common.util.IntSerDeUtils;
-import org.apache.hyracks.dataflow.std.sort.Utility;
-import org.apache.hyracks.dataflow.std.structures.TuplePointer;
-
-public class VariableTupleMemoryManagerTest {
-    ISerializerDeserializer[] fieldsSerDer = new ISerializerDeserializer[] {
-            IntegerSerializerDeserializer.INSTANCE, new UTF8StringSerializerDeserializer() };
-    RecordDescriptor recordDescriptor = new RecordDescriptor(fieldsSerDer);
-    ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(recordDescriptor.getFieldCount());
-    VariableTupleMemoryManager tupleMemoryManager;
-    FrameTupleAccessor inFTA = new FrameTupleAccessor(recordDescriptor);
-    Random random = new Random(System.currentTimeMillis());
-
-    @Before
-    public void setup() {
-        VariableFramePool framePool = new VariableFramePool(Common.commonFrameManager, Common.BUDGET);
-        tupleMemoryManager = new VariableTupleMemoryManager(framePool, recordDescriptor);
-    }
-
-    @Test
-    public void testInsertTupleToMemoryManager() throws HyracksDataException {
-        int iTuplePerFrame = 3;
-        Map<Integer, Integer> mapPrepare = prepareFixedSizeTuples(iTuplePerFrame);
-        Map<TuplePointer, Integer> mapInserted = insertInFTAToBufferShouldAllSuccess();
-        assertEachTupleInFTAIsInBuffer(mapPrepare, mapInserted);
-    }
-
-    @Test
-    public void testReset() throws HyracksDataException {
-        testInsertVariableSizeTupleToMemoryManager();
-        tupleMemoryManager.reset();
-        testInsertTupleToMemoryManager();
-        tupleMemoryManager.reset();
-        testInsertVariableSizeTupleToMemoryManager();
-    }
-
-    @Test
-    public void testDeleteTupleInMemoryManager() throws HyracksDataException {
-        int iTuplePerFrame = 3;
-        Map<Integer, Integer> map = prepareFixedSizeTuples(iTuplePerFrame);
-        Map<TuplePointer, Integer> mapInserted = insertInFTAToBufferShouldAllSuccess();
-        deleteRandomSelectedTuples(map, mapInserted, 1);
-        assertEachTupleInFTAIsInBuffer(map, mapInserted);
-    }
-
-    @Test
-    public void testReOrganizeSpace() throws HyracksDataException {
-        int iTuplePerFrame = 3;
-        Map<Integer, Integer> map = prepareFixedSizeTuples(iTuplePerFrame);
-        Map<Integer, Integer> copyMap = new HashMap<>(map);
-        Map<TuplePointer, Integer> mapInserted = insertInFTAToBufferShouldAllSuccess();
-        ByteBuffer buffer = deleteRandomSelectedTuples(map, mapInserted, map.size() / 2);
-        inFTA.reset(buffer);
-        Map<TuplePointer, Integer> mapInserted2 = insertInFTAToBufferShouldAllSuccess();
-        Map<TuplePointer, Integer> mergedMap = new HashMap<>(mapInserted);
-        mergedMap.putAll(mapInserted2);
-        assertEachTupleInFTAIsInBuffer(copyMap, mergedMap);
-    }
-
-    @Test
-    public void testReOrganizeVariableSizeTuple() throws HyracksDataException {
-        Map<Integer, Integer> map = prepareVariableSizeTuples();
-        Map<TuplePointer, Integer> mapInserted = insertInFTAToBufferCouldFailForLargerTuples(map);
-        Map<Integer, Integer> copyMap = new HashMap<>(map);
-
-        ByteBuffer buffer = deleteRandomSelectedTuples(map, mapInserted, map.size() / 2);
-        inFTA.reset(buffer);
-
-        Map<TuplePointer, Integer> mapInserted2 = insertInFTAToBufferCouldFailForLargerTuples(copyMap);
-        Map<TuplePointer, Integer> mergedMap = new HashMap<>(mapInserted);
-        mergedMap.putAll(mapInserted2);
-
-        assertEachTupleInFTAIsInBuffer(copyMap, mergedMap);
-    }
-
-    @Test
-    public void testInsertVariableSizeTupleToMemoryManager() throws HyracksDataException {
-        Map<Integer, Integer> map = prepareVariableSizeTuples();
-        Map<TuplePointer, Integer> mapInserted = insertInFTAToBufferCouldFailForLargerTuples(map);
-        assertEachTupleInFTAIsInBuffer(map, mapInserted);
-    }
-
-    private void assertEachTupleInFTAIsInBuffer(Map<Integer, Integer> map, Map<TuplePointer, Integer> mapInserted) {
-        ITupleBufferAccessor accessor = tupleMemoryManager.getTupleAccessor();
-        for (Map.Entry<TuplePointer, Integer> entry : mapInserted.entrySet()) {
-            accessor.reset(entry.getKey());
-            int dataLength = map.get(entry.getValue());
-            assertEquals((int) entry.getValue(),
-                    IntSerDeUtils.getInt(accessor.getTupleBuffer().array(), accessor.getAbsFieldStartOffset(0)));
-            assertEquals(dataLength, accessor.getTupleLength());
-        }
-        assertEquals(map.size(), mapInserted.size());
-    }
-
-    private Map<Integer, Integer> prepareFixedSizeTuples(int tuplePerFrame) throws HyracksDataException {
-        Map<Integer, Integer> dataSet = new HashMap<>();
-        ByteBuffer buffer = ByteBuffer.allocate(Common.BUDGET);
-        FixedSizeFrame frame = new FixedSizeFrame(buffer);
-        FrameTupleAppender appender = new FrameTupleAppender();
-        appender.reset(frame, true);
-
-        int sizePerTuple = (Common.MIN_FRAME_SIZE - 1 - 4 - tuplePerFrame * 4 - 4) / tuplePerFrame;
-        int sizeChar = sizePerTuple - fieldsSerDer.length * 4 - 4 - 4;
-        assert (sizeChar > 0);
-        for (int i = 0; i < Common.NUM_MIN_FRAME * tuplePerFrame; i++) {
-            tupleBuilder.reset();
-            tupleBuilder.addField(fieldsSerDer[0], i);
-            tupleBuilder.addField(fieldsSerDer[1], Utility.repeatString('a', sizeChar));
-            assertTrue(appender.append(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray(), 0,
-                    tupleBuilder.getSize()));
-            dataSet.put(i, tupleBuilder.getSize() + tupleBuilder.getFieldEndOffsets().length * 4);
-        }
-        inFTA.reset(buffer);
-        return dataSet;
-    }
-
-    private Map<Integer, Integer> prepareVariableSizeTuples() throws HyracksDataException {
-        Map<Integer, Integer> dataSet = new HashMap<>();
-        ByteBuffer buffer = ByteBuffer.allocate(Common.BUDGET);
-        FixedSizeFrame frame = new FixedSizeFrame(buffer);
-        FrameTupleAppender appender = new FrameTupleAppender();
-        appender.reset(frame, true);
-
-        for (int i = 0; true; i++) {
-            tupleBuilder.reset();
-            tupleBuilder.addField(fieldsSerDer[0], i);
-            tupleBuilder.addField(fieldsSerDer[1], Utility.repeatString('a', i));
-            if (!appender.append(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray(), 0,
-                    tupleBuilder.getSize())) {
-                break;
-            }
-            dataSet.put(i, tupleBuilder.getSize() + tupleBuilder.getFieldEndOffsets().length * 4);
-        }
-        inFTA.reset(buffer);
-        return dataSet;
-    }
-
-    private Map<TuplePointer, Integer> insertInFTAToBufferShouldAllSuccess() throws HyracksDataException {
-        Map<TuplePointer, Integer> tuplePointerIntegerMap = new HashMap<>();
-        for (int i = 0; i < inFTA.getTupleCount(); i++) {
-            TuplePointer tuplePointer = new TuplePointer();
-            assertTrue(tupleMemoryManager.insertTuple(inFTA, i, tuplePointer));
-            tuplePointerIntegerMap.put(tuplePointer,
-                    IntSerDeUtils.getInt(inFTA.getBuffer().array(), inFTA.getAbsoluteFieldStartOffset(i, 0)));
-        }
-        return tuplePointerIntegerMap;
-    }
-
-    private Map<TuplePointer, Integer> insertInFTAToBufferCouldFailForLargerTuples(Map<Integer, Integer> map)
-            throws HyracksDataException {
-        Map<TuplePointer, Integer> tuplePointerIdMap = new HashMap<>();
-        int i = 0;
-        for (; i < inFTA.getTupleCount(); i++) {
-            TuplePointer tuplePointer = new TuplePointer();
-            if (!tupleMemoryManager.insertTuple(inFTA, i, tuplePointer)) {
-                break;
-            }
-            tuplePointerIdMap.put(tuplePointer,
-                    IntSerDeUtils.getInt(inFTA.getBuffer().array(), inFTA.getAbsoluteFieldStartOffset(i, 0)));
-        }
-        for (; i < inFTA.getTupleCount(); i++) {
-            map.remove(IntSerDeUtils.getInt(inFTA.getBuffer().array(), inFTA.getAbsoluteFieldStartOffset(i, 0)));
-        }
-        return tuplePointerIdMap;
-    }
-
-    private ByteBuffer deleteRandomSelectedTuples(Map<Integer, Integer> map, Map<TuplePointer, Integer> mapInserted,
-            int minNumOfRecordTobeDeleted)
-            throws HyracksDataException {
-        ByteBuffer buffer = ByteBuffer.allocate(Common.BUDGET);
-        FixedSizeFrame frame = new FixedSizeFrame(buffer);
-        FrameTupleAppender appender = new FrameTupleAppender();
-        appender.reset(frame, true);
-
-        assert (minNumOfRecordTobeDeleted < mapInserted.size());
-        int countDeleted = minNumOfRecordTobeDeleted + random.nextInt(mapInserted.size() - minNumOfRecordTobeDeleted);
-
-        ITupleBufferAccessor accessor = tupleMemoryManager.getTupleAccessor();
-        for (int i = 0; i < countDeleted; i++) {
-            Iterator<Map.Entry<TuplePointer, Integer>> iter = mapInserted.entrySet().iterator();
-            assert (iter.hasNext());
-            Map.Entry<TuplePointer, Integer> pair = iter.next();
-            accessor.reset(pair.getKey());
-            appender.append(accessor.getTupleBuffer().array(), accessor.getTupleStartOffset(),
-                    accessor.getTupleLength());
-            map.remove(pair.getValue());
-            tupleMemoryManager.deleteTuple(pair.getKey());
-            iter.remove();
-        }
-        return buffer;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb-hyracks/blob/6abc63e2/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/structures/SerializableHashTableTest.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/structures/SerializableHashTableTest.java b/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/structures/SerializableHashTableTest.java
new file mode 100644
index 0000000..5893f35
--- /dev/null
+++ b/hyracks/hyracks-dataflow-std/src/test/java/org/apache/hyracks/dataflow/std/structures/SerializableHashTableTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.dataflow.std.structures;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.control.nc.resources.memory.FrameManager;
+import org.junit.Before;
+import org.junit.Test;
+
+public class SerializableHashTableTest {
+
+    SerializableHashTable nsTable;
+    final int NUM_PART = 101;
+    TuplePointer pointer = new TuplePointer(0, 0);
+    final int num = 1000;
+
+    @Before
+    public void setup() throws HyracksDataException {
+        nsTable = new SerializableHashTable(NUM_PART, new FrameManager(256));
+    }
+
+    @Test
+    public void testBatchDeletePartition() throws Exception {
+        testInsert();
+        for (int i = 0; i < NUM_PART; i++) {
+            nsTable.delete(i);
+            assertFalse(nsTable.getTuplePointer(i, 0, pointer));
+            assertEquals(0, nsTable.getTupleCount(i));
+
+            for (int j = i; j < num; j += NUM_PART) {
+                pointer.reset(j, j);
+                nsTable.insert(i, pointer);
+            }
+
+            assertGetValue();
+        }
+    }
+
+    @Test
+    public void testInsert() throws Exception {
+        for (int i = 0; i < num; i++) {
+            pointer.reset(i, i);
+            nsTable.insert(i % NUM_PART, pointer);
+        }
+        assertGetValue();
+    }
+
+    private void assertGetValue() {
+        int loop = 0;
+        for (int i = 0; i < num; i++) {
+            assertTrue(nsTable.getTuplePointer(i % NUM_PART, loop, pointer));
+            assertTrue(pointer.frameIndex == i);
+            if (i % NUM_PART == NUM_PART - 1) {
+                loop++;
+            }
+        }
+        for (int i = 0; i < NUM_PART; i++) {
+            assertTrue(nsTable.getTupleCount(i) == 10 || nsTable.getTupleCount(i) == 9);
+        }
+
+    }
+
+    @Test
+    public void testGetCount() throws Exception {
+        assertAllPartitionsCountIsZero();
+    }
+
+    private void assertAllPartitionsCountIsZero() {
+        for (int i = 0; i < NUM_PART; i++) {
+            assertEquals(0, nsTable.getTupleCount(i));
+        }
+    }
+}
\ No newline at end of file


Mime
View raw message