asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amo...@apache.org
Subject [1/6] asterixdb git commit: Fix upsert deadlock and upsert with filtered primary only
Date Thu, 25 May 2017 06:04:08 GMT
Repository: asterixdb
Updated Branches:
  refs/heads/master d9b36ff7c -> 639fe8cb3


http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/DummyLSMIndexFileManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/DummyLSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/DummyLSMIndexFileManager.java
deleted file mode 100644
index e03f765..0000000
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/DummyLSMIndexFileManager.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.lsm.common;
-
-import java.io.File;
-import java.io.FilenameFilter;
-import java.util.ArrayList;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.api.io.IIOManager;
-import org.apache.hyracks.storage.am.common.api.ITreeIndex;
-import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
-import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
-import org.apache.hyracks.storage.common.file.IFileMapProvider;
-
-public class DummyLSMIndexFileManager extends AbstractLSMIndexFileManager {
-
-    public DummyLSMIndexFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file,
-            TreeIndexFactory<? extends ITreeIndex> treeFactory) {
-        super(ioManager, fileMapProvider, file, treeFactory);
-    }
-
-    @Override
-    protected void cleanupAndGetValidFilesInternal(FilenameFilter filter,
-            TreeIndexFactory<? extends ITreeIndex> treeFactory, ArrayList<ComparableFileName> allFiles)
-            throws HyracksDataException {
-        File dir = new File(baseDir);
-        String[] files = dir.list(filter);
-        for (String fileName : files) {
-            File file = new File(dir.getPath() + File.separator + fileName);
-            FileReference fileRef = ioManager.resolveAbsolutePath(file.getAbsolutePath());
-            allFiles.add(new ComparableFileName(fileRef));
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/DummyTreeFactory.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/DummyTreeFactory.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/DummyTreeFactory.java
deleted file mode 100644
index dafd6b9..0000000
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/DummyTreeFactory.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.lsm.common;
-
-import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.storage.am.common.api.ITreeIndex;
-import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
-
-public class DummyTreeFactory extends TreeIndexFactory<ITreeIndex> {
-
-    public DummyTreeFactory() {
-        super(null, null, null, null, null, null, null, 0);
-    }
-
-    @Override
-    public ITreeIndex createIndexInstance(FileReference file) {
-        return null;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/LSMComponentFilterReferenceTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/LSMComponentFilterReferenceTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/LSMComponentFilterReferenceTest.java
deleted file mode 100644
index 0257bad..0000000
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/LSMComponentFilterReferenceTest.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.storage.am.lsm.common;
-
-import java.nio.ByteBuffer;
-
-import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
-import org.apache.hyracks.api.dataflow.value.ITypeTraits;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.data.std.primitive.IntegerPointable;
-import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import org.apache.hyracks.dataflow.common.utils.TupleUtils;
-import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriter;
-import org.apache.hyracks.storage.am.lsm.common.frames.LSMComponentFilterReference;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class LSMComponentFilterReferenceTest {
-
-    @Test
-    public void test() throws HyracksDataException {
-        LSMComponentFilterReference filter = new LSMComponentFilterReference(
-                new TypeAwareTupleWriter(new ITypeTraits[] { IntegerPointable.TYPE_TRAITS }));
-        Assert.assertEquals(filter.getLength(), 0);
-        Assert.assertFalse(filter.isMaxTupleSet() || filter.isMinTupleSet());
-        filter.writeMaxTuple(TupleUtils.createIntegerTuple(false, Integer.MAX_VALUE));
-        Assert.assertFalse(filter.isMinTupleSet());
-        Assert.assertTrue(filter.isMaxTupleSet());
-        Assert.assertTrue(filter.getLength() == 11);
-        filter.writeMinTuple(TupleUtils.createIntegerTuple(false, Integer.MIN_VALUE));
-        Assert.assertTrue(filter.isMinTupleSet() && filter.isMaxTupleSet());
-        Assert.assertTrue(filter.getLength() == 20);
-        byte[] serFilter = filter.getByteArray();
-        LSMComponentFilterReference deserFilter = new LSMComponentFilterReference(
-                new TypeAwareTupleWriter((new ITypeTraits[] { IntegerPointable.TYPE_TRAITS })));
-        deserFilter.set(serFilter,0,20);
-        Assert.assertTrue(deserFilter.isMaxTupleSet() && deserFilter.isMinTupleSet());
-        Assert.assertEquals(
-                TupleUtils.deserializeTuple(deserFilter.getMinTuple(),
-                        new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE })[0],
-                Integer.MIN_VALUE);
-        Assert.assertEquals(
-                TupleUtils.deserializeTuple(deserFilter.getMaxTuple(),
-                        new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE })[0],
-                Integer.MAX_VALUE);
-    }
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/LSMIndexFileManagerTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/LSMIndexFileManagerTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/LSMIndexFileManagerTest.java
deleted file mode 100644
index 70e3ed6..0000000
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/LSMIndexFileManagerTest.java
+++ /dev/null
@@ -1,268 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.lsm.common;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.Date;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.Executors;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.api.io.IODeviceHandle;
-import org.apache.hyracks.control.nc.io.DefaultDeviceResolver;
-import org.apache.hyracks.control.nc.io.IOManager;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
-import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
-import org.apache.hyracks.storage.common.file.IFileMapProvider;
-import org.apache.hyracks.test.support.TestStorageManagerComponentHolder;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class LSMIndexFileManagerTest {
-    private static final int DEFAULT_PAGE_SIZE = 256;
-    private static final int DEFAULT_NUM_PAGES = 100;
-    private static final int DEFAULT_MAX_OPEN_FILES = 10;
-    private static final int DEFAULT_IO_DEVICE_ID = 0;
-    protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
-    protected final static String sep = System.getProperty("file.separator");
-    protected IOManager ioManager;
-    protected IFileMapProvider fileMapProvider;
-    protected String baseDir;
-    protected FileReference file;
-
-    @Before
-    public void setUp() throws HyracksDataException {
-        TestStorageManagerComponentHolder.init(DEFAULT_PAGE_SIZE, DEFAULT_NUM_PAGES, DEFAULT_MAX_OPEN_FILES);
-        ioManager = TestStorageManagerComponentHolder.getIOManager();
-        fileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider();
-        baseDir = ioManager.getIODevices().get(DEFAULT_IO_DEVICE_ID).getMount() + sep + "lsm_tree"
-                + simpleDateFormat.format(new Date()) + sep;
-        File f = new File(baseDir);
-        f.mkdirs();
-        file = ioManager.resolveAbsolutePath(f.getAbsolutePath());
-    }
-
-    @After
-    public void tearDown() throws HyracksDataException {
-        File f = new File(baseDir);
-        f.deleteOnExit();
-    }
-
-    public void sortOrderTest(boolean testFlushFileName) throws InterruptedException, HyracksDataException {
-        ILSMIndexFileManager fileManager =
-                new DummyLSMIndexFileManager(ioManager, fileMapProvider, file, new DummyTreeFactory());
-        LinkedList<String> fileNames = new LinkedList<>();
-
-        int numFileNames = 100;
-        long sleepTime = 5;
-        for (int i = 0; i < numFileNames; i++) {
-            String flushFileName =
-                    fileManager.getRelFlushFileReference().getInsertIndexFileReference().getFile().getName();
-            if (testFlushFileName) {
-                fileNames.addFirst(flushFileName);
-            }
-            Thread.sleep(sleepTime);
-            if (!testFlushFileName) {
-                String secondFlushFileName =
-                        fileManager.getRelFlushFileReference().getInsertIndexFileReference().getFile().getName();
-                String mergeFileName = getMergeFileName(fileManager, flushFileName, secondFlushFileName);
-                fileNames.addFirst(mergeFileName);
-                Thread.sleep(sleepTime);
-            }
-        }
-
-        List<String> sortedFileNames = new ArrayList<>();
-        sortedFileNames.addAll(fileNames);
-
-        // Make sure the comparator sorts in the correct order (i.e., the
-        // reverse insertion order in this case).
-        Comparator<String> cmp = fileManager.getFileNameComparator();
-        Collections.sort(sortedFileNames, cmp);
-        for (int i = 0; i < numFileNames; i++) {
-            assertEquals(fileNames.get(i), sortedFileNames.get(i));
-        }
-    }
-
-    @Test
-    public void flushAndMergeFilesSortOrderTest() throws InterruptedException, HyracksDataException {
-        sortOrderTest(true);
-        sortOrderTest(false);
-    }
-
-    public void cleanInvalidFilesTest(IOManager ioManager) throws InterruptedException, IOException {
-        String dirPath = ioManager.getIODevices().get(DEFAULT_IO_DEVICE_ID).getMount() + sep + "lsm_tree"
-                + simpleDateFormat.format(new Date()) + sep;
-        File f = new File(dirPath);
-        f.mkdirs();
-        FileReference file = ioManager.resolveAbsolutePath(f.getAbsolutePath());
-        ILSMIndexFileManager fileManager =
-                new DummyLSMIndexFileManager(ioManager, fileMapProvider, file, new DummyTreeFactory());
-        fileManager.createDirs();
-
-        List<FileReference> flushFiles = new ArrayList<>();
-        List<FileReference> allFiles = new ArrayList<>();
-
-        int numFileNames = 100;
-        long sleepTime = 5;
-        // Generate a bunch of flush files.
-        for (int i = 0; i < numFileNames; i++) {
-            LSMComponentFileReferences relFlushFileRefs = fileManager.getRelFlushFileReference();
-            flushFiles.add(relFlushFileRefs.getInsertIndexFileReference());
-            Thread.sleep(sleepTime);
-        }
-        allFiles.addAll(flushFiles);
-
-        // Simulate merging some of the flush files.
-        // Merge range 0 to 4.
-        FileReference mergeFile1 = simulateMerge(fileManager, flushFiles.get(0), flushFiles.get(4));
-        allFiles.add(mergeFile1);
-        // Merge range 5 to 9.
-        FileReference mergeFile2 = simulateMerge(fileManager, flushFiles.get(5), flushFiles.get(9));
-        allFiles.add(mergeFile2);
-        // Merge range 10 to 19.
-        FileReference mergeFile3 = simulateMerge(fileManager, flushFiles.get(10), flushFiles.get(19));
-        allFiles.add(mergeFile3);
-        // Merge range 20 to 29.
-        FileReference mergeFile4 = simulateMerge(fileManager, flushFiles.get(20), flushFiles.get(29));
-        allFiles.add(mergeFile4);
-        // Merge range 50 to 79.
-        FileReference mergeFile5 = simulateMerge(fileManager, flushFiles.get(50), flushFiles.get(79));
-        allFiles.add(mergeFile5);
-
-        // Simulate merging of merge files.
-        FileReference mergeFile6 = simulateMerge(fileManager, mergeFile1, mergeFile2);
-        allFiles.add(mergeFile6);
-        FileReference mergeFile7 = simulateMerge(fileManager, mergeFile3, mergeFile4);
-        allFiles.add(mergeFile7);
-
-        // Create all files and set delete on exit for all files.
-        for (FileReference fileRef : allFiles) {
-            fileRef.getFile().createNewFile();
-            fileRef.getFile().deleteOnExit();
-        }
-
-        // Populate expected valid flush files.
-        List<String> expectedValidFiles = new ArrayList<>();
-        for (int i = 30; i < 50; i++) {
-            expectedValidFiles.add(flushFiles.get(i).getFile().getName());
-        }
-        for (int i = 80; i < 100; i++) {
-            expectedValidFiles.add(flushFiles.get(i).getFile().getName());
-        }
-
-        // Populate expected valid merge files.
-        expectedValidFiles.add(mergeFile5.getFile().getName());
-        expectedValidFiles.add(mergeFile6.getFile().getName());
-        expectedValidFiles.add(mergeFile7.getFile().getName());
-
-        // Sort expected files.
-        Collections.sort(expectedValidFiles, fileManager.getFileNameComparator());
-
-        // Pass null and a dummy component finalizer. We don't test for physical consistency in this test.
-        List<LSMComponentFileReferences> lsmComonentFileReference = fileManager.cleanupAndGetValidFiles();
-
-        // Check actual files against expected files.
-        assertEquals(expectedValidFiles.size(), lsmComonentFileReference.size());
-        for (int i = 0; i < expectedValidFiles.size(); i++) {
-            assertEquals(expectedValidFiles.get(i),
-                    lsmComonentFileReference.get(i).getInsertIndexFileReference().getFile().getName());
-        }
-
-        // Make sure invalid files were removed from the IODevices.
-        ArrayList<String> remainingFiles = new ArrayList<>();
-        File dir = new File(dirPath);
-        FilenameFilter filter = new FilenameFilter() {
-            @Override
-            public boolean accept(File dir, String name) {
-                return !name.startsWith(".");
-            }
-        };
-        String[] files = dir.list(filter);
-        for (String aFilePath : files) {
-            File aFile = new File(aFilePath);
-            remainingFiles.add(aFile.getName());
-        }
-
-        Collections.sort(remainingFiles, fileManager.getFileNameComparator());
-        // Check actual files in directory against expected files.
-        assertEquals(expectedValidFiles.size(), remainingFiles.size());
-        for (int i = 0; i < expectedValidFiles.size(); i++) {
-            assertEquals(expectedValidFiles.get(i), remainingFiles.get(i));
-        }
-    }
-
-    @Test
-    public void singleIODeviceTest() throws InterruptedException, IOException {
-        IOManager singleDeviceIOManager = createIOManager(1);
-        cleanInvalidFilesTest(singleDeviceIOManager);
-        cleanDirs(singleDeviceIOManager);
-    }
-
-    private void cleanDirs(IOManager ioManager) {
-        File dir = new File(baseDir);
-        FilenameFilter filter = new FilenameFilter() {
-            @Override
-            public boolean accept(File dir, String name) {
-                return !name.startsWith(".");
-            }
-        };
-        String[] files = dir.list(filter);
-        for (String file : files) {
-            File f = new File(file);
-            f.delete();
-        }
-
-    }
-
-    private IOManager createIOManager(int numDevices) throws HyracksDataException {
-        List<IODeviceHandle> devices = new ArrayList<>();
-        for (int i = 0; i < numDevices; i++) {
-            String iodevPath = System.getProperty("java.io.tmpdir") + sep + "test_iodev" + i;
-            devices.add(new IODeviceHandle(new File(iodevPath), "wa"));
-        }
-        return new IOManager(devices, Executors.newCachedThreadPool(), new DefaultDeviceResolver());
-    }
-
-    private FileReference simulateMerge(ILSMIndexFileManager fileManager, FileReference a, FileReference b)
-            throws HyracksDataException {
-        LSMComponentFileReferences relMergeFileRefs =
-                fileManager.getRelMergeFileReference(a.getFile().getName(), b.getFile().getName());
-        return relMergeFileRefs.getInsertIndexFileReference();
-    }
-
-    private String getMergeFileName(ILSMIndexFileManager fileNameManager, String firstFile, String lastFile)
-            throws HyracksDataException {
-        File f1 = new File(firstFile);
-        File f2 = new File(lastFile);
-        return fileNameManager.getRelMergeFileReference(f1.getName(), f2.getName()).getInsertIndexFileReference()
-                .getFile().getName();
-    }
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/VirtualBufferCacheTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/VirtualBufferCacheTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/VirtualBufferCacheTest.java
deleted file mode 100644
index b0d0ecb..0000000
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/VirtualBufferCacheTest.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.hyracks.storage.am.lsm.common;
-
-import static org.junit.Assert.assertTrue;
-
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Random;
-import java.util.Set;
-
-import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.control.nc.io.IOManager;
-import org.apache.hyracks.storage.am.lsm.common.impls.VirtualBufferCache;
-import org.apache.hyracks.storage.common.buffercache.HeapBufferAllocator;
-import org.apache.hyracks.storage.common.buffercache.ICacheMemoryAllocator;
-import org.apache.hyracks.storage.common.buffercache.ICachedPage;
-import org.apache.hyracks.storage.common.file.BufferedFileHandle;
-import org.apache.hyracks.test.support.TestStorageManagerComponentHolder;
-import org.junit.Test;
-
-public class VirtualBufferCacheTest {
-    private static final long SEED = 123456789L;
-    private static final int NUM_OVERPIN = 128;
-    private static final int PAGE_SIZE = 256;
-    private static final int NUM_FILES = 10;
-    private static final int NUM_PAGES = 1000;
-
-    private final Random random;
-    private final FileState[] fileStates;
-
-    private VirtualBufferCache vbc;
-    private IOManager ioManager;
-
-    public VirtualBufferCacheTest() {
-        fileStates = new FileState[NUM_FILES];
-        for (int i = 0; i < NUM_FILES; i++) {
-            fileStates[i] = new FileState();
-        }
-        random = new Random(SEED);
-        vbc = null;
-    }
-
-    private static class FileState {
-        private int fileId;
-        private FileReference fileRef;
-        private int pinCount;
-        private Set<ICachedPage> pinnedPages;
-
-        public FileState() {
-            fileId = -1;
-            fileRef = null;
-            pinCount = 0;
-            pinnedPages = new HashSet<>();
-        }
-    }
-
-    /**
-     * Pins NUM_PAGES randomly distributed across NUM_FILES and checks that each
-     * set of cached pages pinned on behalf of a file are disjoint from all other sets of
-     * cached pages pinned on behalf of other files.
-     * Additionally, the test perform the same test when pinning over soft cap (NUM_PAGES)
-     * of pages.
-     */
-    @Test
-    public void test01() throws Exception {
-        TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, NUM_FILES);
-        ioManager = TestStorageManagerComponentHolder.getIOManager();
-        ICacheMemoryAllocator allocator = new HeapBufferAllocator();
-        vbc = new VirtualBufferCache(allocator, PAGE_SIZE, NUM_PAGES);
-        vbc.open();
-        createFiles();
-
-        kPins(NUM_PAGES);
-        assertTrue(pagesDisjointed());
-
-        kPins(NUM_OVERPIN);
-        assertTrue(pagesDisjointed());
-
-        deleteFiles();
-        vbc.close();
-    }
-
-    private boolean pagesDisjointed() {
-        boolean disjoint = true;
-        for (int i = 0; i < NUM_FILES; i++) {
-            FileState fi = fileStates[i];
-            for (int j = i + 1; j < NUM_FILES; j++) {
-                FileState fj = fileStates[j];
-                disjoint = disjoint && Collections.disjoint(fi.pinnedPages, fj.pinnedPages);
-            }
-        }
-        return disjoint;
-    }
-
-    private void createFiles() throws Exception {
-        for (int i = 0; i < NUM_FILES; i++) {
-            FileState f = fileStates[i];
-            String fName = String.format("f%d", i);
-            f.fileRef = ioManager.resolve(fName);
-            vbc.createFile(f.fileRef);
-            f.fileId = vbc.getFileMapProvider().lookupFileId(f.fileRef);
-        }
-    }
-
-    private void deleteFiles() throws Exception {
-        for (int i = 0; i < NUM_FILES; i++) {
-            vbc.deleteFile(fileStates[i].fileId, false);
-        }
-    }
-
-    private void kPins(int k) throws Exception {
-        int numPinned = 0;
-        while (numPinned < k) {
-            int fsIdx = random.nextInt(NUM_FILES);
-            FileState f = fileStates[fsIdx];
-            ICachedPage p = vbc.pin(BufferedFileHandle.getDiskPageId(f.fileId, f.pinCount), true);
-            f.pinnedPages.add(p);
-            ++f.pinCount;
-            ++numPinned;
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/VirtualFreePageManagerTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/VirtualFreePageManagerTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/VirtualFreePageManagerTest.java
deleted file mode 100644
index 764a82f..0000000
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/VirtualFreePageManagerTest.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hyracks.storage.am.lsm.common;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.File;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.api.io.IODeviceHandle;
-import org.apache.hyracks.storage.am.lsm.common.freepage.VirtualFreePageManager;
-import org.apache.hyracks.storage.am.lsm.common.impls.VirtualBufferCache;
-import org.apache.hyracks.storage.common.buffercache.HeapBufferAllocator;
-import org.junit.Test;
-
-public class VirtualFreePageManagerTest {
-
-    private final int NUM_PAGES = 100;
-
-    private void testInMemoryFreePageManager(VirtualFreePageManager virtualFreePageManager) throws HyracksDataException {
-        // The first two pages are reserved for the BTree's metadata page and
-        // root page.
-        int capacity = NUM_PAGES - 2;
-        for (int i = 0; i < capacity; i++) {
-            int pageId = virtualFreePageManager.takePage(null);
-            // The free pages start from page 2;
-            assertEquals(i + 2, pageId);
-        }
-        // Start asking for 100 pages above the capacity.
-        // Asking for pages above the capacity should be very rare, but
-        // nevertheless succeed.
-        // We expect isFull() to return true.
-        for (int i = 0; i < 100; i++) {
-            int pageId = virtualFreePageManager.takePage(null);
-            assertEquals(capacity + i + 2, pageId);
-        }
-    }
-
-    @Test
-    public void test01() throws HyracksDataException {
-        VirtualBufferCache bufferCache = new VirtualBufferCache(new HeapBufferAllocator(), 4096, 128);
-        bufferCache.open();
-        FileReference fileRef = new FileReference(new IODeviceHandle(new File("target"), "workspace"), "tempfile.tmp");
-        bufferCache.createFile(fileRef);
-        int fileId = bufferCache.getFileMapProvider().lookupFileId(fileRef);
-        bufferCache.openFile(fileId);
-        VirtualFreePageManager virtualFreePageManager = new VirtualFreePageManager(bufferCache);
-        virtualFreePageManager.open(fileId);
-        virtualFreePageManager.init(null, null);
-        testInMemoryFreePageManager(virtualFreePageManager);
-        // We expect exactly the same behavior after a reset().
-        virtualFreePageManager.init(null, null);
-        testInMemoryFreePageManager(virtualFreePageManager);
-    }
-}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java
new file mode 100644
index 0000000..9d8f208
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.common.component;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.util.ArrayList;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.storage.am.common.api.ITreeIndex;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
+import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
+
+public class TestLsmIndexFileManager extends AbstractLSMIndexFileManager {
+
+    public TestLsmIndexFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file) {
+        super(ioManager, fileMapProvider, file, null);
+    }
+
+    @Override
+    protected void cleanupAndGetValidFilesInternal(FilenameFilter filter,
+            TreeIndexFactory<? extends ITreeIndex> treeFactory, ArrayList<ComparableFileName> allFiles)
+            throws HyracksDataException {
+        File dir = new File(baseDir);
+        String[] files = dir.list(filter);
+        for (String fileName : files) {
+            File file = new File(dir.getPath() + File.separator + fileName);
+            FileReference fileRef = ioManager.resolveAbsolutePath(file.getAbsolutePath());
+            allFiles.add(new ComparableFileName(fileRef));
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/LSMComponentFilterReferenceTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/LSMComponentFilterReferenceTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/LSMComponentFilterReferenceTest.java
new file mode 100644
index 0000000..af37d80
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/LSMComponentFilterReferenceTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.common.test;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.dataflow.value.ITypeTraits;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.primitive.IntegerPointable;
+import org.apache.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import org.apache.hyracks.dataflow.common.utils.TupleUtils;
+import org.apache.hyracks.storage.am.common.tuples.TypeAwareTupleWriter;
+import org.apache.hyracks.storage.am.lsm.common.frames.LSMComponentFilterReference;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class LSMComponentFilterReferenceTest {
+
+    @Test
+    public void test() throws HyracksDataException {
+        LSMComponentFilterReference filter = new LSMComponentFilterReference(
+                new TypeAwareTupleWriter(new ITypeTraits[] { IntegerPointable.TYPE_TRAITS }));
+        Assert.assertEquals(filter.getLength(), 0);
+        Assert.assertFalse(filter.isMaxTupleSet() || filter.isMinTupleSet());
+        filter.writeMaxTuple(TupleUtils.createIntegerTuple(false, Integer.MAX_VALUE));
+        Assert.assertFalse(filter.isMinTupleSet());
+        Assert.assertTrue(filter.isMaxTupleSet());
+        Assert.assertTrue(filter.getLength() == 11);
+        filter.writeMinTuple(TupleUtils.createIntegerTuple(false, Integer.MIN_VALUE));
+        Assert.assertTrue(filter.isMinTupleSet() && filter.isMaxTupleSet());
+        Assert.assertTrue(filter.getLength() == 20);
+        byte[] serFilter = filter.getByteArray();
+        LSMComponentFilterReference deserFilter = new LSMComponentFilterReference(
+                new TypeAwareTupleWriter((new ITypeTraits[] { IntegerPointable.TYPE_TRAITS })));
+        deserFilter.set(serFilter,0,20);
+        Assert.assertTrue(deserFilter.isMaxTupleSet() && deserFilter.isMinTupleSet());
+        Assert.assertEquals(
+                TupleUtils.deserializeTuple(deserFilter.getMinTuple(),
+                        new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE })[0],
+                Integer.MIN_VALUE);
+        Assert.assertEquals(
+                TupleUtils.deserializeTuple(deserFilter.getMaxTuple(),
+                        new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE })[0],
+                Integer.MAX_VALUE);
+    }
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/LSMIndexFileManagerTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/LSMIndexFileManagerTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/LSMIndexFileManagerTest.java
new file mode 100644
index 0000000..67a6f69
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/LSMIndexFileManagerTest.java
@@ -0,0 +1,267 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.common.test;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.Executors;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IODeviceHandle;
+import org.apache.hyracks.control.nc.io.DefaultDeviceResolver;
+import org.apache.hyracks.control.nc.io.IOManager;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
+import org.apache.hyracks.storage.am.lsm.common.component.TestLsmIndexFileManager;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
+import org.apache.hyracks.storage.common.file.IFileMapProvider;
+import org.apache.hyracks.test.support.TestStorageManagerComponentHolder;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class LSMIndexFileManagerTest {
+    private static final int DEFAULT_PAGE_SIZE = 256;
+    private static final int DEFAULT_NUM_PAGES = 100;
+    private static final int DEFAULT_MAX_OPEN_FILES = 10;
+    private static final int DEFAULT_IO_DEVICE_ID = 0;
+    protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
+    protected final static String sep = System.getProperty("file.separator");
+    protected IOManager ioManager;
+    protected IFileMapProvider fileMapProvider;
+    protected String baseDir;
+    protected FileReference file;
+
+    @Before
+    public void setUp() throws HyracksDataException {
+        TestStorageManagerComponentHolder.init(DEFAULT_PAGE_SIZE, DEFAULT_NUM_PAGES, DEFAULT_MAX_OPEN_FILES);
+        ioManager = TestStorageManagerComponentHolder.getIOManager();
+        fileMapProvider = TestStorageManagerComponentHolder.getFileMapProvider();
+        baseDir = ioManager.getIODevices().get(DEFAULT_IO_DEVICE_ID).getMount() + sep + "lsm_tree"
+                + simpleDateFormat.format(new Date()) + sep;
+        File f = new File(baseDir);
+        f.mkdirs();
+        file = ioManager.resolveAbsolutePath(f.getAbsolutePath());
+    }
+
+    @After
+    public void tearDown() throws HyracksDataException {
+        File f = new File(baseDir);
+        f.deleteOnExit();
+    }
+
+    public void sortOrderTest(boolean testFlushFileName) throws InterruptedException, HyracksDataException {
+        ILSMIndexFileManager fileManager = new TestLsmIndexFileManager(ioManager, fileMapProvider, file);
+        LinkedList<String> fileNames = new LinkedList<>();
+
+        int numFileNames = 100;
+        long sleepTime = 5;
+        for (int i = 0; i < numFileNames; i++) {
+            String flushFileName =
+                    fileManager.getRelFlushFileReference().getInsertIndexFileReference().getFile().getName();
+            if (testFlushFileName) {
+                fileNames.addFirst(flushFileName);
+            }
+            Thread.sleep(sleepTime);
+            if (!testFlushFileName) {
+                String secondFlushFileName =
+                        fileManager.getRelFlushFileReference().getInsertIndexFileReference().getFile().getName();
+                String mergeFileName = getMergeFileName(fileManager, flushFileName, secondFlushFileName);
+                fileNames.addFirst(mergeFileName);
+                Thread.sleep(sleepTime);
+            }
+        }
+
+        List<String> sortedFileNames = new ArrayList<>();
+        sortedFileNames.addAll(fileNames);
+
+        // Make sure the comparator sorts in the correct order (i.e., the
+        // reverse insertion order in this case).
+        Comparator<String> cmp = fileManager.getFileNameComparator();
+        Collections.sort(sortedFileNames, cmp);
+        for (int i = 0; i < numFileNames; i++) {
+            assertEquals(fileNames.get(i), sortedFileNames.get(i));
+        }
+    }
+
+    @Test
+    public void flushAndMergeFilesSortOrderTest() throws InterruptedException, HyracksDataException {
+        sortOrderTest(true);
+        sortOrderTest(false);
+    }
+
+    public void cleanInvalidFilesTest(IOManager ioManager) throws InterruptedException, IOException {
+        String dirPath = ioManager.getIODevices().get(DEFAULT_IO_DEVICE_ID).getMount() + sep + "lsm_tree"
+                + simpleDateFormat.format(new Date()) + sep;
+        File f = new File(dirPath);
+        f.mkdirs();
+        FileReference file = ioManager.resolveAbsolutePath(f.getAbsolutePath());
+        ILSMIndexFileManager fileManager = new TestLsmIndexFileManager(ioManager, fileMapProvider, file);
+        fileManager.createDirs();
+
+        List<FileReference> flushFiles = new ArrayList<>();
+        List<FileReference> allFiles = new ArrayList<>();
+
+        int numFileNames = 100;
+        long sleepTime = 5;
+        // Generate a bunch of flush files.
+        for (int i = 0; i < numFileNames; i++) {
+            LSMComponentFileReferences relFlushFileRefs = fileManager.getRelFlushFileReference();
+            flushFiles.add(relFlushFileRefs.getInsertIndexFileReference());
+            Thread.sleep(sleepTime);
+        }
+        allFiles.addAll(flushFiles);
+
+        // Simulate merging some of the flush files.
+        // Merge range 0 to 4.
+        FileReference mergeFile1 = simulateMerge(fileManager, flushFiles.get(0), flushFiles.get(4));
+        allFiles.add(mergeFile1);
+        // Merge range 5 to 9.
+        FileReference mergeFile2 = simulateMerge(fileManager, flushFiles.get(5), flushFiles.get(9));
+        allFiles.add(mergeFile2);
+        // Merge range 10 to 19.
+        FileReference mergeFile3 = simulateMerge(fileManager, flushFiles.get(10), flushFiles.get(19));
+        allFiles.add(mergeFile3);
+        // Merge range 20 to 29.
+        FileReference mergeFile4 = simulateMerge(fileManager, flushFiles.get(20), flushFiles.get(29));
+        allFiles.add(mergeFile4);
+        // Merge range 50 to 79.
+        FileReference mergeFile5 = simulateMerge(fileManager, flushFiles.get(50), flushFiles.get(79));
+        allFiles.add(mergeFile5);
+
+        // Simulate merging of merge files.
+        FileReference mergeFile6 = simulateMerge(fileManager, mergeFile1, mergeFile2);
+        allFiles.add(mergeFile6);
+        FileReference mergeFile7 = simulateMerge(fileManager, mergeFile3, mergeFile4);
+        allFiles.add(mergeFile7);
+
+        // Create all files and set delete on exit for all files.
+        for (FileReference fileRef : allFiles) {
+            fileRef.getFile().createNewFile();
+            fileRef.getFile().deleteOnExit();
+        }
+
+        // Populate expected valid flush files.
+        List<String> expectedValidFiles = new ArrayList<>();
+        for (int i = 30; i < 50; i++) {
+            expectedValidFiles.add(flushFiles.get(i).getFile().getName());
+        }
+        for (int i = 80; i < 100; i++) {
+            expectedValidFiles.add(flushFiles.get(i).getFile().getName());
+        }
+
+        // Populate expected valid merge files.
+        expectedValidFiles.add(mergeFile5.getFile().getName());
+        expectedValidFiles.add(mergeFile6.getFile().getName());
+        expectedValidFiles.add(mergeFile7.getFile().getName());
+
+        // Sort expected files.
+        Collections.sort(expectedValidFiles, fileManager.getFileNameComparator());
+
+        // Pass null and a dummy component finalizer. We don't test for physical consistency in this test.
+        List<LSMComponentFileReferences> lsmComonentFileReference = fileManager.cleanupAndGetValidFiles();
+
+        // Check actual files against expected files.
+        assertEquals(expectedValidFiles.size(), lsmComonentFileReference.size());
+        for (int i = 0; i < expectedValidFiles.size(); i++) {
+            assertEquals(expectedValidFiles.get(i),
+                    lsmComonentFileReference.get(i).getInsertIndexFileReference().getFile().getName());
+        }
+
+        // Make sure invalid files were removed from the IODevices.
+        ArrayList<String> remainingFiles = new ArrayList<>();
+        File dir = new File(dirPath);
+        FilenameFilter filter = new FilenameFilter() {
+            @Override
+            public boolean accept(File dir, String name) {
+                return !name.startsWith(".");
+            }
+        };
+        String[] files = dir.list(filter);
+        for (String aFilePath : files) {
+            File aFile = new File(aFilePath);
+            remainingFiles.add(aFile.getName());
+        }
+
+        Collections.sort(remainingFiles, fileManager.getFileNameComparator());
+        // Check actual files in directory against expected files.
+        assertEquals(expectedValidFiles.size(), remainingFiles.size());
+        for (int i = 0; i < expectedValidFiles.size(); i++) {
+            assertEquals(expectedValidFiles.get(i), remainingFiles.get(i));
+        }
+    }
+
+    @Test
+    public void singleIODeviceTest() throws InterruptedException, IOException {
+        IOManager singleDeviceIOManager = createIOManager(1);
+        cleanInvalidFilesTest(singleDeviceIOManager);
+        cleanDirs(singleDeviceIOManager);
+    }
+
+    private void cleanDirs(IOManager ioManager) {
+        File dir = new File(baseDir);
+        FilenameFilter filter = new FilenameFilter() {
+            @Override
+            public boolean accept(File dir, String name) {
+                return !name.startsWith(".");
+            }
+        };
+        String[] files = dir.list(filter);
+        for (String file : files) {
+            File f = new File(file);
+            f.delete();
+        }
+
+    }
+
+    private IOManager createIOManager(int numDevices) throws HyracksDataException {
+        List<IODeviceHandle> devices = new ArrayList<>();
+        for (int i = 0; i < numDevices; i++) {
+            String iodevPath = System.getProperty("java.io.tmpdir") + sep + "test_iodev" + i;
+            devices.add(new IODeviceHandle(new File(iodevPath), "wa"));
+        }
+        return new IOManager(devices, Executors.newCachedThreadPool(), new DefaultDeviceResolver());
+    }
+
+    private FileReference simulateMerge(ILSMIndexFileManager fileManager, FileReference a, FileReference b)
+            throws HyracksDataException {
+        LSMComponentFileReferences relMergeFileRefs =
+                fileManager.getRelMergeFileReference(a.getFile().getName(), b.getFile().getName());
+        return relMergeFileRefs.getInsertIndexFileReference();
+    }
+
+    private String getMergeFileName(ILSMIndexFileManager fileNameManager, String firstFile, String lastFile)
+            throws HyracksDataException {
+        File f1 = new File(firstFile);
+        File f2 = new File(lastFile);
+        return fileNameManager.getRelMergeFileReference(f1.getName(), f2.getName()).getInsertIndexFileReference()
+                .getFile().getName();
+    }
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/VirtualBufferCacheTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/VirtualBufferCacheTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/VirtualBufferCacheTest.java
new file mode 100644
index 0000000..59afa19
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/VirtualBufferCacheTest.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.common.test;
+
+import static org.junit.Assert.assertTrue;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.control.nc.io.IOManager;
+import org.apache.hyracks.storage.am.lsm.common.impls.VirtualBufferCache;
+import org.apache.hyracks.storage.common.buffercache.HeapBufferAllocator;
+import org.apache.hyracks.storage.common.buffercache.ICacheMemoryAllocator;
+import org.apache.hyracks.storage.common.buffercache.ICachedPage;
+import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.hyracks.test.support.TestStorageManagerComponentHolder;
+import org.junit.Test;
+
+public class VirtualBufferCacheTest {
+    private static final long SEED = 123456789L;
+    private static final int NUM_OVERPIN = 128;
+    private static final int PAGE_SIZE = 256;
+    private static final int NUM_FILES = 10;
+    private static final int NUM_PAGES = 1000;
+
+    private final Random random;
+    private final FileState[] fileStates;
+
+    private VirtualBufferCache vbc;
+    private IOManager ioManager;
+
+    public VirtualBufferCacheTest() {
+        fileStates = new FileState[NUM_FILES];
+        for (int i = 0; i < NUM_FILES; i++) {
+            fileStates[i] = new FileState();
+        }
+        random = new Random(SEED);
+        vbc = null;
+    }
+
+    private static class FileState {
+        private int fileId;
+        private FileReference fileRef;
+        private int pinCount;
+        private Set<ICachedPage> pinnedPages;
+
+        public FileState() {
+            fileId = -1;
+            fileRef = null;
+            pinCount = 0;
+            pinnedPages = new HashSet<>();
+        }
+    }
+
+    /**
+     * Pins NUM_PAGES randomly distributed across NUM_FILES and checks that each
+     * set of cached pages pinned on behalf of a file are disjoint from all other sets of
+     * cached pages pinned on behalf of other files.
+     * Additionally, the test perform the same test when pinning over soft cap (NUM_PAGES)
+     * of pages.
+     */
+    @Test
+    public void test01() throws Exception {
+        TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, NUM_FILES);
+        ioManager = TestStorageManagerComponentHolder.getIOManager();
+        ICacheMemoryAllocator allocator = new HeapBufferAllocator();
+        vbc = new VirtualBufferCache(allocator, PAGE_SIZE, NUM_PAGES);
+        vbc.open();
+        createFiles();
+
+        kPins(NUM_PAGES);
+        assertTrue(pagesDisjointed());
+
+        kPins(NUM_OVERPIN);
+        assertTrue(pagesDisjointed());
+
+        deleteFiles();
+        vbc.close();
+    }
+
+    private boolean pagesDisjointed() {
+        boolean disjoint = true;
+        for (int i = 0; i < NUM_FILES; i++) {
+            FileState fi = fileStates[i];
+            for (int j = i + 1; j < NUM_FILES; j++) {
+                FileState fj = fileStates[j];
+                disjoint = disjoint && Collections.disjoint(fi.pinnedPages, fj.pinnedPages);
+            }
+        }
+        return disjoint;
+    }
+
+    private void createFiles() throws Exception {
+        for (int i = 0; i < NUM_FILES; i++) {
+            FileState f = fileStates[i];
+            String fName = String.format("f%d", i);
+            f.fileRef = ioManager.resolve(fName);
+            vbc.createFile(f.fileRef);
+            f.fileId = vbc.getFileMapProvider().lookupFileId(f.fileRef);
+        }
+    }
+
+    private void deleteFiles() throws Exception {
+        for (int i = 0; i < NUM_FILES; i++) {
+            vbc.deleteFile(fileStates[i].fileId, false);
+        }
+    }
+
+    private void kPins(int k) throws Exception {
+        int numPinned = 0;
+        while (numPinned < k) {
+            int fsIdx = random.nextInt(NUM_FILES);
+            FileState f = fileStates[fsIdx];
+            ICachedPage p = vbc.pin(BufferedFileHandle.getDiskPageId(f.fileId, f.pinCount), true);
+            f.pinnedPages.add(p);
+            ++f.pinCount;
+            ++numPinned;
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/VirtualFreePageManagerTest.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/VirtualFreePageManagerTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/VirtualFreePageManagerTest.java
new file mode 100644
index 0000000..f4ec55d
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/test/VirtualFreePageManagerTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.common.test;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IODeviceHandle;
+import org.apache.hyracks.storage.am.lsm.common.freepage.VirtualFreePageManager;
+import org.apache.hyracks.storage.am.lsm.common.impls.VirtualBufferCache;
+import org.apache.hyracks.storage.common.buffercache.HeapBufferAllocator;
+import org.junit.Test;
+
+public class VirtualFreePageManagerTest {
+
+    private final int NUM_PAGES = 100;
+
+    private void testInMemoryFreePageManager(VirtualFreePageManager virtualFreePageManager) throws HyracksDataException {
+        // The first two pages are reserved for the BTree's metadata page and
+        // root page.
+        int capacity = NUM_PAGES - 2;
+        for (int i = 0; i < capacity; i++) {
+            int pageId = virtualFreePageManager.takePage(null);
+            // The free pages start from page 2;
+            assertEquals(i + 2, pageId);
+        }
+        // Start asking for 100 pages above the capacity.
+        // Asking for pages above the capacity should be very rare, but
+        // nevertheless succeed.
+        // We expect isFull() to return true.
+        for (int i = 0; i < 100; i++) {
+            int pageId = virtualFreePageManager.takePage(null);
+            assertEquals(capacity + i + 2, pageId);
+        }
+    }
+
+    @Test
+    public void test01() throws HyracksDataException {
+        VirtualBufferCache bufferCache = new VirtualBufferCache(new HeapBufferAllocator(), 4096, 128);
+        bufferCache.open();
+        FileReference fileRef = new FileReference(new IODeviceHandle(new File("target"), "workspace"), "tempfile.tmp");
+        bufferCache.createFile(fileRef);
+        int fileId = bufferCache.getFileMapProvider().lookupFileId(fileRef);
+        bufferCache.openFile(fileId);
+        VirtualFreePageManager virtualFreePageManager = new VirtualFreePageManager(bufferCache);
+        virtualFreePageManager.open(fileId);
+        virtualFreePageManager.init(null, null);
+        testInMemoryFreePageManager(virtualFreePageManager);
+        // We expect exactly the same behavior after a reset().
+        virtualFreePageManager.init(null, null);
+        testInMemoryFreePageManager(virtualFreePageManager);
+    }
+}

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeTestWorker.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeTestWorker.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeTestWorker.java
index 22ccf88..95b4f44 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeTestWorker.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeTestWorker.java
@@ -30,7 +30,8 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
 import org.apache.hyracks.storage.am.common.datagen.DataGenThread;
 import org.apache.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallbackFactory;
 import org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTree;
-import org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTree.LSMRTreeAccessor;
+import org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTreeAccessor;
+import org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTreeOpContext;
 import org.apache.hyracks.storage.am.rtree.impls.SearchPredicate;
 import org.apache.hyracks.storage.common.IIndex;
 import org.apache.hyracks.storage.common.MultiComparator;
@@ -54,7 +55,8 @@ public class LSMRTreeTestWorker extends AbstractIndexTestWorker {
     public void performOp(ITupleReference tuple, TestOperation op) throws HyracksDataException {
         LSMRTreeAccessor accessor = (LSMRTreeAccessor) indexAccessor;
         ITreeIndexCursor searchCursor = accessor.createSearchCursor(false);
-        MultiComparator cmp = accessor.getMultiComparator();
+        LSMRTreeOpContext concreteCtx = (LSMRTreeOpContext) accessor.getCtx();
+        MultiComparator cmp = concreteCtx.getCurrentRTreeOpContext().getCmp();
         SearchPredicate rangePred = new SearchPredicate(tuple, cmp);
 
         switch (op) {

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesTestWorker.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesTestWorker.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesTestWorker.java
index 4f043bd..3917203 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesTestWorker.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/multithread/LSMRTreeWithAntiMatterTuplesTestWorker.java
@@ -25,9 +25,10 @@ import org.apache.hyracks.storage.am.common.TestOperationSelector;
 import org.apache.hyracks.storage.am.common.TestOperationSelector.TestOperation;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexCursor;
 import org.apache.hyracks.storage.am.common.datagen.DataGenThread;
+import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
 import org.apache.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallbackFactory;
 import org.apache.hyracks.storage.am.lsm.rtree.impls.AbstractLSMRTree;
-import org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTreeWithAntiMatterTuples.LSMRTreeWithAntiMatterTuplesAccessor;
+import org.apache.hyracks.storage.am.lsm.rtree.impls.LSMRTreeOpContext;
 import org.apache.hyracks.storage.am.rtree.impls.SearchPredicate;
 import org.apache.hyracks.storage.common.IIndex;
 import org.apache.hyracks.storage.common.MultiComparator;
@@ -41,9 +42,10 @@ public class LSMRTreeWithAntiMatterTuplesTestWorker extends AbstractLSMRTreeTest
 
     @Override
     public void performOp(ITupleReference tuple, TestOperation op) throws HyracksDataException {
-        LSMRTreeWithAntiMatterTuplesAccessor accessor = (LSMRTreeWithAntiMatterTuplesAccessor) indexAccessor;
+        LSMTreeIndexAccessor accessor = (LSMTreeIndexAccessor) indexAccessor;
         ITreeIndexCursor searchCursor = accessor.createSearchCursor(false);
-        MultiComparator cmp = accessor.getMultiComparator();
+        LSMRTreeOpContext concreteCtx = (LSMRTreeOpContext) accessor.getCtx();
+        MultiComparator cmp = concreteCtx.getCurrentRTreeOpContext().getCmp();
         SearchPredicate rangePred = new SearchPredicate(tuple, cmp);
 
         switch (op) {


Mime
View raw message