Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id A7847200CB0 for ; Fri, 23 Jun 2017 20:56:36 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id A64F0160BE5; Fri, 23 Jun 2017 18:56:36 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 80756160BE2 for ; Fri, 23 Jun 2017 20:56:34 +0200 (CEST) Received: (qmail 24034 invoked by uid 500); 23 Jun 2017 18:56:33 -0000 Mailing-List: contact commits-help@asterixdb.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@asterixdb.apache.org Delivered-To: mailing list commits@asterixdb.apache.org Received: (qmail 24025 invoked by uid 99); 23 Jun 2017 18:56:33 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 23 Jun 2017 18:56:33 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 1B3E9DFC25; Fri, 23 Jun 2017 18:56:33 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: amoudi@apache.org To: commits@asterixdb.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: asterixdb git commit: [ASTERIXDB-1953][STO] Cleanup LSM File Managers Date: Fri, 23 Jun 2017 18:56:33 +0000 (UTC) archived-at: Fri, 23 Jun 2017 18:56:36 -0000 Repository: asterixdb Updated Branches: refs/heads/master a32a852be -> e0cf97012 [ASTERIXDB-1953][STO] Cleanup LSM File Managers - user model changes: no - storage format changes: no - interface changes: no Details: - use FileReference instead of String absolute path - user error codes Change-Id: I97bab76888790ca282ad9508ce8416f7c7a52fb7 Reviewed-on: https://asterix-gerrit.ics.uci.edu/1849 Sonar-Qube: Jenkins Tested-by: Jenkins Integration-Tests: Jenkins BAD: Jenkins Reviewed-by: Ian Maxon Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/e0cf9701 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/e0cf9701 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/e0cf9701 Branch: refs/heads/master Commit: e0cf970124baed319800d58d107c4bfbe98ae84a Parents: a32a852 Author: Abdullah Alamoudi Authored: Fri Jun 23 10:06:03 2017 -0700 Committer: abdullah alamoudi Committed: Fri Jun 23 11:56:17 2017 -0700 ---------------------------------------------------------------------- .../hyracks/api/exceptions/ErrorCode.java | 4 + .../apache/hyracks/api/io/FileReference.java | 4 + .../src/main/resources/errormsg/en.properties | 4 + .../am/lsm/btree/impls/ExternalBTree.java | 2 +- .../lsm/btree/impls/ExternalBTreeWithBuddy.java | 3 +- .../storage/am/lsm/btree/impls/LSMBTree.java | 4 +- .../am/lsm/btree/impls/LSMBTreeFileManager.java | 84 +++++---------- .../impls/LSMBTreeWithBuddyFileManager.java | 97 ++++++----------- .../am/lsm/common/api/ILSMIndexFileManager.java | 3 +- .../impls/AbstractLSMIndexFileManager.java | 106 ++++++------------- .../invertedindex/impls/LSMInvertedIndex.java | 4 +- .../impls/LSMInvertedIndexFileManager.java | 73 +++++-------- .../am/lsm/rtree/impls/ExternalRTree.java | 2 +- .../storage/am/lsm/rtree/impls/LSMRTree.java | 4 +- .../am/lsm/rtree/impls/LSMRTreeFileManager.java | 93 ++++++---------- .../impls/LSMRTreeWithAntiMatterTuples.java | 4 +- .../component/TestLsmIndexFileManager.java | 7 +- 17 files changed, 185 insertions(+), 313 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/e0cf9701/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java index 2e6c8a3..7dbade2 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java @@ -117,6 +117,10 @@ public class ErrorCode { public static final int FILE_ALREADY_MAPPED = 81; public static final int FILE_ALREADY_EXISTS = 82; public static final int NO_INDEX_FOUND_WITH_RESOURCE_ID = 83; + public static final int FOUND_OVERLAPPING_LSM_FILES = 84; + public static final int FOUND_MULTIPLE_TRANSACTIONS = 85; + public static final int UNRECOGNIZED_INDEX_COMPONENT_FILE = 86; + public static final int UNEQUAL_NUM_FILTERS_TREES = 87; // Compilation error codes. public static final int RULECOLLECTION_NOT_INSTANCE_OF_LIST = 10000; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/e0cf9701/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java index b48ba14..0375e9e 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/FileReference.java @@ -86,4 +86,8 @@ public final class FileReference implements Serializable { public String getAbsolutePath() { return file.getAbsolutePath(); } + + public FileReference getChild(String name) { + return new FileReference(dev, path + File.separator + name); + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/e0cf9701/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties index 6389ffa..cd38917 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties +++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties @@ -100,5 +100,9 @@ 81 = File %1$s is already mapped 82 = Failed to create the file %1$s because it already exists 83 = No index found with resourceID %1$s +84 = Files with overlapping non-contained timestamp intervals were found in %1$s +85 = Found more than one transaction file in %1$s +86 = Found an unrecognized index file %1$s +87 = Unequal number of trees and filters found in %1$s 10000 = The given rule collection %1$s is not an instance of the List class. http://git-wip-us.apache.org/repos/asf/asterixdb/blob/e0cf9701/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java index c06edfd..86d926f 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTree.java @@ -197,7 +197,7 @@ public class ExternalBTree extends LSMBTree implements ITwoPCIndex { ILSMIndexAccessor accessor = new LSMTreeIndexAccessor(getLsmHarness(), opCtx, cursorFactory); ioScheduler.scheduleOperation(new LSMBTreeMergeOperation(accessor, mergingComponents, cursor, relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getBloomFilterFileReference(), - callback, fileManager.getBaseDir())); + callback, fileManager.getBaseDir().getAbsolutePath())); } // This function should only be used when a transaction fail. it doesn't http://git-wip-us.apache.org/repos/asf/asterixdb/blob/e0cf9701/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java index 4245687..7462c7a 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java @@ -366,7 +366,8 @@ public class ExternalBTreeWithBuddy extends AbstractLSMIndex implements ITreeInd ioScheduler.scheduleOperation(new LSMBTreeWithBuddyMergeOperation(accessor, mergingComponents, cursor, relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getDeleteIndexFileReference(), - relMergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir(), keepDeleteTuples)); + relMergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir().getAbsolutePath(), + keepDeleteTuples)); } // This method creates the appropriate opContext for the targeted version http://git-wip-us.apache.org/repos/asf/asterixdb/blob/e0cf9701/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java index b437fda..d241dc6 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java @@ -470,7 +470,7 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex { ILSMIOOperationCallback callback) { ILSMIndexAccessor accessor = createAccessor(opCtx); return new LSMBTreeFlushOperation(accessor, flushingComponent, componentFileRefs.getInsertIndexFileReference(), - componentFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir()); + componentFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir().getAbsolutePath()); } @Override @@ -625,6 +625,6 @@ public class LSMBTree extends AbstractLSMIndex implements ITreeIndex { ITreeIndexCursor cursor = new LSMBTreeRangeSearchCursor(opCtx, returnDeletedTuples); return new LSMBTreeMergeOperation(accessor, mergingComponents, cursor, mergeFileRefs.getInsertIndexFileReference(), mergeFileRefs.getBloomFilterFileReference(), callback, - fileManager.getBaseDir()); + fileManager.getBaseDir().getAbsolutePath()); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/e0cf9701/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java index 432a2b8..54bc1fe 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeFileManager.java @@ -22,17 +22,17 @@ package org.apache.hyracks.storage.am.lsm.btree.impls; import java.io.File; import java.io.FilenameFilter; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; import java.util.Iterator; import java.util.List; +import org.apache.hyracks.api.exceptions.ErrorCode; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.IIOManager; +import org.apache.hyracks.api.util.IoUtil; import org.apache.hyracks.storage.am.common.api.ITreeIndex; import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; @@ -40,6 +40,8 @@ import org.apache.hyracks.storage.am.lsm.common.impls.TreeIndexFactory; public class LSMBTreeFileManager extends AbstractLSMIndexFileManager { + private static final FilenameFilter btreeFilter = + (dir, name) -> !name.startsWith(".") && name.endsWith(BTREE_SUFFIX); private final TreeIndexFactory btreeFactory; private final boolean hasBloomFilter; @@ -53,10 +55,10 @@ public class LSMBTreeFileManager extends AbstractLSMIndexFileManager { @Override public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException { String ts = getCurrentTimestamp(); - String baseName = baseDir + ts + DELIMITER + ts; + String baseName = ts + DELIMITER + ts; // Begin timestamp and end timestamp are identical since it is a flush - return new LSMComponentFileReferences(createFlushFile(baseName + DELIMITER + BTREE_SUFFIX), null, - hasBloomFilter ? createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX) : null); + return new LSMComponentFileReferences(baseDir.getChild(baseName + DELIMITER + BTREE_SUFFIX), null, + hasBloomFilter ? baseDir.getChild(baseName + DELIMITER + BLOOM_FILTER_SUFFIX) : null); } @Override @@ -65,30 +67,19 @@ public class LSMBTreeFileManager extends AbstractLSMIndexFileManager { String[] firstTimestampRange = firstFileName.split(DELIMITER); String[] lastTimestampRange = lastFileName.split(DELIMITER); - String baseName = baseDir + firstTimestampRange[0] + DELIMITER + lastTimestampRange[1]; + String baseName = firstTimestampRange[0] + DELIMITER + lastTimestampRange[1]; // Get the range of timestamps by taking the earliest and the latest timestamps - return new LSMComponentFileReferences(createMergeFile(baseName + DELIMITER + BTREE_SUFFIX), null, - hasBloomFilter ? createMergeFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX) : null); + return new LSMComponentFileReferences(baseDir.getChild(baseName + DELIMITER + BTREE_SUFFIX), null, + hasBloomFilter ? baseDir.getChild(baseName + DELIMITER + BLOOM_FILTER_SUFFIX) : null); } - private static FilenameFilter btreeFilter = new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return !name.startsWith(".") && name.endsWith(BTREE_SUFFIX); - } - }; - @Override public List cleanupAndGetValidFiles() throws HyracksDataException { List validFiles = new ArrayList<>(); ArrayList allBTreeFiles = new ArrayList<>(); ArrayList allBloomFilterFiles = new ArrayList<>(); - // create transaction filter FilenameFilter transactionFilter = getTransactionFileFilter(false); - - // Gather files - // List of valid BTree files. cleanupAndGetValidFilesInternal(getCompoundFilter(transactionFilter, btreeFilter), btreeFactory, allBTreeFiles); HashSet btreeFilesSet = new HashSet<>(); @@ -102,8 +93,7 @@ public class LSMBTreeFileManager extends AbstractLSMIndexFileManager { null); // Sanity check. if (allBTreeFiles.size() != allBloomFilterFiles.size()) { - throw new HyracksDataException( - "Unequal number of valid BTree and bloom filter files found. Aborting cleanup."); + throw HyracksDataException.create(ErrorCode.UNEQUAL_NUM_FILTERS_TREES, baseDir); } } @@ -137,7 +127,7 @@ public class LSMBTreeFileManager extends AbstractLSMIndexFileManager { validComparableBloomFilterFiles.add(lastBloomFilter); } - ComparableFileName currentBTree = null; + ComparableFileName currentBTree; ComparableFileName currentBloomFilter = null; for (int i = 1; i < allBTreeFiles.size(); i++) { currentBTree = allBTreeFiles.get(i); @@ -158,16 +148,13 @@ public class LSMBTreeFileManager extends AbstractLSMIndexFileManager { && (!hasBloomFilter || (currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[0]) >= 0 && currentBloomFilter.interval[1].compareTo(lastBloomFilter.interval[1]) <= 0))) { // Invalid files are completely contained in last interval. - File invalidBTreeFile = new File(currentBTree.fullPath); - invalidBTreeFile.delete(); + IoUtil.delete(new File(currentBTree.fullPath)); if (hasBloomFilter) { - File invalidBloomFilterFile = new File(currentBloomFilter.fullPath); - invalidBloomFilterFile.delete(); + IoUtil.delete(new File(currentBloomFilter.fullPath)); } } else { // This scenario should not be possible. - throw new HyracksDataException( - "Found LSM files with overlapping but not contained timetamp intervals."); + throw HyracksDataException.create(ErrorCode.FOUND_OVERLAPPING_LSM_FILES, baseDir); } } @@ -198,53 +185,40 @@ public class LSMBTreeFileManager extends AbstractLSMIndexFileManager { public LSMComponentFileReferences getNewTransactionFileReference() throws IOException { String ts = getCurrentTimestamp(); // Create transaction lock file - Files.createFile(Paths.get(baseDir + TXN_PREFIX + ts)); - - String baseName = baseDir + ts + DELIMITER + ts; + IoUtil.create(baseDir.getChild(TXN_PREFIX + ts)); + String baseName = ts + DELIMITER + ts; // Begin timestamp and end timestamp are identical since it is a transaction - return new LSMComponentFileReferences(createFlushFile(baseName + DELIMITER + BTREE_SUFFIX), null, - createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); + return new LSMComponentFileReferences(baseDir.getChild(baseName + DELIMITER + BTREE_SUFFIX), null, + baseDir.getChild(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); } @Override public LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException { FilenameFilter transactionFilter; - File dir = new File(baseDir); - String[] files = dir.list(txnFileNameFilter); + String[] files = baseDir.getFile().list(txnFileNameFilter); if (files.length == 0) { return null; } if (files.length != 1) { - throw new HyracksDataException("More than one transaction lock found:" + files.length); + throw HyracksDataException.create(ErrorCode.FOUND_MULTIPLE_TRANSACTIONS, baseDir); } else { transactionFilter = getTransactionFileFilter(true); - String txnFileName = dir.getPath() + File.separator + files[0]; + FileReference txnFile = baseDir.getChild(files[0]); // get the actual transaction files - files = dir.list(transactionFilter); - if (files.length < 2) { - throw new HyracksDataException("LSM Btree transaction has less than 2 files :" + files.length); - } - try { - Files.delete(Paths.get(txnFileName)); - } catch (IOException e) { - throw new HyracksDataException("Failed to delete transaction lock :" + txnFileName); - } + files = baseDir.getFile().list(transactionFilter); + IoUtil.delete(txnFile); } - File bTreeFile = null; - File bloomFilterFile = null; - + FileReference bTreeFileRef = null; + FileReference bloomFilterFileRef = null; for (String fileName : files) { if (fileName.endsWith(BTREE_SUFFIX)) { - bTreeFile = new File(dir.getPath() + File.separator + fileName); + bTreeFileRef = baseDir.getChild(fileName); } else if (fileName.endsWith(BLOOM_FILTER_SUFFIX)) { - bloomFilterFile = new File(dir.getPath() + File.separator + fileName); + bloomFilterFileRef = baseDir.getChild(fileName); } else { - throw new HyracksDataException("unrecognized file found = " + fileName); + throw HyracksDataException.create(ErrorCode.UNRECOGNIZED_INDEX_COMPONENT_FILE, fileName); } } - FileReference bTreeFileRef = ioManager.resolveAbsolutePath(bTreeFile.getAbsolutePath()); - FileReference bloomFilterFileRef = ioManager.resolveAbsolutePath(bloomFilterFile.getAbsolutePath()); - return new LSMComponentFileReferences(bTreeFileRef, null, bloomFilterFileRef); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/e0cf9701/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java index c866a45..f61d783 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyFileManager.java @@ -29,9 +29,11 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; +import org.apache.hyracks.api.exceptions.ErrorCode; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.IIOManager; +import org.apache.hyracks.api.util.IoUtil; import org.apache.hyracks.storage.am.common.api.ITreeIndex; import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; @@ -42,19 +44,10 @@ public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager { private final TreeIndexFactory btreeFactory; private final TreeIndexFactory buddyBtreeFactory; - private static FilenameFilter btreeFilter = new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return !name.startsWith(".") && name.endsWith(BTREE_SUFFIX); - } - }; + private static FilenameFilter btreeFilter = (dir, name) -> !name.startsWith(".") && name.endsWith(BTREE_SUFFIX); - private static FilenameFilter buddyBtreeFilter = new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return !name.startsWith(".") && name.endsWith(DELETE_TREE_SUFFIX); - } - }; + private static FilenameFilter buddyBtreeFilter = + (dir, name) -> !name.startsWith(".") && name.endsWith(DELETE_TREE_SUFFIX); public LSMBTreeWithBuddyFileManager(IIOManager ioManager, FileReference file, TreeIndexFactory btreeFactory, @@ -67,11 +60,11 @@ public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager { @Override public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException { String ts = getCurrentTimestamp(); - String baseName = baseDir + ts + DELIMITER + ts; + String baseName = ts + DELIMITER + ts; // Begin timestamp and end timestamp are identical since it is a flush - return new LSMComponentFileReferences(createFlushFile(baseName + DELIMITER + BTREE_SUFFIX), - createFlushFile(baseName + DELIMITER + DELETE_TREE_SUFFIX), - createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); + return new LSMComponentFileReferences(baseDir.getChild(baseName + DELIMITER + BTREE_SUFFIX), + baseDir.getChild(baseName + DELIMITER + DELETE_TREE_SUFFIX), + baseDir.getChild(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); } @Override @@ -80,12 +73,12 @@ public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager { String[] firstTimestampRange = firstFileName.split(DELIMITER); String[] lastTimestampRange = lastFileName.split(DELIMITER); - String baseName = baseDir + firstTimestampRange[0] + DELIMITER + lastTimestampRange[1]; + String baseName = firstTimestampRange[0] + DELIMITER + lastTimestampRange[1]; // Get the range of timestamps by taking the earliest and the latest // timestamps - return new LSMComponentFileReferences(createMergeFile(baseName + DELIMITER + BTREE_SUFFIX), - createMergeFile(baseName + DELIMITER + DELETE_TREE_SUFFIX), - createMergeFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); + return new LSMComponentFileReferences(baseDir.getChild(baseName + DELIMITER + BTREE_SUFFIX), + baseDir.getChild(baseName + DELIMITER + DELETE_TREE_SUFFIX), + baseDir.getChild(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); } @Override @@ -94,10 +87,8 @@ public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager { ArrayList allBTreeFiles = new ArrayList<>(); ArrayList allBuddyBTreeFiles = new ArrayList<>(); ArrayList allBloomFilterFiles = new ArrayList<>(); - // Create transaction file filter FilenameFilter transactionFilefilter = getTransactionFileFilter(false); - // Gather files. cleanupAndGetValidFilesInternal(getCompoundFilter(btreeFilter, transactionFilefilter), btreeFactory, allBTreeFiles); @@ -110,18 +101,14 @@ public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager { buddyBtreeFactory); validateFiles(btreeFilesSet, allBloomFilterFiles, getCompoundFilter(bloomFilterFilter, transactionFilefilter), null); - // Sanity check. if (allBTreeFiles.size() != allBuddyBTreeFiles.size() || allBTreeFiles.size() != allBloomFilterFiles.size()) { - throw new HyracksDataException( - "Unequal number of valid BTree, Buddy BTree, and Bloom Filter files found. Aborting cleanup."); + throw HyracksDataException.create(ErrorCode.UNEQUAL_NUM_FILTERS_TREES, baseDir); } - // Trivial cases. if (allBTreeFiles.isEmpty() || allBuddyBTreeFiles.isEmpty() || allBloomFilterFiles.isEmpty()) { return validFiles; } - if (allBTreeFiles.size() == 1 && allBuddyBTreeFiles.size() == 1 && allBloomFilterFiles.size() == 1) { validFiles.add(new LSMComponentFileReferences(allBTreeFiles.get(0).fileRef, allBuddyBTreeFiles.get(0).fileRef, allBloomFilterFiles.get(0).fileRef)); @@ -166,16 +153,12 @@ public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager { && currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[0]) >= 0 && currentBloomFilter.interval[1].compareTo(lastBloomFilter.interval[1]) <= 0) { // Invalid files are completely contained in last interval. - File invalidBTreeFile = new File(currentBTree.fullPath); - invalidBTreeFile.delete(); - File invalidBuddyBTreeFile = new File(currentBuddyBTree.fullPath); - invalidBuddyBTreeFile.delete(); - File invalidBloomFilterFile = new File(currentBloomFilter.fullPath); - invalidBloomFilterFile.delete(); + IoUtil.delete(new File(currentBTree.fullPath)); + IoUtil.delete(new File(currentBuddyBTree.fullPath)); + IoUtil.delete(new File(currentBloomFilter.fullPath)); } else { // This scenario should not be possible. - throw new HyracksDataException( - "Found LSM files with overlapping but not contained timetamp intervals."); + throw HyracksDataException.create(ErrorCode.FOUND_OVERLAPPING_LSM_FILES, baseDir); } } @@ -204,55 +187,45 @@ public class LSMBTreeWithBuddyFileManager extends AbstractLSMIndexFileManager { String ts = getCurrentTimestamp(); // Create transaction lock file Files.createFile(Paths.get(baseDir + TXN_PREFIX + ts)); - - String baseName = baseDir + ts + DELIMITER + ts; - return new LSMComponentFileReferences(createFlushFile(baseName + DELIMITER + BTREE_SUFFIX), - createFlushFile(baseName + DELIMITER + DELETE_TREE_SUFFIX), - createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); + String baseName = ts + DELIMITER + ts; + return new LSMComponentFileReferences(baseDir.getChild(baseName + DELIMITER + BTREE_SUFFIX), + baseDir.getChild(baseName + DELIMITER + DELETE_TREE_SUFFIX), + baseDir.getChild(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); } @Override public LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException { FilenameFilter transactionFilter; - File dir = new File(baseDir); - String[] files = dir.list(txnFileNameFilter); + String[] files = baseDir.getFile().list(txnFileNameFilter); if (files.length == 0) { return null; } if (files.length != 1) { - throw new HyracksDataException("More than one transaction lock found:" + files.length); + throw HyracksDataException.create(ErrorCode.FOUND_MULTIPLE_TRANSACTIONS, baseDir); } else { transactionFilter = getTransactionFileFilter(true); - String txnFileName = dir.getPath() + File.separator + files[0]; // get the actual transaction files - files = dir.list(transactionFilter); + files = baseDir.getFile().list(transactionFilter); if (files.length < 3) { - throw new HyracksDataException( - "LSM Btree with buddy transaction has less than 3 files :" + files.length); - } - try { - Files.delete(Paths.get(txnFileName)); - } catch (IOException e) { - throw new HyracksDataException("Failed to delete transaction lock :" + txnFileName); + throw HyracksDataException.create(ErrorCode.UNEQUAL_NUM_FILTERS_TREES, baseDir); } + IoUtil.delete(baseDir.getChild(files[0])); } - File bTreeFile = null; - File buddyBTreeFile = null; - File bloomFilterFile = null; + FileReference bTreeFileRef = null; + FileReference buddyBTreeFileRef = null; + FileReference bloomFilterFileRef = null; for (String fileName : files) { if (fileName.endsWith(BTREE_SUFFIX)) { - bTreeFile = new File(dir.getPath() + File.separator + fileName); + bTreeFileRef = baseDir.getChild(fileName); } else if (fileName.endsWith(DELETE_TREE_SUFFIX)) { - buddyBTreeFile = new File(dir.getPath() + File.separator + fileName); + buddyBTreeFileRef = baseDir.getChild(fileName); } else if (fileName.endsWith(BLOOM_FILTER_SUFFIX)) { - bloomFilterFile = new File(dir.getPath() + File.separator + fileName); + bloomFilterFileRef = baseDir.getChild(fileName); } else { - throw new HyracksDataException("unrecognized file found = " + fileName); + throw HyracksDataException.create(ErrorCode.UNRECOGNIZED_INDEX_COMPONENT_FILE, fileName); } } - FileReference bTreeFileRef = ioManager.resolveAbsolutePath(bTreeFile.getAbsolutePath()); - FileReference buddyBTreeFileRef = ioManager.resolveAbsolutePath(buddyBTreeFile.getAbsolutePath()); - FileReference bloomFilterFileRef = ioManager.resolveAbsolutePath(bloomFilterFile.getAbsolutePath()); + return new LSMComponentFileReferences(bTreeFileRef, buddyBTreeFileRef, bloomFilterFileRef); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/e0cf9701/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java index 18d9a67..5580387 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndexFileManager.java @@ -24,6 +24,7 @@ import java.util.Comparator; import java.util.List; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; /** @@ -43,7 +44,7 @@ public interface ILSMIndexFileManager { LSMComponentFileReferences getRelMergeFileReference(String firstFileName, String lastFileName) throws HyracksDataException; - String getBaseDir(); + FileReference getBaseDir(); // Deletes invalid files, and returns list of valid files from baseDir. // The returned valid files are correctly sorted (based on the recency of data). http://git-wip-us.apache.org/repos/asf/asterixdb/blob/e0cf9701/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java index ead9db4..d3c056d 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java @@ -22,8 +22,6 @@ package org.apache.hyracks.storage.am.lsm.common.impls; import java.io.File; import java.io.FilenameFilter; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; import java.text.Format; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -88,7 +86,7 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage protected final IIOManager ioManager; // baseDir should reflect dataset name and partition name and be absolute - protected final String baseDir; + protected final FileReference baseDir; protected final Format formatter = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss-SSS"); protected final Comparator recencyCmp = new RecencyComparator(); protected final TreeIndexFactory treeFactory; @@ -97,8 +95,7 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage public AbstractLSMIndexFileManager(IIOManager ioManager, FileReference file, TreeIndexFactory treeFactory) { this.ioManager = ioManager; - this.baseDir = file.getFile().getAbsolutePath().endsWith(File.separator) ? file.getFile().getAbsolutePath() - : file.getFile().getAbsolutePath() + File.separator; + this.baseDir = file; this.treeFactory = treeFactory; } @@ -136,9 +133,8 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage TreeIndexFactory treeFactory, ArrayList allFiles) throws HyracksDataException { String[] files = listDirFiles(baseDir, filter); - File dir = new File(baseDir); for (String fileName : files) { - FileReference fileRef = ioManager.resolveAbsolutePath(dir.getPath() + File.separator + fileName); + FileReference fileRef = baseDir.getChild(fileName); if (treeFactory == null) { allFiles.add(new ComparableFileName(fileRef)); continue; @@ -152,21 +148,20 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage } } - static String[] listDirFiles(String path, FilenameFilter filter) throws HyracksDataException { - File dir = new File(path); + static String[] listDirFiles(FileReference dir, FilenameFilter filter) throws HyracksDataException { /* * Returns null if this abstract pathname does not denote a directory, or if an I/O error occurs. */ - String[] files = dir.list(filter); + String[] files = dir.getFile().list(filter); if (files == null) { - if (!dir.canRead()) { - throw HyracksDataException.create(ErrorCode.CANNOT_READ_FILE, path); - } else if (!dir.exists()) { - throw HyracksDataException.create(ErrorCode.FILE_DOES_NOT_EXIST, path); - } else if (!dir.isDirectory()) { - throw HyracksDataException.create(ErrorCode.FILE_IS_NOT_DIRECTORY, path); + if (!dir.getFile().canRead()) { + throw HyracksDataException.create(ErrorCode.CANNOT_READ_FILE, dir); + } else if (!dir.getFile().exists()) { + throw HyracksDataException.create(ErrorCode.FILE_DOES_NOT_EXIST, dir); + } else if (!dir.getFile().isDirectory()) { + throw HyracksDataException.create(ErrorCode.FILE_IS_NOT_DIRECTORY, dir); } - throw HyracksDataException.create(ErrorCode.UNIDENTIFIED_IO_ERROR_READING_FILE, path); + throw HyracksDataException.create(ErrorCode.UNIDENTIFIED_IO_ERROR_READING_FILE, dir); } return files; } @@ -189,31 +184,22 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage @Override public void createDirs() throws HyracksDataException { - File f = new File(baseDir); - if (f.exists()) { + if (baseDir.getFile().exists()) { throw HyracksDataException.create(ErrorCode.CANNOT_CREATE_EXISTING_INDEX); } - f.mkdirs(); + baseDir.getFile().mkdirs(); } @Override public void deleteDirs() throws HyracksDataException { - IoUtil.delete(new File(baseDir)); - } - - protected FileReference createFlushFile(String flushFileName) throws HyracksDataException { - return ioManager.resolveAbsolutePath(flushFileName); - } - - protected FileReference createMergeFile(String mergeFileName) throws HyracksDataException { - return createFlushFile(mergeFileName); + IoUtil.delete(baseDir); } @Override public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException { String ts = getCurrentTimestamp(); // Begin timestamp and end timestamp are identical since it is a flush - return new LSMComponentFileReferences(createFlushFile(baseDir + ts + DELIMITER + ts), null, null); + return new LSMComponentFileReferences(baseDir.getChild(ts + DELIMITER + ts), null, null); } @Override @@ -223,7 +209,7 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage String[] lastTimestampRange = lastFileName.split(DELIMITER); // Get the range of timestamps by taking the earliest and the latest timestamps return new LSMComponentFileReferences( - createMergeFile(baseDir + firstTimestampRange[0] + DELIMITER + lastTimestampRange[1]), null, null); + baseDir.getChild(firstTimestampRange[0] + DELIMITER + lastTimestampRange[1]), null, null); } @Override @@ -267,8 +253,7 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage current.fileRef.delete(); } else { // This scenario should not be possible since timestamps are monotonically increasing. - throw new HyracksDataException("Found LSM files with overlapping timestamp intervals, " - + "but the intervals were not contained by another file."); + throw HyracksDataException.create(ErrorCode.FOUND_OVERLAPPING_LSM_FILES, baseDir); } } @@ -302,24 +287,19 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage } @Override - public String getBaseDir() { + public FileReference getBaseDir() { return baseDir; } @Override public void recoverTransaction() throws HyracksDataException { String[] files = listDirFiles(baseDir, txnFileNameFilter); - File dir = new File(baseDir); - try { - if (files.length == 0) { - // Do nothing - } else if (files.length > 1) { - throw new HyracksDataException("Found more than one transaction"); - } else { - Files.delete(Paths.get(dir.getPath() + File.separator + files[0])); - } - } catch (IOException e) { - throw new HyracksDataException("Failed to recover transaction", e); + if (files.length == 0) { + // Do nothing + } else if (files.length > 1) { + throw HyracksDataException.create(ErrorCode.FOUND_MULTIPLE_TRANSACTIONS, baseDir); + } else { + IoUtil.delete(baseDir.getChild(files[0])); } } @@ -366,27 +346,17 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage if (files.length == 0) { // Do nothing } else if (files.length > 1) { - throw new HyracksDataException("Found more than one transaction"); + throw HyracksDataException.create(ErrorCode.FOUND_MULTIPLE_TRANSACTIONS, baseDir); } else { - File dir = new File(baseDir); //create transaction filter FilenameFilter transactionFilter = createTransactionFilter(files[0], true); String[] componentsFiles = listDirFiles(baseDir, transactionFilter); for (String fileName : componentsFiles) { - try { - String absFileName = dir.getPath() + File.separator + fileName; - Files.delete(Paths.get(absFileName)); - } catch (IOException e) { - throw new HyracksDataException("Failed to delete transaction files", e); - } + FileReference file = baseDir.getChild(fileName); + IoUtil.delete(file); } // delete the txn lock file - String absFileName = dir.getPath() + File.separator + files[0]; - try { - Files.delete(Paths.get(absFileName)); - } catch (IOException e) { - throw new HyracksDataException("Failed to delete transaction files", e); - } + IoUtil.delete(baseDir.getChild(files[0])); } } @@ -403,16 +373,7 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage protected static FilenameFilter createTransactionFilter(String transactionFileName, final boolean inclusive) { final String timeStamp = transactionFileName.substring(transactionFileName.indexOf(TXN_PREFIX) + TXN_PREFIX.length()); - return new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - if (inclusive) { - return name.startsWith(timeStamp); - } else { - return !name.startsWith(timeStamp); - } - } - }; + return (dir, name) -> inclusive ? name.startsWith(timeStamp) : !name.startsWith(timeStamp); } protected FilenameFilter getTransactionFileFilter(boolean inclusive) throws HyracksDataException { @@ -425,12 +386,7 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage } protected FilenameFilter getCompoundFilter(final FilenameFilter filter1, final FilenameFilter filter2) { - return new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return (filter1.accept(dir, name) && filter2.accept(dir, name)); - } - }; + return (dir, name) -> filter1.accept(dir, name) && filter2.accept(dir, name); } /** http://git-wip-us.apache.org/repos/asf/asterixdb/blob/e0cf9701/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java index 4a60db0..0f862ac 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java @@ -724,7 +724,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex return new LSMInvertedIndexFlushOperation(new LSMInvertedIndexAccessor(getLsmHarness(), opCtx), flushingComponent, componentFileRefs.getInsertIndexFileReference(), componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), - callback, fileManager.getBaseDir()); + callback, fileManager.getBaseDir().getAbsolutePath()); } @Override @@ -735,7 +735,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex IIndexCursor cursor = new LSMInvertedIndexRangeSearchCursor(opCtx); return new LSMInvertedIndexMergeOperation(accessor, mergingComponents, cursor, mergeFileRefs.getInsertIndexFileReference(), mergeFileRefs.getDeleteIndexFileReference(), - mergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir()); + mergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir().getAbsolutePath()); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/e0cf9701/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java index 9254c45..c4a2f73 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java @@ -27,9 +27,11 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; +import org.apache.hyracks.api.exceptions.ErrorCode; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.IIOManager; +import org.apache.hyracks.api.util.IoUtil; import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager; import org.apache.hyracks.storage.am.lsm.common.impls.BTreeFactory; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; @@ -43,27 +45,12 @@ public class LSMInvertedIndexFileManager extends AbstractLSMIndexFileManager imp // We only need a BTree factory because the inverted indexes consistency is validated against its dictionary BTree. private final BTreeFactory btreeFactory; - - private static FilenameFilter dictBTreeFilter = new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return !name.startsWith(".") && name.endsWith(DICT_BTREE_SUFFIX); - } - }; - - private static FilenameFilter invListFilter = new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return !name.startsWith(".") && name.endsWith(INVLISTS_SUFFIX); - } - }; - - private static FilenameFilter deletedKeysBTreeFilter = new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return !name.startsWith(".") && name.endsWith(DELETED_KEYS_BTREE_SUFFIX); - } - }; + private static final FilenameFilter dictBTreeFilter = + (dir, name) -> !name.startsWith(".") && name.endsWith(DICT_BTREE_SUFFIX); + private static final FilenameFilter invListFilter = + (dir, name) -> !name.startsWith(".") && name.endsWith(INVLISTS_SUFFIX); + private static final FilenameFilter deletedKeysBTreeFilter = + (dir, name) -> !name.startsWith(".") && name.endsWith(DELETED_KEYS_BTREE_SUFFIX); public LSMInvertedIndexFileManager(IIOManager ioManager, FileReference file, BTreeFactory btreeFactory) { super(ioManager, file, null); @@ -73,11 +60,11 @@ public class LSMInvertedIndexFileManager extends AbstractLSMIndexFileManager imp @Override public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException { String ts = getCurrentTimestamp(); - String baseName = baseDir + ts + DELIMITER + ts; + String baseName = ts + DELIMITER + ts; // Begin timestamp and end timestamp are identical since it is a flush - return new LSMComponentFileReferences(createFlushFile(baseName + DELIMITER + DICT_BTREE_SUFFIX), - createFlushFile(baseName + DELIMITER + DELETED_KEYS_BTREE_SUFFIX), - createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); + return new LSMComponentFileReferences(baseDir.getChild(baseName + DELIMITER + DICT_BTREE_SUFFIX), + baseDir.getChild(baseName + DELIMITER + DELETED_KEYS_BTREE_SUFFIX), + baseDir.getChild(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); } @Override @@ -85,12 +72,11 @@ public class LSMInvertedIndexFileManager extends AbstractLSMIndexFileManager imp throws HyracksDataException { String[] firstTimestampRange = firstFileName.split(DELIMITER); String[] lastTimestampRange = lastFileName.split(DELIMITER); - - String baseName = baseDir + firstTimestampRange[0] + DELIMITER + lastTimestampRange[1]; + String baseName = firstTimestampRange[0] + DELIMITER + lastTimestampRange[1]; // Get the range of timestamps by taking the earliest and the latest timestamps - return new LSMComponentFileReferences(createMergeFile(baseName + DELIMITER + DICT_BTREE_SUFFIX), - createMergeFile(baseName + DELIMITER + DELETED_KEYS_BTREE_SUFFIX), - createMergeFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); + return new LSMComponentFileReferences(baseDir.getChild(baseName + DELIMITER + DICT_BTREE_SUFFIX), + baseDir.getChild(baseName + DELIMITER + DELETED_KEYS_BTREE_SUFFIX), + baseDir.getChild(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); } @Override @@ -118,8 +104,7 @@ public class LSMInvertedIndexFileManager extends AbstractLSMIndexFileManager imp if (allDictBTreeFiles.size() != allInvListsFiles.size() || allDictBTreeFiles.size() != allDeletedKeysBTreeFiles.size() || allDictBTreeFiles.size() != allBloomFilterFiles.size()) { - throw new HyracksDataException( - "Unequal number of valid Dictionary BTree, Inverted Lists, Deleted BTree, and Bloom Filter files found. Aborting cleanup."); + throw HyracksDataException.create(ErrorCode.UNEQUAL_NUM_FILTERS_TREES, baseDir); } // Trivial cases. @@ -154,35 +139,31 @@ public class LSMInvertedIndexFileManager extends AbstractLSMIndexFileManager imp for (int i = 1; i < allDictBTreeFiles.size(); i++) { ComparableFileName currentDeletedKeysBTree = allDeletedKeysBTreeFiles.get(i); - ComparableFileName CurrentDictBTree = allDictBTreeFiles.get(i); + ComparableFileName currentDictBTree = allDictBTreeFiles.get(i); ComparableFileName currentBloomFilter = allBloomFilterFiles.get(i); // Current start timestamp is greater than last stop timestamp. if (currentDeletedKeysBTree.interval[0].compareTo(lastDeletedKeysBTree.interval[1]) > 0 - && CurrentDictBTree.interval[0].compareTo(lastDictBTree.interval[1]) > 0 + && currentDictBTree.interval[0].compareTo(lastDictBTree.interval[1]) > 0 && currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[1]) > 0) { - validComparableDictBTreeFiles.add(CurrentDictBTree); + validComparableDictBTreeFiles.add(currentDictBTree); validComparableDeletedKeysBTreeFiles.add(currentDeletedKeysBTree); validComparableBloomFilterFiles.add(currentBloomFilter); - lastDictBTree = CurrentDictBTree; + lastDictBTree = currentDictBTree; lastDeletedKeysBTree = currentDeletedKeysBTree; lastBloomFilter = currentBloomFilter; } else if (currentDeletedKeysBTree.interval[0].compareTo(lastDeletedKeysBTree.interval[0]) >= 0 && currentDeletedKeysBTree.interval[1].compareTo(lastDeletedKeysBTree.interval[1]) <= 0 - && CurrentDictBTree.interval[0].compareTo(lastDictBTree.interval[0]) >= 0 - && CurrentDictBTree.interval[1].compareTo(lastDictBTree.interval[1]) <= 0 + && currentDictBTree.interval[0].compareTo(lastDictBTree.interval[0]) >= 0 + && currentDictBTree.interval[1].compareTo(lastDictBTree.interval[1]) <= 0 && currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[0]) >= 0 && currentBloomFilter.interval[1].compareTo(lastBloomFilter.interval[1]) <= 0) { // Invalid files are completely contained in last interval. - File invalidDeletedBTreeFile = new File(currentDeletedKeysBTree.fullPath); - invalidDeletedBTreeFile.delete(); - File invalidDictBTreeFile = new File(CurrentDictBTree.fullPath); - invalidDictBTreeFile.delete(); - File invalidBloomFilterFile = new File(currentBloomFilter.fullPath); - invalidBloomFilterFile.delete(); + IoUtil.delete(new File(currentDeletedKeysBTree.fullPath)); + IoUtil.delete(new File(currentDictBTree.fullPath)); + IoUtil.delete(new File(currentBloomFilter.fullPath)); } else { // This scenario should not be possible. - throw new HyracksDataException( - "Found LSM files with overlapping but not contained timetamp intervals."); + throw HyracksDataException.create(ErrorCode.FOUND_OVERLAPPING_LSM_FILES, baseDir); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/e0cf9701/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java index 512e35c..e39c3f9 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java @@ -642,7 +642,7 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex { // create the merge operation. LSMRTreeMergeOperation mergeOp = new LSMRTreeMergeOperation(accessor, mergingComponents, cursor, relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getDeleteIndexFileReference(), - relMergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir()); + relMergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir().getAbsolutePath()); ioScheduler.scheduleOperation(mergeOp); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/e0cf9701/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java index 07d48de..0f291df 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTree.java @@ -431,7 +431,7 @@ public class LSMRTree extends AbstractLSMRTree { LSMRTreeAccessor accessor = new LSMRTreeAccessor(getLsmHarness(), opCtx, buddyBTreeFields); return new LSMRTreeFlushOperation(accessor, flushingComponent, componentFileRefs.getInsertIndexFileReference(), componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(), - callback, fileManager.getBaseDir()); + callback, fileManager.getBaseDir().getAbsolutePath()); } @Override @@ -442,6 +442,6 @@ public class LSMRTree extends AbstractLSMRTree { ILSMIndexAccessor accessor = new LSMRTreeAccessor(getLsmHarness(), opCtx, buddyBTreeFields); return new LSMRTreeMergeOperation(accessor, mergingComponents, cursor, mergeFileRefs.getInsertIndexFileReference(), mergeFileRefs.getDeleteIndexFileReference(), - mergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir()); + mergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir().getAbsolutePath()); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/e0cf9701/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java index 4d5eb5b..001228c 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeFileManager.java @@ -30,9 +30,11 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; +import org.apache.hyracks.api.exceptions.ErrorCode; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.IIOManager; +import org.apache.hyracks.api.util.IoUtil; import org.apache.hyracks.storage.am.common.api.ITreeIndex; import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager; import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences; @@ -43,19 +45,10 @@ public class LSMRTreeFileManager extends AbstractLSMIndexFileManager { private final TreeIndexFactory rtreeFactory; private final TreeIndexFactory btreeFactory; - private static FilenameFilter btreeFilter = new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return !name.startsWith(".") && name.endsWith(BTREE_SUFFIX); - } - }; - - private static FilenameFilter rtreeFilter = new FilenameFilter() { - @Override - public boolean accept(File dir, String name) { - return !name.startsWith(".") && name.endsWith(RTREE_SUFFIX); - } - }; + private static final FilenameFilter btreeFilter = + (dir, name) -> !name.startsWith(".") && name.endsWith(BTREE_SUFFIX); + private static final FilenameFilter rtreeFilter = + (dir, name) -> !name.startsWith(".") && name.endsWith(RTREE_SUFFIX); public LSMRTreeFileManager(IIOManager ioManager, FileReference file, TreeIndexFactory rtreeFactory, TreeIndexFactory btreeFactory) { @@ -67,11 +60,11 @@ public class LSMRTreeFileManager extends AbstractLSMIndexFileManager { @Override public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException { String ts = getCurrentTimestamp(); - String baseName = baseDir + ts + DELIMITER + ts; + String baseName = ts + DELIMITER + ts; // Begin timestamp and end timestamp are identical since it is a flush - return new LSMComponentFileReferences(createFlushFile(baseName + DELIMITER + RTREE_SUFFIX), - createFlushFile(baseName + DELIMITER + BTREE_SUFFIX), - createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); + return new LSMComponentFileReferences(baseDir.getChild(baseName + DELIMITER + RTREE_SUFFIX), + baseDir.getChild(baseName + DELIMITER + BTREE_SUFFIX), + baseDir.getChild(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); } @Override @@ -79,13 +72,12 @@ public class LSMRTreeFileManager extends AbstractLSMIndexFileManager { throws HyracksDataException { String[] firstTimestampRange = firstFileName.split(DELIMITER); String[] lastTimestampRange = lastFileName.split(DELIMITER); - - String baseName = baseDir + firstTimestampRange[0] + DELIMITER + lastTimestampRange[1]; + String baseName = firstTimestampRange[0] + DELIMITER + lastTimestampRange[1]; // Get the range of timestamps by taking the earliest and the latest // timestamps - return new LSMComponentFileReferences(createMergeFile(baseName + DELIMITER + RTREE_SUFFIX), - createMergeFile(baseName + DELIMITER + BTREE_SUFFIX), - createMergeFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); + return new LSMComponentFileReferences(baseDir.getChild(baseName + DELIMITER + RTREE_SUFFIX), + baseDir.getChild(baseName + DELIMITER + BTREE_SUFFIX), + baseDir.getChild(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); } @Override @@ -111,8 +103,7 @@ public class LSMRTreeFileManager extends AbstractLSMIndexFileManager { // Sanity check. if (allRTreeFiles.size() != allBTreeFiles.size() || allBTreeFiles.size() != allBloomFilterFiles.size()) { - throw new HyracksDataException( - "Unequal number of valid RTree, BTree, and Bloom Filter files found. Aborting cleanup."); + throw HyracksDataException.create(ErrorCode.UNEQUAL_NUM_FILTERS_TREES, baseDir); } // Trivial cases. @@ -164,16 +155,12 @@ public class LSMRTreeFileManager extends AbstractLSMIndexFileManager { && currentBloomFilter.interval[0].compareTo(lastBloomFilter.interval[0]) >= 0 && currentBloomFilter.interval[1].compareTo(lastBloomFilter.interval[1]) <= 0) { // Invalid files are completely contained in last interval. - File invalidRTreeFile = new File(currentRTree.fullPath); - invalidRTreeFile.delete(); - File invalidBTreeFile = new File(currentBTree.fullPath); - invalidBTreeFile.delete(); - File invalidBloomFilterFile = new File(currentBloomFilter.fullPath); - invalidBloomFilterFile.delete(); + IoUtil.delete(new File(currentRTree.fullPath)); + IoUtil.delete(new File(currentBTree.fullPath)); + IoUtil.delete(new File(currentBloomFilter.fullPath)); } else { // This scenario should not be possible. - throw new HyracksDataException( - "Found LSM files with overlapping but not contained timetamp intervals."); + throw HyracksDataException.create(ErrorCode.FOUND_OVERLAPPING_LSM_FILES, baseDir); } } @@ -201,54 +188,44 @@ public class LSMRTreeFileManager extends AbstractLSMIndexFileManager { String ts = getCurrentTimestamp(); // Create transaction lock file Files.createFile(Paths.get(baseDir + TXN_PREFIX + ts)); - - String baseName = baseDir + ts + DELIMITER + ts; - return new LSMComponentFileReferences(createFlushFile(baseName + DELIMITER + RTREE_SUFFIX), - createFlushFile(baseName + DELIMITER + BTREE_SUFFIX), - createFlushFile(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); + String baseName = ts + DELIMITER + ts; + return new LSMComponentFileReferences(baseDir.getChild(baseName + DELIMITER + RTREE_SUFFIX), + baseDir.getChild(baseName + DELIMITER + BTREE_SUFFIX), + baseDir.getChild(baseName + DELIMITER + BLOOM_FILTER_SUFFIX)); } @Override public LSMComponentFileReferences getTransactionFileReferenceForCommit() throws HyracksDataException { FilenameFilter transactionFilter; - File dir = new File(baseDir); - String[] files = dir.list(txnFileNameFilter); + String[] files = baseDir.getFile().list(txnFileNameFilter); if (files.length == 0) { return null; } if (files.length != 1) { - throw new HyracksDataException("More than one transaction lock found:" + files.length); + throw HyracksDataException.create(ErrorCode.FOUND_MULTIPLE_TRANSACTIONS, baseDir); } else { transactionFilter = getTransactionFileFilter(true); - String txnFileName = dir.getPath() + File.separator + files[0]; // get the actual transaction files - files = dir.list(transactionFilter); + files = baseDir.getFile().list(transactionFilter); if (files.length < 3) { - throw new HyracksDataException("LSM Rtree transaction has less than 3 files :" + files.length); - } - try { - Files.delete(Paths.get(txnFileName)); - } catch (IOException e) { - throw new HyracksDataException("Failed to delete transaction lock :" + txnFileName); + throw HyracksDataException.create(ErrorCode.UNEQUAL_NUM_FILTERS_TREES, baseDir); } + IoUtil.delete(baseDir.getChild(files[0])); } - File rTreeFile = null; - File bTreeFile = null; - File bloomFilterFile = null; + FileReference rTreeFileRef = null; + FileReference bTreeFileRef = null; + FileReference bloomFilterFileRef = null; for (String fileName : files) { if (fileName.endsWith(BTREE_SUFFIX)) { - bTreeFile = new File(dir.getPath() + File.separator + fileName); + bTreeFileRef = baseDir.getChild(fileName); } else if (fileName.endsWith(RTREE_SUFFIX)) { - rTreeFile = new File(dir.getPath() + File.separator + fileName); + rTreeFileRef = baseDir.getChild(fileName); } else if (fileName.endsWith(BLOOM_FILTER_SUFFIX)) { - bloomFilterFile = new File(dir.getPath() + File.separator + fileName); + bloomFilterFileRef = baseDir.getChild(fileName); } else { - throw new HyracksDataException("unrecognized file found = " + fileName); + throw HyracksDataException.create(ErrorCode.UNRECOGNIZED_INDEX_COMPONENT_FILE, fileName); } } - FileReference rTreeFileRef = ioManager.resolveAbsolutePath(rTreeFile.getAbsolutePath()); - FileReference bTreeFileRef = ioManager.resolveAbsolutePath(bTreeFile.getAbsolutePath()); - FileReference bloomFilterFileRef = ioManager.resolveAbsolutePath(bloomFilterFile.getAbsolutePath()); return new LSMComponentFileReferences(rTreeFileRef, bTreeFileRef, bloomFilterFileRef); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/e0cf9701/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java index 2b61f1e..58d4421 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeWithAntiMatterTuples.java @@ -339,7 +339,7 @@ public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree { ILSMIOOperationCallback callback) throws HyracksDataException { ILSMIndexAccessor accessor = new LSMTreeIndexAccessor(getLsmHarness(), opCtx, cursorFactory); return new LSMRTreeFlushOperation(accessor, flushingComponent, componentFileRefs.getInsertIndexFileReference(), - null, null, callback, fileManager.getBaseDir()); + null, null, callback, fileManager.getBaseDir().getAbsolutePath()); } @Override @@ -353,6 +353,6 @@ public class LSMRTreeWithAntiMatterTuples extends AbstractLSMRTree { ITreeIndexCursor cursor = new LSMRTreeWithAntiMatterTuplesSearchCursor(opCtx, returnDeletedTuples); ILSMIndexAccessor accessor = new LSMTreeIndexAccessor(getLsmHarness(), opCtx, cursorFactory); return new MergeOperation(accessor, mergeFileRefs.getInsertIndexFileReference(), callback, - fileManager.getBaseDir(), mergingComponents, cursor); + fileManager.getBaseDir().getAbsolutePath(), mergingComponents, cursor); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/e0cf9701/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java index 1c75387..87fb11a 100644 --- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-common-test/src/test/java/org/apache/hyracks/storage/am/lsm/common/component/TestLsmIndexFileManager.java @@ -19,7 +19,6 @@ package org.apache.hyracks.storage.am.lsm.common.component; -import java.io.File; import java.io.FilenameFilter; import java.util.ArrayList; @@ -40,11 +39,9 @@ public class TestLsmIndexFileManager extends AbstractLSMIndexFileManager { protected void cleanupAndGetValidFilesInternal(FilenameFilter filter, TreeIndexFactory treeFactory, ArrayList allFiles) throws HyracksDataException { - File dir = new File(baseDir); - String[] files = dir.list(filter); + String[] files = baseDir.getFile().list(filter); for (String fileName : files) { - File file = new File(dir.getPath() + File.separator + fileName); - FileReference fileRef = ioManager.resolveAbsolutePath(file.getAbsolutePath()); + FileReference fileRef = baseDir.getChild(fileName); allFiles.add(new ComparableFileName(fileRef)); } }