Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id D65B9200BCB for ; Thu, 24 Nov 2016 07:30:08 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id D2B76160B1F; Thu, 24 Nov 2016 06:30:08 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id ACCE8160B21 for ; Thu, 24 Nov 2016 07:30:06 +0100 (CET) Received: (qmail 11780 invoked by uid 500); 24 Nov 2016 06:30:05 -0000 Mailing-List: contact commits-help@asterixdb.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@asterixdb.apache.org Delivered-To: mailing list commits@asterixdb.apache.org Received: (qmail 11698 invoked by uid 99); 24 Nov 2016 06:30:05 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 24 Nov 2016 06:30:05 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 9B28EEEE7B; Thu, 24 Nov 2016 06:30:05 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: amoudi@apache.org To: commits@asterixdb.apache.org Date: Thu, 24 Nov 2016 06:30:09 -0000 Message-Id: <0a5f84c631344f6fa8a0cba64838fa58@git.apache.org> In-Reply-To: <2b17756e807e450ba109228f089d9061@git.apache.org> References: <2b17756e807e450ba109228f089d9061@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [05/12] asterixdb git commit: Cleanup FileSplit and FileReference archived-at: Thu, 24 Nov 2016 06:30:09 -0000 http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IVirtualBufferCacheProvider.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IVirtualBufferCacheProvider.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IVirtualBufferCacheProvider.java index 2e2bb2c..ca61075 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IVirtualBufferCacheProvider.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IVirtualBufferCacheProvider.java @@ -22,8 +22,10 @@ import java.io.Serializable; import java.util.List; import org.apache.hyracks.api.context.IHyracksTaskContext; +import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.dataflow.std.file.IFileSplitProvider; public interface IVirtualBufferCacheProvider extends Serializable { - public List getVirtualBufferCaches(IHyracksTaskContext ctx, IFileSplitProvider fileSplitProvider); + public List getVirtualBufferCaches(IHyracksTaskContext ctx, + IFileSplitProvider fileSplitProvider) throws HyracksDataException; } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelper.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelper.java index f0d1c75..fa5b623 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelper.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/AbstractLSMIndexDataflowHelper.java @@ -24,6 +24,7 @@ import java.util.List; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.dataflow.value.ITypeTraits; +import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor; import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelper; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory; @@ -51,7 +52,8 @@ public abstract class AbstractLSMIndexDataflowHelper extends IndexDataflowHelper List virtualBufferCaches, ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory, ITypeTraits[] filterTypeTraits, - IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, boolean durable) { + IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, boolean durable) + throws HyracksDataException { this(opDesc, ctx, partition, virtualBufferCaches, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackFactory, filterTypeTraits, filterCmpFactories, filterFields, durable); @@ -62,7 +64,7 @@ public abstract class AbstractLSMIndexDataflowHelper extends IndexDataflowHelper ILSMMergePolicy mergePolicy, ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, - boolean durable) { + boolean durable) throws HyracksDataException { super(opDesc, ctx, partition, durable); this.virtualBufferCaches = virtualBufferCaches; this.bloomFilterFalsePositiveRate = bloomFilterFalsePositiveRate; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMIndexCompactOperatorNodePushable.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMIndexCompactOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMIndexCompactOperatorNodePushable.java index 8171c58..48f59f1 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMIndexCompactOperatorNodePushable.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMIndexCompactOperatorNodePushable.java @@ -33,7 +33,8 @@ import org.apache.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallback; public class LSMIndexCompactOperatorNodePushable extends AbstractOperatorNodePushable { private final IIndexDataflowHelper indexHelper; - public LSMIndexCompactOperatorNodePushable(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition) { + public LSMIndexCompactOperatorNodePushable(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition) + throws HyracksDataException { this.indexHelper = opDesc.getIndexDataflowHelperFactory().createIndexDataflowHelper(opDesc, ctx, partition); } @@ -56,7 +57,7 @@ public class LSMIndexCompactOperatorNodePushable extends AbstractOperatorNodePus public void initialize() throws HyracksDataException { indexHelper.open(); ILSMIndex index = (ILSMIndex) indexHelper.getIndexInstance(); - ILSMIndexAccessor accessor = (ILSMIndexAccessor) index.createAccessor(NoOpOperationCallback.INSTANCE, + ILSMIndexAccessor accessor = index.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE); try { accessor.scheduleFullMerge(NoOpIOOperationCallback.INSTANCE); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMIndexInsertUpdateDeleteOperatorNodePushable.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMIndexInsertUpdateDeleteOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMIndexInsertUpdateDeleteOperatorNodePushable.java index de1719c..b51d132 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMIndexInsertUpdateDeleteOperatorNodePushable.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMIndexInsertUpdateDeleteOperatorNodePushable.java @@ -43,7 +43,8 @@ public class LSMIndexInsertUpdateDeleteOperatorNodePushable extends IndexInsertU } public LSMIndexInsertUpdateDeleteOperatorNodePushable(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, - int partition, int[] fieldPermutation, IRecordDescriptorProvider recordDescProvider, IndexOperation op) { + int partition, int[] fieldPermutation, IRecordDescriptorProvider recordDescProvider, IndexOperation op) + throws HyracksDataException { super(opDesc, ctx, partition, fieldPermutation, recordDescProvider, op); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMTreeIndexCompactOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMTreeIndexCompactOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMTreeIndexCompactOperatorDescriptor.java index 6835d33..e97c904 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMTreeIndexCompactOperatorDescriptor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/dataflow/LSMTreeIndexCompactOperatorDescriptor.java @@ -24,6 +24,7 @@ import org.apache.hyracks.api.dataflow.IOperatorNodePushable; import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider; import org.apache.hyracks.api.dataflow.value.ITypeTraits; +import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.job.IOperatorDescriptorRegistry; import org.apache.hyracks.dataflow.std.file.IFileSplitProvider; import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider; @@ -52,7 +53,7 @@ public class LSMTreeIndexCompactOperatorDescriptor extends AbstractTreeIndexOper @Override public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, - IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) { + IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException { return new LSMIndexCompactOperatorNodePushable(this, ctx, partition); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java index 8988797..090ca68 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbortRecoverLSMIndexFileManager.java @@ -19,9 +19,10 @@ package org.apache.hyracks.storage.am.lsm.common.impls; import org.apache.hyracks.api.io.FileReference; +import org.apache.hyracks.api.io.IIOManager; public class AbortRecoverLSMIndexFileManager extends AbstractLSMIndexFileManager { - public AbortRecoverLSMIndexFileManager(FileReference file) { - super(null, file, null); + public AbortRecoverLSMIndexFileManager(IIOManager ioManager, FileReference file) { + super(ioManager, null, file, null); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java index cb3a2db..981a2c3 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java @@ -29,6 +29,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.api.replication.IReplicationJob.ReplicationExecutionType; import org.apache.hyracks.api.replication.IReplicationJob.ReplicationOperation; import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter; @@ -51,7 +52,7 @@ import org.apache.hyracks.storage.common.file.IFileMapProvider; public abstract class AbstractLSMIndex implements ILSMIndexInternal { protected final ILSMHarness lsmHarness; - + protected final IIOManager ioManager; protected final ILSMIOOperationScheduler ioScheduler; protected final ILSMIOOperationCallback ioOpCallback; @@ -71,16 +72,17 @@ public abstract class AbstractLSMIndex implements ILSMIndexInternal { protected final LSMComponentFilterManager filterManager; protected final int[] filterFields; protected final boolean durable; - protected boolean isActivated; protected final AtomicBoolean[] flushRequests; protected boolean memoryComponentsAllocated = false; - public AbstractLSMIndex(List virtualBufferCaches, IBufferCache diskBufferCache, + public AbstractLSMIndex(IIOManager ioManager, List virtualBufferCaches, + IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager, int[] filterFields, boolean durable) { + this.ioManager = ioManager; this.virtualBufferCaches = virtualBufferCaches; this.diskBufferCache = diskBufferCache; this.diskFileMapProvider = diskFileMapProvider; @@ -92,12 +94,12 @@ public abstract class AbstractLSMIndex implements ILSMIndexInternal { this.filterFrameFactory = filterFrameFactory; this.filterManager = filterManager; this.filterFields = filterFields; - this.inactiveDiskComponents = new LinkedList(); + this.inactiveDiskComponents = new LinkedList<>(); this.durable = durable; lsmHarness = new LSMHarness(this, mergePolicy, opTracker, diskBufferCache.isReplicationEnabled()); isActivated = false; - diskComponents = new ArrayList(); - memoryComponents = new ArrayList(); + diskComponents = new ArrayList<>(); + memoryComponents = new ArrayList<>(); currentMutableComponentId = new AtomicInteger(); flushRequests = new AtomicBoolean[virtualBufferCaches.size()]; for (int i = 0; i < virtualBufferCaches.size(); i++) { @@ -106,10 +108,11 @@ public abstract class AbstractLSMIndex implements ILSMIndexInternal { } // The constructor used by external indexes - public AbstractLSMIndex(IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, + public AbstractLSMIndex(IIOManager ioManager, IBufferCache diskBufferCache, ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider, double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, boolean durable) { + this.ioManager = ioManager; this.diskBufferCache = diskBufferCache; this.diskFileMapProvider = diskFileMapProvider; this.fileManager = fileManager; @@ -119,8 +122,8 @@ public abstract class AbstractLSMIndex implements ILSMIndexInternal { this.durable = durable; lsmHarness = new ExternalIndexHarness(this, mergePolicy, opTracker, diskBufferCache.isReplicationEnabled()); isActivated = false; - diskComponents = new LinkedList(); - this.inactiveDiskComponents = new LinkedList(); + diskComponents = new LinkedList<>(); + this.inactiveDiskComponents = new LinkedList<>(); // Memory related objects are nulled this.virtualBufferCaches = null; memoryComponents = null; @@ -260,7 +263,7 @@ public abstract class AbstractLSMIndex implements ILSMIndexInternal { public void scheduleReplication(ILSMIndexOperationContext ctx, List lsmComponents, boolean bulkload, ReplicationOperation operation, LSMOperationType opType) throws HyracksDataException { //get set of files to be replicated for this component - Set componentFiles = new HashSet(); + Set componentFiles = new HashSet<>(); //get set of files to be replicated for each component for (ILSMComponent lsmComponent : lsmComponents) { @@ -275,8 +278,8 @@ public abstract class AbstractLSMIndex implements ILSMIndexInternal { } //create replication job and submit it - LSMIndexReplicationJob job = - new LSMIndexReplicationJob(this, ctx, componentFiles, operation, executionType, opType); + LSMIndexReplicationJob job = new LSMIndexReplicationJob(this, ctx, componentFiles, operation, executionType, + opType); try { diskBufferCache.getIOReplicationManager().submitJob(job); } catch (IOException e) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java index ed9cad0..b9f4cad 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexFileManager.java @@ -35,6 +35,7 @@ import java.util.List; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; +import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.storage.am.common.api.ITreeIndex; import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame; import org.apache.hyracks.storage.am.common.api.IndexException; @@ -56,9 +57,9 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage VALID } + protected final IIOManager ioManager; protected final IFileMapProvider fileMapProvider; - - // baseDir should reflect dataset name and partition name. + // baseDir should reflect dataset name and partition name and be absolute protected String baseDir; protected final Format formatter = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss-SSS"); protected final Comparator cmp = new FileNameComparator(); @@ -67,9 +68,10 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage private String prevTimestamp = null; - public AbstractLSMIndexFileManager(IFileMapProvider fileMapProvider, FileReference file, + public AbstractLSMIndexFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file, TreeIndexFactory treeFactory) { - this.baseDir = file.getFile().getPath(); + this.ioManager = ioManager; + this.baseDir = file.getFile().getAbsolutePath(); if (!baseDir.endsWith(System.getProperty("file.separator"))) { baseDir += System.getProperty("file.separator"); } @@ -122,8 +124,7 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage File dir = new File(baseDir); String[] files = dir.list(filter); for (String fileName : files) { - File file = new File(dir.getPath() + File.separator + fileName); - FileReference fileRef = new FileReference(file); + FileReference fileRef = ioManager.getFileRef(dir.getPath() + File.separator + fileName, false); if (treeFactory == null) { allFiles.add(new ComparableFileName(fileRef)); continue; @@ -132,7 +133,7 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage if (idxState == TreeIndexState.VALID) { allFiles.add(new ComparableFileName(fileRef)); } else if (idxState == TreeIndexState.INVALID) { - file.delete(); + fileRef.delete(); } } } @@ -141,7 +142,7 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage FilenameFilter filter, TreeIndexFactory treeFactory ) throws HyracksDataException, IndexException { - ArrayList tmpAllInvListsFiles = new ArrayList(); + ArrayList tmpAllInvListsFiles = new ArrayList<>(); cleanupAndGetValidFilesInternal(filter, treeFactory, tmpAllInvListsFiles); for (ComparableFileName cmpFileName : tmpAllInvListsFiles) { int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING); @@ -183,19 +184,19 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage } }; - protected FileReference createFlushFile(String relFlushFileName) { - return new FileReference(new File(relFlushFileName)); + protected FileReference createFlushFile(String relFlushFileName, boolean relative) throws HyracksDataException { + return ioManager.getFileRef(relFlushFileName, relative); } - protected FileReference createMergeFile(String relMergeFileName) { - return createFlushFile(relMergeFileName); + protected FileReference createMergeFile(String relMergeFileName, boolean relative) throws HyracksDataException { + return createFlushFile(relMergeFileName, relative); } @Override - public LSMComponentFileReferences getRelFlushFileReference() { + public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException { String ts = getCurrentTimestamp(); // Begin timestamp and end timestamp are identical since it is a flush - return new LSMComponentFileReferences(createFlushFile(baseDir + ts + SPLIT_STRING + ts), null, null); + return new LSMComponentFileReferences(createFlushFile(baseDir + ts + SPLIT_STRING + ts, false), null, null); } @Override @@ -205,13 +206,13 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage String[] lastTimestampRange = lastFileName.split(SPLIT_STRING); // Get the range of timestamps by taking the earliest and the latest timestamps return new LSMComponentFileReferences(createMergeFile(baseDir + firstTimestampRange[0] + SPLIT_STRING - + lastTimestampRange[1]), null, null); + + lastTimestampRange[1], false), null, null); } @Override public List cleanupAndGetValidFiles() throws HyracksDataException, IndexException { - List validFiles = new ArrayList(); - ArrayList allFiles = new ArrayList(); + List validFiles = new ArrayList<>(); + ArrayList allFiles = new ArrayList<>(); // Gather files and delete invalid files // There are two types of invalid files: @@ -232,7 +233,7 @@ public abstract class AbstractLSMIndexFileManager implements ILSMIndexFileManage // Sorts files names from earliest to latest timestamp. Collections.sort(allFiles); - List validComparableFiles = new ArrayList(); + List validComparableFiles = new ArrayList<>(); ComparableFileName last = allFiles.get(0); validComparableFiles.add(last); for (int i = 1; i < allFiles.size(); i++) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java index c6712c1..9d17b7e 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/BTreeFactory.java @@ -22,6 +22,7 @@ package org.apache.hyracks.storage.am.lsm.common.impls; import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; +import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.storage.am.btree.impls.BTree; import org.apache.hyracks.storage.am.common.api.IMetadataManagerFactory; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory; @@ -31,10 +32,10 @@ import org.apache.hyracks.storage.common.file.IFileMapProvider; public class BTreeFactory extends TreeIndexFactory { - public BTreeFactory(IBufferCache bufferCache, IFileMapProvider fileMapProvider, + public BTreeFactory(IIOManager ioManager, IBufferCache bufferCache, IFileMapProvider fileMapProvider, IMetadataManagerFactory freePageManagerFactory, ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory, IBinaryComparatorFactory[] cmpFactories, int fieldCount) { - super(bufferCache, fileMapProvider, freePageManagerFactory, interiorFrameFactory, leafFrameFactory, + super(ioManager, bufferCache, fileMapProvider, freePageManagerFactory, interiorFrameFactory, leafFrameFactory, cmpFactories, fieldCount); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java index 8897a2f..7ab70b6 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/IndexFactory.java @@ -19,27 +19,31 @@ package org.apache.hyracks.storage.am.lsm.common.impls; +import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; -import org.apache.hyracks.storage.am.common.api.IMetadataManagerFactory; +import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.storage.am.common.api.IIndex; +import org.apache.hyracks.storage.am.common.api.IMetadataManagerFactory; import org.apache.hyracks.storage.am.common.api.IndexException; import org.apache.hyracks.storage.common.buffercache.IBufferCache; import org.apache.hyracks.storage.common.file.IFileMapProvider; public abstract class IndexFactory { + protected final IIOManager ioManager; protected final IBufferCache bufferCache; protected final IFileMapProvider fileMapProvider; protected final IMetadataManagerFactory freePageManagerFactory; - public IndexFactory(IBufferCache bufferCache, IFileMapProvider fileMapProvider, + public IndexFactory(IIOManager ioManager, IBufferCache bufferCache, IFileMapProvider fileMapProvider, IMetadataManagerFactory freePageManagerFactory) { + this.ioManager = ioManager; this.bufferCache = bufferCache; this.fileMapProvider = fileMapProvider; this.freePageManagerFactory = freePageManagerFactory; } - public abstract T createIndexInstance(FileReference file) throws IndexException; + public abstract T createIndexInstance(FileReference file) throws IndexException, HyracksDataException; public IBufferCache getBufferCache() { return bufferCache; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java index 7f11a8d..ff04398 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TreeIndexFactory.java @@ -20,6 +20,7 @@ package org.apache.hyracks.storage.am.lsm.common.impls; import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; +import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.storage.am.common.api.IMetadataManagerFactory; import org.apache.hyracks.storage.am.common.api.ITreeIndex; import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory; @@ -33,10 +34,10 @@ public abstract class TreeIndexFactory extends IndexFactor protected final IBinaryComparatorFactory[] cmpFactories; protected final int fieldCount; - public TreeIndexFactory(IBufferCache bufferCache, IFileMapProvider fileMapProvider, + public TreeIndexFactory(IIOManager ioManager, IBufferCache bufferCache, IFileMapProvider fileMapProvider, IMetadataManagerFactory freePageManagerFactory, ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory, IBinaryComparatorFactory[] cmpFactories, int fieldCount) { - super(bufferCache, fileMapProvider, freePageManagerFactory); + super(ioManager, bufferCache, fileMapProvider, freePageManagerFactory); this.interiorFrameFactory = interiorFrameFactory; this.leafFrameFactory = leafFrameFactory; this.cmpFactories = cmpFactories; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexBulkLoadOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexBulkLoadOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexBulkLoadOperatorDescriptor.java index f1070f5..721d4aa 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexBulkLoadOperatorDescriptor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexBulkLoadOperatorDescriptor.java @@ -25,6 +25,7 @@ import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider; import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.api.dataflow.value.RecordDescriptor; +import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.job.IOperatorDescriptorRegistry; import org.apache.hyracks.dataflow.std.file.IFileSplitProvider; import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider; @@ -64,7 +65,7 @@ public class LSMInvertedIndexBulkLoadOperatorDescriptor extends AbstractLSMInver @Override public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, - IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) { + IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException { return new IndexBulkLoadOperatorNodePushable(this, ctx, partition, fieldPermutation, 1.0f, verifyInput, numElementsHint, checkIfEmptyIndex, recordDescProvider); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexCompactOperator.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexCompactOperator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexCompactOperator.java index 613eaf0..0956c2e 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexCompactOperator.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexCompactOperator.java @@ -24,6 +24,7 @@ import org.apache.hyracks.api.dataflow.IOperatorNodePushable; import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider; import org.apache.hyracks.api.dataflow.value.ITypeTraits; +import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.job.IOperatorDescriptorRegistry; import org.apache.hyracks.dataflow.std.file.IFileSplitProvider; import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider; @@ -53,7 +54,7 @@ public class LSMInvertedIndexCompactOperator extends AbstractLSMInvertedIndexOpe @Override public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, - IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) { + IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException { return new LSMIndexCompactOperatorNodePushable(this, ctx, partition); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexCreateOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexCreateOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexCreateOperatorDescriptor.java index f55777f..75290fb 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexCreateOperatorDescriptor.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexCreateOperatorDescriptor.java @@ -24,6 +24,7 @@ import org.apache.hyracks.api.dataflow.IOperatorNodePushable; import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider; import org.apache.hyracks.api.dataflow.value.ITypeTraits; +import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.job.IOperatorDescriptorRegistry; import org.apache.hyracks.dataflow.std.file.IFileSplitProvider; import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider; @@ -55,7 +56,7 @@ public class LSMInvertedIndexCreateOperatorDescriptor extends AbstractLSMInverte @Override public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, - IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) { + IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException { return new IndexCreateOperatorNodePushable(this, ctx, partition); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java index 5abf4ef..8f9e843 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelper.java @@ -24,9 +24,11 @@ import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.storage.am.common.api.IIndex; import org.apache.hyracks.storage.am.common.api.IndexException; import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor; +import org.apache.hyracks.storage.am.common.util.IndexFileNameUtil; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler; import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy; @@ -50,7 +52,8 @@ public final class LSMInvertedIndexDataflowHelper extends AbstractLSMIndexDatafl ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory, int[] invertedIndexFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, - int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable) { + int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable) + throws HyracksDataException { this(opDesc, ctx, partition, virtualBufferCaches, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackFactory, invertedIndexFields, filterTypeTraits, filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, @@ -63,7 +66,7 @@ public final class LSMInvertedIndexDataflowHelper extends AbstractLSMIndexDatafl ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory, int[] invertedIndexFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, - boolean durable) { + boolean durable) throws HyracksDataException { super(opDesc, ctx, partition, virtualBufferCaches, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackFactory, filterTypeTraits, filterCmpFactories, filterFields, durable); this.invertedIndexFields = invertedIndexFields; @@ -77,11 +80,14 @@ public final class LSMInvertedIndexDataflowHelper extends AbstractLSMIndexDatafl try { IBufferCache diskBufferCache = opDesc.getStorageManager().getBufferCache(ctx); IFileMapProvider diskFileMapProvider = opDesc.getStorageManager().getFileMapProvider(ctx); - LSMInvertedIndex invIndex = InvertedIndexUtils.createLSMInvertedIndex(virtualBufferCaches, + FileReference fileRef = IndexFileNameUtil.getIndexAbsoluteFileRef(invIndexOpDesc, ctx.getTaskAttemptId() + .getTaskId().getPartition(), ctx.getIOManager()); + LSMInvertedIndex invIndex = InvertedIndexUtils.createLSMInvertedIndex(ctx.getIOManager(), + virtualBufferCaches, diskFileMapProvider, invIndexOpDesc.getInvListsTypeTraits(), invIndexOpDesc.getInvListsComparatorFactories(), invIndexOpDesc.getTokenTypeTraits(), invIndexOpDesc.getTokenComparatorFactories(), invIndexOpDesc.getTokenizerFactory(), - diskBufferCache, file.getFile().getPath(), bloomFilterFalsePositiveRate, mergePolicy, + diskBufferCache, fileRef.getFile().getPath(), bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory.getOperationTracker(ctx), ioScheduler, ioOpCallbackFactory.createIOOperationCallback(), invertedIndexFields, filterTypeTraits, filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps, http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java index b30bf5b..f1cc153 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexDataflowHelperFactory.java @@ -24,6 +24,7 @@ import java.util.Map; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.dataflow.value.ITypeTraits; +import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor; import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelper; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory; @@ -57,7 +58,7 @@ public class LSMInvertedIndexDataflowHelperFactory extends AbstractLSMIndexDataf @Override public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, - int partition) { + int partition) throws HyracksDataException { return new LSMInvertedIndexDataflowHelper(opDesc, ctx, partition, virtualBufferCacheProvider.getVirtualBufferCaches(ctx, opDesc.getFileSplitProvider()), bloomFilterFalsePositiveRate, mergePolicyFactory.createMergePolicy(mergePolicyProperties, ctx), opTrackerFactory, http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexInsertUpdateDeleteOperator.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexInsertUpdateDeleteOperator.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexInsertUpdateDeleteOperator.java index 69770e6..252a4fb 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexInsertUpdateDeleteOperator.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexInsertUpdateDeleteOperator.java @@ -25,6 +25,7 @@ import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider; import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.api.dataflow.value.RecordDescriptor; +import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.job.IOperatorDescriptorRegistry; import org.apache.hyracks.dataflow.std.file.IFileSplitProvider; import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider; @@ -62,7 +63,7 @@ public class LSMInvertedIndexInsertUpdateDeleteOperator extends AbstractLSMInver @Override public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, - IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) { + IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException { return new LSMIndexInsertUpdateDeleteOperatorNodePushable(this, ctx, partition, fieldPermutation, recordDescProvider, op); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorNodePushable.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorNodePushable.java index f218845..09893fb 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorNodePushable.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexSearchOperatorNodePushable.java @@ -19,7 +19,6 @@ package org.apache.hyracks.storage.am.lsm.invertedindex.dataflow; -import org.apache.hyracks.api.comm.IFrame; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider; import org.apache.hyracks.api.exceptions.HyracksDataException; @@ -38,7 +37,8 @@ public class LSMInvertedIndexSearchOperatorNodePushable extends IndexSearchOpera public LSMInvertedIndexSearchOperatorNodePushable(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition, IRecordDescriptorProvider recordDescProvider, int queryFieldIndex, - IInvertedIndexSearchModifier searchModifier, int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes) { + IInvertedIndexSearchModifier searchModifier, int[] minFilterFieldIndexes, int[] maxFilterFieldIndexes) + throws HyracksDataException { super(opDesc, ctx, partition, recordDescProvider, minFilterFieldIndexes, maxFilterFieldIndexes); this.searchModifier = searchModifier; this.queryFieldIndex = queryFieldIndex; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java index a7489e7..c8ad60b 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelper.java @@ -24,9 +24,11 @@ import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.storage.am.common.api.IIndex; import org.apache.hyracks.storage.am.common.api.IndexException; import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor; +import org.apache.hyracks.storage.am.common.util.IndexFileNameUtil; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler; import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy; @@ -50,7 +52,8 @@ public final class PartitionedLSMInvertedIndexDataflowHelper extends AbstractLSM ILSMOperationTrackerProvider opTrackerFactory, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory, int[] invertedIndexFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, - int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable) { + int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable) + throws HyracksDataException { this(opDesc, ctx, partition, virtualBufferCache, DEFAULT_BLOOM_FILTER_FALSE_POSITIVE_RATE, mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackFactory, invertedIndexFields, filterTypeTraits, filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, @@ -63,7 +66,7 @@ public final class PartitionedLSMInvertedIndexDataflowHelper extends AbstractLSM ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory, int[] invertedIndexFields, ITypeTraits[] filterTypeTraits, IBinaryComparatorFactory[] filterCmpFactories, int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, - boolean durable) { + boolean durable) throws HyracksDataException { super(opDesc, ctx, partition, virtualBufferCaches, bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory, ioScheduler, ioOpCallbackFactory, filterTypeTraits, filterCmpFactories, filterFields, durable); this.invertedIndexFields = invertedIndexFields; @@ -77,11 +80,14 @@ public final class PartitionedLSMInvertedIndexDataflowHelper extends AbstractLSM try { IBufferCache diskBufferCache = opDesc.getStorageManager().getBufferCache(ctx); IFileMapProvider diskFileMapProvider = opDesc.getStorageManager().getFileMapProvider(ctx); - PartitionedLSMInvertedIndex invIndex = InvertedIndexUtils.createPartitionedLSMInvertedIndex( + FileReference fileRef = IndexFileNameUtil.getIndexAbsoluteFileRef(invIndexOpDesc, ctx.getTaskAttemptId() + .getTaskId().getPartition(), ctx.getIOManager()); + PartitionedLSMInvertedIndex invIndex = InvertedIndexUtils.createPartitionedLSMInvertedIndex(ctx + .getIOManager(), virtualBufferCaches, diskFileMapProvider, invIndexOpDesc.getInvListsTypeTraits(), invIndexOpDesc.getInvListsComparatorFactories(), invIndexOpDesc.getTokenTypeTraits(), invIndexOpDesc.getTokenComparatorFactories(), invIndexOpDesc.getTokenizerFactory(), - diskBufferCache, file.getFile().getPath(), bloomFilterFalsePositiveRate, mergePolicy, + diskBufferCache, fileRef.getFile().getAbsolutePath(), bloomFilterFalsePositiveRate, mergePolicy, opTrackerFactory.getOperationTracker(ctx), ioScheduler, ioOpCallbackFactory.createIOOperationCallback(), invertedIndexFields, filterTypeTraits, filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps, http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java index 4a6194b..bc1fc5b 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/PartitionedLSMInvertedIndexDataflowHelperFactory.java @@ -24,6 +24,7 @@ import java.util.Map; import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.dataflow.value.ITypeTraits; +import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor; import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelper; import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory; @@ -57,7 +58,7 @@ public class PartitionedLSMInvertedIndexDataflowHelperFactory extends AbstractLS @Override public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, - int partition) { + int partition) throws HyracksDataException { return new PartitionedLSMInvertedIndexDataflowHelper(opDesc, ctx, partition, virtualBufferCacheProvider.getVirtualBufferCaches(ctx, opDesc.getFileSplitProvider()), bloomFilterFalsePositiveRate, mergePolicyFactory.createMergePolicy(mergePolicyProperties, ctx), opTrackerFactory, http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java index bf6e9b5..4478feb 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java @@ -18,7 +18,6 @@ */ package org.apache.hyracks.storage.am.lsm.invertedindex.impls; -import java.io.File; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -28,6 +27,7 @@ import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.dataflow.value.ITypeTraits; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; +import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.data.std.primitive.IntegerPointable; import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference; import org.apache.hyracks.storage.am.bloomfilter.impls.BloomCalculations; @@ -105,7 +105,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex private final int[] filterFieldsForNonBulkLoadOps; private final int[] invertedIndexFieldsForNonBulkLoadOps; - public LSMInvertedIndex(List virtualBufferCaches, + public LSMInvertedIndex(IIOManager ioManager, List virtualBufferCaches, OnDiskInvertedIndexFactory diskInvIndexFactory, BTreeFactory deletedKeysBTreeFactory, BloomFilterFactory bloomFilterFactory, ILSMComponentFilterFactory filterFactory, ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager, @@ -115,8 +115,8 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex IBinaryTokenizerFactory tokenizerFactory, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, - boolean durable) throws IndexException { - super(virtualBufferCaches, diskInvIndexFactory.getBufferCache(), fileManager, diskFileMapProvider, + boolean durable) throws IndexException, HyracksDataException { + super(ioManager, virtualBufferCaches, diskInvIndexFactory.getBufferCache(), fileManager, diskFileMapProvider, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory, filterManager, filterFields, durable); @@ -139,8 +139,8 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex BTree deleteKeysBTree = BTreeUtils.createBTree(virtualBufferCache, new VirtualMetaDataPageManager(virtualBufferCache.getNumPages()), virtualBufferCache.getFileMapProvider(), invListTypeTraits, invListCmpFactories, - BTreeLeafFrameType.REGULAR_NSM, - new FileReference(new File(fileManager.getBaseDir() + "_virtual_del_" + i))); + BTreeLeafFrameType.REGULAR_NSM, ioManager.getFileRef(fileManager.getBaseDir() + "_virtual_del_" + i, + false)); LSMInvertedIndexMemoryComponent mutableComponent = new LSMInvertedIndexMemoryComponent(memInvIndex, deleteKeysBTree, virtualBufferCache, i == 0 ? true : false, filterFactory == null ? null : filterFactory.createLSMComponentFilter()); @@ -375,8 +375,8 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex List operationalComponents = ictx.getComponentHolder(); int numComponents = operationalComponents.size(); boolean includeMutableComponent = false; - ArrayList indexAccessors = new ArrayList(numComponents); - ArrayList deletedKeysBTreeAccessors = new ArrayList(numComponents); + ArrayList indexAccessors = new ArrayList<>(numComponents); + ArrayList deletedKeysBTreeAccessors = new ArrayList<>(numComponents); for (int i = 0; i < operationalComponents.size(); i++) { ILSMComponent component = operationalComponents.get(i); @@ -491,7 +491,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex scanCursor.close(); } if (component.getLSMComponentFilter() != null) { - List filterTuples = new ArrayList(); + List filterTuples = new ArrayList<>(); filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple()); filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple()); filterManager.updateFilterInfo(component.getLSMComponentFilter(), filterTuples); @@ -643,7 +643,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex cursor.close(); } if (component.getLSMComponentFilter() != null) { - List filterTuples = new ArrayList(); + List filterTuples = new ArrayList<>(); for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) { ITupleReference min = mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple(); ITupleReference max = mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple(); @@ -801,10 +801,10 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex } protected InMemoryInvertedIndex createInMemoryInvertedIndex(IVirtualBufferCache virtualBufferCache, - IVirtualMetaDataPageManager virtualFreePageManager, int id) throws IndexException { + IVirtualMetaDataPageManager virtualFreePageManager, int id) throws IndexException, HyracksDataException { return InvertedIndexUtils.createInMemoryBTreeInvertedindex(virtualBufferCache, virtualFreePageManager, - invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, - new FileReference(new File(fileManager.getBaseDir() + "_virtual_vocab_" + id))); + invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, ioManager + .getFileRef(fileManager.getBaseDir() + "_virtual_vocab_" + id, false)); } protected LSMInvertedIndexDiskComponent createDiskInvIndexComponent(ILSMComponentFactory factory, @@ -929,7 +929,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex @Override public Set getLSMComponentPhysicalFiles(ILSMComponent lsmComponent) { - Set files = new HashSet(); + Set files = new HashSet<>(); LSMInvertedIndexDiskComponent invIndexComponent = (LSMInvertedIndexDiskComponent) lsmComponent; OnDiskInvertedIndex invIndex = (OnDiskInvertedIndex) invIndexComponent.getInvIndex(); files.add(invIndex.getInvListsFile().getFile().getAbsolutePath()); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java index 6c699da..63772d2 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFileManager.java @@ -29,6 +29,7 @@ import java.util.List; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; +import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.storage.am.common.api.IndexException; import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager; import org.apache.hyracks.storage.am.lsm.common.impls.BTreeFactory; @@ -66,19 +67,20 @@ public class LSMInvertedIndexFileManager extends AbstractLSMIndexFileManager imp } }; - public LSMInvertedIndexFileManager(IFileMapProvider fileMapProvider, FileReference file, BTreeFactory btreeFactory) { - super(fileMapProvider, file, null); + public LSMInvertedIndexFileManager(IIOManager ioManager, IFileMapProvider fileMapProvider, FileReference file, + BTreeFactory btreeFactory) { + super(ioManager, fileMapProvider, file, null); this.btreeFactory = btreeFactory; } @Override - public LSMComponentFileReferences getRelFlushFileReference() { + public LSMComponentFileReferences getRelFlushFileReference() throws HyracksDataException { String ts = getCurrentTimestamp(); String baseName = baseDir + ts + SPLIT_STRING + ts; // Begin timestamp and end timestamp are identical since it is a flush - return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + DICT_BTREE_SUFFIX), - createFlushFile(baseName + SPLIT_STRING + DELETED_KEYS_BTREE_SUFFIX), createFlushFile(baseName - + SPLIT_STRING + BLOOM_FILTER_STRING)); + return new LSMComponentFileReferences(createFlushFile(baseName + SPLIT_STRING + DICT_BTREE_SUFFIX, false), + createFlushFile(baseName + SPLIT_STRING + DELETED_KEYS_BTREE_SUFFIX, false), createFlushFile(baseName + + SPLIT_STRING + BLOOM_FILTER_STRING, false)); } @Override @@ -89,22 +91,22 @@ public class LSMInvertedIndexFileManager extends AbstractLSMIndexFileManager imp String baseName = baseDir + firstTimestampRange[0] + SPLIT_STRING + lastTimestampRange[1]; // Get the range of timestamps by taking the earliest and the latest timestamps - return new LSMComponentFileReferences(createMergeFile(baseName + SPLIT_STRING + DICT_BTREE_SUFFIX), - createMergeFile(baseName + SPLIT_STRING + DELETED_KEYS_BTREE_SUFFIX), createMergeFile(baseName - + SPLIT_STRING + BLOOM_FILTER_STRING)); + return new LSMComponentFileReferences(createMergeFile(baseName + SPLIT_STRING + DICT_BTREE_SUFFIX, false), + createMergeFile(baseName + SPLIT_STRING + DELETED_KEYS_BTREE_SUFFIX, false), createMergeFile(baseName + + SPLIT_STRING + BLOOM_FILTER_STRING, false)); } @Override public List cleanupAndGetValidFiles() throws HyracksDataException, IndexException { - List validFiles = new ArrayList(); - ArrayList allDictBTreeFiles = new ArrayList(); - ArrayList allInvListsFiles = new ArrayList(); - ArrayList allDeletedKeysBTreeFiles = new ArrayList(); - ArrayList allBloomFilterFiles = new ArrayList(); + List validFiles = new ArrayList<>(); + ArrayList allDictBTreeFiles = new ArrayList<>(); + ArrayList allInvListsFiles = new ArrayList<>(); + ArrayList allDeletedKeysBTreeFiles = new ArrayList<>(); + ArrayList allBloomFilterFiles = new ArrayList<>(); // Gather files. cleanupAndGetValidFilesInternal(deletedKeysBTreeFilter, btreeFactory, allDeletedKeysBTreeFiles); - HashSet deletedKeysBTreeFilesSet = new HashSet(); + HashSet deletedKeysBTreeFilesSet = new HashSet<>(); for (ComparableFileName cmpFileName : allDeletedKeysBTreeFiles) { int index = cmpFileName.fileName.lastIndexOf(SPLIT_STRING); deletedKeysBTreeFilesSet.add(cmpFileName.fileName.substring(0, index)); @@ -141,15 +143,15 @@ public class LSMInvertedIndexFileManager extends AbstractLSMIndexFileManager imp Collections.sort(allDictBTreeFiles); Collections.sort(allBloomFilterFiles); - List validComparableDictBTreeFiles = new ArrayList(); + List validComparableDictBTreeFiles = new ArrayList<>(); ComparableFileName lastDictBTree = allDictBTreeFiles.get(0); validComparableDictBTreeFiles.add(lastDictBTree); - List validComparableDeletedKeysBTreeFiles = new ArrayList(); + List validComparableDeletedKeysBTreeFiles = new ArrayList<>(); ComparableFileName lastDeletedKeysBTree = allDeletedKeysBTreeFiles.get(0); validComparableDeletedKeysBTreeFiles.add(lastDeletedKeysBTree); - List validComparableBloomFilterFiles = new ArrayList(); + List validComparableBloomFilterFiles = new ArrayList<>(); ComparableFileName lastBloomFilter = allBloomFilterFiles.get(0); validComparableBloomFilterFiles.add(lastBloomFilter); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java index 7aecbd1..382d065 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/PartitionedLSMInvertedIndex.java @@ -19,12 +19,12 @@ package org.apache.hyracks.storage.am.lsm.invertedindex.impls; -import java.io.File; import java.util.List; import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.dataflow.value.ITypeTraits; -import org.apache.hyracks.api.io.FileReference; +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilterFactory; import org.apache.hyracks.storage.am.common.api.IVirtualMetaDataPageManager; import org.apache.hyracks.storage.am.common.api.IndexException; @@ -46,7 +46,7 @@ import org.apache.hyracks.storage.common.file.IFileMapProvider; public class PartitionedLSMInvertedIndex extends LSMInvertedIndex { - public PartitionedLSMInvertedIndex(List virtualBufferCaches, + public PartitionedLSMInvertedIndex(IIOManager ioManager, List virtualBufferCaches, OnDiskInvertedIndexFactory diskInvIndexFactory, BTreeFactory deletedKeysBTreeFactory, BloomFilterFactory bloomFilterFactory, ILSMComponentFilterFactory filterFactory, ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager filterManager, @@ -57,8 +57,9 @@ public class PartitionedLSMInvertedIndex extends LSMInvertedIndex { ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback, int[] invertedIndexFields, int[] filterFields, int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps, boolean durable) - throws IndexException { - super(virtualBufferCaches, diskInvIndexFactory, deletedKeysBTreeFactory, bloomFilterFactory, filterFactory, + throws IndexException, HyracksDataException { + super(ioManager, virtualBufferCaches, diskInvIndexFactory, deletedKeysBTreeFactory, bloomFilterFactory, + filterFactory, filterFrameFactory, filterManager, bloomFilterFalsePositiveRate, fileManager, diskFileMapProvider, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, mergePolicy, opTracker, ioScheduler, ioOpCallback, invertedIndexFields, filterFields, @@ -67,10 +68,10 @@ public class PartitionedLSMInvertedIndex extends LSMInvertedIndex { @Override protected InMemoryInvertedIndex createInMemoryInvertedIndex(IVirtualBufferCache virtualBufferCache, - IVirtualMetaDataPageManager virtualFreePageManager, int id) throws IndexException { + IVirtualMetaDataPageManager virtualFreePageManager, int id) throws IndexException, HyracksDataException { return InvertedIndexUtils.createPartitionedInMemoryBTreeInvertedindex(virtualBufferCache, virtualFreePageManager, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, - tokenizerFactory, new FileReference(new File(fileManager.getBaseDir() + "_virtual_vocab_" + id))); + tokenizerFactory, ioManager.getFileRef(fileManager.getBaseDir() + "_virtual_vocab_" + id, false)); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java index f926bba..ab852ed 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndexFactory.java @@ -18,11 +18,11 @@ */ package org.apache.hyracks.storage.am.lsm.invertedindex.ondisk; -import java.io.File; - import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.dataflow.value.ITypeTraits; +import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; +import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.storage.am.common.api.IndexException; import org.apache.hyracks.storage.am.lsm.common.impls.IndexFactory; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex; @@ -41,11 +41,11 @@ public class OnDiskInvertedIndexFactory extends IndexFactory { protected final IBinaryComparatorFactory[] tokenCmpFactories; protected final IInvertedIndexFileNameMapper fileNameMapper; - public OnDiskInvertedIndexFactory(IBufferCache bufferCache, IFileMapProvider fileMapProvider, + public OnDiskInvertedIndexFactory(IIOManager ioManager, IBufferCache bufferCache, IFileMapProvider fileMapProvider, IInvertedListBuilderFactory invListBuilderFactory, ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, IInvertedIndexFileNameMapper fileNameMapper) { - super(bufferCache, fileMapProvider, null); + super(ioManager, bufferCache, fileMapProvider, null); this.invListBuilderFactory = invListBuilderFactory; this.invListTypeTraits = invListTypeTraits; this.invListCmpFactories = invListCmpFactories; @@ -55,9 +55,9 @@ public class OnDiskInvertedIndexFactory extends IndexFactory { } @Override - public IInvertedIndex createIndexInstance(FileReference dictBTreeFile) throws IndexException { + public IInvertedIndex createIndexInstance(FileReference dictBTreeFile) throws IndexException, HyracksDataException { String invListsFilePath = fileNameMapper.getInvListsFilePath(dictBTreeFile.getFile().getPath()); - FileReference invListsFile = new FileReference(new File(invListsFilePath)); + FileReference invListsFile = ioManager.getFileRef(invListsFilePath, false); IInvertedListBuilder invListBuilder = invListBuilderFactory.create(); return new OnDiskInvertedIndex(bufferCache, fileMapProvider, invListBuilder, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, dictBTreeFile, invListsFile); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/163c3be3/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java index b43cc39..85adb45 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/PartitionedOnDiskInvertedIndexFactory.java @@ -18,11 +18,11 @@ */ package org.apache.hyracks.storage.am.lsm.invertedindex.ondisk; -import java.io.File; - import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory; import org.apache.hyracks.api.dataflow.value.ITypeTraits; +import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; +import org.apache.hyracks.api.io.IIOManager; import org.apache.hyracks.storage.am.common.api.IndexException; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndex; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexFileNameMapper; @@ -33,18 +33,20 @@ import org.apache.hyracks.storage.common.file.IFileMapProvider; public class PartitionedOnDiskInvertedIndexFactory extends OnDiskInvertedIndexFactory { - public PartitionedOnDiskInvertedIndexFactory(IBufferCache bufferCache, IFileMapProvider fileMapProvider, + public PartitionedOnDiskInvertedIndexFactory(IIOManager ioManager, IBufferCache bufferCache, + IFileMapProvider fileMapProvider, IInvertedListBuilderFactory invListBuilderFactory, ITypeTraits[] invListTypeTraits, IBinaryComparatorFactory[] invListCmpFactories, ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories, IInvertedIndexFileNameMapper fileNameMapper) { - super(bufferCache, fileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories, tokenTypeTraits, + super(ioManager, bufferCache, fileMapProvider, invListBuilderFactory, invListTypeTraits, invListCmpFactories, + tokenTypeTraits, tokenCmpFactories, fileNameMapper); } @Override - public IInvertedIndex createIndexInstance(FileReference dictBTreeFile) throws IndexException { - String invListsFilePath = fileNameMapper.getInvListsFilePath(dictBTreeFile.getFile().getPath()); - FileReference invListsFile = new FileReference(new File(invListsFilePath)); + public IInvertedIndex createIndexInstance(FileReference dictBTreeFile) throws IndexException, HyracksDataException { + String invListsFilePath = fileNameMapper.getInvListsFilePath(dictBTreeFile.getFile().getAbsolutePath()); + FileReference invListsFile = ioManager.getFileRef(invListsFilePath, false); IInvertedListBuilder invListBuilder = invListBuilderFactory.create(); return new PartitionedOnDiskInvertedIndex(bufferCache, fileMapProvider, invListBuilder, invListTypeTraits, invListCmpFactories, tokenTypeTraits, tokenCmpFactories, dictBTreeFile, invListsFile);