asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From amo...@apache.org
Subject [3/6] asterixdb git commit: Fix upsert deadlock and upsert with filtered primary only
Date Thu, 25 May 2017 06:04:10 GMT
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index 3d7b716..89d8f0b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -42,9 +42,7 @@ import org.apache.hyracks.storage.am.btree.impls.BTree.BTreeAccessor;
 import org.apache.hyracks.storage.am.btree.impls.RangePredicate;
 import org.apache.hyracks.storage.am.btree.util.BTreeUtils;
 import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
-import org.apache.hyracks.storage.am.common.impls.AbstractSearchPredicate;
 import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
-import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
 import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.LSMComponentType;
@@ -58,14 +56,15 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
 import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
 import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
 import org.apache.hyracks.storage.am.lsm.common.freepage.VirtualFreePageManager;
 import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
+import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext;
 import org.apache.hyracks.storage.am.lsm.common.impls.BTreeFactory;
-import org.apache.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
 import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
 import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager;
 import org.apache.hyracks.storage.am.lsm.common.impls.LSMIndexSearchCursor;
@@ -103,7 +102,6 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
     protected final IBinaryComparatorFactory[] invListCmpFactories;
     protected final ITypeTraits[] tokenTypeTraits;
     protected final IBinaryComparatorFactory[] tokenCmpFactories;
-    private final int[] invertedIndexFields;
     private final int[] filterFieldsForNonBulkLoadOps;
     private final int[] invertedIndexFieldsForNonBulkLoadOps;
 
@@ -120,14 +118,13 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
             boolean durable) throws HyracksDataException {
         super(ioManager, virtualBufferCaches, diskInvIndexFactory.getBufferCache(), fileManager, diskFileMapProvider,
                 bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory,
-                filterManager, filterFields, durable);
+                filterManager, filterFields, durable, filterFactory, invertedIndexFields);
 
         this.tokenizerFactory = tokenizerFactory;
         this.invListTypeTraits = invListTypeTraits;
         this.invListCmpFactories = invListCmpFactories;
         this.tokenTypeTraits = tokenTypeTraits;
         this.tokenCmpFactories = tokenCmpFactories;
-        this.invertedIndexFields = invertedIndexFields;
         this.filterFieldsForNonBulkLoadOps = filterFieldsForNonBulkLoadOps;
         this.invertedIndexFieldsForNonBulkLoadOps = invertedIndexFieldsForNonBulkLoadOps;
 
@@ -151,151 +148,53 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
     }
 
     @Override
-    public synchronized void create() throws HyracksDataException {
-        if (isActivated) {
-            throw new HyracksDataException("Failed to create the index since it is activated.");
-        }
-
-        fileManager.deleteDirs();
-        fileManager.createDirs();
-        diskComponents.clear();
-    }
-
-    @Override
-    public synchronized void activate() throws HyracksDataException {
-        if (isActivated) {
-            throw new HyracksDataException("Failed to activate the index since it is already activated.");
-        }
-        List<ILSMDiskComponent> immutableComponents = diskComponents;
-        immutableComponents.clear();
-        List<LSMComponentFileReferences> validFileReferences = fileManager.cleanupAndGetValidFiles();
-        for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) {
-            LSMInvertedIndexDiskComponent component;
-            component = createDiskInvIndexComponent(componentFactory,
-                    lsmComonentFileReference.getInsertIndexFileReference(),
-                    lsmComonentFileReference.getDeleteIndexFileReference(),
-                    lsmComonentFileReference.getBloomFilterFileReference(), false);
-            immutableComponents.add(component);
-        }
-        isActivated = true;
+    protected ILSMDiskComponent loadComponent(LSMComponentFileReferences refs) throws HyracksDataException {
+        return createDiskInvIndexComponent(componentFactory, refs.getInsertIndexFileReference(),
+                refs.getDeleteIndexFileReference(), refs.getBloomFilterFileReference(), false);
     }
 
     @Override
-    public void clear() throws HyracksDataException {
-        if (!isActivated) {
-            throw new HyracksDataException("Failed to clear the index since it is not activated.");
-        }
-
-        clearMemoryComponents();
-        List<ILSMDiskComponent> immutableComponents = diskComponents;
-        for (ILSMDiskComponent c : immutableComponents) {
-            LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c;
-            component.getBloomFilter().deactivate();
-            component.getInvIndex().deactivate();
-            component.getDeletedKeysBTree().deactivate();
-            component.getBloomFilter().destroy();
-            component.getInvIndex().destroy();
-            component.getDeletedKeysBTree().destroy();
-        }
-        immutableComponents.clear();
+    protected void clearDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+        LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c;
+        component.getBloomFilter().deactivate();
+        component.getInvIndex().deactivate();
+        component.getDeletedKeysBTree().deactivate();
+        component.getBloomFilter().destroy();
+        component.getInvIndex().destroy();
+        component.getDeletedKeysBTree().destroy();
     }
 
     @Override
-    public synchronized void deactivate(boolean flushOnExit) throws HyracksDataException {
-        if (!isActivated) {
-            throw new HyracksDataException("Failed to deactivate the index since it is already deactivated.");
-        }
-        if (flushOnExit) {
-            BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(ioOpCallback);
-            ILSMIndexAccessor accessor = createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
-            accessor.scheduleFlush(cb);
-            try {
-                cb.waitForIO();
-            } catch (InterruptedException e) {
-                throw new HyracksDataException(e);
-            }
-        }
-
-        List<ILSMDiskComponent> immutableComponents = diskComponents;
-        for (ILSMDiskComponent c : immutableComponents) {
-            LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c;
-            component.getBloomFilter().deactivate();
-            component.getInvIndex().deactivate();
-            component.getDeletedKeysBTree().deactivate();
-        }
-        deallocateMemoryComponents();
-        isActivated = false;
+    protected void deactivateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+        LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
+        mutableComponent.getInvIndex().deactivate();
+        mutableComponent.getDeletedKeysBTree().deactivate();
+        mutableComponent.getInvIndex().destroy();
+        mutableComponent.getDeletedKeysBTree().destroy();
+        ((IVirtualBufferCache) mutableComponent.getInvIndex().getBufferCache()).close();
     }
 
     @Override
-    public synchronized void deactivate() throws HyracksDataException {
-        deactivate(true);
+    protected void deactivateDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+        LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c;
+        component.getBloomFilter().deactivate();
+        component.getInvIndex().deactivate();
+        component.getDeletedKeysBTree().deactivate();
     }
 
     @Override
-    public synchronized void destroy() throws HyracksDataException {
-        if (isActivated) {
-            throw new HyracksDataException("Failed to destroy the index since it is activated.");
-        }
-
-        for (ILSMComponent c : memoryComponents) {
-            LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
-            mutableComponent.getInvIndex().destroy();
-            mutableComponent.getDeletedKeysBTree().destroy();
-        }
-        List<ILSMDiskComponent> immutableComponents = diskComponents;
-        for (ILSMDiskComponent c : immutableComponents) {
-            LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c;
-            component.getInvIndex().destroy();
-            component.getDeletedKeysBTree().destroy();
-            component.getBloomFilter().destroy();
-        }
-        fileManager.deleteDirs();
+    protected void destroyDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+        LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c;
+        component.getInvIndex().destroy();
+        component.getDeletedKeysBTree().destroy();
+        component.getBloomFilter().destroy();
     }
 
     @Override
-    public void getOperationalComponents(ILSMIndexOperationContext ctx) throws HyracksDataException {
-        List<ILSMDiskComponent> immutableComponents = diskComponents;
-        List<ILSMComponent> operationalComponents = ctx.getComponentHolder();
-        int cmc = currentMutableComponentId.get();
-        ctx.setCurrentMutableComponentId(cmc);
-        operationalComponents.clear();
-        switch (ctx.getOperation()) {
-            case FLUSH:
-            case DELETE:
-            case INSERT:
-            case UPSERT:
-                operationalComponents.add(memoryComponents.get(cmc));
-                break;
-            case SEARCH:
-                if (memoryComponentsAllocated) {
-                    addOperationalMutableComponents(operationalComponents);
-                }
-                if (filterManager != null) {
-                    for (ILSMComponent c : immutableComponents) {
-                        if (c.getLSMComponentFilter().satisfy(
-                                ((AbstractSearchPredicate) ctx.getSearchPredicate()).getMinFilterTuple(),
-                                ((AbstractSearchPredicate) ctx.getSearchPredicate()).getMaxFilterTuple(),
-                                ((LSMInvertedIndexOpContext) ctx).getFilterCmp())) {
-                            operationalComponents.add(c);
-                        }
-                    }
-                } else {
-                    operationalComponents.addAll(immutableComponents);
-                }
-                break;
-            case MERGE:
-                operationalComponents.addAll(ctx.getComponentsToBeMerged());
-                break;
-            case FULL_MERGE:
-                operationalComponents.addAll(immutableComponents);
-                break;
-            case REPLICATE:
-                operationalComponents.addAll(ctx.getComponentsToBeReplicated());
-                break;
-            default:
-                throw new UnsupportedOperationException("Operation " + ctx.getOperation() + " not supported.");
-        }
+    protected void destroyMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+        LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
+        mutableComponent.getInvIndex().destroy();
+        mutableComponent.getDeletedKeysBTree().destroy();
     }
 
     /**
@@ -333,12 +232,11 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
         ctx.getModificationCallback().before(indexTuple);
         ctx.getModificationCallback().found(null, indexTuple);
         switch (ctx.getOperation()) {
-            case INSERT: {
+            case INSERT:
                 // Insert into the in-memory inverted index.
                 ctx.getCurrentMutableInvIndexAccessors().insert(indexTuple);
                 break;
-            }
-            case DELETE: {
+            case DELETE:
                 // First remove all entries in the in-memory inverted index (if any).
                 ctx.getCurrentMutableInvIndexAccessors().delete(indexTuple);
                 // Insert key into the deleted-keys BTree.
@@ -353,16 +251,10 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
                     }
                 }
                 break;
-            }
-            default: {
+            default:
                 throw new UnsupportedOperationException("Operation " + ctx.getOperation() + " not supported.");
-            }
-        }
-        if (ctx.getFilterTuple() != null) {
-            ctx.getFilterTuple().reset(tuple);
-            memoryComponents.get(currentMutableComponentId.get()).getLSMComponentFilter().update(ctx.getFilterTuple(),
-                    ctx.getFilterCmp());
         }
+        updateFilter(ctx, tuple);
     }
 
     @Override
@@ -402,7 +294,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
     private ICursorInitialState createCursorInitialState(ISearchPredicate pred, IIndexOperationContext ictx,
             boolean includeMutableComponent, ArrayList<IIndexAccessor> indexAccessors,
             ArrayList<IIndexAccessor> deletedKeysBTreeAccessors, List<ILSMComponent> operationalComponents) {
-        ICursorInitialState initState = null;
+        ICursorInitialState initState;
         PermutingTupleReference keysOnlyTuple = createKeysOnlyTupleReference();
         MultiComparator keyCmp = MultiComparator.create(invListCmpFactories);
 
@@ -413,7 +305,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
                     deletedKeysBTreeAccessors,
                     ((LSMInvertedIndexMemoryComponent) memoryComponents.get(currentMutableComponentId.get()))
                             .getDeletedKeysBTree().getLeafFrameFactory(),
-                    ictx, includeMutableComponent, lsmHarness, operationalComponents);
+                    ictx, includeMutableComponent, getLsmHarness(), operationalComponents);
         } else {
             LSMInvertedIndexMemoryComponent mutableComponent =
                     (LSMInvertedIndexMemoryComponent) memoryComponents.get(currentMutableComponentId.get());
@@ -422,7 +314,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
             initState = new LSMInvertedIndexRangeSearchCursorInitialState(tokensAndKeysCmp, keyCmp, keysOnlyTuple,
                     ((LSMInvertedIndexMemoryComponent) memoryComponents.get(currentMutableComponentId.get()))
                             .getDeletedKeysBTree().getLeafFrameFactory(),
-                    includeMutableComponent, lsmHarness, indexAccessors, deletedKeysBTreeAccessors, pred,
+                    includeMutableComponent, getLsmHarness(), indexAccessors, deletedKeysBTreeAccessors, pred,
                     operationalComponents);
         }
         return initState;
@@ -443,28 +335,12 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
     }
 
     @Override
-    public void scheduleFlush(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
-            throws HyracksDataException {
-        ILSMComponent flushingComponent = ctx.getComponentHolder().get(0);
-        LSMComponentFileReferences componentFileRefs = fileManager.getRelFlushFileReference();
-        LSMInvertedIndexOpContext opCtx =
-                createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
-        opCtx.setOperation(IndexOperation.FLUSH);
-        opCtx.getComponentHolder().add(flushingComponent);
-        ioScheduler.scheduleOperation(new LSMInvertedIndexFlushOperation(
-                new LSMInvertedIndexAccessor(lsmHarness, opCtx), flushingComponent,
-                componentFileRefs.getInsertIndexFileReference(), componentFileRefs.getDeleteIndexFileReference(),
-                componentFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir()));
-    }
-
-    @Override
     public ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException {
         LSMInvertedIndexFlushOperation flushOp = (LSMInvertedIndexFlushOperation) operation;
 
         // Create an inverted index instance to be bulk loaded.
-        LSMInvertedIndexDiskComponent component =
-                createDiskInvIndexComponent(componentFactory, flushOp.getDictBTreeFlushTarget(),
-                        flushOp.getDeletedKeysBTreeFlushTarget(), flushOp.getBloomFilterFlushTarget(), true);
+        LSMInvertedIndexDiskComponent component = createDiskInvIndexComponent(componentFactory, flushOp.getTarget(),
+                flushOp.getDeletedKeysBTreeTarget(), flushOp.getBloomFilterTarget(), true);
         IInvertedIndex diskInvertedIndex = component.getInvIndex();
 
         // Create a scan cursor on the BTree underlying the in-memory inverted index.
@@ -491,8 +367,8 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
             List<ITupleReference> filterTuples = new ArrayList<>();
             filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple());
             filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple());
-            filterManager.updateFilter(component.getLSMComponentFilter(), filterTuples);
-            filterManager.writeFilter(component.getLSMComponentFilter(),
+            getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
+            getFilterManager().writeFilter(component.getLSMComponentFilter(),
                     ((OnDiskInvertedIndex) component.getInvIndex()).getBTree());
         }
         flushingComponent.getMetadata().copy(component.getMetadata());
@@ -545,31 +421,6 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
     }
 
     @Override
-    public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback)
-            throws HyracksDataException {
-        LSMInvertedIndexOpContext ictx =
-                createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
-        ictx.setOperation(IndexOperation.MERGE);
-        List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
-        IIndexCursor cursor = new LSMInvertedIndexRangeSearchCursor(ictx);
-
-        LSMInvertedIndexDiskComponent firstComponent = (LSMInvertedIndexDiskComponent) mergingComponents.get(0);
-        OnDiskInvertedIndex firstInvIndex = (OnDiskInvertedIndex) firstComponent.getInvIndex();
-        String firstFileName = firstInvIndex.getBTree().getFileReference().getFile().getName();
-
-        LSMInvertedIndexDiskComponent lastComponent =
-                (LSMInvertedIndexDiskComponent) mergingComponents.get(mergingComponents.size() - 1);
-        OnDiskInvertedIndex lastInvIndex = (OnDiskInvertedIndex) lastComponent.getInvIndex();
-        String lastFileName = lastInvIndex.getBTree().getFileReference().getFile().getName();
-
-        LSMComponentFileReferences relMergeFileRefs = fileManager.getRelMergeFileReference(firstFileName, lastFileName);
-        ILSMIndexAccessor accessor = new LSMInvertedIndexAccessor(lsmHarness, ctx);
-        ioScheduler.scheduleOperation(new LSMInvertedIndexMergeOperation(accessor, mergingComponents, cursor,
-                relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getDeleteIndexFileReference(),
-                relMergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir()));
-    }
-
-    @Override
     public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException {
         LSMInvertedIndexMergeOperation mergeOp = (LSMInvertedIndexMergeOperation) operation;
         IIndexCursor cursor = mergeOp.getCursor();
@@ -581,9 +432,8 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
         search(opCtx, cursor, mergePred);
 
         // Create an inverted index instance.
-        LSMInvertedIndexDiskComponent component =
-                createDiskInvIndexComponent(componentFactory, mergeOp.getDictBTreeMergeTarget(),
-                        mergeOp.getDeletedKeysBTreeMergeTarget(), mergeOp.getBloomFilterMergeTarget(), true);
+        LSMInvertedIndexDiskComponent component = createDiskInvIndexComponent(componentFactory, mergeOp.getTarget(),
+                mergeOp.getDeletedKeysBTreeTarget(), mergeOp.getBloomFilterTarget(), true);
 
         IInvertedIndex mergedDiskInvertedIndex = component.getInvIndex();
 
@@ -651,8 +501,8 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
                     filterTuples.add(max);
                 }
             }
-            filterManager.updateFilter(component.getLSMComponentFilter(), filterTuples);
-            filterManager.writeFilter(component.getLSMComponentFilter(),
+            getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
+            getFilterManager().writeFilter(component.getLSMComponentFilter(),
                     ((OnDiskInvertedIndex) component.getInvIndex()).getBTree());
         }
         invIndexBulkLoader.end();
@@ -661,9 +511,9 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
     }
 
     @Override
-    public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
-            boolean checkIfEmptyIndex) throws HyracksDataException {
-        return new LSMInvertedIndexBulkLoader(fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex);
+    public IIndexBulkLoader createBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint)
+            throws HyracksDataException {
+        return new LSMInvertedIndexBulkLoader(fillFactor, verifyInput, numElementsHint);
     }
 
     public class LSMInvertedIndexBulkLoader implements IIndexBulkLoader {
@@ -676,11 +526,8 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
         public final PermutingTupleReference filterTuple;
         public final MultiComparator filterCmp;
 
-        public LSMInvertedIndexBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
-                boolean checkIfEmptyIndex) throws HyracksDataException {
-            if (checkIfEmptyIndex && !isEmptyIndex()) {
-                throw HyracksDataException.create(ErrorCode.LOAD_NON_EMPTY_INDEX);
-            }
+        public LSMInvertedIndexBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint)
+                throws HyracksDataException {
             // Note that by using a flush target file name, we state that the
             // new bulk loaded tree is "newer" than any other merged tree.
             component = createBulkLoadTarget();
@@ -691,10 +538,10 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
             deletedKeysBTreeBulkLoader = ((LSMInvertedIndexDiskComponent) component).getDeletedKeysBTree()
                     .createBulkLoader(fillFactor, verifyInput, numElementsHint, false);
 
-            if (filterFields != null) {
-                indexTuple = new PermutingTupleReference(invertedIndexFields);
+            if (getFilterFields() != null) {
+                indexTuple = new PermutingTupleReference(getTreeFields());
                 filterCmp = MultiComparator.create(component.getLSMComponentFilter().getFilterCmpFactories());
-                filterTuple = new PermutingTupleReference(filterFields);
+                filterTuple = new PermutingTupleReference(getFilterFields());
             } else {
                 indexTuple = null;
                 filterCmp = null;
@@ -745,9 +592,9 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
         public void end() throws HyracksDataException {
             if (!cleanedUpArtifacts) {
                 if (component.getLSMComponentFilter() != null) {
-                    filterManager.writeFilter(component.getLSMComponentFilter(),
-                            (((OnDiskInvertedIndex) ((LSMInvertedIndexDiskComponent) component).getInvIndex())
-                                    .getBTree()));
+                    getFilterManager().writeFilter(component.getLSMComponentFilter(),
+                            ((OnDiskInvertedIndex) ((LSMInvertedIndexDiskComponent) component).getInvIndex())
+                                    .getBTree());
                 }
                 invIndexBulkLoader.end();
                 deletedKeysBTreeBulkLoader.end();
@@ -756,7 +603,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
                     cleanupArtifacts();
                 } else {
                     ioOpCallback.afterOperation(LSMOperationType.FLUSH, null, component);
-                    lsmHarness.addBulkLoadedComponent(component);
+                    getLsmHarness().addBulkLoadedComponent(component);
                 }
             }
         }
@@ -802,7 +649,7 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
         component.getBloomFilter().activate();
         // Will be closed during cleanup of merge().
         if (component.getLSMComponentFilter() != null && !create) {
-            filterManager.readFilter(component.getLSMComponentFilter(),
+            getFilterManager().readFilter(component.getLSMComponentFilter(),
                     ((OnDiskInvertedIndex) component.getInvIndex()).getBTree());
         }
         return component;
@@ -811,13 +658,14 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
     @Override
     public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
             ISearchOperationCallback searchCallback) throws HyracksDataException {
-        return new LSMInvertedIndexAccessor(lsmHarness, createOpContext(modificationCallback, searchCallback));
+        return new LSMInvertedIndexAccessor(getLsmHarness(), createOpContext(modificationCallback, searchCallback));
     }
 
-    private LSMInvertedIndexOpContext createOpContext(IModificationOperationCallback modificationCallback,
+    @Override
+    protected LSMInvertedIndexOpContext createOpContext(IModificationOperationCallback modificationCallback,
             ISearchOperationCallback searchCallback) throws HyracksDataException {
         return new LSMInvertedIndexOpContext(memoryComponents, modificationCallback, searchCallback,
-                invertedIndexFieldsForNonBulkLoadOps, filterFieldsForNonBulkLoadOps);
+                invertedIndexFieldsForNonBulkLoadOps, filterFieldsForNonBulkLoadOps, getFilterCmpFactories());
     }
 
     @Override
@@ -842,17 +690,6 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
     }
 
     @Override
-    public long getMemoryAllocationSize() {
-        long size = 0;
-        for (ILSMComponent c : memoryComponents) {
-            LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
-            IBufferCache virtualBufferCache = mutableComponent.getInvIndex().getBufferCache();
-            size += virtualBufferCache.getNumPages() * virtualBufferCache.getPageSize();
-        }
-        return size;
-    }
-
-    @Override
     public ITypeTraits[] getTokenTypeTraits() {
         return tokenTypeTraits;
     }
@@ -882,27 +719,11 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
     }
 
     @Override
-    public void validate() throws HyracksDataException {
-        validateMemoryComponents();
-        List<ILSMDiskComponent> immutableComponents = diskComponents;
-        for (ILSMDiskComponent c : immutableComponents) {
-            LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c;
-            component.getInvIndex().validate();
-            component.getDeletedKeysBTree().validate();
-        }
-    }
-
-    @Override
     public String toString() {
         return "LSMInvertedIndex [" + fileManager.getBaseDir() + "]";
     }
 
     @Override
-    public boolean hasMemoryComponents() {
-        return true;
-    }
-
-    @Override
     public boolean isPrimaryIndex() {
         return false;
     }
@@ -920,71 +741,74 @@ public class LSMInvertedIndex extends AbstractLSMIndex implements IInvertedIndex
     }
 
     @Override
-    public synchronized void allocateMemoryComponents() throws HyracksDataException {
-        if (!isActivated) {
-            throw new HyracksDataException("Failed to allocate memory components since the index is not active.");
-        }
-        if (memoryComponentsAllocated) {
-            return;
-        }
-        for (ILSMComponent c : memoryComponents) {
-            LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
-            ((IVirtualBufferCache) mutableComponent.getInvIndex().getBufferCache()).open();
-            mutableComponent.getInvIndex().create();
-            mutableComponent.getInvIndex().activate();
-            mutableComponent.getDeletedKeysBTree().create();
-            mutableComponent.getDeletedKeysBTree().activate();
-        }
-        memoryComponentsAllocated = true;
-    }
-
-    private void addOperationalMutableComponents(List<ILSMComponent> operationalComponents) {
-        int cmc = currentMutableComponentId.get();
-        int numMutableComponents = memoryComponents.size();
-        for (int i = 0; i < numMutableComponents - 1; i++) {
-            ILSMComponent c = memoryComponents.get((cmc + i + 1) % numMutableComponents);
-            LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
-            if (mutableComponent.isReadable()) {
-                // Make sure newest components are added first
-                operationalComponents.add(0, mutableComponent);
-            }
-        }
-        // The current mutable component is always added
-        operationalComponents.add(0, memoryComponents.get(cmc));
+    protected void clearMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+        LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
+        mutableComponent.getInvIndex().clear();
+        mutableComponent.getDeletedKeysBTree().clear();
+        mutableComponent.reset();
     }
 
-    private synchronized void clearMemoryComponents() throws HyracksDataException {
-        if (memoryComponentsAllocated) {
-            for (ILSMComponent c : memoryComponents) {
-                LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
-                mutableComponent.getInvIndex().clear();
-                mutableComponent.getDeletedKeysBTree().clear();
-                mutableComponent.reset();
-            }
-        }
+    @Override
+    protected long getMemoryComponentSize(ILSMMemoryComponent c) {
+        LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
+        IBufferCache virtualBufferCache = mutableComponent.getInvIndex().getBufferCache();
+        return ((long) virtualBufferCache.getNumPages()) * virtualBufferCache.getPageSize();
     }
 
-    private synchronized void validateMemoryComponents() throws HyracksDataException {
-        if (memoryComponentsAllocated) {
-            for (ILSMComponent c : memoryComponents) {
-                LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
-                mutableComponent.getInvIndex().validate();
-                mutableComponent.getDeletedKeysBTree().validate();
-            }
-        }
+    @Override
+    protected void validateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+        LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
+        mutableComponent.getInvIndex().validate();
+        mutableComponent.getDeletedKeysBTree().validate();
     }
 
-    private synchronized void deallocateMemoryComponents() throws HyracksDataException {
-        if (memoryComponentsAllocated) {
-            for (ILSMComponent c : memoryComponents) {
-                LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
-                mutableComponent.getInvIndex().deactivate();
-                mutableComponent.getDeletedKeysBTree().deactivate();
-                mutableComponent.getInvIndex().destroy();
-                mutableComponent.getDeletedKeysBTree().destroy();
-                ((IVirtualBufferCache) mutableComponent.getInvIndex().getBufferCache()).close();
-            }
-            memoryComponentsAllocated = false;
-        }
+    @Override
+    protected void validateDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+        LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c;
+        component.getInvIndex().validate();
+        component.getDeletedKeysBTree().validate();
+    }
+
+    @Override
+    protected void allocateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+        LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent) c;
+        ((IVirtualBufferCache) mutableComponent.getInvIndex().getBufferCache()).open();
+        mutableComponent.getInvIndex().create();
+        mutableComponent.getInvIndex().activate();
+        mutableComponent.getDeletedKeysBTree().create();
+        mutableComponent.getDeletedKeysBTree().activate();
+    }
+
+    @Override
+    protected LSMComponentFileReferences getMergeFileReferences(ILSMDiskComponent firstComponent,
+            ILSMDiskComponent lastComponent) throws HyracksDataException {
+        LSMInvertedIndexDiskComponent first = (LSMInvertedIndexDiskComponent) firstComponent;
+        OnDiskInvertedIndex firstInvIndex = (OnDiskInvertedIndex) first.getInvIndex();
+        String firstFileName = firstInvIndex.getBTree().getFileReference().getFile().getName();
+        LSMInvertedIndexDiskComponent last = (LSMInvertedIndexDiskComponent) lastComponent;
+        OnDiskInvertedIndex lastInvIndex = (OnDiskInvertedIndex) last.getInvIndex();
+        String lastFileName = lastInvIndex.getBTree().getFileReference().getFile().getName();
+        return fileManager.getRelMergeFileReference(firstFileName, lastFileName);
+    }
+
+    @Override
+    protected ILSMIOOperation createFlushOperation(AbstractLSMIndexOperationContext opCtx,
+            ILSMMemoryComponent flushingComponent, LSMComponentFileReferences componentFileRefs,
+            ILSMIOOperationCallback callback) throws HyracksDataException {
+        return new LSMInvertedIndexFlushOperation(new LSMInvertedIndexAccessor(getLsmHarness(), opCtx),
+                flushingComponent, componentFileRefs.getInsertIndexFileReference(),
+                componentFileRefs.getDeleteIndexFileReference(), componentFileRefs.getBloomFilterFileReference(),
+                callback, fileManager.getBaseDir());
+    }
+
+    @Override
+    protected ILSMIOOperation createMergeOperation(AbstractLSMIndexOperationContext opCtx,
+            List<ILSMComponent> mergingComponents, LSMComponentFileReferences mergeFileRefs,
+            ILSMIOOperationCallback callback) throws HyracksDataException {
+        ILSMIndexAccessor accessor = new LSMInvertedIndexAccessor(getLsmHarness(), opCtx);
+        IIndexCursor cursor = new LSMInvertedIndexRangeSearchCursor(opCtx);
+        return new LSMInvertedIndexMergeOperation(accessor, mergingComponents, cursor,
+                mergeFileRefs.getInsertIndexFileReference(), mergeFileRefs.getDeleteIndexFileReference(),
+                mergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir());
     }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
index edae439..2482103 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexAccessor.java
@@ -203,4 +203,22 @@ public class LSMInvertedIndexAccessor implements ILSMIndexAccessor, IInvertedInd
         throw new UnsupportedOperationException("Upsert not supported by lsm inverted index.");
     }
 
+    /**
+     * enter the memory component for modification
+     */
+    @Override
+    public void enter() throws HyracksDataException {
+        ctx.setOperation(IndexOperation.UPSERT);
+        lsmHarness.enter(ctx);
+    }
+
+    /**
+     * exit the memory component
+     */
+    @Override
+    public void exit() throws HyracksDataException {
+        ctx.setOperation(IndexOperation.UPSERT);
+        lsmHarness.exit(ctx);
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java
index 26eed04..df4f095 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexFlushOperation.java
@@ -19,93 +19,29 @@
 
 package org.apache.hyracks.storage.am.lsm.invertedindex.impls;
 
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Set;
-
-import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.api.io.IODeviceHandle;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
+import org.apache.hyracks.storage.am.lsm.common.impls.FlushOperation;
 
-public class LSMInvertedIndexFlushOperation implements ILSMIOOperation, Comparable<LSMInvertedIndexFlushOperation> {
-    private final ILSMIndexAccessor accessor;
-    private final ILSMComponent flushingComponent;
-    private final FileReference dictBTreeFlushTarget;
+public class LSMInvertedIndexFlushOperation extends FlushOperation {
     private final FileReference deletedKeysBTreeFlushTarget;
     private final FileReference bloomFilterFlushTarget;
-    private final ILSMIOOperationCallback callback;
-    private final String indexIdentifier;
 
-    public LSMInvertedIndexFlushOperation(ILSMIndexAccessor accessor, ILSMComponent flushingComponent,
-            FileReference dictBTreeFlushTarget, FileReference deletedKeysBTreeFlushTarget,
-            FileReference bloomFilterFlushTarget, ILSMIOOperationCallback callback, String indexIdentifier) {
-        this.accessor = accessor;
-        this.flushingComponent = flushingComponent;
-        this.dictBTreeFlushTarget = dictBTreeFlushTarget;
+    public LSMInvertedIndexFlushOperation(ILSMIndexAccessor accessor, ILSMMemoryComponent flushingComponent,
+            FileReference flushTarget, FileReference deletedKeysBTreeFlushTarget, FileReference bloomFilterFlushTarget,
+            ILSMIOOperationCallback callback, String indexIdentifier) {
+        super(accessor, flushingComponent, flushTarget, callback, indexIdentifier);
         this.deletedKeysBTreeFlushTarget = deletedKeysBTreeFlushTarget;
         this.bloomFilterFlushTarget = bloomFilterFlushTarget;
-        this.callback = callback;
-        this.indexIdentifier = indexIdentifier;
-    }
-
-    @Override
-    public Set<IODeviceHandle> getReadDevices() {
-        return Collections.emptySet();
-    }
-
-    @Override
-    public Set<IODeviceHandle> getWriteDevices() {
-        Set<IODeviceHandle> devs = new HashSet<>();
-        devs.add(dictBTreeFlushTarget.getDeviceHandle());
-        devs.add(deletedKeysBTreeFlushTarget.getDeviceHandle());
-        devs.add(bloomFilterFlushTarget.getDeviceHandle());
-        return devs;
-
-    }
-
-    @Override
-    public Boolean call() throws HyracksDataException {
-        accessor.flush(this);
-        return true;
-    }
-
-    @Override
-    public ILSMIOOperationCallback getCallback() {
-        return callback;
     }
 
-    public FileReference getDictBTreeFlushTarget() {
-        return dictBTreeFlushTarget;
-    }
-
-    public FileReference getDeletedKeysBTreeFlushTarget() {
+    public FileReference getDeletedKeysBTreeTarget() {
         return deletedKeysBTreeFlushTarget;
     }
 
-    public FileReference getBloomFilterFlushTarget() {
+    public FileReference getBloomFilterTarget() {
         return bloomFilterFlushTarget;
     }
-
-    public ILSMComponent getFlushingComponent() {
-        return flushingComponent;
-    }
-
-    @Override
-    public String getIndexUniqueIdentifier() {
-        return indexIdentifier;
-    }
-
-    @Override
-    public LSMIOOpertionType getIOOpertionType() {
-        return LSMIOOpertionType.FLUSH;
-    }
-
-    @Override
-    public int compareTo(LSMInvertedIndexFlushOperation o) {
-        return dictBTreeFlushTarget.getFile().getName().compareTo(o.getDictBTreeFlushTarget().getFile().getName());
-    }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java
index c3c8c8b..da374dc 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexMergeOperation.java
@@ -19,103 +19,32 @@
 
 package org.apache.hyracks.storage.am.lsm.invertedindex.impls;
 
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 
-import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.io.FileReference;
-import org.apache.hyracks.api.io.IODeviceHandle;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
-import org.apache.hyracks.storage.am.lsm.invertedindex.ondisk.OnDiskInvertedIndex;
+import org.apache.hyracks.storage.am.lsm.common.impls.MergeOperation;
 import org.apache.hyracks.storage.common.IIndexCursor;
 
-public class LSMInvertedIndexMergeOperation implements ILSMIOOperation {
-    private final ILSMIndexAccessor accessor;
-    private final List<ILSMComponent> mergingComponents;
-    private final IIndexCursor cursor;
-    private final FileReference dictBTreeMergeTarget;
+public class LSMInvertedIndexMergeOperation extends MergeOperation {
     private final FileReference deletedKeysBTreeMergeTarget;
     private final FileReference bloomFilterMergeTarget;
-    private final ILSMIOOperationCallback callback;
-    private final String indexIdentifier;
 
     public LSMInvertedIndexMergeOperation(ILSMIndexAccessor accessor, List<ILSMComponent> mergingComponents,
-            IIndexCursor cursor, FileReference dictBTreeMergeTarget, FileReference deletedKeysBTreeMergeTarget,
+            IIndexCursor cursor, FileReference target, FileReference deletedKeysBTreeMergeTarget,
             FileReference bloomFilterMergeTarget, ILSMIOOperationCallback callback, String indexIdentifier) {
-        this.accessor = accessor;
-        this.mergingComponents = mergingComponents;
-        this.cursor = cursor;
-        this.dictBTreeMergeTarget = dictBTreeMergeTarget;
+        super(accessor, target, callback, indexIdentifier, mergingComponents, cursor);
         this.deletedKeysBTreeMergeTarget = deletedKeysBTreeMergeTarget;
         this.bloomFilterMergeTarget = bloomFilterMergeTarget;
-        this.callback = callback;
-        this.indexIdentifier = indexIdentifier;
     }
 
-    @Override
-    public Set<IODeviceHandle> getReadDevices() {
-        Set<IODeviceHandle> devs = new HashSet<>();
-        for (Object o : mergingComponents) {
-            LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) o;
-            OnDiskInvertedIndex invIndex = (OnDiskInvertedIndex) component.getInvIndex();
-            devs.add(invIndex.getBTree().getFileReference().getDeviceHandle());
-            devs.add(component.getDeletedKeysBTree().getFileReference().getDeviceHandle());
-            devs.add(component.getBloomFilter().getFileReference().getDeviceHandle());
-        }
-        return devs;
-    }
-
-    @Override
-    public Set<IODeviceHandle> getWriteDevices() {
-        Set<IODeviceHandle> devs = new HashSet<>();
-        devs.add(dictBTreeMergeTarget.getDeviceHandle());
-        devs.add(deletedKeysBTreeMergeTarget.getDeviceHandle());
-        devs.add(bloomFilterMergeTarget.getDeviceHandle());
-        return devs;
-    }
-
-    @Override
-    public Boolean call() throws HyracksDataException {
-        accessor.merge(this);
-        return true;
-    }
-
-    @Override
-    public ILSMIOOperationCallback getCallback() {
-        return callback;
-    }
-
-    public FileReference getDictBTreeMergeTarget() {
-        return dictBTreeMergeTarget;
-    }
-
-    public FileReference getDeletedKeysBTreeMergeTarget() {
+    public FileReference getDeletedKeysBTreeTarget() {
         return deletedKeysBTreeMergeTarget;
     }
 
-    public FileReference getBloomFilterMergeTarget() {
+    public FileReference getBloomFilterTarget() {
         return bloomFilterMergeTarget;
     }
-
-    public IIndexCursor getCursor() {
-        return cursor;
-    }
-
-    public List<ILSMComponent> getMergingComponents() {
-        return mergingComponents;
-    }
-
-    @Override
-    public String getIndexUniqueIdentifier() {
-        return indexIdentifier;
-    }
-
-    @Override
-    public LSMIOOpertionType getIOOpertionType() {
-        return LSMIOOpertionType.MERGE;
-    }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
index 44c1c13..29f98b4 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexOpContext.java
@@ -19,65 +19,39 @@
 
 package org.apache.hyracks.storage.am.lsm.invertedindex.impls;
 
-import java.util.LinkedList;
 import java.util.List;
 
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
-import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
 import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
 import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexOperationContext;
 import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexAccessor;
 import org.apache.hyracks.storage.common.IIndexAccessor;
 import org.apache.hyracks.storage.common.IModificationOperationCallback;
 import org.apache.hyracks.storage.common.ISearchOperationCallback;
-import org.apache.hyracks.storage.common.ISearchPredicate;
-import org.apache.hyracks.storage.common.MultiComparator;
 
 public class LSMInvertedIndexOpContext extends AbstractLSMIndexOperationContext {
 
     private static final int NUM_DOCUMENT_FIELDS = 1;
 
-    private IndexOperation op;
-    private final List<ILSMComponent> componentHolder;
-    private final List<ILSMDiskComponent> componentsToBeMerged;
-    private final List<ILSMDiskComponent> componentsToBeReplicated;
-
-    private IModificationOperationCallback modificationCallback;
-    private ISearchOperationCallback searchCallback;
-
     // Tuple that only has the inverted-index elements (aka keys), projecting away the document fields.
     private PermutingTupleReference keysOnlyTuple;
-
     // Accessor to the in-memory inverted indexes.
     private IInvertedIndexAccessor[] mutableInvIndexAccessors;
     // Accessor to the deleted-keys BTrees.
     private IIndexAccessor[] deletedKeysBTreeAccessors;
-
     private IInvertedIndexAccessor currentMutableInvIndexAccessors;
     private IIndexAccessor currentDeletedKeysBTreeAccessors;
 
-    private final PermutingTupleReference indexTuple;
-    private final MultiComparator filterCmp;
-    private final PermutingTupleReference filterTuple;
-
-    private ISearchPredicate searchPredicate;
-
     public LSMInvertedIndexOpContext(List<ILSMMemoryComponent> mutableComponents,
             IModificationOperationCallback modificationCallback, ISearchOperationCallback searchCallback,
-            int[] invertedIndexFields, int[] filterFields) throws HyracksDataException {
-        this.componentHolder = new LinkedList<>();
-        this.componentsToBeMerged = new LinkedList<>();
-        this.componentsToBeReplicated = new LinkedList<>();
-        this.modificationCallback = modificationCallback;
-        this.searchCallback = searchCallback;
-
+            int[] invertedIndexFields, int[] filterFields, IBinaryComparatorFactory[] filterComparatorFactories)
+            throws HyracksDataException {
+        super(invertedIndexFields, filterFields, filterComparatorFactories, searchCallback, modificationCallback);
         mutableInvIndexAccessors = new IInvertedIndexAccessor[mutableComponents.size()];
         deletedKeysBTreeAccessors = new IIndexAccessor[mutableComponents.size()];
-
         for (int i = 0; i < mutableComponents.size(); i++) {
             LSMInvertedIndexMemoryComponent mutableComponent =
                     (LSMInvertedIndexMemoryComponent) mutableComponents.get(i);
@@ -86,9 +60,6 @@ public class LSMInvertedIndexOpContext extends AbstractLSMIndexOperationContext
             deletedKeysBTreeAccessors[i] = mutableComponent.getDeletedKeysBTree()
                     .createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
         }
-
-        assert mutableComponents.size() > 0;
-
         // Project away the document fields, leaving only the key fields.
         LSMInvertedIndexMemoryComponent c = (LSMInvertedIndexMemoryComponent) mutableComponents.get(0);
         int numKeyFields = c.getInvIndex().getInvListTypeTraits().length;
@@ -97,51 +68,6 @@ public class LSMInvertedIndexOpContext extends AbstractLSMIndexOperationContext
             keyFieldPermutation[i] = NUM_DOCUMENT_FIELDS + i;
         }
         keysOnlyTuple = new PermutingTupleReference(keyFieldPermutation);
-
-        if (filterFields != null) {
-            indexTuple = new PermutingTupleReference(invertedIndexFields);
-            filterCmp = MultiComparator.create(c.getLSMComponentFilter().getFilterCmpFactories());
-            filterTuple = new PermutingTupleReference(filterFields);
-        } else {
-            indexTuple = null;
-            filterCmp = null;
-            filterTuple = null;
-        }
-    }
-
-    @Override
-    public void reset() {
-        super.reset();
-        componentHolder.clear();
-        componentsToBeMerged.clear();
-        componentsToBeReplicated.clear();
-    }
-
-    @Override
-    // TODO: Ignore opcallback for now.
-    public void setOperation(IndexOperation newOp) throws HyracksDataException {
-        reset();
-        op = newOp;
-    }
-
-    @Override
-    public IndexOperation getOperation() {
-        return op;
-    }
-
-    @Override
-    public List<ILSMComponent> getComponentHolder() {
-        return componentHolder;
-    }
-
-    @Override
-    public ISearchOperationCallback getSearchOperationCallback() {
-        return searchCallback;
-    }
-
-    @Override
-    public IModificationOperationCallback getModificationCallback() {
-        return modificationCallback;
     }
 
     @Override
@@ -150,34 +76,6 @@ public class LSMInvertedIndexOpContext extends AbstractLSMIndexOperationContext
         currentDeletedKeysBTreeAccessors = deletedKeysBTreeAccessors[currentMutableComponentId];
     }
 
-    @Override
-    public List<ILSMDiskComponent> getComponentsToBeMerged() {
-        return componentsToBeMerged;
-    }
-
-    @Override
-    public void setSearchPredicate(ISearchPredicate searchPredicate) {
-        this.searchPredicate = searchPredicate;
-    }
-
-    @Override
-    public ISearchPredicate getSearchPredicate() {
-        return searchPredicate;
-    }
-
-    @Override
-    public List<ILSMDiskComponent> getComponentsToBeReplicated() {
-        return componentsToBeReplicated;
-    }
-
-    public MultiComparator getFilterCmp() {
-        return filterCmp;
-    }
-
-    public PermutingTupleReference getIndexTuple() {
-        return indexTuple;
-    }
-
     public IInvertedIndexAccessor getCurrentMutableInvIndexAccessors() {
         return currentMutableInvIndexAccessors;
     }
@@ -189,8 +87,4 @@ public class LSMInvertedIndexOpContext extends AbstractLSMIndexOperationContext
     public IIndexAccessor getCurrentDeletedKeysBTreeAccessors() {
         return currentDeletedKeysBTreeAccessors;
     }
-
-    public PermutingTupleReference getFilterTuple() {
-        return filterTuple;
-    }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
index 9c56dec..a9eca4d 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
@@ -33,25 +33,21 @@ import org.apache.hyracks.storage.am.common.api.IIndexOperationContext;
 import org.apache.hyracks.storage.am.common.api.IPageManager;
 import org.apache.hyracks.storage.am.common.api.ITreeIndex;
 import org.apache.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import org.apache.hyracks.storage.am.common.impls.AbstractSearchPredicate;
-import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
 import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFactory;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentFactory;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallback;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexFileManager;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicy;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
 import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
 import org.apache.hyracks.storage.am.lsm.common.freepage.VirtualFreePageManager;
 import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
-import org.apache.hyracks.storage.am.lsm.common.impls.BlockingIOOperationCallbackWrapper;
 import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFileReferences;
 import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentFilterManager;
 import org.apache.hyracks.storage.am.rtree.impls.RTree;
@@ -81,7 +77,6 @@ public abstract class AbstractLSMRTree extends AbstractLSMIndex implements ITree
     protected final ITreeIndexFrameFactory btreeInteriorFrameFactory;
     protected final ITreeIndexFrameFactory rtreeLeafFrameFactory;
     protected final ITreeIndexFrameFactory btreeLeafFrameFactory;
-    protected final int[] rtreeFields;
 
     public AbstractLSMRTree(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
             ITreeIndexFrameFactory rtreeInteriorFrameFactory, ITreeIndexFrameFactory rtreeLeafFrameFactory,
@@ -97,7 +92,7 @@ public abstract class AbstractLSMRTree extends AbstractLSMIndex implements ITree
             throws HyracksDataException {
         super(ioManager, virtualBufferCaches, diskBufferCache, fileManager, diskFileMapProvider,
                 bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback, filterFrameFactory,
-                filterManager, filterFields, durable);
+                filterManager, filterFields, durable, filterFactory, rtreeFields);
         int i = 0;
         for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) {
             RTree memRTree = new RTree(virtualBufferCache, virtualBufferCache.getFileMapProvider(),
@@ -125,7 +120,6 @@ public abstract class AbstractLSMRTree extends AbstractLSMIndex implements ITree
         this.linearizer = linearizer;
         this.comparatorFields = comparatorFields;
         this.linearizerArray = linearizerArray;
-        this.rtreeFields = rtreeFields;
         this.isPointMBR = isPointMBR;
     }
 
@@ -153,126 +147,16 @@ public abstract class AbstractLSMRTree extends AbstractLSMIndex implements ITree
         this.linearizer = linearizer;
         this.comparatorFields = comparatorFields;
         this.linearizerArray = linearizerArray;
-        this.rtreeFields = null;
         this.isPointMBR = isPointMBR;
     }
 
     @Override
-    public synchronized void create() throws HyracksDataException {
-        if (isActivated) {
-            throw new HyracksDataException("Failed to create the index since it is activated.");
-        }
-
-        fileManager.deleteDirs();
-        fileManager.createDirs();
-        diskComponents.clear();
-    }
-
-    @Override
-    public synchronized void activate() throws HyracksDataException {
-        if (isActivated) {
-            throw new HyracksDataException("Failed to activate the index since it is already activated.");
-        }
-    }
-
-    @Override
-    public synchronized void deactivate(boolean flushOnExit) throws HyracksDataException {
-        if (!isActivated) {
-            throw new HyracksDataException("Failed to deactivate the index since it is already deactivated.");
-        }
-
-        if (flushOnExit) {
-            BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(ioOpCallback);
-            ILSMIndexAccessor accessor = createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
-            accessor.scheduleFlush(cb);
-            try {
-                cb.waitForIO();
-            } catch (InterruptedException e) {
-                throw new HyracksDataException(e);
-            }
-        }
-        deallocateMemoryComponents();
-    }
-
-    @Override
-    public synchronized void destroy() throws HyracksDataException {
-        if (isActivated) {
-            throw new HyracksDataException("Failed to destroy the index since it is activated.");
-        }
-    }
-
-    @Override
-    public synchronized void clear() throws HyracksDataException {
-        if (!isActivated) {
-            throw new HyracksDataException("Failed to clear the index since it is not activated.");
-        }
-        clearMemoryComponents();
-    }
-
-    @Override
-    public void getOperationalComponents(ILSMIndexOperationContext ctx) throws HyracksDataException {
-        List<ILSMComponent> operationalComponents = ctx.getComponentHolder();
-        List<ILSMDiskComponent> immutableComponents = diskComponents;
-        int cmc = currentMutableComponentId.get();
-        ctx.setCurrentMutableComponentId(cmc);
-        operationalComponents.clear();
-        switch (ctx.getOperation()) {
-            case INSERT:
-            case DELETE:
-            case FLUSH:
-            case UPSERT:
-                operationalComponents.add(memoryComponents.get(cmc));
-                break;
-            case SEARCH:
-                if (memoryComponentsAllocated) {
-                    addOperationalMutableComponents(operationalComponents);
-                }
-                if (filterManager != null) {
-                    for (ILSMComponent c : immutableComponents) {
-                        if (c.getLSMComponentFilter().satisfy(
-                                ((AbstractSearchPredicate) ctx.getSearchPredicate()).getMinFilterTuple(),
-                                ((AbstractSearchPredicate) ctx.getSearchPredicate()).getMaxFilterTuple(),
-                                ((LSMRTreeOpContext) ctx).getFilterCmp())) {
-                            operationalComponents.add(c);
-                        }
-                    }
-                } else {
-                    operationalComponents.addAll(immutableComponents);
-                }
-                break;
-            case MERGE:
-                operationalComponents.addAll(ctx.getComponentsToBeMerged());
-                break;
-            case FULL_MERGE:
-                operationalComponents.addAll(immutableComponents);
-                break;
-            case REPLICATE:
-                operationalComponents.addAll(ctx.getComponentsToBeReplicated());
-                break;
-            default:
-                throw new UnsupportedOperationException("Operation " + ctx.getOperation() + " not supported.");
-        }
-    }
-
-    @Override
     public void search(ILSMIndexOperationContext ictx, IIndexCursor cursor, ISearchPredicate pred)
             throws HyracksDataException {
         LSMRTreeOpContext ctx = (LSMRTreeOpContext) ictx;
         cursor.open(ctx.getSearchInitialState(), pred);
     }
 
-    protected LSMComponentFileReferences getMergeTargetFileName(List<ILSMComponent> mergingDiskComponents)
-            throws HyracksDataException {
-        RTree firstTree = ((LSMRTreeDiskComponent) mergingDiskComponents.get(0)).getRTree();
-        RTree lastTree =
-                ((LSMRTreeDiskComponent) mergingDiskComponents.get(mergingDiskComponents.size() - 1)).getRTree();
-        FileReference firstFile = firstTree.getFileReference();
-        FileReference lastFile = lastTree.getFileReference();
-        LSMComponentFileReferences fileRefs =
-                fileManager.getRelMergeFileReference(firstFile.getFile().getName(), lastFile.getFile().getName());
-        return fileRefs;
-    }
-
     protected LSMRTreeDiskComponent createDiskComponent(ILSMDiskComponentFactory factory, FileReference insertFileRef,
             FileReference deleteFileRef, FileReference bloomFilterFileRef, boolean createComponent)
             throws HyracksDataException {
@@ -293,7 +177,7 @@ public abstract class AbstractLSMRTree extends AbstractLSMIndex implements ITree
             component.getBloomFilter().activate();
         }
         if (component.getLSMComponentFilter() != null && !createComponent) {
-            filterManager.readFilter(component.getLSMComponentFilter(), component.getRTree());
+            getFilterManager().readFilter(component.getLSMComponentFilter(), component.getRTree());
         }
         return component;
     }
@@ -373,18 +257,15 @@ public abstract class AbstractLSMRTree extends AbstractLSMIndex implements ITree
                 }
             }
         }
-        if (ctx.getFilterTuple() != null) {
-            ctx.getFilterTuple().reset(tuple);
-            memoryComponents.get(currentMutableComponentId.get()).getLSMComponentFilter().update(ctx.getFilterTuple(),
-                    ctx.getFilterCmp());
-        }
+        updateFilter(ctx, tuple);
     }
 
+    @Override
     protected LSMRTreeOpContext createOpContext(IModificationOperationCallback modCallback,
             ISearchOperationCallback searchCallback) {
         return new LSMRTreeOpContext(memoryComponents, rtreeLeafFrameFactory, rtreeInteriorFrameFactory,
-                btreeLeafFrameFactory, modCallback, searchCallback, rtreeFields, filterFields, lsmHarness,
-                comparatorFields, linearizerArray);
+                btreeLeafFrameFactory, modCallback, searchCallback, getTreeFields(), getFilterFields(), getLsmHarness(),
+                comparatorFields, linearizerArray, getFilterCmpFactories());
     }
 
     @Override
@@ -393,19 +274,20 @@ public abstract class AbstractLSMRTree extends AbstractLSMIndex implements ITree
     }
 
     @Override
-    public void validate() throws HyracksDataException {
+    protected void validateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
         throw new UnsupportedOperationException("Validation not implemented for LSM R-Trees.");
     }
 
     @Override
-    public long getMemoryAllocationSize() {
-        long size = 0;
-        for (ILSMComponent c : memoryComponents) {
-            LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
-            IBufferCache virtualBufferCache = mutableComponent.getRTree().getBufferCache();
-            size += virtualBufferCache.getNumPages() * virtualBufferCache.getPageSize();
-        }
-        return size;
+    protected void validateDiskComponent(ILSMDiskComponent c) throws HyracksDataException {
+        throw new UnsupportedOperationException("Validation not implemented for LSM R-Trees.");
+    }
+
+    @Override
+    protected long getMemoryComponentSize(ILSMMemoryComponent c) {
+        LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
+        IBufferCache virtualBufferCache = mutableComponent.getRTree().getBufferCache();
+        return virtualBufferCache.getNumPages() * (long) virtualBufferCache.getPageSize();
     }
 
     @Override
@@ -419,61 +301,47 @@ public abstract class AbstractLSMRTree extends AbstractLSMIndex implements ITree
     }
 
     @Override
-    public synchronized void allocateMemoryComponents() throws HyracksDataException {
-        if (!isActivated) {
-            throw new HyracksDataException("Failed to allocate memory components since the index is not active.");
-        }
-        if (memoryComponentsAllocated) {
-            return;
-        }
-        for (ILSMComponent c : memoryComponents) {
-            LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
-            ((IVirtualBufferCache) mutableComponent.getRTree().getBufferCache()).open();
-            mutableComponent.getRTree().create();
-            mutableComponent.getBTree().create();
-            mutableComponent.getRTree().activate();
-            mutableComponent.getBTree().activate();
-        }
-        memoryComponentsAllocated = true;
+    protected void allocateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+        LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
+        ((IVirtualBufferCache) mutableComponent.getRTree().getBufferCache()).open();
+        mutableComponent.getRTree().create();
+        mutableComponent.getBTree().create();
+        mutableComponent.getRTree().activate();
+        mutableComponent.getBTree().activate();
     }
 
-    private void addOperationalMutableComponents(List<ILSMComponent> operationalComponents) {
-        int cmc = currentMutableComponentId.get();
-        int numMutableComponents = memoryComponents.size();
-        for (int i = 0; i < numMutableComponents - 1; i++) {
-            ILSMComponent c = memoryComponents.get((cmc + i + 1) % numMutableComponents);
-            LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
-            if (mutableComponent.isReadable()) {
-                // Make sure newest components are added first
-                operationalComponents.add(0, mutableComponent);
-            }
-        }
-        // The current mutable component is always added
-        operationalComponents.add(0, memoryComponents.get(cmc));
+    @Override
+    protected void deactivateMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+        LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
+        mutableComponent.getRTree().deactivate();
+        mutableComponent.getBTree().deactivate();
+        mutableComponent.getRTree().destroy();
+        mutableComponent.getBTree().destroy();
+        ((IVirtualBufferCache) mutableComponent.getRTree().getBufferCache()).close();
     }
 
-    private synchronized void clearMemoryComponents() throws HyracksDataException {
-        if (memoryComponentsAllocated) {
-            for (ILSMComponent c : memoryComponents) {
-                LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
-                mutableComponent.getRTree().clear();
-                mutableComponent.getBTree().clear();
-                mutableComponent.reset();
-            }
-        }
+    @Override
+    protected void clearMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+        LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
+        mutableComponent.getRTree().clear();
+        mutableComponent.getBTree().clear();
+        mutableComponent.reset();
     }
 
-    private synchronized void deallocateMemoryComponents() throws HyracksDataException {
-        if (memoryComponentsAllocated) {
-            for (ILSMComponent c : memoryComponents) {
-                LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
-                mutableComponent.getRTree().deactivate();
-                mutableComponent.getBTree().deactivate();
-                mutableComponent.getRTree().destroy();
-                mutableComponent.getBTree().destroy();
-                ((IVirtualBufferCache) mutableComponent.getRTree().getBufferCache()).close();
-            }
-            memoryComponentsAllocated = false;
-        }
+    @Override
+    protected void destroyMemoryComponent(ILSMMemoryComponent c) throws HyracksDataException {
+        LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
+        mutableComponent.getRTree().destroy();
+        mutableComponent.getBTree().destroy();
+    }
+
+    @Override
+    protected LSMComponentFileReferences getMergeFileReferences(ILSMDiskComponent firstComponent,
+            ILSMDiskComponent lastComponent) throws HyracksDataException {
+        RTree firstTree = ((LSMRTreeDiskComponent) firstComponent).getRTree();
+        RTree lastTree = ((LSMRTreeDiskComponent) lastComponent).getRTree();
+        FileReference firstFile = firstTree.getFileReference();
+        FileReference lastFile = lastTree.getFileReference();
+        return fileManager.getRelMergeFileReference(firstFile.getFile().getName(), lastFile.getFile().getName());
     }
 }

http://git-wip-us.apache.org/repos/asf/asterixdb/blob/639fe8cb/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
----------------------------------------------------------------------
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
index d6b94ec..987aec7 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/ExternalRTree.java
@@ -24,7 +24,6 @@ import java.util.List;
 
 import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
 import org.apache.hyracks.api.dataflow.value.ILinearizeComparatorFactory;
-import org.apache.hyracks.api.exceptions.ErrorCode;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.io.IIOManager;
 import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
@@ -102,6 +101,11 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex {
         this.fieldCount = fieldCount;
     }
 
+    @Override
+    public ExternalIndexHarness getLsmHarness() {
+        return (ExternalIndexHarness) super.getLsmHarness();
+    }
+
     // The subsume merged components is overridden to account for:
     // 1. the number of readers of components
     // 2. maintaining two versions of the index
@@ -189,7 +193,7 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex {
 
     @Override
     public synchronized void activate() throws HyracksDataException {
-        if (isActivated) {
+        if (isActive) {
             throw new HyracksDataException("Failed to activate the index since it is already activated.");
         }
 
@@ -206,7 +210,7 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex {
                 diskComponents.add(component);
                 secondDiskComponents.add(component);
             }
-            ((ExternalIndexHarness) lsmHarness).indexFirstTimeActivated();
+            getLsmHarness().indexFirstTimeActivated();
         } else {
             // This index has been opened before or is brand new with no components
             // components. It should also maintain the version pointer
@@ -232,7 +236,7 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex {
                 }
             }
         }
-        isActivated = true;
+        isActive = true;
     }
 
     @Override
@@ -258,20 +262,29 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex {
     @Override
     public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException {
         LSMRTreeMergeOperation mergeOp = (LSMRTreeMergeOperation) operation;
-        ITreeIndexCursor cursor = mergeOp.getCursor();
+        IIndexCursor cursor = mergeOp.getCursor();
         ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null);
         ILSMIndexOperationContext opCtx = ((LSMRTreeSortedCursor) cursor).getOpCtx();
         opCtx.getComponentHolder().addAll(mergeOp.getMergingComponents());
         search(opCtx, cursor, rtreeSearchPred);
 
-        LSMRTreeDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getRTreeMergeTarget(),
-                mergeOp.getBTreeMergeTarget(), mergeOp.getBloomFilterMergeTarget(), true);
+        LSMRTreeDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getTarget(),
+                mergeOp.getBTreeTarget(), mergeOp.getBloomFilterTarget(), true);
 
         // In case we must keep the deleted-keys BTrees, then they must be
         // merged *before* merging the r-trees so that
         // lsmHarness.endSearch() is called once when the r-trees have been
         // merged.
-        if (mergeOp.isKeepDeletedTuples()) {
+        boolean keepDeleteTuples = false;
+        if (version == 0) {
+            keepDeleteTuples = mergeOp.getMergingComponents()
+                    .get(mergeOp.getMergingComponents().size() - 1) != diskComponents.get(diskComponents.size() - 1);
+        } else {
+            keepDeleteTuples = mergeOp.getMergingComponents()
+                    .get(mergeOp.getMergingComponents().size() - 1) != secondDiskComponents
+                            .get(secondDiskComponents.size() - 1);
+        }
+        if (keepDeleteTuples) {
             // Keep the deleted tuples since the oldest disk component is not
             // included in the merge operation
 
@@ -323,7 +336,7 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex {
 
     @Override
     public void deactivate(boolean flushOnExit) throws HyracksDataException {
-        if (!isActivated) {
+        if (!isActive) {
             throw new HyracksDataException("Failed to deactivate the index since it is already deactivated.");
         }
 
@@ -353,17 +366,17 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex {
                 bloomFilter.deactivate();
             }
         }
-        isActivated = false;
+        isActive = false;
     }
 
     // The clear method is not used anywhere in AsterixDB! we override it anyway
     // to exit components first and clear the two lists
     @Override
     public void clear() throws HyracksDataException {
-        if (!isActivated) {
+        if (!isActive) {
             throw new HyracksDataException("Failed to clear the index since it is not activated.");
         }
-        ((ExternalIndexHarness) lsmHarness).indexClear();
+        getLsmHarness().indexClear();
 
         for (ILSMComponent c : diskComponents) {
             LSMRTreeDiskComponent component = (LSMRTreeDiskComponent) c;
@@ -394,7 +407,7 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex {
 
     @Override
     public void destroy() throws HyracksDataException {
-        if (isActivated) {
+        if (isActive) {
             throw new HyracksDataException("Failed to destroy the index since it is activated.");
         }
         for (ILSMComponent c : diskComponents) {
@@ -479,16 +492,16 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex {
 
     // For initial load
     @Override
-    public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint,
-            boolean checkIfEmptyIndex) throws HyracksDataException {
-        return new LSMTwoPCRTreeBulkLoader(fillLevel, verifyInput, 0, checkIfEmptyIndex, false);
+    public IIndexBulkLoader createBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint)
+            throws HyracksDataException {
+        return new LSMTwoPCRTreeBulkLoader(fillLevel, verifyInput, 0, false);
     }
 
     // For transaction bulk load <- could consolidate with the above method ->
     @Override
-    public IIndexBulkLoader createTransactionBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint,
-            boolean checkIfEmptyIndex) throws HyracksDataException {
-        return new LSMTwoPCRTreeBulkLoader(fillLevel, verifyInput, numElementsHint, checkIfEmptyIndex, true);
+    public IIndexBulkLoader createTransactionBulkLoader(float fillLevel, boolean verifyInput, long numElementsHint)
+            throws HyracksDataException {
+        return new LSMTwoPCRTreeBulkLoader(fillLevel, verifyInput, numElementsHint, true);
     }
 
     // The bulk loader used for both initial loading and transaction
@@ -504,15 +517,12 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex {
         private final boolean isTransaction;
 
         public LSMTwoPCRTreeBulkLoader(float fillFactor, boolean verifyInput, long numElementsHint,
-                boolean checkIfEmptyIndex, boolean isTransaction) throws HyracksDataException {
+                boolean isTransaction) throws HyracksDataException {
             this.isTransaction = isTransaction;
             // Create the appropriate target
             if (isTransaction) {
                 component = createTransactionTarget();
             } else {
-                if (checkIfEmptyIndex && !isEmptyIndex()) {
-                    throw HyracksDataException.create(ErrorCode.LOAD_NON_EMPTY_INDEX);
-                }
                 component = createBulkLoadTarget();
             }
 
@@ -589,7 +599,7 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex {
                     btree.deactivate();
                     bloomFilter.deactivate();
                 } else {
-                    lsmHarness.addBulkLoadedComponent(component);
+                    getLsmHarness().addBulkLoadedComponent(component);
                 }
             }
         }
@@ -646,37 +656,28 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex {
         rctx.setOperation(IndexOperation.MERGE);
         List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
         ITreeIndexCursor cursor = new LSMRTreeSortedCursor(rctx, linearizer, buddyBTreeFields);
-        LSMComponentFileReferences relMergeFileRefs = getMergeTargetFileName(mergingComponents);
-        ILSMIndexAccessor accessor = new LSMRTreeAccessor(lsmHarness, rctx);
+        LSMComponentFileReferences relMergeFileRefs =
+                getMergeFileReferences((ILSMDiskComponent) mergingComponents.get(0),
+                        (ILSMDiskComponent) mergingComponents.get(mergingComponents.size() - 1));
+        ILSMIndexAccessor accessor = new LSMRTreeAccessor(getLsmHarness(), rctx, buddyBTreeFields);
         // create the merge operation.
         LSMRTreeMergeOperation mergeOp = new LSMRTreeMergeOperation(accessor, mergingComponents, cursor,
                 relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getDeleteIndexFileReference(),
                 relMergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir());
-        // set the keepDeletedTuples flag
-        boolean keepDeleteTuples = false;
-        if (version == 0) {
-            keepDeleteTuples = mergeOp.getMergingComponents()
-                    .get(mergeOp.getMergingComponents().size() - 1) != diskComponents.get(diskComponents.size() - 1);
-        } else {
-            keepDeleteTuples = mergeOp.getMergingComponents()
-                    .get(mergeOp.getMergingComponents().size() - 1) != secondDiskComponents
-                            .get(secondDiskComponents.size() - 1);
-        }
-        mergeOp.setKeepDeletedTuples(keepDeleteTuples);
-
         ioScheduler.scheduleOperation(mergeOp);
     }
 
     @Override
     public ILSMIndexAccessor createAccessor(ISearchOperationCallback searchCallback, int targetIndexVersion)
             throws HyracksDataException {
-        return new LSMRTreeAccessor(lsmHarness, createOpContext(searchCallback, targetIndexVersion));
+        return new LSMRTreeAccessor(getLsmHarness(), createOpContext(searchCallback, targetIndexVersion),
+                buddyBTreeFields);
     }
 
     // This method creates the appropriate opContext for the targeted version
     public ExternalRTreeOpContext createOpContext(ISearchOperationCallback searchCallback, int targetVersion) {
         return new ExternalRTreeOpContext(rtreeCmpFactories, btreeCmpFactories, searchCallback, targetVersion,
-                lsmHarness, comparatorFields, linearizerArray, rtreeLeafFrameFactory, rtreeInteriorFrameFactory,
+                getLsmHarness(), comparatorFields, linearizerArray, rtreeLeafFrameFactory, rtreeInteriorFrameFactory,
                 btreeLeafFrameFactory);
     }
 
@@ -685,7 +686,7 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex {
     @Override
     public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
             ISearchOperationCallback searchCallback) {
-        return new LSMRTreeAccessor(lsmHarness, createOpContext(searchCallback, version));
+        return new LSMRTreeAccessor(getLsmHarness(), createOpContext(searchCallback, version), buddyBTreeFields);
     }
 
     @Override
@@ -717,7 +718,7 @@ public class ExternalRTree extends LSMRTree implements ITwoPCIndex {
                     componentFileRefrences.getDeleteIndexFileReference(),
                     componentFileRefrences.getBloomFilterFileReference(), false);
         }
-        ((ExternalIndexHarness) lsmHarness).addTransactionComponents(component);
+        getLsmHarness().addTransactionComponents(component);
     }
 
     @Override


Mime
View raw message