asterixdb-notifications mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Murtadha Hubail (Code Review)" <do-not-re...@asterixdb.incubator.apache.org>
Subject Change in hyracks[master]: ASTERIXDB-1058: Lazy LSM memory components allocation
Date Fri, 25 Sep 2015 07:55:49 GMT
Murtadha Hubail has uploaded a new change for review.

  https://asterix-gerrit.ics.uci.edu/405

Change subject: ASTERIXDB-1058: Lazy LSM memory components allocation
......................................................................

ASTERIXDB-1058: Lazy LSM memory components allocation

Change-Id: I476e756f8d71260ea614c8c072fc9503053866c9
---
M hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IIndexLifecycleManager.java
M hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexLifecycleManager.java
M hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
M hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
M hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
M hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
M hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
M hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
8 files changed, 263 insertions(+), 131 deletions(-)


  git pull ssh://asterix-gerrit.ics.uci.edu:29418/hyracks refs/changes/05/405/1

diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IIndexLifecycleManager.java
b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IIndexLifecycleManager.java
index ad0fcef..1605f50 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IIndexLifecycleManager.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/api/IIndexLifecycleManager.java
@@ -34,5 +34,8 @@
     public void close(long resourceID) throws HyracksDataException;
 
     public List<IIndex> getOpenIndexes();
-    
+
+    public void allocateMemory(long resourceID) throws HyracksDataException;
+
+    public void deallocateMemory(long resourceID) throws HyracksDataException;
 }
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexLifecycleManager.java
b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexLifecycleManager.java
index b3213d1..4fbb31e 100644
--- a/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexLifecycleManager.java
+++ b/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexLifecycleManager.java
@@ -25,6 +25,9 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.lifecycle.ILifeCycleComponent;
@@ -32,6 +35,7 @@
 import org.apache.hyracks.storage.am.common.api.IIndexLifecycleManager;
 
 public class IndexLifecycleManager implements IIndexLifecycleManager, ILifeCycleComponent
{
+    private static final Logger LOGGER = Logger.getLogger(IndexLifecycleManager.class.getName());
     private static final long DEFAULT_MEMORY_BUDGET = 1024 * 1024 * 100; // 100 megabytes
 
     private final Map<Long, IndexInfo> indexInfos;
@@ -60,8 +64,17 @@
             return false;
         }
 
+        //find resource id to deallocate its memory
+        //TODO replace this when resource lookup by name is merged.
+        long candidiateIndexResourceId = -1;
+        for (Entry<Long, IndexInfo> entry : indexInfos.entrySet()) {
+            if (entry.getValue() == info) {
+                candidiateIndexResourceId = entry.getKey();
+                break;
+            }
+        }
         info.index.deactivate();
-        memoryUsed -= info.index.getMemoryAllocationSize();
+        deallocateMemory(candidiateIndexResourceId);
         info.isOpen = false;
 
         return true;
@@ -84,20 +97,20 @@
 
     @Override
     public void unregister(long resourceID) throws HyracksDataException {
-        IndexInfo info = indexInfos.remove(resourceID);
+        IndexInfo info = indexInfos.get(resourceID);
         if (info == null) {
             throw new HyracksDataException("Index with resource ID " + resourceID + " does
not exist.");
         }
 
         if (info.referenceCount != 0) {
-            indexInfos.put(resourceID, info);
             throw new HyracksDataException("Cannot remove index while it is open.");
         }
 
         if (info.isOpen) {
             info.index.deactivate();
-            memoryUsed -= info.index.getMemoryAllocationSize();
+            deallocateMemory(resourceID);
         }
+        indexInfos.remove(resourceID);
     }
 
     @Override
@@ -109,15 +122,9 @@
         }
 
         if (!info.isOpen) {
-            long inMemorySize = info.index.getMemoryAllocationSize();
-            while (memoryUsed + inMemorySize > memoryBudget) {
-                if (!evictCandidateIndex()) {
-                    throw new HyracksDataException("Cannot activate index since memory budget
would be exceeded.");
-                }
-            }
+            allocateMemory(resourceID);
             info.index.activate();
             info.isOpen = true;
-            memoryUsed += inMemorySize;
         }
         info.touch();
     }
@@ -132,12 +139,14 @@
         private int referenceCount;
         private long lastAccess;
         private boolean isOpen;
+        private boolean memoryAllocated;
 
         public IndexInfo(IIndex index) {
             this.index = index;
             this.lastAccess = -1;
             this.referenceCount = 0;
             this.isOpen = false;
+            this.memoryAllocated = false;
         }
 
         public void touch() {
@@ -235,4 +244,41 @@
         }
         os.write(sb.toString().getBytes());
     }
+
+    @Override
+    public void allocateMemory(long resourceID) throws HyracksDataException {
+        IndexInfo info = indexInfos.get(resourceID);
+        if (info == null) {
+            throw new HyracksDataException("Failed to allocate memory for index with resource
ID " + resourceID
+                    + " since it does not exist.");
+        }
+        if (!info.memoryAllocated) {
+            long inMemorySize = info.index.getMemoryAllocationSize();
+            while (memoryUsed + inMemorySize > memoryBudget) {
+                if (!evictCandidateIndex()) {
+                    throw new HyracksDataException(
+                            "Cannot allocate memory for index since memory budget would be
exceeded.");
+                }
+            }
+            memoryUsed += inMemorySize;
+            info.memoryAllocated = true;
+            LOGGER.log(Level.INFO, "Allocated memory (" + inMemorySize + ") bytes for resource(
" + resourceID
+                    + "). Total Used (" + memoryUsed + ") bytes.");
+        }
+    }
+
+    @Override
+    public void deallocateMemory(long resourceID) throws HyracksDataException {
+        IndexInfo info = indexInfos.get(resourceID);
+        if (info == null) {
+            throw new HyracksDataException("Failed to deallocate memory for index with resource
ID " + resourceID
+                    + " since it does not exist.");
+        }
+        if (info.isOpen && info.memoryAllocated) {
+            memoryUsed -= info.index.getMemoryAllocationSize();
+            info.memoryAllocated = false;
+            LOGGER.log(Level.INFO, "Deallocated memory for resource( " + resourceID + ").
Total Used (" + memoryUsed
+                    + ") bytes.");
+        }
+    }
 }
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
index ad78f6b..702b17f 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/ExternalBTreeWithBuddy.java
@@ -898,4 +898,9 @@
 
         return files;
     }
+
+    @Override
+    public void allocateMemoryComponents() throws HyracksDataException {
+        //do nothing since external index never use memory components
+    }
 }
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
index 663d2f7..9247061 100644
--- a/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
+++ b/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTree.java
@@ -113,6 +113,9 @@
         super(virtualBufferCaches, diskBTreeFactory.getBufferCache(), fileManager, diskFileMapProvider,
                 bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallback,
filterFrameFactory,
                 filterManager, filterFields, durable);
+        this.insertLeafFrameFactory = insertLeafFrameFactory;
+        this.deleteLeafFrameFactory = deleteLeafFrameFactory;
+        this.cmpFactories = cmpFactories;
         int i = 0;
         for (IVirtualBufferCache virtualBufferCache : virtualBufferCaches) {
             LSMBTreeMemoryComponent mutableComponent = new LSMBTreeMemoryComponent(new BTree(virtualBufferCache,
@@ -124,10 +127,6 @@
             memoryComponents.add(mutableComponent);
             ++i;
         }
-
-        this.insertLeafFrameFactory = insertLeafFrameFactory;
-        this.deleteLeafFrameFactory = deleteLeafFrameFactory;
-        this.cmpFactories = cmpFactories;
         componentFactory = new LSMBTreeDiskComponentFactory(diskBTreeFactory, bloomFilterFactory,
filterFactory);
         bulkLoadComponentFactory = new LSMBTreeDiskComponentFactory(bulkLoadBTreeFactory,
bloomFilterFactory,
                 filterFactory);
@@ -169,12 +168,6 @@
     public synchronized void activate() throws HyracksDataException {
         if (isActivated) {
             throw new HyracksDataException("Failed to activate the index since it is already
activated.");
-        }
-        for (ILSMComponent c : memoryComponents) {
-            LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
-            ((IVirtualBufferCache) mutableComponent.getBTree().getBufferCache()).open();
-            mutableComponent.getBTree().create();
-            mutableComponent.getBTree().activate();
         }
         List<ILSMComponent> immutableComponents = diskComponents;
         immutableComponents.clear();
@@ -223,12 +216,7 @@
             btree.deactivate();
             bloomFilter.deactivate();
         }
-        for (ILSMComponent c : memoryComponents) {
-            LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
-            mutableComponent.getBTree().deactivate();
-            mutableComponent.getBTree().destroy();
-            ((IVirtualBufferCache) mutableComponent.getBTree().getBufferCache()).close();
-        }
+        deallocateMemoryComponents();
         isActivated = false;
     }
 
@@ -263,10 +251,12 @@
         }
 
         List<ILSMComponent> immutableComponents = diskComponents;
-        for (ILSMComponent c : memoryComponents) {
-            LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
-            mutableComponent.getBTree().clear();
-            mutableComponent.reset();
+        if (memoryComponentsAllocated) {
+            for (ILSMComponent c : memoryComponents) {
+                LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
+                mutableComponent.getBTree().clear();
+                mutableComponent.reset();
+            }
         }
         for (ILSMComponent c : immutableComponents) {
             LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) c;
@@ -308,16 +298,19 @@
                 operationalComponents.addAll(immutableComponents);
                 break;
             case SEARCH:
-                for (int i = 0; i < numMutableComponents - 1; i++) {
-                    ILSMComponent c = memoryComponents.get((cmc + i + 1) % numMutableComponents);
-                    LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent)
c;
-                    if (mutableComponent.isReadable()) {
-                        // Make sure newest components are added first
-                        operationalComponents.add(0, mutableComponent);
+                if (memoryComponentsAllocated) {
+                    for (int i = 0; i < numMutableComponents - 1; i++) {
+                        ILSMComponent c = memoryComponents.get((cmc + i + 1) % numMutableComponents);
+                        LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent)
c;
+                        if (mutableComponent.isReadable()) {
+                            // Make sure newest components are added first
+                            operationalComponents.add(0, mutableComponent);
+                        }
                     }
+                    // The current mutable component is always added
+                    operationalComponents.add(0, memoryComponents.get(cmc));
                 }
-                // The current mutable component is always added
-                operationalComponents.add(0, memoryComponents.get(cmc));
+
                 if (filterManager != null) {
                     for (ILSMComponent c : immutableComponents) {
                         if (c.getLSMComponentFilter().satisfy(
@@ -834,9 +827,11 @@
 
     @Override
     public void validate() throws HyracksDataException {
-        for (ILSMComponent c : memoryComponents) {
-            LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
-            mutableComponent.getBTree().validate();
+        if (memoryComponentsAllocated) {
+            for (ILSMComponent c : memoryComponents) {
+                LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
+                mutableComponent.getBTree().validate();
+            }
         }
         List<ILSMComponent> immutableComponents = diskComponents;
         for (ILSMComponent c : immutableComponents) {
@@ -857,13 +852,42 @@
 
     @Override
     public Set<String> getLSMComponentPhysicalFiles(ILSMComponent lsmComponent) {
-
         Set<String> files = new HashSet<String>();
         LSMBTreeDiskComponent component = (LSMBTreeDiskComponent) lsmComponent;
 
         files.add(component.getBTree().getFileReference().toString());
         files.add(component.getBloomFilter().getFileReference().toString());
-        
+
         return files;
     }
-}
+
+    @Override
+    public synchronized void allocateMemoryComponents() throws HyracksDataException {
+        if (!isActivated) {
+            throw new HyracksDataException("Failed to allocate memory components since the
index is not active");
+        }
+        if (memoryComponentsAllocated) {
+            return;
+        }
+        for (ILSMComponent c : memoryComponents) {
+            LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
+            ((IVirtualBufferCache) mutableComponent.getBTree().getBufferCache()).open();
+            mutableComponent.getBTree().create();
+            mutableComponent.getBTree().activate();
+        }
+        memoryComponentsAllocated = true;
+    }
+
+    private synchronized void deallocateMemoryComponents() throws HyracksDataException {
+        if (!memoryComponentsAllocated) {
+            return;
+        }
+        for (ILSMComponent c : memoryComponents) {
+            LSMBTreeMemoryComponent mutableComponent = (LSMBTreeMemoryComponent) c;
+            mutableComponent.getBTree().deactivate();
+            mutableComponent.getBTree().destroy();
+            ((IVirtualBufferCache) mutableComponent.getBTree().getBufferCache()).close();
+        }
+        memoryComponentsAllocated = false;
+    }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index 3a7c632..c54536e 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -75,6 +75,7 @@
 
     protected boolean isActivated;
     protected final AtomicBoolean[] flushRequests;
+    protected boolean memoryComponentsAllocated = false;
 
     public AbstractLSMIndex(List<IVirtualBufferCache> virtualBufferCaches, IBufferCache
diskBufferCache,
             ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider,
@@ -334,4 +335,10 @@
         }
 
     }
-}
+
+    public abstract void allocateMemoryComponents() throws HyracksDataException;
+
+    public boolean isMemoryComponentsAllocated() {
+        return memoryComponentsAllocated;
+    }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index f78e6d3..6bb50c6 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@ -55,8 +55,9 @@
     protected final AtomicBoolean fullMergeIsRequested;
     protected final boolean replicationEnabled;
     protected List<ILSMComponent> componentsToBeReplicated;
-    
-    public LSMHarness(ILSMIndexInternal lsmIndex, ILSMMergePolicy mergePolicy, ILSMOperationTracker
opTracker, boolean replicationEnabled) {
+
+    public LSMHarness(ILSMIndexInternal lsmIndex, ILSMMergePolicy mergePolicy, ILSMOperationTracker
opTracker,
+            boolean replicationEnabled) {
         this.lsmIndex = lsmIndex;
         this.opTracker = opTracker;
         this.mergePolicy = mergePolicy;
@@ -298,7 +299,7 @@
                         lsmIndex.scheduleReplication(null, inactiveDiskComponentsToBeDeleted,
false,
                                 ReplicationOperation.DELETE);
                     }
-                    
+
                     for (ILSMComponent c : inactiveDiskComponentsToBeDeleted) {
                         ((AbstractDiskLSMComponent) c).destroy();
                     }
@@ -325,8 +326,11 @@
         return modify(ctx, tryOperation, tuple, opType);
     }
 
-    protected boolean modify(ILSMIndexOperationContext ctx, boolean tryOperation, ITupleReference
tuple,
+    private boolean modify(ILSMIndexOperationContext ctx, boolean tryOperation, ITupleReference
tuple,
             LSMOperationType opType) throws HyracksDataException, IndexException {
+        if (!((AbstractLSMIndex) lsmIndex).isMemoryComponentsAllocated()) {
+            ((AbstractLSMIndex) lsmIndex).allocateMemoryComponents();
+        }
         boolean failedOperation = false;
         if (!getAndEnterComponents(ctx, opType, tryOperation)) {
             return false;
@@ -470,13 +474,13 @@
     public ILSMOperationTracker getOperationTracker() {
         return opTracker;
     }
-    
+
     protected void triggerReplication(List<ILSMComponent> lsmComponents, boolean bulkload)
throws HyracksDataException {
         ILSMIndexAccessorInternal accessor = lsmIndex.createAccessor(NoOpOperationCallback.INSTANCE,
                 NoOpOperationCallback.INSTANCE);
         accessor.scheduleReplication(lsmComponents, bulkload);
     }
-    
+
     @Override
     public void scheduleReplication(ILSMIndexOperationContext ctx, List<ILSMComponent>
lsmComponents, boolean bulkload)
             throws HyracksDataException {
@@ -497,4 +501,4 @@
             throw new HyracksDataException(e);
         }
     }
-}
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
index 28b96db..bfe8ab3 100644
--- a/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
+++ b/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndex.java
@@ -168,15 +168,6 @@
 
         try {
             List<ILSMComponent> immutableComponents = diskComponents;
-            for (ILSMComponent c : memoryComponents) {
-                LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent)
c;
-                ((IVirtualBufferCache) mutableComponent.getInvIndex().getBufferCache()).open();
-                mutableComponent.getInvIndex().create();
-                mutableComponent.getInvIndex().activate();
-                mutableComponent.getDeletedKeysBTree().create();
-                mutableComponent.getDeletedKeysBTree().activate();
-            }
-
             immutableComponents.clear();
             List<LSMComponentFileReferences> validFileReferences = fileManager.cleanupAndGetValidFiles();
             for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences)
{
@@ -192,7 +183,6 @@
                 immutableComponents.add(component);
             }
             isActivated = true;
-            // TODO: Maybe we can make activate throw an index exception?
         } catch (IndexException e) {
             throw new HyracksDataException(e);
         }
@@ -204,12 +194,15 @@
             throw new HyracksDataException("Failed to clear the index since it is not activated.");
         }
 
-        for (ILSMComponent c : memoryComponents) {
-            LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent)
c;
-            mutableComponent.getInvIndex().clear();
-            mutableComponent.getDeletedKeysBTree().clear();
-            mutableComponent.reset();
+        if (memoryComponentsAllocated) {
+            for (ILSMComponent c : memoryComponents) {
+                LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent)
c;
+                mutableComponent.getInvIndex().clear();
+                mutableComponent.getDeletedKeysBTree().clear();
+                mutableComponent.reset();
+            }
         }
+
         List<ILSMComponent> immutableComponents = diskComponents;
         for (ILSMComponent c : immutableComponents) {
             LSMInvertedIndexDiskComponent component = (LSMInvertedIndexDiskComponent) c;
@@ -228,8 +221,6 @@
         if (!isActivated) {
             throw new HyracksDataException("Failed to deactivate the index since it is already
deactivated.");
         }
-
-        isActivated = false;
         if (flushOnExit) {
             BlockingIOOperationCallbackWrapper cb = new BlockingIOOperationCallbackWrapper(ioOpCallback);
             ILSMIndexAccessor accessor = createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
@@ -248,14 +239,8 @@
             component.getInvIndex().deactivate();
             component.getDeletedKeysBTree().deactivate();
         }
-        for (ILSMComponent c : memoryComponents) {
-            LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent)
c;
-            mutableComponent.getInvIndex().deactivate();
-            mutableComponent.getDeletedKeysBTree().deactivate();
-            mutableComponent.getInvIndex().destroy();
-            mutableComponent.getDeletedKeysBTree().destroy();
-            ((IVirtualBufferCache) mutableComponent.getInvIndex().getBufferCache()).close();
-        }
+        deallocateMemoryComponents();
+        isActivated = false;
     }
 
     @Override
@@ -299,17 +284,18 @@
                 operationalComponents.add(memoryComponents.get(cmc));
                 break;
             case SEARCH:
-                for (int i = 0; i < numMutableComponents - 1; i++) {
-                    ILSMComponent c = memoryComponents.get((cmc + i + 1) % numMutableComponents);
-                    LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent)
c;
-                    if (mutableComponent.isReadable()) {
-                        // Make sure newest components are added first
-                        operationalComponents.add(0, mutableComponent);
+                if (memoryComponentsAllocated) {
+                    for (int i = 0; i < numMutableComponents - 1; i++) {
+                        ILSMComponent c = memoryComponents.get((cmc + i + 1) % numMutableComponents);
+                        LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent)
c;
+                        if (mutableComponent.isReadable()) {
+                            // Make sure newest components are added first
+                            operationalComponents.add(0, mutableComponent);
+                        }
                     }
+                    // The current mutable component is always added
+                    operationalComponents.add(0, memoryComponents.get(cmc));
                 }
-                // The current mutable component is always added
-                operationalComponents.add(0, memoryComponents.get(cmc));
-
                 if (filterManager != null) {
                     for (ILSMComponent c : immutableComponents) {
                         if (c.getLSMComponentFilter().satisfy(
@@ -406,7 +392,6 @@
             throws HyracksDataException, IndexException {
         List<ILSMComponent> operationalComponents = ictx.getComponentHolder();
         int numComponents = operationalComponents.size();
-        assert numComponents > 0;
         boolean includeMutableComponent = false;
         ArrayList<IIndexAccessor> indexAccessors = new ArrayList<IIndexAccessor>(numComponents);
         ArrayList<IIndexAccessor> deletedKeysBTreeAccessors = new ArrayList<IIndexAccessor>(numComponents);
@@ -863,10 +848,12 @@
     @Override
     public long getMemoryAllocationSize() {
         long size = 0;
-        for (ILSMComponent c : memoryComponents) {
-            LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent)
c;
-            IBufferCache virtualBufferCache = mutableComponent.getInvIndex().getBufferCache();
-            size += virtualBufferCache.getNumPages() * virtualBufferCache.getPageSize();
+        if (memoryComponentsAllocated) {
+            for (ILSMComponent c : memoryComponents) {
+                LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent)
c;
+                IBufferCache virtualBufferCache = mutableComponent.getInvIndex().getBufferCache();
+                size += virtualBufferCache.getNumPages() * virtualBufferCache.getPageSize();
+            }
         }
         return size;
     }
@@ -916,10 +903,12 @@
 
     @Override
     public void validate() throws HyracksDataException {
-        for (ILSMComponent c : memoryComponents) {
-            LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent)
c;
-            mutableComponent.getInvIndex().validate();
-            mutableComponent.getDeletedKeysBTree().validate();
+        if (memoryComponentsAllocated) {
+            for (ILSMComponent c : memoryComponents) {
+                LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent)
c;
+                mutableComponent.getInvIndex().validate();
+                mutableComponent.getDeletedKeysBTree().validate();
+            }
         }
         List<ILSMComponent> immutableComponents = diskComponents;
         for (ILSMComponent c : immutableComponents) {
@@ -958,4 +947,38 @@
 
         return files;
     }
-}
+
+    @Override
+    public synchronized void allocateMemoryComponents() throws HyracksDataException {
+        if (!isActivated) {
+            throw new HyracksDataException("Failed to allocate memory components since the
index is not active.");
+        }
+        if (memoryComponentsAllocated) {
+            return;
+        }
+        for (ILSMComponent c : memoryComponents) {
+            LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent)
c;
+            ((IVirtualBufferCache) mutableComponent.getInvIndex().getBufferCache()).open();
+            mutableComponent.getInvIndex().create();
+            mutableComponent.getInvIndex().activate();
+            mutableComponent.getDeletedKeysBTree().create();
+            mutableComponent.getDeletedKeysBTree().activate();
+        }
+        memoryComponentsAllocated = true;
+    }
+
+    private synchronized void deallocateMemoryComponents() throws HyracksDataException {
+        if (!memoryComponentsAllocated) {
+            return;
+        }
+        for (ILSMComponent c : memoryComponents) {
+            LSMInvertedIndexMemoryComponent mutableComponent = (LSMInvertedIndexMemoryComponent)
c;
+            mutableComponent.getInvIndex().deactivate();
+            mutableComponent.getDeletedKeysBTree().deactivate();
+            mutableComponent.getInvIndex().destroy();
+            mutableComponent.getDeletedKeysBTree().destroy();
+            ((IVirtualBufferCache) mutableComponent.getInvIndex().getBufferCache()).close();
+        }
+        memoryComponentsAllocated = false;
+    }
+}
\ No newline at end of file
diff --git a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
index 533dfac..3ee28ca 100644
--- a/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
+++ b/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/AbstractLSMRTree.java
@@ -169,15 +169,6 @@
         if (isActivated) {
             throw new HyracksDataException("Failed to activate the index since it is already
activated.");
         }
-
-        for (ILSMComponent c : memoryComponents) {
-            LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
-            ((IVirtualBufferCache) mutableComponent.getRTree().getBufferCache()).open();
-            mutableComponent.getRTree().create();
-            mutableComponent.getBTree().create();
-            mutableComponent.getRTree().activate();
-            mutableComponent.getBTree().activate();
-        }
     }
 
     @Override
@@ -196,15 +187,7 @@
                 throw new HyracksDataException(e);
             }
         }
-
-        for (ILSMComponent c : memoryComponents) {
-            LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
-            mutableComponent.getRTree().deactivate();
-            mutableComponent.getBTree().deactivate();
-            mutableComponent.getRTree().destroy();
-            mutableComponent.getBTree().destroy();
-            ((IVirtualBufferCache) mutableComponent.getRTree().getBufferCache()).close();
-        }
+        deallocateMemoryComponents();
     }
 
     @Override
@@ -219,12 +202,13 @@
         if (!isActivated) {
             throw new HyracksDataException("Failed to clear the index since it is not activated.");
         }
-
-        for (ILSMComponent c : memoryComponents) {
-            LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
-            mutableComponent.getRTree().clear();
-            mutableComponent.getBTree().clear();
-            mutableComponent.reset();
+        if (memoryComponentsAllocated) {
+            for (ILSMComponent c : memoryComponents) {
+                LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
+                mutableComponent.getRTree().clear();
+                mutableComponent.getBTree().clear();
+                mutableComponent.reset();
+            }
         }
     }
 
@@ -243,16 +227,18 @@
                 operationalComponents.add(memoryComponents.get(cmc));
                 break;
             case SEARCH:
-                for (int i = 0; i < numMutableComponents - 1; i++) {
-                    ILSMComponent c = memoryComponents.get((cmc + i + 1) % numMutableComponents);
-                    LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent)
c;
-                    if (mutableComponent.isReadable()) {
-                        // Make sure newest components are added first
-                        operationalComponents.add(0, mutableComponent);
+                if (memoryComponentsAllocated) {
+                    for (int i = 0; i < numMutableComponents - 1; i++) {
+                        ILSMComponent c = memoryComponents.get((cmc + i + 1) % numMutableComponents);
+                        LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent)
c;
+                        if (mutableComponent.isReadable()) {
+                            // Make sure newest components are added first
+                            operationalComponents.add(0, mutableComponent);
+                        }
                     }
+                    // The current mutable component is always added
+                    operationalComponents.add(0, memoryComponents.get(cmc));
                 }
-                // The current mutable component is always added
-                operationalComponents.add(0, memoryComponents.get(cmc));
 
                 if (filterManager != null) {
                     for (ILSMComponent c : immutableComponents) {
@@ -449,4 +435,38 @@
     public String toString() {
         return "LSMRTree [" + fileManager.getBaseDir() + "]";
     }
-}
+
+    @Override
+    public synchronized void allocateMemoryComponents() throws HyracksDataException {
+        if (!isActivated) {
+            throw new HyracksDataException("Failed to allocate memory components since the
index is not active.");
+        }
+        if (memoryComponentsAllocated) {
+            return;
+        }
+        for (ILSMComponent c : memoryComponents) {
+            LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
+            ((IVirtualBufferCache) mutableComponent.getRTree().getBufferCache()).open();
+            mutableComponent.getRTree().create();
+            mutableComponent.getBTree().create();
+            mutableComponent.getRTree().activate();
+            mutableComponent.getBTree().activate();
+        }
+        memoryComponentsAllocated = true;
+    }
+
+    private synchronized void deallocateMemoryComponents() throws HyracksDataException {
+        if (!memoryComponentsAllocated) {
+            return;
+        }
+        for (ILSMComponent c : memoryComponents) {
+            LSMRTreeMemoryComponent mutableComponent = (LSMRTreeMemoryComponent) c;
+            mutableComponent.getRTree().deactivate();
+            mutableComponent.getBTree().deactivate();
+            mutableComponent.getRTree().destroy();
+            mutableComponent.getBTree().destroy();
+            ((IVirtualBufferCache) mutableComponent.getRTree().getBufferCache()).close();
+        }
+        memoryComponentsAllocated = false;
+    }
+}
\ No newline at end of file

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/405
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: I476e756f8d71260ea614c8c072fc9503053866c9
Gerrit-PatchSet: 1
Gerrit-Project: hyracks
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <hubailmor@gmail.com>

Mime
View raw message