asterixdb-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ima...@apache.org
Subject [03/50] [abbrv] incubator-asterixdb git commit: Exclude Temporary Indexes From Replication
Date Thu, 07 Apr 2016 14:59:38 GMT
Exclude Temporary Indexes From Replication

- Exclude non-durable LSM indexes from replication.
- Stop heartbeat task after NC has stopped to avoid false failures detection.

Change-Id: Icce91a203e04cb068a7a5aa541720bbd0289eacb
Reviewed-on: https://asterix-gerrit.ics.uci.edu/777
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <bamousaa@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/commit/08aa051a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/tree/08aa051a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/diff/08aa051a

Branch: refs/heads/master
Commit: 08aa051a34d5c2b757cc0e0b2e018725e164d0f7
Parents: b2e9d08
Author: Murtadha Hubail <mhubail@uci.edu>
Authored: Sat Apr 2 20:55:44 2016 -0700
Committer: Murtadha Hubail <hubailmor@gmail.com>
Committed: Sun Apr 3 00:18:32 2016 -0700

----------------------------------------------------------------------
 .../control/nc/NodeControllerService.java       |  9 +++++--
 .../storage/am/lsm/common/api/ILSMIndex.java    |  6 +++++
 .../am/lsm/common/impls/AbstractLSMIndex.java   | 25 +++++++++++---------
 .../storage/am/lsm/common/impls/LSMHarness.java |  3 ++-
 4 files changed, 29 insertions(+), 14 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/08aa051a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
index 7b5758c..598d6db 100644
--- a/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
+++ b/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
@@ -338,12 +338,17 @@ public class NodeControllerService implements IControllerService {
             }
             partitionManager.close();
             datasetPartitionManager.close();
-            heartbeatTask.cancel();
             netManager.stop();
             datasetNetworkManager.stop();
             queue.stop();
-            if (ncAppEntryPoint != null)
+            if (ncAppEntryPoint != null) {
                 ncAppEntryPoint.stop();
+            }
+            /**
+             * Stop heartbeat after NC has stopped to avoid false node failure detection
+             * on CC if an NC takes a long time to stop.
+             */
+            heartbeatTask.cancel();
             LOGGER.log(Level.INFO, "Stopped NodeControllerService");
             shuttedDown = true;
         }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/08aa051a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
index c1cef2d..11b933d 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMIndex.java
@@ -39,6 +39,7 @@ public interface ILSMIndex extends IIndex {
 
     public void deactivate(boolean flushOnExit) throws HyracksDataException;
 
+    @Override
     public ILSMIndexAccessor createAccessor(IModificationOperationCallback modificationCallback,
             ISearchOperationCallback searchCallback) throws HyracksDataException;
 
@@ -51,4 +52,9 @@ public interface ILSMIndex extends IIndex {
     public List<ILSMComponent> getImmutableComponents();
 
     public boolean isPrimaryIndex();
+
+    /**
+     * @return true if the index is durable. Otherwise false.
+     */
+    public boolean isDurable();
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/08aa051a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
index 441dda1..440ad31 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndex.java
@@ -33,7 +33,6 @@ import org.apache.hyracks.api.replication.IReplicationJob.ReplicationExecutionTy
 import org.apache.hyracks.api.replication.IReplicationJob.ReplicationOperation;
 import org.apache.hyracks.storage.am.bloomfilter.impls.BloomFilter;
 import org.apache.hyracks.storage.am.common.api.ITreeIndex;
-import org.apache.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent.ComponentState;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentFilterFrameFactory;
@@ -48,8 +47,6 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
 import org.apache.hyracks.storage.am.lsm.common.api.IVirtualBufferCache;
 import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
 import org.apache.hyracks.storage.common.buffercache.IBufferCache;
-import org.apache.hyracks.storage.common.buffercache.ICachedPage;
-import org.apache.hyracks.storage.common.file.BufferedFileHandle;
 import org.apache.hyracks.storage.common.file.IFileMapProvider;
 
 public abstract class AbstractLSMIndex implements ILSMIndexInternal {
@@ -80,11 +77,10 @@ public abstract class AbstractLSMIndex implements ILSMIndexInternal {
     protected boolean memoryComponentsAllocated = false;
 
     public AbstractLSMIndex(List<IVirtualBufferCache> virtualBufferCaches, IBufferCache
diskBufferCache,
-            ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider,
-            double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker
opTracker,
-            ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallback ioOpCallback,
-            ILSMComponentFilterFrameFactory filterFrameFactory, LSMComponentFilterManager
filterManager,
-            int[] filterFields, boolean durable) {
+            ILSMIndexFileManager fileManager, IFileMapProvider diskFileMapProvider, double
bloomFilterFalsePositiveRate,
+            ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker, ILSMIOOperationScheduler
ioScheduler,
+            ILSMIOOperationCallback ioOpCallback, ILSMComponentFilterFrameFactory filterFrameFactory,
+            LSMComponentFilterManager filterManager, int[] filterFields, boolean durable)
{
         this.virtualBufferCaches = virtualBufferCaches;
         this.diskBufferCache = diskBufferCache;
         this.diskFileMapProvider = diskFileMapProvider;
@@ -149,8 +145,8 @@ public abstract class AbstractLSMIndex implements ILSMIndexInternal {
     }
 
     protected void markAsValidInternal(IBufferCache bufferCache, BloomFilter filter) throws
HyracksDataException {
-        if(durable){
-            bufferCache.force(filter.getFileId(),true);
+        if (durable) {
+            bufferCache.force(filter.getFileId(), true);
         }
     }
 
@@ -208,7 +204,7 @@ public abstract class AbstractLSMIndex implements ILSMIndexInternal {
         return diskBufferCache;
     }
 
-    public boolean isEmptyIndex() throws HyracksDataException {
+    public boolean isEmptyIndex() {
         boolean isModified = false;
         for (ILSMComponent c : memoryComponents) {
             AbstractMemoryLSMComponent mutableComponent = (AbstractMemoryLSMComponent) c;
@@ -288,9 +284,16 @@ public abstract class AbstractLSMIndex implements ILSMIndexInternal {
         }
     }
 
+    @Override
     public abstract void allocateMemoryComponents() throws HyracksDataException;
 
+    @Override
     public boolean isMemoryComponentsAllocated() {
         return memoryComponentsAllocated;
     }
+
+    @Override
+    public boolean isDurable() {
+        return durable;
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-asterixdb/blob/08aa051a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
----------------------------------------------------------------------
diff --git a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index 21b0d8a..0224c5c 100644
--- a/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++ b/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@ -63,7 +63,8 @@ public class LSMHarness implements ILSMHarness {
         this.opTracker = opTracker;
         this.mergePolicy = mergePolicy;
         fullMergeIsRequested = new AtomicBoolean();
-        this.replicationEnabled = replicationEnabled;
+        //only durable indexes are replicated
+        this.replicationEnabled = replicationEnabled && lsmIndex.isDurable();
         if (replicationEnabled) {
             this.componentsToBeReplicated = new ArrayList<ILSMComponent>();
         }


Mime
View raw message