geode-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From upthewatersp...@apache.org
Subject [1/2] incubator-geode git commit: GEODE-657: Prevent two backups from happening at the same time
Date Fri, 12 Feb 2016 01:37:45 GMT
Repository: incubator-geode
Updated Branches:
  refs/heads/develop 401280d73 -> 3adb0b8e1


GEODE-657: Prevent two backups from happening at the same time

Adding a dlock to prevent two backups from occuring at the same time.


Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/2b2e619e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/2b2e619e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/2b2e619e

Branch: refs/heads/develop
Commit: 2b2e619e35a44c6e5f1f9881b5fba46d884a3934
Parents: 401280d
Author: Barry Oglesby <boglesby@pivotal.io>
Authored: Thu Dec 10 16:41:14 2015 -0800
Committer: Dan Smith <upthewaterspout@apache.org>
Committed: Thu Feb 11 16:42:27 2016 -0800

----------------------------------------------------------------------
 .../internal/AdminDistributedSystemImpl.java    | 25 +++++---
 .../admin/internal/BackupDataStoreHelper.java   | 60 ++++++++++++++++++++
 .../admin/internal/BackupDataStoreResult.java   | 41 +++++++++++++
 .../gemfire/internal/i18n/LocalizedStrings.java |  1 +
 .../internal/beans/DistributedSystemBridge.java | 35 ++++++------
 5 files changed, 136 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/2b2e619e/gemfire-core/src/main/java/com/gemstone/gemfire/admin/internal/AdminDistributedSystemImpl.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/admin/internal/AdminDistributedSystemImpl.java
b/gemfire-core/src/main/java/com/gemstone/gemfire/admin/internal/AdminDistributedSystemImpl.java
index 5709fce..4e84936 100755
--- a/gemfire-core/src/main/java/com/gemstone/gemfire/admin/internal/AdminDistributedSystemImpl.java
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/admin/internal/AdminDistributedSystemImpl.java
@@ -2421,31 +2421,38 @@ implements com.gemstone.gemfire.admin.AdminDistributedSystem,
 
   public static BackupStatus backupAllMembers(DM dm, File targetDir, File baselineDir)
       throws AdminException {
+    BackupStatus status = null;
+    if (BackupDataStoreHelper.obtainLock(dm)) {
+    try {
     Set<PersistentID> missingMembers = getMissingPersistentMembers(dm);
     Set recipients = dm.getOtherDistributionManagerIds();
     
     SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");
     targetDir = new File(targetDir, format.format(new Date()));
-    FlushToDiskRequest.send(dm, recipients);
-    Map<DistributedMember, Set<PersistentID>> existingDataStores 
-        = PrepareBackupRequest.send(dm, recipients);
-    Map<DistributedMember, Set<PersistentID>> successfulMembers 
-        = FinishBackupRequest.send(dm, recipients, targetDir, baselineDir);
+    BackupDataStoreResult result = BackupDataStoreHelper.backupAllMembers(
+        dm, recipients, targetDir, baselineDir);
     
     // It's possible that when calling getMissingPersistentMembers, some members are 
     // still creating/recovering regions, and at FinishBackupRequest.send, the 
     // regions at the members are ready. Logically, since the members in successfulMembers
     // should override the previous missingMembers
-    for(Set<PersistentID> onlineMembersIds : successfulMembers.values()) {
+    for(Set<PersistentID> onlineMembersIds : result.getSuccessfulMembers().values())
{
       missingMembers.removeAll(onlineMembersIds);
     }
     
-    existingDataStores.keySet().removeAll(successfulMembers.keySet());
-    for(Set<PersistentID> lostMembersIds : existingDataStores.values()) {
+    result.getExistingDataStores().keySet().removeAll(result.getSuccessfulMembers().keySet());
+    for(Set<PersistentID> lostMembersIds : result.getExistingDataStores().values())
{
       missingMembers.addAll(lostMembersIds);
     }
     
-    return new BackupStatusImpl(successfulMembers, missingMembers);
+    status = new BackupStatusImpl(result.getSuccessfulMembers(), missingMembers);
+    } finally {
+      BackupDataStoreHelper.releaseLock(dm);
+    }
+    } else {
+      throw new AdminException(LocalizedStrings.DistributedSystem_BACKUP_ALREADY_IN_PROGRESS.toLocalizedString());
+    }
+    return status;
   }
   
   public Map<DistributedMember, Set<PersistentID>> compactAllDiskStores() throws
AdminException {

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/2b2e619e/gemfire-core/src/main/java/com/gemstone/gemfire/admin/internal/BackupDataStoreHelper.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/admin/internal/BackupDataStoreHelper.java
b/gemfire-core/src/main/java/com/gemstone/gemfire/admin/internal/BackupDataStoreHelper.java
new file mode 100644
index 0000000..26a70a8
--- /dev/null
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/admin/internal/BackupDataStoreHelper.java
@@ -0,0 +1,60 @@
+package com.gemstone.gemfire.admin.internal;
+
+import java.io.File;
+import java.util.Map;
+import java.util.Set;
+
+import com.gemstone.gemfire.cache.persistence.PersistentID;
+import com.gemstone.gemfire.distributed.DistributedLockService;
+import com.gemstone.gemfire.distributed.DistributedMember;
+import com.gemstone.gemfire.distributed.internal.DM;
+import com.gemstone.gemfire.internal.Assert;
+
+public class BackupDataStoreHelper {
+
+  private static String LOCK_SERVICE_NAME = BackupDataStoreHelper.class.getSimpleName();
+
+  private static String LOCK_NAME = LOCK_SERVICE_NAME + "_token";
+  
+  private static Object LOCK_SYNC = new Object();
+
+  @SuppressWarnings("rawtypes")
+  public static BackupDataStoreResult backupAllMembers(
+      DM dm, Set recipients, File targetDir, File baselineDir) {
+    FlushToDiskRequest.send(dm, recipients);
+
+    boolean abort= true;
+    Map<DistributedMember, Set<PersistentID>> successfulMembers;
+    Map<DistributedMember, Set<PersistentID>> existingDataStores;
+    try {
+      existingDataStores = PrepareBackupRequest.send(dm, recipients);
+      abort = false;
+    } finally {
+      successfulMembers = FinishBackupRequest.send(dm, recipients, targetDir, baselineDir,
abort);
+    }
+    return new BackupDataStoreResult(existingDataStores, successfulMembers);
+  }
+  
+  private static DistributedLockService getLockService(DM dm) {
+    DistributedLockService dls = DistributedLockService.getServiceNamed(LOCK_SERVICE_NAME);
+    if (dls == null) {
+      synchronized (LOCK_SYNC) {
+        dls = DistributedLockService.getServiceNamed(LOCK_SERVICE_NAME);
+        if (dls == null) {
+          // Create the DistributedLockService
+          dls = DistributedLockService.create(LOCK_SERVICE_NAME, dm.getSystem());
+        }
+      }
+    }
+    Assert.assertTrue(dls != null);
+    return dls;
+  }
+  
+  public static boolean obtainLock(DM dm) {
+    return getLockService(dm).lock(LOCK_NAME, 0, -1);
+  }
+  
+  public static void releaseLock(DM dm) {
+    getLockService(dm).unlock(LOCK_NAME);
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/2b2e619e/gemfire-core/src/main/java/com/gemstone/gemfire/admin/internal/BackupDataStoreResult.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/admin/internal/BackupDataStoreResult.java
b/gemfire-core/src/main/java/com/gemstone/gemfire/admin/internal/BackupDataStoreResult.java
new file mode 100644
index 0000000..f8a8911
--- /dev/null
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/admin/internal/BackupDataStoreResult.java
@@ -0,0 +1,41 @@
+package com.gemstone.gemfire.admin.internal;
+
+import java.util.Map;
+import java.util.Set;
+
+import com.gemstone.gemfire.cache.persistence.PersistentID;
+import com.gemstone.gemfire.distributed.DistributedMember;
+
+public class BackupDataStoreResult {
+  
+  private Map<DistributedMember, Set<PersistentID>> existingDataStores;
+
+  private Map<DistributedMember, Set<PersistentID>> successfulMembers;
+
+  public BackupDataStoreResult(
+      Map<DistributedMember, Set<PersistentID>> existingDataStores,
+      Map<DistributedMember, Set<PersistentID>> successfulMembers) {
+    this.existingDataStores = existingDataStores;
+    this.successfulMembers = successfulMembers;
+  }
+
+  public Map<DistributedMember, Set<PersistentID>> getExistingDataStores() {
+    return this.existingDataStores;
+  }
+
+  public Map<DistributedMember, Set<PersistentID>> getSuccessfulMembers() {
+    return this.successfulMembers;
+  }
+  
+  public String toString() {
+    return new StringBuilder()
+      .append(getClass().getSimpleName())
+      .append("[")
+      .append("existingDataStores=")
+      .append(this.existingDataStores)
+      .append("; successfulMembers=")
+      .append(this.successfulMembers)
+      .append("]")
+      .toString();
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/2b2e619e/gemfire-core/src/main/java/com/gemstone/gemfire/internal/i18n/LocalizedStrings.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/i18n/LocalizedStrings.java
b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/i18n/LocalizedStrings.java
index f6edb9b..8147718 100644
--- a/gemfire-core/src/main/java/com/gemstone/gemfire/internal/i18n/LocalizedStrings.java
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/internal/i18n/LocalizedStrings.java
@@ -2144,6 +2144,7 @@ public class LocalizedStrings extends ParentLocalizedStrings {
   public static final StringId AUTH_PEER_AUTHENTICATION_MISSING_CREDENTIALS = new StringId(6610,
"Failed to find credentials from [{0}] using Authenticator [{1}]");
   public static final StringId AUTH_FAILED_TO_ACQUIRE_AUTHINITIALIZE_INSTANCE = new StringId(6611,
"AuthInitialize instance could not be obtained");
   public static final StringId AUTH_FAILED_TO_OBTAIN_CREDENTIALS_IN_0_USING_AUTHINITIALIZE_1_2
= new StringId(6612, "Failed to obtain credentials using AuthInitialize [{1}]. {2}");
+  public static final StringId DistributedSystem_BACKUP_ALREADY_IN_PROGRESS = new StringId(6613,
"A backup is already in progress.");
   
   /** Testing strings, messageId 90000-99999 **/
   

http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/2b2e619e/gemfire-core/src/main/java/com/gemstone/gemfire/management/internal/beans/DistributedSystemBridge.java
----------------------------------------------------------------------
diff --git a/gemfire-core/src/main/java/com/gemstone/gemfire/management/internal/beans/DistributedSystemBridge.java
b/gemfire-core/src/main/java/com/gemstone/gemfire/management/internal/beans/DistributedSystemBridge.java
index 5034f60..9427831 100644
--- a/gemfire-core/src/main/java/com/gemstone/gemfire/management/internal/beans/DistributedSystemBridge.java
+++ b/gemfire-core/src/main/java/com/gemstone/gemfire/management/internal/beans/DistributedSystemBridge.java
@@ -44,9 +44,8 @@ import javax.management.ObjectName;
 
 import org.apache.logging.log4j.Logger;
 
-import com.gemstone.gemfire.admin.internal.FinishBackupRequest;
-import com.gemstone.gemfire.admin.internal.FlushToDiskRequest;
-import com.gemstone.gemfire.admin.internal.PrepareBackupRequest;
+import com.gemstone.gemfire.admin.internal.BackupDataStoreHelper;
+import com.gemstone.gemfire.admin.internal.BackupDataStoreResult;
 import com.gemstone.gemfire.cache.persistence.PersistentID;
 import com.gemstone.gemfire.distributed.DistributedMember;
 import com.gemstone.gemfire.distributed.internal.DM;
@@ -59,6 +58,7 @@ import com.gemstone.gemfire.internal.admin.remote.RevokePersistentIDRequest;
 import com.gemstone.gemfire.internal.admin.remote.ShutdownAllRequest;
 import com.gemstone.gemfire.internal.cache.GemFireCacheImpl;
 import com.gemstone.gemfire.internal.cache.persistence.PersistentMemberPattern;
+import com.gemstone.gemfire.internal.i18n.LocalizedStrings;
 import com.gemstone.gemfire.internal.logging.LogService;
 import com.gemstone.gemfire.internal.logging.log4j.LocalizedMessage;
 import com.gemstone.gemfire.management.CacheServerMXBean;
@@ -521,6 +521,7 @@ public class DistributedSystemBridge {
    */
   public DiskBackupStatus backupAllMembers(String targetDirPath, String baselineDirPath)
       throws Exception {
+    if (BackupDataStoreHelper.obtainLock(dm)) {
     try {
       
       if(targetDirPath == null || targetDirPath.isEmpty()){
@@ -540,26 +541,21 @@ public class DistributedSystemBridge {
       Set<PersistentID> missingMembers = MissingPersistentIDsRequest.send(dm);
       Set recipients = dm.getOtherDistributionManagerIds();
 
-      FlushToDiskRequest.send(dm, recipients);
+      BackupDataStoreResult result = BackupDataStoreHelper.backupAllMembers(dm, recipients,
targetDir, baselineDir);
 
-      Map<DistributedMember, Set<PersistentID>> existingDataStores = PrepareBackupRequest
-          .send(dm, recipients);
-      Map<DistributedMember, Set<PersistentID>> successfulMembers = FinishBackupRequest
-          .send(dm, recipients, targetDir, baselineDir);
-
-      Iterator<DistributedMember> it = successfulMembers.keySet().iterator();
+      Iterator<DistributedMember> it = result.getSuccessfulMembers().keySet().iterator();
 
       Map<String, String[]> backedUpDiskStores = new HashMap<String, String[]>();
       while (it.hasNext()) {
         DistributedMember member = it.next();
-        Set<PersistentID> setOfDisk = successfulMembers.get(member);
+        Set<PersistentID> setOfDisk = result.getSuccessfulMembers().get(member);
         String[] setOfDiskStr = new String[setOfDisk.size()];
         int j = 0;
         for (PersistentID id : setOfDisk) {
           setOfDiskStr[j] = id.getDirectory();
           j++;
         }
-        backedUpDiskStores.put(member.getName(), setOfDiskStr);
+        backedUpDiskStores.put(member.getId(), setOfDiskStr);
       }
 
       // It's possible that when calling getMissingPersistentMembers, some
@@ -569,17 +565,17 @@ public class DistributedSystemBridge {
       // regions at the members are ready. Logically, since the members in
       // successfulMembers
       // should override the previous missingMembers
-      for (Set<PersistentID> onlineMembersIds : successfulMembers.values()) {
+      for (Set<PersistentID> onlineMembersIds : result.getSuccessfulMembers().values())
{
         missingMembers.removeAll(onlineMembersIds);
       }
 
-      existingDataStores.keySet().removeAll(successfulMembers.keySet());
+      result.getExistingDataStores().keySet().removeAll(result.getSuccessfulMembers().keySet());
       String[] setOfMissingDiskStr = null;
 
-      if (existingDataStores.size() > 0) {
-        setOfMissingDiskStr = new String[existingDataStores.size()];
+      if (result.getExistingDataStores().size() > 0) {
+        setOfMissingDiskStr = new String[result.getExistingDataStores().size()];
         int j = 0;
-        for (Set<PersistentID> lostMembersIds : existingDataStores.values()) {
+        for (Set<PersistentID> lostMembersIds : result.getExistingDataStores().values())
{
           for (PersistentID id : lostMembersIds) {
             setOfMissingDiskStr[j] = id.getDirectory();
             j++;
@@ -595,6 +591,11 @@ public class DistributedSystemBridge {
 
     } catch (Exception e) {
       throw new Exception(e.getLocalizedMessage());
+    } finally {
+      BackupDataStoreHelper.releaseLock(dm);
+    }
+    } else {
+      throw new Exception(LocalizedStrings.DistributedSystem_BACKUP_ALREADY_IN_PROGRESS.toLocalizedString());
     }
   }
 


Mime
View raw message