zookeeper-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From fang...@apache.org
Subject [zookeeper] branch master updated: 948
Date Tue, 21 May 2019 21:41:44 GMT
This is an automated email from the ASF dual-hosted git repository.

fangmin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zookeeper.git


The following commit(s) were added to refs/heads/master by this push:
     new 39a316a  948
39a316a is described below

commit 39a316a3bab747d879ef974308ad888808377983
Author: Brian Nixon <nixon@fb.com>
AuthorDate: Tue May 21 14:41:39 2019 -0700

    948
    
    …e been tried
    
    Author: Brian Nixon <nixon@fb.com>
    
    Reviewers: eolivelli@apache.org, fangmin@apache.org
    
    Closes #948 from enixon/obsr-elect-delay
---
 .../src/main/resources/markdown/zookeeperAdmin.md  | 14 +++++++++
 .../apache/zookeeper/server/quorum/Observer.java   | 33 ++++++++++++++++++++--
 .../apache/zookeeper/server/quorum/QuorumBean.java | 10 +++++++
 .../zookeeper/server/quorum/QuorumMXBean.java      | 10 +++++++
 .../apache/zookeeper/server/quorum/QuorumPeer.java | 21 ++++++++++++--
 5 files changed, 84 insertions(+), 4 deletions(-)

diff --git a/zookeeper-docs/src/main/resources/markdown/zookeeperAdmin.md b/zookeeper-docs/src/main/resources/markdown/zookeeperAdmin.md
index 0c36bb4..454aa06 100644
--- a/zookeeper-docs/src/main/resources/markdown/zookeeperAdmin.md
+++ b/zookeeper-docs/src/main/resources/markdown/zookeeperAdmin.md
@@ -1005,6 +1005,20 @@ As an example, this will enable all four letter word commands:
     keepalive for more information.  Defaults to
     **false**.
 
+* *observer.reconnectDelayMs* :
+    (Java system property: **zookeeper.observer.reconnectDelayMs**)
+    When observer loses its connection with the leader, it waits for the
+    specified value before trying to reconnect with the leader so that
+    the entire observer fleet won't try to run leader election and reconnect
+    to the leader at once.
+    Defaults to 0 ms.
+
+* *observer.election.DelayMs* :
+    (Java system property: **zookeeper.observer.election.DelayMs**)
+    Delay the observer's participation in a leader election upon disconnect
+    so as to prevent unexpected additional load on the voting peers during
+    the process. Defaults to 200 ms.
+
 <a name="sc_authOptions"></a>
 
 #### Encryption, Authentication, Authorization Options
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Observer.java
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Observer.java
index 5e086b7..551c8fe 100644
--- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Observer.java
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Observer.java
@@ -57,11 +57,23 @@ public class Observer extends Learner{
     public static final String OBSERVER_RECONNECT_DELAY_MS =
             "zookeeper.observer.reconnectDelayMs";
 
+    /**
+     * Delay the Observer's participation in a leader election upon disconnect
+     * so as to prevent unexpected additional load on the voting peers during
+     * the process. Default value is 200.
+     */
+    public static final String OBSERVER_ELECTION_DELAY_MS =
+            "zookeeper.observer.election.DelayMs";
+
     private static final long reconnectDelayMs;
 
+    private static volatile long observerElectionDelayMs;
+
     static {
         reconnectDelayMs = Long.getLong(OBSERVER_RECONNECT_DELAY_MS, 0);
         LOG.info(OBSERVER_RECONNECT_DELAY_MS + " = " + reconnectDelayMs);
+        observerElectionDelayMs = Long.getLong(OBSERVER_ELECTION_DELAY_MS, 200);
+        LOG.info(OBSERVER_ELECTION_DELAY_MS + " = " + observerElectionDelayMs);
     }
 
     /**
@@ -214,8 +226,16 @@ public class Observer extends Learner{
     }
 
     static void waitForReconnectDelay() {
-        if (reconnectDelayMs > 0) {
-            long randomDelay = (long) (reconnectDelayMs * Math.random());
+        waitForReconnectDelayHelper(reconnectDelayMs);
+    }
+
+    static void waitForObserverElectionDelay(){
+        waitForReconnectDelayHelper(observerElectionDelayMs);
+    }
+
+    private static void waitForReconnectDelayHelper(long delayValueMs){
+        if (delayValueMs > 0) {
+            long randomDelay = (long) (delayValueMs * Math.random());
             LOG.info("Waiting for " + randomDelay
                     + " ms before reconnecting with the leader");
             try {
@@ -255,5 +275,14 @@ public class Observer extends Learner{
     public QuorumPeer.QuorumServer getCurrentLearnerMaster() {
         return currentLearnerMaster;
     }
+
+    public static long getObserverElectionDelayMs(){
+        return observerElectionDelayMs;
+    }
+
+    public static void setObserverElectionDelayMs(long electionDelayMs) {
+        observerElectionDelayMs = electionDelayMs;
+        LOG.info(OBSERVER_ELECTION_DELAY_MS + " = " + observerElectionDelayMs);
+    }
 }
 
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumBean.java
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumBean.java
index bab3c17..41e32c5 100644
--- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumBean.java
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumBean.java
@@ -70,4 +70,14 @@ public class QuorumBean implements QuorumMXBean, ZKMBeanInfo {
     public boolean isPortUnification() {
         return peer.shouldUsePortUnification();
     }
+
+    @Override
+    public long getObserverElectionDelayMS() {
+        return Observer.getObserverElectionDelayMs();
+    }
+
+    @Override
+    public void setObserverElectionDelayMS(long delayMS) {
+        Observer.setObserverElectionDelayMs(delayMS);
+    }
 }
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumMXBean.java
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumMXBean.java
index 3c24941..9296563 100644
--- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumMXBean.java
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumMXBean.java
@@ -61,4 +61,14 @@ public interface QuorumMXBean {
      * @return SSL communication between quorum members enabled
      */
     public boolean isPortUnification();
+
+    /**
+     * @return Observer Leader Election Reconnect Delay time in MS
+     */
+    public long getObserverElectionDelayMS();
+
+    /**
+     * Set the Observer Leader Election Reconnect Delay time in MS
+     */
+    public void setObserverElectionDelayMS(long delayMS);
 }
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java
index 401c7b0..34b5d94 100644
--- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java
@@ -1275,7 +1275,7 @@ public class QuorumPeer extends ZooKeeperThread implements QuorumStats.Provider
                         // Add delay jitter before we switch to LOOKING
                         // state to reduce the load of ObserverMaster
                         if (isRunning()) {
-                            Observer.waitForReconnectDelay();
+                            Observer.waitForObserverElectionDelay();
                         }
                     }
                     break;
@@ -2073,6 +2073,9 @@ public class QuorumPeer extends ZooKeeperThread implements QuorumStats.Provider
         }
         LOG.info("Updated learner master list to be {}", sb.toString());
         Collections.shuffle(observerMasters);
+        // Reset the internal index of the observerMaster when
+        // the observerMaster List is refreshed
+        nextObserverMaster = 0;
     }
 
     private boolean useObserverMasters() {
@@ -2083,12 +2086,26 @@ public class QuorumPeer extends ZooKeeperThread implements QuorumStats.Provider
     private QuorumServer nextObserverMaster() {
         if (nextObserverMaster >= observerMasters.size()) {
             nextObserverMaster = 0;
+            // Add a reconnect delay only after the observer
+            // has exhausted trying to connect to all the masters
+            // from the observerMasterList
+            if (isRunning()) {
+                Observer.waitForReconnectDelay();
+            }
         }
         return observerMasters.get(nextObserverMaster++);
     }
 
     QuorumServer findLearnerMaster(QuorumServer leader) {
-        return useObserverMasters() ? nextObserverMaster() : leader;
+        if (useObserverMasters()) {
+            return nextObserverMaster();
+        } else {
+            // Add delay jitter to reduce the load on the leader
+            if (isRunning()) {
+                Observer.waitForReconnectDelay();
+            }
+            return leader;
+        }
     }
 
     /**


Mime
View raw message