This is an automated email from the ASF dual-hosted git repository.
fangmin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zookeeper.git
The following commit(s) were added to refs/heads/master by this push:
new 39a316a 948
39a316a is described below
commit 39a316a3bab747d879ef974308ad888808377983
Author: Brian Nixon <nixon@fb.com>
AuthorDate: Tue May 21 14:41:39 2019 -0700
948
…e been tried
Author: Brian Nixon <nixon@fb.com>
Reviewers: eolivelli@apache.org, fangmin@apache.org
Closes #948 from enixon/obsr-elect-delay
---
.../src/main/resources/markdown/zookeeperAdmin.md | 14 +++++++++
.../apache/zookeeper/server/quorum/Observer.java | 33 ++++++++++++++++++++--
.../apache/zookeeper/server/quorum/QuorumBean.java | 10 +++++++
.../zookeeper/server/quorum/QuorumMXBean.java | 10 +++++++
.../apache/zookeeper/server/quorum/QuorumPeer.java | 21 ++++++++++++--
5 files changed, 84 insertions(+), 4 deletions(-)
diff --git a/zookeeper-docs/src/main/resources/markdown/zookeeperAdmin.md b/zookeeper-docs/src/main/resources/markdown/zookeeperAdmin.md
index 0c36bb4..454aa06 100644
--- a/zookeeper-docs/src/main/resources/markdown/zookeeperAdmin.md
+++ b/zookeeper-docs/src/main/resources/markdown/zookeeperAdmin.md
@@ -1005,6 +1005,20 @@ As an example, this will enable all four letter word commands:
keepalive for more information. Defaults to
**false**.
+* *observer.reconnectDelayMs* :
+ (Java system property: **zookeeper.observer.reconnectDelayMs**)
+ When observer loses its connection with the leader, it waits for the
+ specified value before trying to reconnect with the leader so that
+ the entire observer fleet won't try to run leader election and reconnect
+ to the leader at once.
+ Defaults to 0 ms.
+
+* *observer.election.DelayMs* :
+ (Java system property: **zookeeper.observer.election.DelayMs**)
+ Delay the observer's participation in a leader election upon disconnect
+ so as to prevent unexpected additional load on the voting peers during
+ the process. Defaults to 200 ms.
+
<a name="sc_authOptions"></a>
#### Encryption, Authentication, Authorization Options
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Observer.java
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Observer.java
index 5e086b7..551c8fe 100644
--- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Observer.java
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/Observer.java
@@ -57,11 +57,23 @@ public class Observer extends Learner{
public static final String OBSERVER_RECONNECT_DELAY_MS =
"zookeeper.observer.reconnectDelayMs";
+ /**
+ * Delay the Observer's participation in a leader election upon disconnect
+ * so as to prevent unexpected additional load on the voting peers during
+ * the process. Default value is 200.
+ */
+ public static final String OBSERVER_ELECTION_DELAY_MS =
+ "zookeeper.observer.election.DelayMs";
+
private static final long reconnectDelayMs;
+ private static volatile long observerElectionDelayMs;
+
static {
reconnectDelayMs = Long.getLong(OBSERVER_RECONNECT_DELAY_MS, 0);
LOG.info(OBSERVER_RECONNECT_DELAY_MS + " = " + reconnectDelayMs);
+ observerElectionDelayMs = Long.getLong(OBSERVER_ELECTION_DELAY_MS, 200);
+ LOG.info(OBSERVER_ELECTION_DELAY_MS + " = " + observerElectionDelayMs);
}
/**
@@ -214,8 +226,16 @@ public class Observer extends Learner{
}
static void waitForReconnectDelay() {
- if (reconnectDelayMs > 0) {
- long randomDelay = (long) (reconnectDelayMs * Math.random());
+ waitForReconnectDelayHelper(reconnectDelayMs);
+ }
+
+ static void waitForObserverElectionDelay(){
+ waitForReconnectDelayHelper(observerElectionDelayMs);
+ }
+
+ private static void waitForReconnectDelayHelper(long delayValueMs){
+ if (delayValueMs > 0) {
+ long randomDelay = (long) (delayValueMs * Math.random());
LOG.info("Waiting for " + randomDelay
+ " ms before reconnecting with the leader");
try {
@@ -255,5 +275,14 @@ public class Observer extends Learner{
public QuorumPeer.QuorumServer getCurrentLearnerMaster() {
return currentLearnerMaster;
}
+
+ public static long getObserverElectionDelayMs(){
+ return observerElectionDelayMs;
+ }
+
+ public static void setObserverElectionDelayMs(long electionDelayMs) {
+ observerElectionDelayMs = electionDelayMs;
+ LOG.info(OBSERVER_ELECTION_DELAY_MS + " = " + observerElectionDelayMs);
+ }
}
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumBean.java
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumBean.java
index bab3c17..41e32c5 100644
--- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumBean.java
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumBean.java
@@ -70,4 +70,14 @@ public class QuorumBean implements QuorumMXBean, ZKMBeanInfo {
public boolean isPortUnification() {
return peer.shouldUsePortUnification();
}
+
+ @Override
+ public long getObserverElectionDelayMS() {
+ return Observer.getObserverElectionDelayMs();
+ }
+
+ @Override
+ public void setObserverElectionDelayMS(long delayMS) {
+ Observer.setObserverElectionDelayMs(delayMS);
+ }
}
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumMXBean.java
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumMXBean.java
index 3c24941..9296563 100644
--- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumMXBean.java
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumMXBean.java
@@ -61,4 +61,14 @@ public interface QuorumMXBean {
* @return SSL communication between quorum members enabled
*/
public boolean isPortUnification();
+
+ /**
+ * @return Observer Leader Election Reconnect Delay time in MS
+ */
+ public long getObserverElectionDelayMS();
+
+ /**
+ * Set the Observer Leader Election Reconnect Delay time in MS
+ */
+ public void setObserverElectionDelayMS(long delayMS);
}
diff --git a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java
b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java
index 401c7b0..34b5d94 100644
--- a/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java
+++ b/zookeeper-server/src/main/java/org/apache/zookeeper/server/quorum/QuorumPeer.java
@@ -1275,7 +1275,7 @@ public class QuorumPeer extends ZooKeeperThread implements QuorumStats.Provider
// Add delay jitter before we switch to LOOKING
// state to reduce the load of ObserverMaster
if (isRunning()) {
- Observer.waitForReconnectDelay();
+ Observer.waitForObserverElectionDelay();
}
}
break;
@@ -2073,6 +2073,9 @@ public class QuorumPeer extends ZooKeeperThread implements QuorumStats.Provider
}
LOG.info("Updated learner master list to be {}", sb.toString());
Collections.shuffle(observerMasters);
+ // Reset the internal index of the observerMaster when
+ // the observerMaster List is refreshed
+ nextObserverMaster = 0;
}
private boolean useObserverMasters() {
@@ -2083,12 +2086,26 @@ public class QuorumPeer extends ZooKeeperThread implements QuorumStats.Provider
private QuorumServer nextObserverMaster() {
if (nextObserverMaster >= observerMasters.size()) {
nextObserverMaster = 0;
+ // Add a reconnect delay only after the observer
+ // has exhausted trying to connect to all the masters
+ // from the observerMasterList
+ if (isRunning()) {
+ Observer.waitForReconnectDelay();
+ }
}
return observerMasters.get(nextObserverMaster++);
}
QuorumServer findLearnerMaster(QuorumServer leader) {
- return useObserverMasters() ? nextObserverMaster() : leader;
+ if (useObserverMasters()) {
+ return nextObserverMaster();
+ } else {
+ // Add delay jitter to reduce the load on the leader
+ if (isRunning()) {
+ Observer.waitForReconnectDelay();
+ }
+ return leader;
+ }
}
/**
|