aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wfar...@apache.org
Subject aurora git commit: Create an ELB-friendly endpoint to detect the leading scheduler.
Date Tue, 05 Apr 2016 16:52:40 GMT
Repository: aurora
Updated Branches:
  refs/heads/master f729fd4d2 -> 663d13f23


Create an ELB-friendly endpoint to detect the leading scheduler.

Reviewed at https://reviews.apache.org/r/45511/


Project: http://git-wip-us.apache.org/repos/asf/aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/aurora/commit/663d13f2
Tree: http://git-wip-us.apache.org/repos/asf/aurora/tree/663d13f2
Diff: http://git-wip-us.apache.org/repos/asf/aurora/diff/663d13f2

Branch: refs/heads/master
Commit: 663d13f232a7de73892729449c576ad6ff234aaa
Parents: f729fd4
Author: Ashwin Murthy <ashwinmurthy@gmail.com>
Authored: Tue Apr 5 09:52:40 2016 -0700
Committer: Bill Farner <wfarner@apache.org>
Committed: Tue Apr 5 09:52:40 2016 -0700

----------------------------------------------------------------------
 RELEASE-NOTES.md                                |  4 +-
 docs/reference/scheduler-endpoints.md           | 19 ++++
 .../scheduler/http/JettyServerModule.java       |  1 +
 .../aurora/scheduler/http/LeaderHealth.java     | 93 ++++++++++++++++++++
 src/main/resources/scheduler/assets/index.html  |  1 +
 .../aurora/scheduler/http/LeaderHealthTest.java | 63 +++++++++++++
 .../sh/org/apache/aurora/e2e/test_end_to_end.sh |  1 +
 7 files changed, 181 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/aurora/blob/663d13f2/RELEASE-NOTES.md
----------------------------------------------------------------------
diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md
index 1df0345..2f935da 100644
--- a/RELEASE-NOTES.md
+++ b/RELEASE-NOTES.md
@@ -1,9 +1,11 @@
-0.13.0
+0.13.0 (Not yet released)
 ------
 
 ### New/updated:
 
 - Upgraded Mesos to 0.26.0
+- Added a new health endpoint (/leaderhealth) which can be used for load balancer health
+  checks to always forward requests to the leading scheduler.
 - Added a new `aurora job add` client command to scale out an existing job.
 - Upgraded the scheduler ZooKeeper client from 3.4.6 to 3.4.8.
 - Added support for dedicated constraints not exclusive to a particular role.

http://git-wip-us.apache.org/repos/asf/aurora/blob/663d13f2/docs/reference/scheduler-endpoints.md
----------------------------------------------------------------------
diff --git a/docs/reference/scheduler-endpoints.md b/docs/reference/scheduler-endpoints.md
new file mode 100644
index 0000000..d302e90
--- /dev/null
+++ b/docs/reference/scheduler-endpoints.md
@@ -0,0 +1,19 @@
+# HTTP endpoints
+
+There are a number of HTTP endpoints that the Aurora scheduler exposes. These allow various
+operational tasks to be performed on the scheduler. Below is the list of all such endpoints
+and a brief explanation of what they do.
+
+## Leader health
+The /leaderhealth endpoint enables performing health checks on the scheduler instances inorder
+to forward requests to the leading scheduler. This is typically used by a load balancer such
as
+HAProxy or AWS ELB.
+
+When a HTTP GET request is issued on this endpoint, it responds as follows:
+
+- If the instance that received the GET request is the leading scheduler, a HTTP status code
of
+  200 (OK) is returned.
+- If the instance that received the GET request is not the leading scheduler but a leader
does
+  exist, a HTTP status code of 503 (SERVICE_UNAVAILABLE) is returned.
+- If no leader currently exists or the leader is unknown, a HTTP status code of 502
+  (BAD_GATEWAY) is returned.
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/aurora/blob/663d13f2/src/main/java/org/apache/aurora/scheduler/http/JettyServerModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/JettyServerModule.java b/src/main/java/org/apache/aurora/scheduler/http/JettyServerModule.java
index df649ff..a5446e3 100644
--- a/src/main/java/org/apache/aurora/scheduler/http/JettyServerModule.java
+++ b/src/main/java/org/apache/aurora/scheduler/http/JettyServerModule.java
@@ -216,6 +216,7 @@ public class JettyServerModule extends AbstractModule {
           .put(ContentionPrinter.class, "contention")
           .put(Cron.class, "cron")
           .put(HealthHandler.class, "health")
+          .put(LeaderHealth.class, "leaderhealth")
           .put(Locks.class, "locks")
           .put(LogConfig.class, "logconfig")
           .put(Maintenance.class, "maintenance")

http://git-wip-us.apache.org/repos/asf/aurora/blob/663d13f2/src/main/java/org/apache/aurora/scheduler/http/LeaderHealth.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/LeaderHealth.java b/src/main/java/org/apache/aurora/scheduler/http/LeaderHealth.java
new file mode 100644
index 0000000..8a7886f
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/http/LeaderHealth.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.scheduler.http;
+
+import java.util.Map;
+import java.util.Objects;
+
+import javax.inject.Inject;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+
+import org.apache.aurora.common.collections.Pair;
+
+import static javax.servlet.http.HttpServletResponse.SC_BAD_GATEWAY;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
+import static javax.servlet.http.HttpServletResponse.SC_SERVICE_UNAVAILABLE;
+
+import static org.apache.aurora.scheduler.http.LeaderRedirect.LeaderStatus;
+import static org.apache.aurora.scheduler.http.LeaderRedirect.LeaderStatus.LEADING;
+import static org.apache.aurora.scheduler.http.LeaderRedirect.LeaderStatus.NOT_LEADING;
+import static org.apache.aurora.scheduler.http.LeaderRedirect.LeaderStatus.NO_LEADER;
+
+/**
+ * Servlet that exposes the leader health. It is meant to be used by load balancers such
as
+ * ELB to always forward to the leading scheduler.
+ */
+@Path("/leaderhealth")
+public class LeaderHealth {
+
+  private static final Map<LeaderStatus, Pair<Integer, String>> RESPONSE_CODES
= ImmutableMap.of(
+      LEADING, Pair.of(SC_OK, "This is the leading scheduler."),
+      NOT_LEADING, Pair.of(SC_SERVICE_UNAVAILABLE, "Another instance is the leading scheduler."),
+      NO_LEADER, Pair.of(SC_BAD_GATEWAY, "There is no leading scheduler or the leader is
unknown.")
+  );
+
+  private static final String HELP = generateHelpText();
+
+  private final LeaderRedirect leaderRedirect;
+
+  @Inject
+  LeaderHealth(LeaderRedirect leaderRedirect) {
+    this.leaderRedirect = Objects.requireNonNull(leaderRedirect);
+  }
+
+  @VisibleForTesting
+  static String getHelpText(LeaderStatus status) {
+    return "Current status: " + RESPONSE_CODES.get(status).getSecond()
+        + "\n\n"
+        + HELP;
+  }
+
+  /**
+   * Gets the leadership status of the scheduler.
+   *
+   * @return HTTP response.
+   */
+  @GET
+  public Response get() {
+    LeaderStatus leaderStatus = leaderRedirect.getLeaderStatus();
+    return Response.status(RESPONSE_CODES.get(leaderStatus).getFirst())
+        .type(MediaType.TEXT_PLAIN_TYPE)
+        .entity(getHelpText(leaderStatus))
+        .build();
+  }
+
+  private static String generateHelpText() {
+    Map<Integer, String> details = Maps.transformValues(
+        Maps.uniqueIndex(RESPONSE_CODES.values(), Pair::getFirst),
+        Pair::getSecond);
+
+    return "This endpoint can indicate to a load balancer which among a group of schedulers
"
+        + "is leading.\n\nThe response codes are:\n"
+        + Joiner.on("\n").withKeyValueSeparator(": ").join(details);
+  }
+}

http://git-wip-us.apache.org/repos/asf/aurora/blob/663d13f2/src/main/resources/scheduler/assets/index.html
----------------------------------------------------------------------
diff --git a/src/main/resources/scheduler/assets/index.html b/src/main/resources/scheduler/assets/index.html
index 861d551..5679b19 100644
--- a/src/main/resources/scheduler/assets/index.html
+++ b/src/main/resources/scheduler/assets/index.html
@@ -76,6 +76,7 @@
    <h4> Scheduler Internals</h4>
     <ul class="list-unstyled group-summary">
       <li><a href='/contention'>/contention</a></li>
+      <li><a href='/leaderhealth'>/leaderhealth</a></li>
       <li><a href='/locks'>/locks</a></li>
       <li><a href='/logconfig'>/logconfig</a></li>
       <li><a href='/services'>/services</a></li>

http://git-wip-us.apache.org/repos/asf/aurora/blob/663d13f2/src/test/java/org/apache/aurora/scheduler/http/LeaderHealthTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/http/LeaderHealthTest.java b/src/test/java/org/apache/aurora/scheduler/http/LeaderHealthTest.java
new file mode 100644
index 0000000..d21a38e
--- /dev/null
+++ b/src/test/java/org/apache/aurora/scheduler/http/LeaderHealthTest.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.scheduler.http;
+
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.core.Response;
+
+import org.apache.aurora.common.net.pool.DynamicHostSet.MonitorException;
+import org.apache.aurora.common.testing.easymock.EasyMockTest;
+import org.apache.aurora.scheduler.http.LeaderRedirect.LeaderStatus;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.easymock.EasyMock.expect;
+import static org.junit.Assert.assertEquals;
+
+public class LeaderHealthTest extends EasyMockTest {
+
+  private LeaderRedirect leaderRedirect;
+  private LeaderHealth leaderHealth;
+
+  @Before
+  public void setUp() throws MonitorException {
+    leaderRedirect = createMock(LeaderRedirect.class);
+    leaderHealth = new LeaderHealth(leaderRedirect);
+  }
+
+  private void expectResponse(LeaderStatus status, int responseCode) {
+    expect(leaderRedirect.getLeaderStatus()).andReturn(status);
+
+    control.replay();
+
+    Response response = leaderHealth.get();
+    assertEquals(responseCode, response.getStatus());
+    assertEquals(LeaderHealth.getHelpText(status), response.getEntity().toString());
+  }
+
+  @Test
+  public void testLeader() throws Exception {
+    expectResponse(LeaderStatus.LEADING, HttpServletResponse.SC_OK);
+  }
+
+  @Test
+  public void testNotLeader() throws Exception {
+    expectResponse(LeaderStatus.NOT_LEADING, HttpServletResponse.SC_SERVICE_UNAVAILABLE);
+  }
+
+  @Test
+  public void testNoLeaders() throws Exception {
+    expectResponse(LeaderStatus.NO_LEADER, HttpServletResponse.SC_BAD_GATEWAY);
+  }
+}

http://git-wip-us.apache.org/repos/asf/aurora/blob/663d13f2/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh
----------------------------------------------------------------------
diff --git a/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh b/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh
index 77a4c36..3471756 100755
--- a/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh
+++ b/src/test/sh/org/apache/aurora/e2e/test_end_to_end.sh
@@ -116,6 +116,7 @@ test_scheduler_ui() {
 
   # Check that scheduler UI pages shown
   base_url="localhost:8081"
+  check_url_live "$base_url/leaderhealth"
   check_url_live "$base_url/scheduler"
   check_url_live "$base_url/scheduler/$_role"
   check_url_live "$base_url/scheduler/$_role/$_env/$_job"


Mime
View raw message