aurora-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wfar...@apache.org
Subject git commit: Upgrade to latest in jetty 7.x series.
Date Thu, 04 Sep 2014 18:43:14 GMT
Repository: incubator-aurora
Updated Branches:
  refs/heads/master e060687e9 -> 38d56fbdf


Upgrade to latest in jetty 7.x series.

Bugs closed: AURORA-679

Reviewed at https://reviews.apache.org/r/25285/


Project: http://git-wip-us.apache.org/repos/asf/incubator-aurora/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-aurora/commit/38d56fbd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-aurora/tree/38d56fbd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-aurora/diff/38d56fbd

Branch: refs/heads/master
Commit: 38d56fbdf6f5fd5c81e27ac27cf265343ce82c2f
Parents: e060687
Author: Bill Farner <wfarner@apache.org>
Authored: Thu Sep 4 11:39:53 2014 -0700
Committer: Bill Farner <wfarner@apache.org>
Committed: Thu Sep 4 11:39:53 2014 -0700

----------------------------------------------------------------------
 build.gradle                                    |  12 +-
 .../scheduler/http/JettyServerModule.java       | 121 +++++++++++++----
 .../aurora/scheduler/http/RequestLogger.java    | 131 +++++++++++++++++++
 .../scheduler/http/RequestLoggerTest.java       | 103 +++++++++++++++
 4 files changed, 344 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/38d56fbd/build.gradle
----------------------------------------------------------------------
diff --git a/build.gradle b/build.gradle
index 494c544..3237f8d 100644
--- a/build.gradle
+++ b/build.gradle
@@ -148,6 +148,9 @@ dependencies {
 
   def gsonDep = 'com.google.code.gson:gson:2.2.4'
   def guavaDep = 'com.google.guava:guava:16.0'
+  // NOTE: We are using the jetty 7.x series due to a large number of dependencies impacted
+  // by 8.x and later resulting from using newer javax.servlet servlet-api.
+  def jettyDep = '7.6.15.v20140411'
   def thriftLib = "org.apache.thrift:libthrift:${thriftVersion}"
 
   compile 'aopalliance:aopalliance:1.0'
@@ -170,6 +173,9 @@ dependencies {
   compile 'org.apache.mesos:mesos:0.20.0'
   compile thriftLib
   compile 'org.apache.zookeeper:zookeeper:3.3.4'
+  compile "org.eclipse.jetty:jetty-server:${jettyDep}"
+  compile "org.eclipse.jetty:jetty-servlet:${jettyDep}"
+  compile "org.eclipse.jetty:jetty-servlets:${jettyDep}"
   compile 'org.mybatis:mybatis:3.2.7'
   compile 'org.mybatis:mybatis-guice:3.6'
   compile 'org.quartz-scheduler:quartz:2.2.1'
@@ -183,7 +189,11 @@ dependencies {
   compile 'com.twitter.common.zookeeper:group:0.0.69'
   compile 'com.twitter.common.zookeeper:server-set:1.0.74'
   compile 'com.twitter.common.zookeeper:singleton-service:0.0.85'
-  compile 'com.twitter.common:application-http:0.0.59'
+  compile('com.twitter.common:application-http:0.0.59') {
+    // TODO(wfarner): Contribute a patch to twitter commons that allows us to register /graphview
+    // resources without pulling in a jetty dependency.
+    exclude(group: 'org.mortbay.jetty', module: 'jetty')
+  }
   compile 'com.twitter.common:application-module-applauncher:0.0.51'
   compile 'com.twitter.common:application-module-lifecycle:0.0.48'
   compile 'com.twitter.common:application-module-log:0.0.56'

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/38d56fbd/src/main/java/org/apache/aurora/scheduler/http/JettyServerModule.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/JettyServerModule.java b/src/main/java/org/apache/aurora/scheduler/http/JettyServerModule.java
index de49a1c..83ba0e4 100644
--- a/src/main/java/org/apache/aurora/scheduler/http/JettyServerModule.java
+++ b/src/main/java/org/apache/aurora/scheduler/http/JettyServerModule.java
@@ -13,17 +13,23 @@
  */
 package org.apache.aurora.scheduler.http;
 
+import java.util.EnumSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
+import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import javax.annotation.Nonnegative;
 import javax.inject.Inject;
 import javax.inject.Singleton;
-import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Ordering;
 import com.google.common.io.Resources;
 import com.google.common.net.MediaType;
 import com.google.inject.AbstractModule;
@@ -41,20 +47,20 @@ import com.twitter.common.application.http.HttpFilterConfig;
 import com.twitter.common.application.http.HttpServletConfig;
 import com.twitter.common.application.http.Registration;
 import com.twitter.common.application.modules.LifecycleModule;
+import com.twitter.common.application.modules.LifecycleModule.LaunchException;
 import com.twitter.common.args.Arg;
 import com.twitter.common.args.CmdLine;
 import com.twitter.common.base.Command;
 import com.twitter.common.base.ExceptionalCommand;
 import com.twitter.common.base.ExceptionalSupplier;
 import com.twitter.common.base.MoreSuppliers;
-import com.twitter.common.net.http.HttpServerDispatch;
-import com.twitter.common.net.http.RequestLogger;
 import com.twitter.common.net.http.handlers.AbortHandler;
 import com.twitter.common.net.http.handlers.ContentionPrinter;
 import com.twitter.common.net.http.handlers.HealthHandler;
 import com.twitter.common.net.http.handlers.LogConfig;
 import com.twitter.common.net.http.handlers.QuitHandler;
 import com.twitter.common.net.http.handlers.StringTemplateServlet;
+import com.twitter.common.net.http.handlers.TextResponseHandler;
 import com.twitter.common.net.http.handlers.ThreadStackPrinter;
 import com.twitter.common.net.http.handlers.TimeSeriesDataSource;
 import com.twitter.common.net.http.handlers.VarsHandler;
@@ -62,8 +68,15 @@ import com.twitter.common.net.http.handlers.VarsJsonHandler;
 import com.twitter.common.net.pool.DynamicHostSet.MonitorException;
 
 import org.apache.aurora.scheduler.http.api.ApiBeta;
-import org.mortbay.jetty.RequestLog;
-import org.mortbay.servlet.GzipFilter;
+import org.eclipse.jetty.server.Connector;
+import org.eclipse.jetty.server.DispatcherType;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.handler.HandlerCollection;
+import org.eclipse.jetty.server.handler.RequestLogHandler;
+import org.eclipse.jetty.server.nio.SelectChannelConnector;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jetty.servlets.GzipFilter;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.sun.jersey.api.core.ResourceConfig.PROPERTY_CONTAINER_REQUEST_FILTERS;
@@ -75,7 +88,8 @@ import static com.twitter.common.application.modules.LocalServiceRegistry.LocalS
  * Binding module for scheduler HTTP servlets.
  * <p>
  * TODO(wfarner): Continue improvements here by simplifying serving of static assets.  Jetty's
- * DefaultServlet can take over this responsibility.
+ * DefaultServlet can take over this responsibility, and jetty-rewite can be used to rewrite
+ * requests (for static assets) similar to what we currently do with path specs.
  */
 public class JettyServerModule extends AbstractModule {
 
@@ -114,8 +128,6 @@ public class JettyServerModule extends AbstractModule {
 
     bindConstant().annotatedWith(StringTemplateServlet.CacheTemplates.class).to(true);
 
-    bind(HttpServerDispatch.class).in(Singleton.class);
-    bind(RequestLog.class).to(RequestLogger.class);
     Registration.registerServlet(binder(), "/abortabortabort", AbortHandler.class, true);
     Registration.registerServlet(binder(), "/contention", ContentionPrinter.class, false);
     Registration.registerServlet(binder(), "/graphdata", TimeSeriesDataSource.class, true);
@@ -330,7 +342,6 @@ public class JettyServerModule extends AbstractModule {
 
   // TODO(wfarner): Use guava's Service to enforce the lifecycle of this.
   public static final class HttpServerLauncher implements LifecycleModule.ServiceRunner {
-    private final HttpServerDispatch httpServer;
     private final Set<HttpServletConfig> httpServlets;
     private final Set<HttpAssetConfig> httpAssets;
     private final Set<HttpFilterConfig> httpFilters;
@@ -339,14 +350,12 @@ public class JettyServerModule extends AbstractModule {
 
     @Inject
     HttpServerLauncher(
-        HttpServerDispatch httpServer,
         Set<HttpServletConfig> httpServlets,
         Set<HttpAssetConfig> httpAssets,
         Set<HttpFilterConfig> httpFilters,
         @Registration.IndexLink Set<String> additionalIndexLinks,
         Injector injector) {
 
-      this.httpServer = checkNotNull(httpServer);
       this.httpServlets = checkNotNull(httpServlets);
       this.httpAssets = checkNotNull(httpAssets);
       this.httpFilters = checkNotNull(httpFilters);
@@ -354,37 +363,105 @@ public class JettyServerModule extends AbstractModule {
       this.injector = checkNotNull(injector);
     }
 
+    private String makeRecursivePathSpec(String path) {
+      return path.replaceFirst("/?$", "/*");
+    }
+
     @Override
-    public LocalService launch() {
-      if (!httpServer.listen(HTTP_PORT.get())) {
-        throw new IllegalStateException("Failed to start HTTP server, all servlets disabled.");
-      }
+    public LocalService launch() throws LaunchException {
+      final Server server = new Server();
+      ServletContextHandler servletHandler =
+          new ServletContextHandler(server, "/", ServletContextHandler.NO_SESSIONS);
+
+      ImmutableSet.Builder<String> indexLinks = ImmutableSet.builder();
 
       for (HttpServletConfig config : httpServlets) {
-        HttpServlet handler = injector.getInstance(config.handlerClass);
-        httpServer.registerHandler(config.path, handler, config.params, config.silent);
+        ServletHolder holder = new ServletHolder(injector.getInstance(config.handlerClass));
+        if (config.params != null) {
+          holder.setInitParameters(config.params);
+        }
+        servletHandler.addServlet(holder, makeRecursivePathSpec(config.path));
+        if (!config.silent) {
+          indexLinks.add(config.path);
+        }
       }
 
       for (HttpAssetConfig config : httpAssets) {
-        httpServer.registerHandler(config.path, config.handler, null, config.silent);
+        servletHandler.addServlet(
+            new ServletHolder(config.handler),
+            makeRecursivePathSpec(config.path));
+        if (!config.silent) {
+          indexLinks.add(config.path);
+        }
       }
 
       for (HttpFilterConfig filter : httpFilters) {
-        httpServer.registerFilter(filter.filterClass, filter.pathSpec, filter.dispatch);
+        servletHandler.addFilter(
+            filter.filterClass,
+            filter.pathSpec,
+            EnumSet.of(DispatcherType.REQUEST));
       }
 
       for (String indexLink : additionalIndexLinks) {
-        httpServer.registerIndexLink(indexLink);
+        indexLinks.add(indexLink);
+      }
+
+      servletHandler.addServlet(
+          new ServletHolder(new RootHandler(Ordering.natural().sortedCopy(indexLinks.build()))),
+          "/");
+
+      HandlerCollection rootHandler = new HandlerCollection();
+      RequestLogHandler loghandler = new RequestLogHandler();
+      loghandler.setRequestLog(new RequestLogger());
+      rootHandler.addHandler(loghandler);
+      rootHandler.addHandler(servletHandler);
+
+      Connector connector = new SelectChannelConnector();
+      connector.setPort(HTTP_PORT.get());
+      server.addConnector(connector);
+      server.setHandler(rootHandler);
+
+      try {
+        connector.open();
+        server.start();
+      } catch (Exception e) {
+        throw new LaunchException("Failed to start jetty server: " + e, e);
       }
 
       Command shutdown = new Command() {
         @Override public void execute() {
           LOG.info("Shutting down embedded http server");
-          httpServer.stop();
+          try {
+            server.stop();
+          } catch (Exception e) {
+            LOG.log(Level.INFO, "Failed to stop jetty server: " + e, e);
+          }
         }
       };
 
-      return LocalService.auxiliaryService("http", httpServer.getPort(), shutdown);
+      return LocalService.auxiliaryService("http", connector.getLocalPort(), shutdown);
+    }
+  }
+
+  // TODO(wfarner): Replace this with an index.html asset.
+  private static class RootHandler extends TextResponseHandler {
+    private final List<String> lines;
+
+    public RootHandler(List<String> lines) {
+      super("text/html");
+
+      ImmutableList.Builder<String> builder = ImmutableList.builder();
+      builder.add("<html>");
+      for (String handler : lines) {
+        builder.add(String.format("<a href='%s'>%s</a><br />", handler,
handler));
+      }
+      builder.add("</html>");
+      this.lines = builder.build();
+    }
+
+    @Override
+    public Iterable<String> getLines(HttpServletRequest request) {
+      return lines;
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/38d56fbd/src/main/java/org/apache/aurora/scheduler/http/RequestLogger.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/aurora/scheduler/http/RequestLogger.java b/src/main/java/org/apache/aurora/scheduler/http/RequestLogger.java
new file mode 100644
index 0000000..0442191
--- /dev/null
+++ b/src/main/java/org/apache/aurora/scheduler/http/RequestLogger.java
@@ -0,0 +1,131 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.scheduler.http;
+
+import java.util.Locale;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.twitter.common.util.Clock;
+
+import org.eclipse.jetty.http.HttpHeaders;
+import org.eclipse.jetty.server.Request;
+import org.eclipse.jetty.server.RequestLog;
+import org.eclipse.jetty.server.Response;
+import org.eclipse.jetty.util.DateCache;
+import org.eclipse.jetty.util.component.AbstractLifeCycle;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * A copy of twitter RequestLogger from twitter commons, which is a port of jetty's NCSARequestLog
+ * to use java.util.logging.  This clone exists to allow us to upgrade to jetty 7.
+ * <p>
+ * TODO(wfarner): Replace this with jetty's Slf4jRequestLog once we upgrade to jetty 8.
+ */
+public class RequestLogger extends AbstractLifeCycle implements RequestLog {
+
+  private static final Logger LOGGER = Logger.getLogger(RequestLogger.class.getName());
+
+  private final Clock clock;
+  private final LogSink sink;
+  private final DateCache logDateCache;
+
+  interface LogSink {
+    boolean isLoggable(Level level);
+    void log(Level level, String messagge);
+  }
+
+  RequestLogger() {
+    this(Clock.SYSTEM_CLOCK, new LogSink() {
+      @Override
+      public boolean isLoggable(Level level) {
+        return LOGGER.isLoggable(level);
+      }
+
+      @Override public void log(Level level, String message) {
+        LOGGER.log(level, message);
+      }
+    });
+  }
+
+  @VisibleForTesting
+  RequestLogger(Clock clock, LogSink sink) {
+    this.clock = checkNotNull(clock);
+    this.sink = checkNotNull(sink);
+    logDateCache = new DateCache("dd/MMM/yyyy:HH:mm:ss Z", Locale.getDefault());
+    logDateCache.setTimeZoneID("GMT");
+  }
+
+  private String formatEntry(Request request, Response response) {
+    StringBuilder buf = new StringBuilder();
+
+    buf.append(request.getServerName());
+    buf.append(' ');
+
+    String addr = request.getHeader(HttpHeaders.X_FORWARDED_FOR);
+    if (addr == null) {
+      addr = request.getRemoteAddr();
+    }
+
+    buf.append(addr);
+    buf.append(" [");
+    buf.append(logDateCache.format(request.getTimeStamp()));
+    buf.append("] \"");
+    buf.append(request.getMethod());
+    buf.append(' ');
+    buf.append(request.getUri().toString());
+    buf.append(' ');
+    buf.append(request.getProtocol());
+    buf.append("\" ");
+    buf.append(response.getStatus());
+    buf.append(' ');
+    buf.append(response.getContentCount());
+    buf.append(' ');
+
+    String referer = request.getHeader(HttpHeaders.REFERER);
+    if (referer == null) {
+      buf.append("\"-\" ");
+    } else {
+      buf.append('"');
+      buf.append(referer);
+      buf.append("\" ");
+    }
+
+    String agent = request.getHeader(HttpHeaders.USER_AGENT);
+    if (agent == null) {
+      buf.append("\"-\" ");
+    } else {
+      buf.append('"');
+      buf.append(agent);
+      buf.append('"');
+    }
+
+    buf.append(' ');
+    buf.append(clock.nowMillis() - request.getTimeStamp());
+    return buf.toString();
+  }
+
+  @Override
+  public void log(Request request, Response response) {
+    int statusCategory = response.getStatus() / 100;
+    Level level = statusCategory == 2 || statusCategory == 3 ? Level.FINE : Level.INFO;
+    if (!sink.isLoggable(level)) {
+      return;
+    }
+
+    sink.log(level, formatEntry(request, response));
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-aurora/blob/38d56fbd/src/test/java/org/apache/aurora/scheduler/http/RequestLoggerTest.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/aurora/scheduler/http/RequestLoggerTest.java b/src/test/java/org/apache/aurora/scheduler/http/RequestLoggerTest.java
new file mode 100644
index 0000000..6beaa6b
--- /dev/null
+++ b/src/test/java/org/apache/aurora/scheduler/http/RequestLoggerTest.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.aurora.scheduler.http;
+
+import java.util.logging.Level;
+
+import com.twitter.common.quantity.Amount;
+import com.twitter.common.quantity.Time;
+import com.twitter.common.testing.easymock.EasyMockTest;
+import com.twitter.common.util.testing.FakeClock;
+
+import org.eclipse.jetty.http.HttpHeaders;
+import org.eclipse.jetty.http.HttpURI;
+import org.eclipse.jetty.server.Request;
+import org.eclipse.jetty.server.RequestLog;
+import org.eclipse.jetty.server.Response;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.apache.aurora.scheduler.http.RequestLogger.LogSink;
+import static org.easymock.EasyMock.expect;
+
+public class RequestLoggerTest extends EasyMockTest {
+
+  private FakeClock clock;
+  private LogSink sink;
+  private Request request;
+  private Response response;
+
+  private RequestLog log;
+
+  @Before
+  public void setUp() throws Exception {
+    clock = new FakeClock();
+    sink = createMock(LogSink.class);
+    request = createMock(Request.class);
+    response = createMock(Response.class);
+    log = new RequestLogger(clock, sink);
+  }
+
+  @Test
+  public void testFormat200() throws Exception {
+    clock.advance(Amount.of(40L * 365, Time.DAYS));
+
+    expect(response.getStatus()).andReturn(200).atLeastOnce();
+    expect(request.getServerName()).andReturn("snoopy");
+    expect(request.getHeader(HttpHeaders.X_FORWARDED_FOR)).andReturn(null);
+    expect(request.getMethod()).andReturn("GET");
+    expect(request.getUri()).andReturn(new HttpURI("/"));
+    expect(request.getProtocol()).andReturn("http");
+    expect(response.getContentCount()).andReturn(256L);
+    expect(request.getRemoteAddr()).andReturn("easymock-test");
+    expect(request.getHeader(HttpHeaders.REFERER)).andReturn(null);
+    expect(request.getHeader(HttpHeaders.USER_AGENT)).andReturn("junit");
+    expect(request.getTimeStamp()).andReturn(clock.nowMillis()).atLeastOnce();
+
+    expect(sink.isLoggable(Level.FINE)).andReturn(true);
+    sink.log(Level.FINE, "snoopy easymock-test [22/Dec/2009:00:00:00 +0000]"
+        + " \"GET / http\" 200 256 \"-\" \"junit\" 110");
+
+    control.replay();
+
+    clock.advance(Amount.of(110L, Time.MILLISECONDS));
+    log.log(request, response);
+  }
+
+  @Test
+  public void testFormat500() throws Exception {
+    clock.advance(Amount.of(40L * 365, Time.DAYS));
+
+    expect(response.getStatus()).andReturn(500).atLeastOnce();
+    expect(request.getServerName()).andReturn("woodstock");
+    expect(request.getHeader(HttpHeaders.X_FORWARDED_FOR)).andReturn(null);
+    expect(request.getMethod()).andReturn("POST");
+    expect(request.getUri()).andReturn(new HttpURI("/data"));
+    expect(request.getProtocol()).andReturn("http");
+    expect(response.getContentCount()).andReturn(128L);
+    expect(request.getRemoteAddr()).andReturn("easymock-test");
+    expect(request.getHeader(HttpHeaders.REFERER)).andReturn(null);
+    expect(request.getHeader(HttpHeaders.USER_AGENT)).andReturn("junit");
+    expect(request.getTimeStamp()).andReturn(clock.nowMillis()).atLeastOnce();
+
+    expect(sink.isLoggable(Level.INFO)).andReturn(true);
+    sink.log(Level.INFO, "woodstock easymock-test [22/Dec/2009:00:00:00 +0000]"
+        + " \"POST /data http\" 500 128 \"-\" \"junit\" 500");
+
+    control.replay();
+
+    clock.advance(Amount.of(500L, Time.MILLISECONDS));
+    log.log(request, response);
+  }
+}


Mime
View raw message