flume-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jmhs...@apache.org
Subject svn commit: r1158677 - in /incubator/flume/trunk: flume-core/src/main/java/com/cloudera/flume/agent/ flume-core/src/main/java/com/cloudera/flume/master/ flume-core/src/main/java/com/cloudera/util/ flume-core/src/test/java/com/cloudera/flume/agent/ flum...
Date Wed, 17 Aug 2011 12:54:20 GMT
Author: jmhsieh
Date: Wed Aug 17 12:54:20 2011
New Revision: 1158677

URL: http://svn.apache.org/viewvc?rev=1158677&view=rev
Log:
Revert "FLUME-721: Webapps 'autofindport' feature does not work"

This patch breaks project complilation.

This reverts commit 6c523bd372d093d8f68397fc71d14bf0c80e1964.

Added:
    incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/StatusHttpServer.java
    incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/TestStatusHttpServer.java
Removed:
    incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/HttpServerTestUtils.java
Modified:
    incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/agent/FlumeNode.java
    incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/master/FlumeMaster.java
    incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/InternalHttpServer.java
    incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/agent/TestNodeJersey.java
    incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/master/TestMasterJersey.java
    incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/InternalHttpServerTest.java
    incubator/flume/trunk/flume-node-web/src/test/java/com/cloudera/flume/agent/TestBootstrap.java

Modified: incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/agent/FlumeNode.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/agent/FlumeNode.java?rev=1158677&r1=1158676&r2=1158677&view=diff
==============================================================================
--- incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/agent/FlumeNode.java
(original)
+++ incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/agent/FlumeNode.java
Wed Aug 17 12:54:20 2011
@@ -34,7 +34,6 @@ import org.apache.commons.cli.ParseExcep
 import org.apache.commons.cli.PosixParser;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation;
-import org.mortbay.jetty.handler.ContextHandlerCollection;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -62,7 +61,6 @@ import com.cloudera.flume.util.SystemInf
 import com.cloudera.util.CheckJavaVersion;
 import com.cloudera.util.FileUtil;
 import com.cloudera.util.InternalHttpServer;
-import com.cloudera.util.InternalHttpServer.ContextCreator;
 import com.cloudera.util.NetUtils;
 import com.cloudera.util.Pair;
 import com.cloudera.util.StatusHttpServer.StackServlet;
@@ -260,20 +258,24 @@ public class FlumeNode implements Report
     ReportManager.get().add(this);
 
     if (startHttp) {
-      int nodePort = conf.getNodeStatusPort();
-      String bindAddress = "0.0.0.0";
-      ContextCreator cc = new ContextCreator() {
-        @Override
-        public void addContexts(ContextHandlerCollection handlers) {
-          handlers.addHandler(InternalHttpServer.createLogAppContext());
-          handlers.addHandler(InternalHttpServer.createStackSevletContext());
-          String webAppRoot = FlumeConfiguration.get().getNodeWebappRoot();
-          InternalHttpServer.addHandlersFromPaths(handlers,
-              new File(webAppRoot));
-        }
-      };
-      http = InternalHttpServer.startFindPortHttpServer(cc, bindAddress,
-          nodePort);
+      try {
+        http = new InternalHttpServer();
+
+        http.addHandler(InternalHttpServer.createLogAppContext());
+
+        http.addHandler(InternalHttpServer.createServletContext(
+            StackServlet.class, "/stacks", "/*", "stacks"));
+
+        http.setBindAddress("0.0.0.0");
+        http.setPort(conf.getNodeStatusPort());
+        String webAppRoot = FlumeConfiguration.get().getNodeWebappRoot();
+        http.setWebappDir(new File(webAppRoot));
+        http.setScanForApps(true);
+
+        http.start();
+      } catch (Throwable t) {
+        LOG.error("Unexpected exception/error thrown! " + t.getMessage(), t);
+      }
     }
 
     if (reportPusher != null) {
@@ -369,9 +371,9 @@ public class FlumeNode implements Report
 
   /**
    * This function checks the agent logs dir to make sure that the process has
-   * the ability to the directory if necessary, that the path if it does exist
-   * is a directory, and that it can in fact create files inside of the
-   * directory. If it fails any of these, it throws an exception.
+   * the ability to the directory if necessary, that the path if it does exist is
+   * a directory, and that it can in fact create files inside of the directory.
+   * If it fails any of these, it throws an exception.
    * 
    * Finally, it checks to see if the path is in /tmp and warns the user that
    * this may not be the best idea.

Modified: incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/master/FlumeMaster.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/master/FlumeMaster.java?rev=1158677&r1=1158676&r2=1158677&view=diff
==============================================================================
--- incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/master/FlumeMaster.java
(original)
+++ incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/master/FlumeMaster.java
Wed Aug 17 12:54:20 2011
@@ -35,7 +35,6 @@ import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
 import org.apache.commons.cli.PosixParser;
 import org.apache.thrift.transport.TTransportException;
-import org.mortbay.jetty.handler.ContextHandlerCollection;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -54,7 +53,6 @@ import com.cloudera.flume.util.FlumeVMIn
 import com.cloudera.flume.util.SystemInfo;
 import com.cloudera.util.CheckJavaVersion;
 import com.cloudera.util.InternalHttpServer;
-import com.cloudera.util.InternalHttpServer.ContextCreator;
 import com.cloudera.util.NetUtils;
 import com.cloudera.util.StatusHttpServer.StackServlet;
 
@@ -251,19 +249,18 @@ public class FlumeMaster implements Repo
     ReportManager.get().add(sysInfo);
 
     if (doHttp) {
-      String bindAddress = "0.0.0.0";
-      int port = cfg.getMasterHttpPort();
-      final String webAppRoot = FlumeConfiguration.get().getMasterWebappRoot();
+      http = new InternalHttpServer();
+
+      http.addHandler(InternalHttpServer.createLogAppContext());
+      http.addHandler(InternalHttpServer.createServletContext(
+          StackServlet.class, "/stacks", "/*", "stacks"));
+
+      http.setBindAddress("0.0.0.0");
+      http.setPort(cfg.getMasterHttpPort());
+      String webAppRoot = FlumeConfiguration.get().getMasterWebappRoot();
+      http.setWebappDir(new File(webAppRoot));
       LOG.info("Webserver root directory: " + webAppRoot);
-      ContextCreator cc = new ContextCreator() {
-        @Override
-        public void addContexts(ContextHandlerCollection handlers) {
-          handlers.addHandler(InternalHttpServer.createLogAppContext());
-          handlers.addHandler(InternalHttpServer.createStackSevletContext());
-          InternalHttpServer.addHandlersFromPaths(handlers, new File(webAppRoot));
-        }
-      };
-      http = InternalHttpServer.startHttpServer(cc, bindAddress, port);
+      http.start();
     }
 
     controlServer = new MasterClientServer(this, FlumeConfiguration.get());

Modified: incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/InternalHttpServer.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/InternalHttpServer.java?rev=1158677&r1=1158676&r2=1158677&view=diff
==============================================================================
--- incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/InternalHttpServer.java
(original)
+++ incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/InternalHttpServer.java
Wed Aug 17 12:54:20 2011
@@ -1,38 +1,7 @@
-/**
- * Copyright 2011 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
 package com.cloudera.util;
 
 import java.io.File;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.io.PrintWriter;
-import java.net.BindException;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServlet;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.util.ReflectionUtils;
+
 import org.mortbay.jetty.Connector;
 import org.mortbay.jetty.Server;
 import org.mortbay.jetty.handler.ContextHandlerCollection;
@@ -44,14 +13,63 @@ import org.mortbay.jetty.webapp.WebAppCo
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.cloudera.util.StatusHttpServer.StackServlet;
+import com.google.common.base.Preconditions;
+
 /**
  * <p>
- * An embedded Jetty HTTP server.  It defers addition of contexts/handlers to 
- * a callback so that the we can provide a method that increments ports until 
- * a valid port is found.  This is mostly a thin wrapper around Jetty's 
- * {@link Server} class and behaves as Jetty does.
- *
- * Here is an example usage:
+ * An embedded Jetty HTTP server that support both normal and exploded war file
+ * deployment. Those that wish to expose HTTP services should create an instance
+ * of this class, configure the server via accessor methods, and then call
+ * {@link #start()}.
+ * </p>
+ * <p>
+ * Resources internally are allocated upon the first call to {@link #start()}.
+ * This includes scanning of the configured webapp directory for applications if
+ * {@link #getScanForApps()} is true (the default). Mostly this class is a thin
+ * wrapper around Jetty's {@link Server} class and behaves as Jetty does. Both
+ * traditional and exploded war formats are supported in the webapp directory.
+ * In the case of exploded directories, the directory name is used as the
+ * context. For war files, everything from the first instance of ".war" to the
+ * end of the file name (inclusive) is stripped and the remainder is used for
+ * the context name.
+ * </p>
+ * <p>
+ * Name examples:
+ * </p>
+ * <table>
+ * <tr>
+ * <td>Name</td>
+ * <td>Type</td>
+ * <td>Context</td>
+ * </tr>
+ * <tr>
+ * <td>app.war</td>
+ * <td>file</td>
+ * <td>app</td>
+ * </tr>
+ * <tr>
+ * <td>app</td>
+ * <td>dir</td>
+ * <td>app</td>
+ * </tr>
+ * <tr>
+ * <td>app.war</td>
+ * <td>dir</td>
+ * <td>app.war</td>
+ * </tr>
+ * <tr>
+ * <td>app.war.war</td>
+ * <td>file</td>
+ * <td>app</td>
+ * </tr>
+ * <tr>
+ * <td>
+ * </table>
+ * <p>
+ * Example usage:
+ * </p>
+ * 
  * <pre>
  * InternalHttpServer server = new InternalHttpServer();
  * 
@@ -59,20 +77,9 @@ import org.slf4j.LoggerFactory;
  * server.setWebappDir(new File(applicationHome, &quot;webapps&quot;));
  * server.setPort(8080);
  * server.setBindAddress(&quot;0.0.0.0&quot;);
- * server.setContextCreator(new ContextCreator() {
- *   @Override
- *   public void addContexts(ContextHandlerCollection handlers) {
- *        handlers.addHandler(InternalHttpServer.createLogAppContext());
- *        handlers.addHandler(InternalHttpServer.createStackSevletContext(
- *            StackServlet.class, "/stacks", "/*", "stacks"));
- *        String webAppRoot = FlumeConfiguration.get().getNodeWebappRoot();
- *        InternalHttpServer.addHandlersFromPaths(handlers,
- *            new File(webAppRoot));
- *   }
- * });
- *
+ * 
  * server.start();
- *
+ * 
  * // at some later time...
  * server.stop();
  * </pre>
@@ -83,15 +90,16 @@ public class InternalHttpServer {
       .getLogger(InternalHttpServer.class);
 
   private Server server;
+  private File webappDir;
   private int port;
-  private int boundPort = -1;
   private String bindAddress;
+  private boolean scanForApps;
   private ContextHandlerCollection handlers;
-  private ContextCreator contextCreator = null;
 
   public InternalHttpServer() {
     port = 0;
     bindAddress = "0.0.0.0";
+    scanForApps = true;
     handlers = new ContextHandlerCollection();
   }
 
@@ -105,14 +113,51 @@ public class InternalHttpServer {
       connector.setHost(bindAddress);
 
       server.addConnector(connector);
+      server.addHandler(handlers);
+    }
+  }
 
-      if (contextCreator != null) {
-        contextCreator.addContexts(handlers);
+  protected void registerApplications() {
+    logger.debug("Registering webapps in {}", webappDir);
+
+    if (webappDir.isDirectory()) {
+      for (File entry : webappDir.listFiles()) {
+        tryRegisterApplication(server, entry);
       }
-      server.setHandler(handlers);
+    } else {
+      tryRegisterApplication(server, webappDir);
     }
   }
 
+  private boolean tryRegisterApplication(Server server, File path) {
+    String name;
+
+    logger.debug("checking {}", path);
+
+    if (path.isFile()) {
+      int idx = path.getName().indexOf(".war");
+
+      if (idx > -1) {
+        name = path.getName().substring(0, idx);
+      } else {
+        return false;
+      }
+    } else {
+      name = path.getName();
+    }
+
+    logger.debug("creating context {} -> {}", name, path);
+
+    // WebAppContext is for loading war files.
+    WebAppContext handler = new WebAppContext(path.getPath(), "/" + name);
+
+    handler.setParentLoaderPriority(true);
+
+    handlers.addHandler(handler);
+
+    return true;
+  }
+
   /**
    * <p>
    * Start a configured HTTP server. Users should have already injected all the
@@ -123,22 +168,26 @@ public class InternalHttpServer {
    * The configured webappDir is not scanned for applications until start() is
    * called.
    * </p>
-   *
-   * @throws BindException
+   * 
    * @throws InternalHttpServerException
    */
-  public void start() throws BindException {
+  public void start() {
+    Preconditions.checkState(webappDir != null, "Webapp dir can not be null");
 
     initialize();
 
+    if (scanForApps) {
+      registerApplications();
+    } else {
+      logger.info("Not scanning for webapps");
+    }
+
     logger.info("Starting internal HTTP server");
 
     try {
       server.start();
-      boundPort = server.getConnectors()[0].getLocalPort();
-      logger.info("Server started on port " + boundPort);
-    } catch (BindException be) {
-      throw be;
+
+      logger.info("Server started");
     } catch (Exception e) {
       logger.warn("Caught exception during HTTP server start.", e);
 
@@ -171,8 +220,9 @@ public class InternalHttpServer {
 
   @Override
   public String toString() {
-    return "{ bindAddress:" + bindAddress + " port:" + port + " boundPort:"
-        + boundPort + " server:" + server + " }";
+    return "{ bindAddress:" + bindAddress + " webappDir:" + webappDir
+        + " port:" + port + " scanForApps:" + scanForApps + " server:" + server
+        + " }";
   }
 
   public Server getServer() {
@@ -183,12 +233,16 @@ public class InternalHttpServer {
     this.server = server;
   }
 
-  public int getPort() {
-    return port;
+  public File getWebappDir() {
+    return webappDir;
   }
 
-  public int getBoundPort() {
-    return boundPort;
+  public void setWebappDir(File webappDir) {
+    this.webappDir = webappDir;
+  }
+
+  public int getPort() {
+    return port;
   }
 
   public void setPort(int port) {
@@ -203,6 +257,14 @@ public class InternalHttpServer {
     this.bindAddress = bindAddress;
   }
 
+  public boolean getScanForApps() {
+    return scanForApps;
+  }
+
+  public void setScanForApps(boolean scanForApps) {
+    this.scanForApps = scanForApps;
+  }
+
   public static class InternalHttpServerException extends RuntimeException {
 
     private static final long serialVersionUID = -4936285404574873547L;
@@ -225,19 +287,7 @@ public class InternalHttpServer {
 
   }
 
-  public void setHandlers(ContextHandlerCollection ctx) {
-    if (ctx == null) {
-      logger.warn("Attempting to add null webapp context");
-      return;
-    }
-    handlers = ctx;
-  }
-
-  public ContextHandlerCollection getHandlers() {
-    return handlers;
-  }
-
-  protected void addHandler(Context ctx) {
+  public void addHandler(Context ctx) {
     if (ctx == null) {
       logger.warn("Attempting to add null webapp context");
       return;
@@ -245,117 +295,6 @@ public class InternalHttpServer {
     handlers.addHandler(ctx);
   }
 
-  public void setContextCreator(ContextCreator cc) {
-    this.contextCreator = cc;
-  }
-
-  /**
-   * The jetty server cannot properly reload contexts if it attempts to bind to
-   * a port and fails. To support automatically going finding a new port, we
-   * thus need to parameterize the creation and addition of context. This class
-   * provides a call back that gets a instance of the server's
-   * ContextHandlerCollection, and gives clients the opportunity to populate it.
-   */
-  public abstract static class ContextCreator {
-    public abstract void addContexts(ContextHandlerCollection handlers);
-  }
-
-  public static WebAppContext createWarContext(File path) {
-    logger.debug("checking {}", path);
-
-    String name;
-    if (path.isFile()) {
-      // if not a war file reject
-      int idx = path.getName().indexOf(".war");
-      if (idx < 0) {
-        return null;
-      }
-
-      // drop the .war suffix
-      name = path.getName().substring(0, idx);
-    } else {
-      // is a dir
-      name = path.getName();
-    }
-
-    // WebAppContext is for loading war files.
-    logger.debug("creating context {} -> {}", name, path);
-    WebAppContext handler = new WebAppContext(path.getPath(), "/" + name);
-    handler.setParentLoaderPriority(true);
-    return handler;
-  }
-
-  /**
-   * This method adds support for both normal and exploded war file deployment.
-   * <p>
-   * This scannings the specified webapp directory for applications
-   * Both traditional and exploded war formats are supported in the webapp
-   * directory. In the case of exploded directories, the directory name is used
-   * as the context. For war files, everything from the first instance of ".war"
-   * to the end of the file name (inclusive) is stripped and the remainder is
-   * used for the context name.
-   * </p>
-   * <p>
-   * Name examples:
-   * </p>
-   * <table>
-   * <tr>
-   * <td>Name</td>
-   * <td>Type</td>
-   * <td>Context</td>
-   * </tr>
-   * <tr>
-   * <td>app.war</td>
-   * <td>file</td>
-   * <td>app</td>
-   * </tr>
-   * <tr>
-   * <td>app</td>
-   * <td>dir</td>
-   * <td>app</td>
-   * </tr>
-   * <tr>
-   * <td>app.war</td>
-   * <td>dir</td>
-   * <td>app.war</td>
-   * </tr>
-   * <tr>
-   * <td>app.war.war</td>
-   * <td>file</td>
-   * <td>app</td>
-   * </tr>
-   * <tr>
-   * <td>
-   * </table>
-   * <p>
-   * Example usage:
-   * </p>
-   */
-  public static void addHandlersFromPaths(ContextHandlerCollection handlers,
-      File webappDir) {
-    logger.debug("Registering webapps in {}", webappDir);
-
-    if (webappDir.isDirectory()) {
-      for (File entry : webappDir.listFiles()) {
-        Context ctx = createWarContext(entry);
-        if (ctx != null) {
-          handlers.addHandler(ctx);
-        }
-      }
-    } else {
-      Context ctx = createWarContext(webappDir);
-      if (ctx != null) {
-        handlers.addHandler(ctx);
-      }
-    }
-  }
-
-  /**
-   * This creates file listing servlet context that is used to point to the log
-   * directory of the daemon via the web interface.
-   *
-   * @return
-   */
   public static Context createLogAppContext() {
     Context ctx = new Context();
     // logs applet
@@ -369,39 +308,8 @@ public class InternalHttpServer {
     return ctx;
   }
 
-  /**
-   * A very simple servlet to serve up a text representation of the current
-   * stack traces. It both returns the stacks to the caller and logs them.
-   * Currently the stack traces are done sequentially rather than exactly the
-   * same data.
-   */
-  public static class StackServlet extends HttpServlet {
-    private static final Log LOG = LogFactory.getLog(StatusHttpServer.class
-        .getName());
-    private static final long serialVersionUID = -6284183679759467039L;
-
-    @Override
-    public void doGet(HttpServletRequest request, HttpServletResponse response)
-        throws ServletException, IOException {
-
-      OutputStream outStream = response.getOutputStream();
-      ReflectionUtils.printThreadInfo(new PrintWriter(outStream), "");
-      outStream.close();
-      ReflectionUtils.logThreadInfo(LOG, "jsp requested", 1);
-    }
-  }
-
-  /**
-   * This creates a stack dumping servlet that can be used to debug a running
-   * daemon via the web interface.
-   *
-   * @param sltClz
-   * @param contextPath
-   * @param pathSpec
-   * @param name
-   * @return
-   */
-  public static Context createStackSevletContext() {
+  public static Context createServletContext(Class<?> sltClz, String contextPath,
+      String pathSpec, String name) {
     Context ctx = new Context();
     ServletHolder holder = new ServletHolder(StackServlet.class);
     ctx.setContextPath("/stacks");
@@ -411,50 +319,4 @@ public class InternalHttpServer {
     return ctx;
   }
 
-  /**
-   * If successful returns the port the http server successfully bound to. If it
-   * failed, returns -1
-   */
-  public static InternalHttpServer startFindPortHttpServer(ContextCreator cc,
-      String bindAddress, int nodePort) {
-    do {
-      try {
-        return startHttpServer(cc, bindAddress, nodePort);
-      } catch (BindException be) {
-        logger.error("Unable to start webserver on " + bindAddress + ":"
-            + nodePort + ". Trying next port...");
-        nodePort++;
-      }
-    } while (true);
-  }
-
-  /**
-   * Single attempt to create an http server for the node.
-   * 
-   * @param bindAddress
-   * @param nodePort
-   * @return instance of a started http server or null if failed.
-   * @throws BindException
-   */
-  public static InternalHttpServer startHttpServer(ContextCreator cc,
-      String bindAddress, int nodePort) throws BindException {
-    InternalHttpServer http = null;
-    try {
-      http = new InternalHttpServer();
-      http.setBindAddress(bindAddress);
-      http.setPort(nodePort);
-      http.setContextCreator(cc);
-      http.start();
-      return http;
-    } catch (BindException be) {
-      http.stop();
-      http = null;
-      throw be;
-    } catch (Throwable t) {
-      logger.error("Unexpected exception/error thrown! " + t.getMessage(), t);
-      // if any exception happens bail out and cleanup.
-      http.stop();
-      return null;
-    }
-  }
 }

Added: incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/StatusHttpServer.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/StatusHttpServer.java?rev=1158677&view=auto
==============================================================================
--- incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/StatusHttpServer.java
(added)
+++ incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/StatusHttpServer.java
Wed Aug 17 12:54:20 2011
@@ -0,0 +1,251 @@
+/**
+ * Licensed to Cloudera, Inc. under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  Cloudera, Inc. licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.cloudera.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.net.BindException;
+import java.net.InetSocketAddress;
+
+import javax.servlet.Servlet;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.mortbay.jetty.Server;
+import org.mortbay.jetty.nio.SelectChannelConnector;
+import org.mortbay.jetty.security.SslSocketConnector;
+import org.mortbay.jetty.servlet.ServletHolder;
+import org.mortbay.jetty.webapp.WebAppContext;
+
+import com.google.common.base.Preconditions;
+
+// jon: This is a shamelessly hacked version of the Http status server from the jobtracker,
+// simplified for my needs. Originally from apache licensed hadoop 0.18.3, o.a.h.mapred.StatusHttpServer
+
+/**
+ * Create a Jetty embedded server to answer http requests. The primary goal is
+ * to serve up status information for the server. There are three contexts:
+ * "/logs/" -> points to the log directory "/static/" -> points to common static
+ * files (src/webapps/static) "/" -> the jsp server code from
+ * (src/webapps/<name>)
+ */
+public class StatusHttpServer {
+  private Server webServer;
+  private SelectChannelConnector channelConnector;
+  private SslSocketConnector sslConnector;
+  private boolean findPort;
+  private WebAppContext webAppContext;
+  private static final Log LOG = LogFactory.getLog(StatusHttpServer.class
+      .getName());
+
+  /**
+   * Create a status server on the given port. The jsp scripts are taken from
+   * src/webapps/<name>.
+   * 
+   * @param name
+   *          The name of the server
+   * @param port
+   *          The port to use on the server
+   * @param findPort
+   *          whether the server should start at the given port and increment by
+   *          1 until it finds a free port.
+   */
+  public StatusHttpServer(String name, String webAppsPath, String bindAddress,
+      int port, boolean findPort) throws IOException {
+    webServer = new org.mortbay.jetty.Server();
+    this.findPort = findPort;
+    channelConnector = new SelectChannelConnector();
+    channelConnector.setPort(port);
+    channelConnector.setHost(bindAddress);
+    webServer.addConnector(channelConnector);
+
+    String appDir = webAppsPath;
+    // set up the context for "/" jsp files
+    String webapp = new File(appDir, name).getAbsolutePath();
+    LOG.info("starting web app in directory: " + webapp);
+    webAppContext = new WebAppContext(webapp, "/");
+    webServer.setHandler(webAppContext);
+    addServlet("stacks", "/stacks", StackServlet.class);
+  }
+
+  /**
+   * Sets a value in the webapp context. These values are available to the jsp
+   * pages as "application.getAttribute(name)".
+   * 
+   * @param name
+   *          The name of the attribute
+   * @param value
+   *          The value of the attribute
+   */
+  public void setAttribute(String name, Object value) {
+    webAppContext.setAttribute(name, value);
+  }
+
+  /**
+   * Add a servlet in the server.
+   * 
+   * @param name
+   *          The name of the servlet (can be passed as null)
+   * @param pathSpec
+   *          The path spec for the servlet
+   * @param servletClass
+   *          The servlet class
+   */
+  public <T extends HttpServlet> void addServlet(String name, String pathSpec,
+      Class<T> servletClass) {
+
+    WebAppContext context = webAppContext;
+    if (name == null) {
+      context.addServlet(pathSpec, servletClass.getName());
+    } else {
+      context.addServlet(servletClass, pathSpec);
+    }
+  }
+
+  public void addServlet(Servlet servlet, String pathSpec) {
+    webAppContext.addServlet(new ServletHolder(servlet), pathSpec);
+  }
+
+  /**
+   * Get the value in the webapp context.
+   * 
+   * @param name
+   *          The name of the attribute
+   * @return The value of the attribute
+   */
+  public Object getAttribute(String name) {
+    return webAppContext.getAttribute(name);
+  }
+
+  /**
+   * Get the port that the server is on
+   * 
+   * @return the port
+   */
+  public int getPort() {
+    return channelConnector.getPort();
+  }
+
+  /**
+   * Configure an ssl listener on the server.
+   * 
+   * @param addr
+   *          address to listen on
+   * @param keystore
+   *          location of the keystore
+   * @param storPass
+   *          password for the keystore
+   * @param keyPass
+   *          password for the key
+   */
+  public void addSslListener(InetSocketAddress addr, String keystore,
+      String storPass, String keyPass) throws IOException {
+    if (sslConnector != null || webServer.isStarted()) {
+      throw new IOException("Failed to add ssl listener");
+    }
+    sslConnector = new SslSocketConnector();
+    sslConnector.setHost(addr.getHostName());
+    sslConnector.setPort(addr.getPort());
+    sslConnector.setKeystore(keystore);
+    sslConnector.setPassword(storPass);
+    sslConnector.setKeyPassword(keyPass);
+    webServer.addConnector(sslConnector);
+  }
+
+  /**
+   * Start the server. Does not wait for the server to start.
+   */
+  public void start() throws IOException {
+    try {
+      while (true) {
+        try {
+          webServer.start();
+          break;
+        } catch (BindException ex) {
+          // if the multi exception contains ONLY a bind exception,
+          // then try the next port number.
+          if (!findPort) {
+            throw ex;
+          }
+          // pick another port
+          webServer.stop();
+          channelConnector.setPort(channelConnector.getPort() + 1);
+        }
+      }
+    } catch (Exception e) {
+      IOException ie = new IOException("Problem starting http server");
+      ie.initCause(e);
+      throw ie;
+    }
+  }
+
+  /**
+   * stop the server
+   */
+  public void stop() throws Exception {
+    webServer.stop();
+  }
+
+  /**
+   * A very simple servlet to serve up a text representation of the current
+   * stack traces. It both returns the stacks to the caller and logs them.
+   * Currently the stack traces are done sequentially rather than exactly the
+   * same data.
+   */
+  public static class StackServlet extends HttpServlet {
+
+    private static final long serialVersionUID = -6284183679759467039L;
+
+    @Override
+    public void doGet(HttpServletRequest request, HttpServletResponse response)
+        throws ServletException, IOException {
+
+      OutputStream outStream = response.getOutputStream();
+      ReflectionUtils.printThreadInfo(new PrintWriter(outStream), "");
+      outStream.close();
+      ReflectionUtils.logThreadInfo(LOG, "jsp requested", 1);
+    }
+  }
+
+  /**
+   * Test harness to get precompiled jsps working.
+   * 
+   * @param argv
+   */
+  public static void main(String[] argv) {
+    Preconditions.checkArgument(argv.length == 3);
+    String name = argv[0];
+    String path = argv[1];
+    int port = Integer.parseInt(argv[2]);
+
+    try {
+      StatusHttpServer http = new StatusHttpServer(name, path, "0.0.0.0", port,
+          false);
+      http.start();
+    } catch (IOException ioe) {
+      ioe.printStackTrace();
+    }
+  }
+}

Modified: incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/agent/TestNodeJersey.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/agent/TestNodeJersey.java?rev=1158677&r1=1158676&r2=1158677&view=diff
==============================================================================
--- incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/agent/TestNodeJersey.java
(original)
+++ incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/agent/TestNodeJersey.java
Wed Aug 17 12:54:20 2011
@@ -17,7 +17,7 @@
  */
 package com.cloudera.flume.agent;
 
-import static com.cloudera.util.HttpServerTestUtils.curl;
+import static com.cloudera.flume.master.TestMasterJersey.curl;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 

Modified: incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/master/TestMasterJersey.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/master/TestMasterJersey.java?rev=1158677&r1=1158676&r2=1158677&view=diff
==============================================================================
--- incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/master/TestMasterJersey.java
(original)
+++ incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/master/TestMasterJersey.java
Wed Aug 17 12:54:20 2011
@@ -17,11 +17,14 @@
  */
 package com.cloudera.flume.master;
 
-import static com.cloudera.util.HttpServerTestUtils.curl;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 
+import java.io.BufferedReader;
 import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.net.URLConnection;
 
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
@@ -40,6 +43,33 @@ public class TestMasterJersey extends Se
   public static final Logger LOG = LoggerFactory
       .getLogger(TestMasterJersey.class);
 
+  /**
+   * Gra b a url's contents. Since most are json, this should be small.
+   * 
+   * @param urlString
+   * @return
+   * @throws IOException
+   */
+  public static String curl(String urlString) throws IOException {
+    URL url = new URL(urlString);
+    URLConnection urlConn = url.openConnection();
+    urlConn.setDoInput(true);
+    urlConn.setUseCaches(false);
+
+    int len = urlConn.getContentLength();
+    String type = urlConn.getContentType();
+    LOG.info("pulled " + urlString + "[ type=" + type + " len=" + len + "]");
+    InputStreamReader isr = new InputStreamReader(urlConn.getInputStream());
+    BufferedReader br = new BufferedReader(isr);
+    StringBuilder sb = new StringBuilder();
+    String s;
+    while ((s = br.readLine()) != null) {
+      sb.append(s);
+      sb.append('\n');
+    }
+    return sb.toString();
+  }
+
   @Ignore
   @Test
   public void testMaster() throws IOException, InterruptedException,

Modified: incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/InternalHttpServerTest.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/InternalHttpServerTest.java?rev=1158677&r1=1158676&r2=1158677&view=diff
==============================================================================
--- incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/InternalHttpServerTest.java
(original)
+++ incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/InternalHttpServerTest.java
Wed Aug 17 12:54:20 2011
@@ -1,41 +1,13 @@
-/**
- * Copyright 2011 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
 package com.cloudera.util;
 
-import static com.cloudera.util.HttpServerTestUtils.curlResp;
-import static org.junit.Assert.assertEquals;
-
 import java.io.File;
-import java.io.IOException;
-import java.net.BindException;
 
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
-import org.mortbay.jetty.handler.ContextHandlerCollection;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.cloudera.flume.conf.FlumeConfiguration;
-import com.cloudera.util.InternalHttpServer.ContextCreator;
-
 public class InternalHttpServerTest {
 
   private static final Logger logger = LoggerFactory
@@ -49,16 +21,25 @@ public class InternalHttpServerTest {
   }
 
   @Test
-  public void testStart() throws BindException {
+  public void testStartInvalidState() {
     boolean success = false;
 
-    httpServer.setContextCreator(new ContextCreator() {
-      @Override
-      public void addContexts(ContextHandlerCollection handlers) {
-        InternalHttpServer.addHandlersFromPaths(handlers, new File(getClass()
-            .getClassLoader().getResource("test-webroot").getFile()));
-      }
-    });
+    try {
+      httpServer.start();
+      success = true;
+    } catch (IllegalStateException e) {
+      logger.info("Caught expected exception: {}", e.getMessage());
+    }
+
+    Assert.assertFalse(success);
+  }
+
+  @Test
+  public void testStart() {
+    boolean success = false;
+
+    httpServer.setWebappDir(new File(getClass().getClassLoader()
+        .getResource("test-webroot").getFile()));
 
     try {
       httpServer.start();
@@ -87,40 +68,4 @@ public class InternalHttpServerTest {
     Assert.assertTrue(success);
   }
 
-  /**
-   * This tests to make sure that auto find port works. Two http servers are
-   * assigned to the same port -- the second one should detect the conflict and
-   * then pick the next port to bind and serve from. curl will throw exception
-   * on failure.
-   */
-  @Test
-  public void testAutoFindPort() throws IOException, Exception {
-    int port = FlumeConfiguration.get().getNodeStatusPort();
-    String bindAddress = "0.0.0.0";
-    InternalHttpServer http = InternalHttpServer.startHttpServer(null,
-        bindAddress, port);
-    http.start();
-
-    InternalHttpServer http2 = InternalHttpServer.startFindPortHttpServer(null,
-        bindAddress, port);
-    http2.start();
-
-    // grab something from each server
-    int port1 = http.getBoundPort();
-    int resp1 = curlResp("http://localhost:" + port1);
-    logger.info("http1 port:" + port1);
-    
-    int port2 = http2.getBoundPort();
-    int resp2 = curlResp("http://localhost:" + port2);
-    logger.info("http2 port:" + port2);
-
-    // shutdown
-    http.stop();
-    http2.stop();
-
-    assertEquals(404, resp1);
-    assertEquals(404, resp2);
-    assertEquals(port, port1);
-    assertEquals(port + 1, port2);
-  }
 }

Added: incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/TestStatusHttpServer.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/TestStatusHttpServer.java?rev=1158677&view=auto
==============================================================================
--- incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/TestStatusHttpServer.java
(added)
+++ incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/TestStatusHttpServer.java
Wed Aug 17 12:54:20 2011
@@ -0,0 +1,90 @@
+/**
+ * Licensed to Cloudera, Inc. under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  Cloudera, Inc. licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.cloudera.util;
+
+import static com.cloudera.flume.master.TestMasterJersey.curl;
+
+import java.io.IOException;
+
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.cloudera.flume.agent.FlumeNode;
+import com.cloudera.flume.conf.FlumeConfiguration;
+
+/**
+ * This tests the behavior of rapidly opening and closing the http server.
+ * Things are sane.
+ */
+public class TestStatusHttpServer {
+  public static final Logger LOG = LoggerFactory
+      .getLogger(TestStatusHttpServer.class);
+
+  @Test
+  public void testOpenClose() throws Exception {
+    // Set directory of webapps to build-specific dir
+    FlumeConfiguration.get().set(FlumeConfiguration.WEBAPPS_PATH,
+        "build/webapps");
+
+    FlumeConfiguration conf = FlumeConfiguration.get();
+    String webPath = FlumeNode.getWebPath(conf);
+    int port = FlumeConfiguration.get().getNodeStatusPort();
+    StatusHttpServer http = new StatusHttpServer("flumeagent", webPath,
+        "0.0.0.0", port, false);
+
+    for (int i = 0; i < 50; i++) {
+      http.start();
+      http.stop();
+    }
+  }
+
+  /**
+   * This tests to make sure that auto find port works. Two http servers are
+   * assigned to the same port -- the second one should detect the conflict and
+   * then pick the next port to bind and serve from. curl will throw exception
+   * on failure.
+   */
+  @Test
+  @Ignore
+  public void testAutoFindPort() throws IOException, Exception {
+    // Set directory of webapps to build-specific dir
+    FlumeConfiguration.get().set(FlumeConfiguration.WEBAPPS_PATH,
+        "build/webapps");
+
+    FlumeConfiguration conf = FlumeConfiguration.get();
+    String webPath = FlumeNode.getWebPath(conf);
+    int port = FlumeConfiguration.get().getNodeStatusPort();
+    StatusHttpServer http = new StatusHttpServer("flumeagent", webPath,
+        "0.0.0.0", port, true);
+    http.start();
+
+    StatusHttpServer http2 = new StatusHttpServer("flumeagent", webPath,
+        "0.0.0.0", port, true);
+    http2.start();
+
+    String s1 = curl("http://localhost:35862");
+    LOG.info("http1:" + s1);
+    String s2 = curl("http://localhost:35863");
+    LOG.info("http2:" + s2);
+    http.stop();
+    http2.stop();
+  }
+}

Modified: incubator/flume/trunk/flume-node-web/src/test/java/com/cloudera/flume/agent/TestBootstrap.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-node-web/src/test/java/com/cloudera/flume/agent/TestBootstrap.java?rev=1158677&r1=1158676&r2=1158677&view=diff
==============================================================================
--- incubator/flume/trunk/flume-node-web/src/test/java/com/cloudera/flume/agent/TestBootstrap.java
(original)
+++ incubator/flume/trunk/flume-node-web/src/test/java/com/cloudera/flume/agent/TestBootstrap.java
Wed Aug 17 12:54:20 2011
@@ -1,39 +1,14 @@
-/**
- * Copyright 2011 The Apache Software Foundation
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
 package com.cloudera.flume.agent;
 
-import static org.junit.Assert.assertEquals;
-
 import java.io.File;
-import java.io.IOException;
 
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
-import org.mortbay.jetty.handler.ContextHandlerCollection;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.cloudera.util.HttpServerTestUtils;
 import com.cloudera.util.InternalHttpServer;
-import com.cloudera.util.InternalHttpServer.ContextCreator;
 
 public class TestBootstrap {
 
@@ -48,25 +23,19 @@ public class TestBootstrap {
   }
 
   @Test
-  public void testBootstrap() throws InterruptedException, IOException {
+  public void testBootstrap() throws InterruptedException {
     Assert.assertNotNull(httpServer);
 
     logger.debug("httpServer:{}", httpServer);
 
     httpServer.setPort(0);
-    httpServer.setContextCreator(new ContextCreator() {
-      @Override
-      public void addContexts(ContextHandlerCollection handlers) {
-        InternalHttpServer.addHandlersFromPaths(handlers, new File("src/main"));
-      }
-    });
+    httpServer.setWebappDir(new File("src/main"));
 
     httpServer.start();
-    int port = httpServer.getBoundPort();
-    String url = "http://localhost:" + port;
-    logger.debug("Grabbing http response from " + url);
-    int resp = HttpServerTestUtils.curlResp(url);
+
+    Thread.sleep(3000);
+
     httpServer.stop();
-    assertEquals(resp, 200); // expect ok response code.
   }
+
 }



Mime
View raw message