Mailing-List: contact flume-commits-help@incubator.apache.org; run by ezmlm
Precedence: bulk
Reply-To: flume-dev@incubator.apache.org
Content-Type: text/plain; charset="utf-8"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
Subject: svn commit: r1161532 - in /incubator/flume/trunk:
 flume-core/src/main/java/com/cloudera/flume/agent/
 flume-core/src/main/java/com/cloudera/flume/master/
 flume-core/src/main/java/com/cloudera/util/
 flume-core/src/test/java/com/cloudera/flume/agent/ flum...
Date: Thu, 25 Aug 2011 12:16:33 -0000
To: flume-commits@incubator.apache.org
From: jmhsieh@apache.org
Message-Id: <20110825121633.C453123889DE@eris.apache.org>

Author: jmhsieh
Date: Thu Aug 25 12:16:32 2011
New Revision: 1161532

URL: http://svn.apache.org/viewvc?rev=1161532&view=rev
Log:
FLUME-721: Webapps 'autofindport' feature does not work

This refactors the internal http server so that context are created by a callback object.

Added:
    incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/HttpServerTestUtils.java
Removed:
    incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/StatusHttpServer.java
    incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/TestStatusHttpServer.java
Modified:
    incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/agent/FlumeNode.java
    incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/master/FlumeMaster.java
    incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/InternalHttpServer.java
    incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/agent/TestNodeJersey.java
    incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/master/TestMasterJersey.java
    incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/InternalHttpServerTest.java
    incubator/flume/trunk/flume-node-web/src/test/java/com/cloudera/flume/agent/TestBootstrap.java

Modified: incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/agent/FlumeNode.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/agent/FlumeNode.java?rev=1161532&r1=1161531&r2=1161532&view=diff
==============================================================================
--- incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/agent/FlumeNode.java (original)
+++ incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/agent/FlumeNode.java Thu Aug 25 12:16:32 2011
@@ -34,6 +34,7 @@ import org.apache.commons.cli.ParseExcep
 import org.apache.commons.cli.PosixParser;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.mortbay.jetty.handler.ContextHandlerCollection;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -61,9 +62,9 @@ import com.cloudera.flume.util.SystemInf
 import com.cloudera.util.CheckJavaVersion;
 import com.cloudera.util.FileUtil;
 import com.cloudera.util.InternalHttpServer;
+import com.cloudera.util.InternalHttpServer.ContextCreator;
 import com.cloudera.util.NetUtils;
 import com.cloudera.util.Pair;
-import com.cloudera.util.StatusHttpServer.StackServlet;
 import com.google.common.base.Preconditions;
 
 /**
@@ -258,24 +259,20 @@ public class FlumeNode implements Report
     ReportManager.get().add(this);
 
     if (startHttp) {
-      try {
-        http = new InternalHttpServer();
-
-        http.addHandler(InternalHttpServer.createLogAppContext());
-
-        http.addHandler(InternalHttpServer.createServletContext(
-            StackServlet.class, "/stacks", "/*", "stacks"));
-
-        http.setBindAddress("0.0.0.0");
-        http.setPort(conf.getNodeStatusPort());
-        String webAppRoot = FlumeConfiguration.get().getNodeWebappRoot();
-        http.setWebappDir(new File(webAppRoot));
-        http.setScanForApps(true);
-
-        http.start();
-      } catch (Throwable t) {
-        LOG.error("Unexpected exception/error thrown! " + t.getMessage(), t);
-      }
+      int nodePort = conf.getNodeStatusPort();
+      String bindAddress = "0.0.0.0";
+      ContextCreator cc = new ContextCreator() {
+        @Override
+        public void addContexts(ContextHandlerCollection handlers) {
+          handlers.addHandler(InternalHttpServer.createLogAppContext());
+          handlers.addHandler(InternalHttpServer.createStackSevletContext());
+          String webAppRoot = FlumeConfiguration.get().getNodeWebappRoot();
+          InternalHttpServer.addHandlersFromPaths(handlers,
+              new File(webAppRoot));
+        }
+      };
+      http = InternalHttpServer.startFindPortHttpServer(cc, bindAddress,
+          nodePort);
     }
 
     if (reportPusher != null) {
@@ -371,9 +368,9 @@ public class FlumeNode implements Report
 
   /**
    * This function checks the agent logs dir to make sure that the process has
-   * the ability to the directory if necessary, that the path if it does exist is
-   * a directory, and that it can in fact create files inside of the directory.
-   * If it fails any of these, it throws an exception.
+   * the ability to the directory if necessary, that the path if it does exist
+   * is a directory, and that it can in fact create files inside of the
+   * directory. If it fails any of these, it throws an exception.
    * 
    * Finally, it checks to see if the path is in /tmp and warns the user that
    * this may not be the best idea.

Modified: incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/master/FlumeMaster.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/master/FlumeMaster.java?rev=1161532&r1=1161531&r2=1161532&view=diff
==============================================================================
--- incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/master/FlumeMaster.java (original)
+++ incubator/flume/trunk/flume-core/src/main/java/com/cloudera/flume/master/FlumeMaster.java Thu Aug 25 12:16:32 2011
@@ -35,6 +35,7 @@ import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
 import org.apache.commons.cli.PosixParser;
 import org.apache.thrift.transport.TTransportException;
+import org.mortbay.jetty.handler.ContextHandlerCollection;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -53,8 +54,8 @@ import com.cloudera.flume.util.FlumeVMIn
 import com.cloudera.flume.util.SystemInfo;
 import com.cloudera.util.CheckJavaVersion;
 import com.cloudera.util.InternalHttpServer;
+import com.cloudera.util.InternalHttpServer.ContextCreator;
 import com.cloudera.util.NetUtils;
-import com.cloudera.util.StatusHttpServer.StackServlet;
 
 /**
  * This is a first cut at a server for distributing configurations to different
@@ -249,18 +250,19 @@ public class FlumeMaster implements Repo
     ReportManager.get().add(sysInfo);
 
     if (doHttp) {
-      http = new InternalHttpServer();
-
-      http.addHandler(InternalHttpServer.createLogAppContext());
-      http.addHandler(InternalHttpServer.createServletContext(
-          StackServlet.class, "/stacks", "/*", "stacks"));
-
-      http.setBindAddress("0.0.0.0");
-      http.setPort(cfg.getMasterHttpPort());
-      String webAppRoot = FlumeConfiguration.get().getMasterWebappRoot();
-      http.setWebappDir(new File(webAppRoot));
+      String bindAddress = "0.0.0.0";
+      int port = cfg.getMasterHttpPort();
+      final String webAppRoot = FlumeConfiguration.get().getMasterWebappRoot();
       LOG.info("Webserver root directory: " + webAppRoot);
-      http.start();
+      ContextCreator cc = new ContextCreator() {
+        @Override
+        public void addContexts(ContextHandlerCollection handlers) {
+          handlers.addHandler(InternalHttpServer.createLogAppContext());
+          handlers.addHandler(InternalHttpServer.createStackSevletContext());
+          InternalHttpServer.addHandlersFromPaths(handlers, new File(webAppRoot));
+        }
+      };
+      http = InternalHttpServer.startHttpServer(cc, bindAddress, port);
     }
 
     controlServer = new MasterClientServer(this, FlumeConfiguration.get());

Added: incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/HttpServerTestUtils.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/HttpServerTestUtils.java?rev=1161532&view=auto
==============================================================================
--- incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/HttpServerTestUtils.java (added)
+++ incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/HttpServerTestUtils.java Thu Aug 25 12:16:32 2011
@@ -0,0 +1,82 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.cloudera.util;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.net.URLConnection;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HttpServerTestUtils {
+  public static final Logger LOG = LoggerFactory
+      .getLogger(HttpServerTestUtils.class);
+
+  /**
+   * Grab a url's contents. This assumes that grabbed pages are small
+   *
+   * @param urlString
+   * @return
+   * @throws IOException
+   */
+  public static String curl(String urlString) throws IOException {
+    URL url = new URL(urlString);
+    URLConnection urlConn = url.openConnection();
+    urlConn.setDoInput(true);
+    urlConn.setUseCaches(false);
+
+    int len = urlConn.getContentLength();
+    String type = urlConn.getContentType();
+    LOG.info("pulled " + urlString + " [type=" + type + " len=" + len + "]");
+    InputStreamReader isr = new InputStreamReader(urlConn.getInputStream());
+    BufferedReader br = new BufferedReader(isr);
+    StringBuilder sb = new StringBuilder();
+    String s;
+    while ((s = br.readLine()) != null) {
+      sb.append(s);
+      sb.append('\n');
+    }
+    return sb.toString();
+  }
+
+  /**
+   * Grab a url's http response code. It if fails, it will throw an exception.
+   *
+   * @param urlString
+   * @return
+   * @throws IOException
+   */
+  public static int curlResp(String urlString) throws IOException {
+    URL url = new URL(urlString);
+    HttpURLConnection urlConn = (HttpURLConnection) url.openConnection();
+    urlConn.setDoInput(true);
+    urlConn.setUseCaches(false);
+
+    int len = urlConn.getContentLength();
+    String type = urlConn.getContentType();
+    LOG.info("pulled " + urlString + " [type=" + type + " len=" + len + "]");
+    return urlConn.getResponseCode();
+  }
+
+}

Modified: incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/InternalHttpServer.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/InternalHttpServer.java?rev=1161532&r1=1161531&r2=1161532&view=diff
==============================================================================
--- incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/InternalHttpServer.java (original)
+++ incubator/flume/trunk/flume-core/src/main/java/com/cloudera/util/InternalHttpServer.java Thu Aug 25 12:16:32 2011
@@ -1,7 +1,38 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package com.cloudera.util;
 
 import java.io.File;
-
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.net.BindException;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.util.ReflectionUtils;
 import org.mortbay.jetty.Connector;
 import org.mortbay.jetty.Server;
 import org.mortbay.jetty.handler.ContextHandlerCollection;
@@ -13,63 +44,14 @@ import org.mortbay.jetty.webapp.WebAppCo
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.cloudera.util.StatusHttpServer.StackServlet;
-import com.google.common.base.Preconditions;
-
 /**
  * <p>
- * An embedded Jetty HTTP server that support both normal and exploded war file
- * deployment. Those that wish to expose HTTP services should create an instance
- * of this class, configure the server via accessor methods, and then call
- * {@link #start()}.
- * </p>
- * <p>
- * Resources internally are allocated upon the first call to {@link #start()}.
- * This includes scanning of the configured webapp directory for applications if
- * {@link #getScanForApps()} is true (the default). Mostly this class is a thin
- * wrapper around Jetty's {@link Server} class and behaves as Jetty does. Both
- * traditional and exploded war formats are supported in the webapp directory.
- * In the case of exploded directories, the directory name is used as the
- * context. For war files, everything from the first instance of ".war" to the
- * end of the file name (inclusive) is stripped and the remainder is used for
- * the context name.
- * </p>
- * <p>
- * Name examples:
- * </p>
- * <table>
- * <tr>
- * <td>Name</td>
- * <td>Type</td>
- * <td>Context</td>
- * </tr>
- * <tr>
- * <td>app.war</td>
- * <td>file</td>
- * <td>app</td>
- * </tr>
- * <tr>
- * <td>app</td>
- * <td>dir</td>
- * <td>app</td>
- * </tr>
- * <tr>
- * <td>app.war</td>
- * <td>dir</td>
- * <td>app.war</td>
- * </tr>
- * <tr>
- * <td>app.war.war</td>
- * <td>file</td>
- * <td>app</td>
- * </tr>
- * <tr>
- * <td>
- * </table>
- * <p>
- * Example usage:
- * </p>
- * 
+ * An embedded Jetty HTTP server.  It defers addition of contexts/handlers to 
+ * a callback so that the we can provide a method that increments ports until 
+ * a valid port is found.  This is mostly a thin wrapper around Jetty's 
+ * {@link Server} class and behaves as Jetty does.
+ *
+ * Here is an example usage:
  * <pre>
  * InternalHttpServer server = new InternalHttpServer();
  * 
@@ -77,9 +59,20 @@ import com.google.common.base.Preconditi
  * server.setWebappDir(new File(applicationHome, &quot;webapps&quot;));
  * server.setPort(8080);
  * server.setBindAddress(&quot;0.0.0.0&quot;);
- * 
+ * server.setContextCreator(new ContextCreator() {
+ *   @Override
+ *   public void addContexts(ContextHandlerCollection handlers) {
+ *        handlers.addHandler(InternalHttpServer.createLogAppContext());
+ *        handlers.addHandler(InternalHttpServer.createStackSevletContext(
+ *            StackServlet.class, "/stacks", "/*", "stacks"));
+ *        String webAppRoot = FlumeConfiguration.get().getNodeWebappRoot();
+ *        InternalHttpServer.addHandlersFromPaths(handlers,
+ *            new File(webAppRoot));
+ *   }
+ * });
+ *
  * server.start();
- * 
+ *
  * // at some later time...
  * server.stop();
  * </pre>
@@ -90,16 +83,15 @@ public class InternalHttpServer {
       .getLogger(InternalHttpServer.class);
 
   private Server server;
-  private File webappDir;
   private int port;
+  private int boundPort = -1;
   private String bindAddress;
-  private boolean scanForApps;
   private ContextHandlerCollection handlers;
+  private ContextCreator contextCreator = null;
 
   public InternalHttpServer() {
     port = 0;
     bindAddress = "0.0.0.0";
-    scanForApps = true;
     handlers = new ContextHandlerCollection();
   }
 
@@ -113,51 +105,14 @@ public class InternalHttpServer {
       connector.setHost(bindAddress);
 
       server.addConnector(connector);
-      server.addHandler(handlers);
-    }
-  }
 
-  protected void registerApplications() {
-    logger.debug("Registering webapps in {}", webappDir);
-
-    if (webappDir.isDirectory()) {
-      for (File entry : webappDir.listFiles()) {
-        tryRegisterApplication(server, entry);
+      if (contextCreator != null) {
+        contextCreator.addContexts(handlers);
       }
-    } else {
-      tryRegisterApplication(server, webappDir);
+      server.setHandler(handlers);
     }
   }
 
-  private boolean tryRegisterApplication(Server server, File path) {
-    String name;
-
-    logger.debug("checking {}", path);
-
-    if (path.isFile()) {
-      int idx = path.getName().indexOf(".war");
-
-      if (idx > -1) {
-        name = path.getName().substring(0, idx);
-      } else {
-        return false;
-      }
-    } else {
-      name = path.getName();
-    }
-
-    logger.debug("creating context {} -> {}", name, path);
-
-    // WebAppContext is for loading war files.
-    WebAppContext handler = new WebAppContext(path.getPath(), "/" + name);
-
-    handler.setParentLoaderPriority(true);
-
-    handlers.addHandler(handler);
-
-    return true;
-  }
-
   /**
    * <p>
    * Start a configured HTTP server. Users should have already injected all the
@@ -168,26 +123,22 @@ public class InternalHttpServer {
    * The configured webappDir is not scanned for applications until start() is
    * called.
    * </p>
-   * 
+   *
+   * @throws BindException
    * @throws InternalHttpServerException
    */
-  public void start() {
-    Preconditions.checkState(webappDir != null, "Webapp dir can not be null");
+  public void start() throws BindException {
 
     initialize();
 
-    if (scanForApps) {
-      registerApplications();
-    } else {
-      logger.info("Not scanning for webapps");
-    }
-
     logger.info("Starting internal HTTP server");
 
     try {
       server.start();
-
-      logger.info("Server started");
+      boundPort = server.getConnectors()[0].getLocalPort();
+      logger.info("Server started on port " + boundPort);
+    } catch (BindException be) {
+      throw be;
     } catch (Exception e) {
       logger.warn("Caught exception during HTTP server start.", e);
 
@@ -220,9 +171,8 @@ public class InternalHttpServer {
 
   @Override
   public String toString() {
-    return "{ bindAddress:" + bindAddress + " webappDir:" + webappDir
-        + " port:" + port + " scanForApps:" + scanForApps + " server:" + server
-        + " }";
+    return "{ bindAddress:" + bindAddress + " port:" + port + " boundPort:"
+        + boundPort + " server:" + server + " }";
   }
 
   public Server getServer() {
@@ -233,18 +183,14 @@ public class InternalHttpServer {
     this.server = server;
   }
 
-  public File getWebappDir() {
-    return webappDir;
-  }
-
-  public void setWebappDir(File webappDir) {
-    this.webappDir = webappDir;
-  }
-
   public int getPort() {
     return port;
   }
 
+  public int getBoundPort() {
+    return boundPort;
+  }
+
   public void setPort(int port) {
     this.port = port;
   }
@@ -257,14 +203,6 @@ public class InternalHttpServer {
     this.bindAddress = bindAddress;
   }
 
-  public boolean getScanForApps() {
-    return scanForApps;
-  }
-
-  public void setScanForApps(boolean scanForApps) {
-    this.scanForApps = scanForApps;
-  }
-
   public static class InternalHttpServerException extends RuntimeException {
 
     private static final long serialVersionUID = -4936285404574873547L;
@@ -287,7 +225,19 @@ public class InternalHttpServer {
 
   }
 
-  public void addHandler(Context ctx) {
+  public void setHandlers(ContextHandlerCollection ctx) {
+    if (ctx == null) {
+      logger.warn("Attempting to add null webapp context");
+      return;
+    }
+    handlers = ctx;
+  }
+
+  public ContextHandlerCollection getHandlers() {
+    return handlers;
+  }
+
+  protected void addHandler(Context ctx) {
     if (ctx == null) {
       logger.warn("Attempting to add null webapp context");
       return;
@@ -295,6 +245,117 @@ public class InternalHttpServer {
     handlers.addHandler(ctx);
   }
 
+  public void setContextCreator(ContextCreator cc) {
+    this.contextCreator = cc;
+  }
+
+  /**
+   * The jetty server cannot properly reload contexts if it attempts to bind to
+   * a port and fails. To support automatically going finding a new port, we
+   * thus need to parameterize the creation and addition of context. This class
+   * provides a call back that gets a instance of the server's
+   * ContextHandlerCollection, and gives clients the opportunity to populate it.
+   */
+  public abstract static class ContextCreator {
+    public abstract void addContexts(ContextHandlerCollection handlers);
+  }
+
+  public static WebAppContext createWarContext(File path) {
+    logger.debug("checking {}", path);
+
+    String name;
+    if (path.isFile()) {
+      // if not a war file reject
+      int idx = path.getName().indexOf(".war");
+      if (idx < 0) {
+        return null;
+      }
+
+      // drop the .war suffix
+      name = path.getName().substring(0, idx);
+    } else {
+      // is a dir
+      name = path.getName();
+    }
+
+    // WebAppContext is for loading war files.
+    logger.debug("creating context {} -> {}", name, path);
+    WebAppContext handler = new WebAppContext(path.getPath(), "/" + name);
+    handler.setParentLoaderPriority(true);
+    return handler;
+  }
+
+  /**
+   * This method adds support for both normal and exploded war file deployment.
+   * <p>
+   * This scannings the specified webapp directory for applications
+   * Both traditional and exploded war formats are supported in the webapp
+   * directory. In the case of exploded directories, the directory name is used
+   * as the context. For war files, everything from the first instance of ".war"
+   * to the end of the file name (inclusive) is stripped and the remainder is
+   * used for the context name.
+   * </p>
+   * <p>
+   * Name examples:
+   * </p>
+   * <table>
+   * <tr>
+   * <td>Name</td>
+   * <td>Type</td>
+   * <td>Context</td>
+   * </tr>
+   * <tr>
+   * <td>app.war</td>
+   * <td>file</td>
+   * <td>app</td>
+   * </tr>
+   * <tr>
+   * <td>app</td>
+   * <td>dir</td>
+   * <td>app</td>
+   * </tr>
+   * <tr>
+   * <td>app.war</td>
+   * <td>dir</td>
+   * <td>app.war</td>
+   * </tr>
+   * <tr>
+   * <td>app.war.war</td>
+   * <td>file</td>
+   * <td>app</td>
+   * </tr>
+   * <tr>
+   * <td>
+   * </table>
+   * <p>
+   * Example usage:
+   * </p>
+   */
+  public static void addHandlersFromPaths(ContextHandlerCollection handlers,
+      File webappDir) {
+    logger.debug("Registering webapps in {}", webappDir);
+
+    if (webappDir.isDirectory()) {
+      for (File entry : webappDir.listFiles()) {
+        Context ctx = createWarContext(entry);
+        if (ctx != null) {
+          handlers.addHandler(ctx);
+        }
+      }
+    } else {
+      Context ctx = createWarContext(webappDir);
+      if (ctx != null) {
+        handlers.addHandler(ctx);
+      }
+    }
+  }
+
+  /**
+   * This creates file listing servlet context that is used to point to the log
+   * directory of the daemon via the web interface.
+   *
+   * @return
+   */
   public static Context createLogAppContext() {
     Context ctx = new Context();
     // logs applet
@@ -308,8 +369,39 @@ public class InternalHttpServer {
     return ctx;
   }
 
-  public static Context createServletContext(Class<?> sltClz, String contextPath,
-      String pathSpec, String name) {
+  /**
+   * A very simple servlet to serve up a text representation of the current
+   * stack traces. It both returns the stacks to the caller and logs them.
+   * Currently the stack traces are done sequentially rather than exactly the
+   * same data.
+   */
+  public static class StackServlet extends HttpServlet {
+    private static final Log LOG = LogFactory.getLog(InternalHttpServer.class
+        .getName());
+    private static final long serialVersionUID = -6284183679759467039L;
+
+    @Override
+    public void doGet(HttpServletRequest request, HttpServletResponse response)
+        throws ServletException, IOException {
+
+      OutputStream outStream = response.getOutputStream();
+      ReflectionUtils.printThreadInfo(new PrintWriter(outStream), "");
+      outStream.close();
+      ReflectionUtils.logThreadInfo(LOG, "jsp requested", 1);
+    }
+  }
+
+  /**
+   * This creates a stack dumping servlet that can be used to debug a running
+   * daemon via the web interface.
+   *
+   * @param sltClz
+   * @param contextPath
+   * @param pathSpec
+   * @param name
+   * @return
+   */
+  public static Context createStackSevletContext() {
     Context ctx = new Context();
     ServletHolder holder = new ServletHolder(StackServlet.class);
     ctx.setContextPath("/stacks");
@@ -319,4 +411,50 @@ public class InternalHttpServer {
     return ctx;
   }
 
+  /**
+   * If successful returns the port the http server successfully bound to. If it
+   * failed, returns -1
+   */
+  public static InternalHttpServer startFindPortHttpServer(ContextCreator cc,
+      String bindAddress, int nodePort) {
+    do {
+      try {
+        return startHttpServer(cc, bindAddress, nodePort);
+      } catch (BindException be) {
+        logger.error("Unable to start webserver on " + bindAddress + ":"
+            + nodePort + ". Trying next port...");
+        nodePort++;
+      }
+    } while (true);
+  }
+
+  /**
+   * Single attempt to create an http server for the node.
+   * 
+   * @param bindAddress
+   * @param nodePort
+   * @return instance of a started http server or null if failed.
+   * @throws BindException
+   */
+  public static InternalHttpServer startHttpServer(ContextCreator cc,
+      String bindAddress, int nodePort) throws BindException {
+    InternalHttpServer http = null;
+    try {
+      http = new InternalHttpServer();
+      http.setBindAddress(bindAddress);
+      http.setPort(nodePort);
+      http.setContextCreator(cc);
+      http.start();
+      return http;
+    } catch (BindException be) {
+      http.stop();
+      http = null;
+      throw be;
+    } catch (Throwable t) {
+      logger.error("Unexpected exception/error thrown! " + t.getMessage(), t);
+      // if any exception happens bail out and cleanup.
+      http.stop();
+      return null;
+    }
+  }
 }

Modified: incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/agent/TestNodeJersey.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/agent/TestNodeJersey.java?rev=1161532&r1=1161531&r2=1161532&view=diff
==============================================================================
--- incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/agent/TestNodeJersey.java (original)
+++ incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/agent/TestNodeJersey.java Thu Aug 25 12:16:32 2011
@@ -17,7 +17,7 @@
  */
 package com.cloudera.flume.agent;
 
-import static com.cloudera.flume.master.TestMasterJersey.curl;
+import static com.cloudera.util.HttpServerTestUtils.curl;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 

Modified: incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/master/TestMasterJersey.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/master/TestMasterJersey.java?rev=1161532&r1=1161531&r2=1161532&view=diff
==============================================================================
--- incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/master/TestMasterJersey.java (original)
+++ incubator/flume/trunk/flume-core/src/test/java/com/cloudera/flume/master/TestMasterJersey.java Thu Aug 25 12:16:32 2011
@@ -17,14 +17,11 @@
  */
 package com.cloudera.flume.master;
 
+import static com.cloudera.util.HttpServerTestUtils.curl;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 
-import java.io.BufferedReader;
 import java.io.IOException;
-import java.io.InputStreamReader;
-import java.net.URL;
-import java.net.URLConnection;
 
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
@@ -43,33 +40,6 @@ public class TestMasterJersey extends Se
   public static final Logger LOG = LoggerFactory
       .getLogger(TestMasterJersey.class);
 
-  /**
-   * Gra b a url's contents. Since most are json, this should be small.
-   * 
-   * @param urlString
-   * @return
-   * @throws IOException
-   */
-  public static String curl(String urlString) throws IOException {
-    URL url = new URL(urlString);
-    URLConnection urlConn = url.openConnection();
-    urlConn.setDoInput(true);
-    urlConn.setUseCaches(false);
-
-    int len = urlConn.getContentLength();
-    String type = urlConn.getContentType();
-    LOG.info("pulled " + urlString + "[ type=" + type + " len=" + len + "]");
-    InputStreamReader isr = new InputStreamReader(urlConn.getInputStream());
-    BufferedReader br = new BufferedReader(isr);
-    StringBuilder sb = new StringBuilder();
-    String s;
-    while ((s = br.readLine()) != null) {
-      sb.append(s);
-      sb.append('\n');
-    }
-    return sb.toString();
-  }
-
   @Ignore
   @Test
   public void testMaster() throws IOException, InterruptedException,

Modified: incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/InternalHttpServerTest.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/InternalHttpServerTest.java?rev=1161532&r1=1161531&r2=1161532&view=diff
==============================================================================
--- incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/InternalHttpServerTest.java (original)
+++ incubator/flume/trunk/flume-core/src/test/java/com/cloudera/util/InternalHttpServerTest.java Thu Aug 25 12:16:32 2011
@@ -1,13 +1,41 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package com.cloudera.util;
 
+import static com.cloudera.util.HttpServerTestUtils.curlResp;
+import static org.junit.Assert.assertEquals;
+
 import java.io.File;
+import java.io.IOException;
+import java.net.BindException;
 
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.mortbay.jetty.handler.ContextHandlerCollection;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.cloudera.flume.conf.FlumeConfiguration;
+import com.cloudera.util.InternalHttpServer.ContextCreator;
+
 public class InternalHttpServerTest {
 
   private static final Logger logger = LoggerFactory
@@ -21,25 +49,16 @@ public class InternalHttpServerTest {
   }
 
   @Test
-  public void testStartInvalidState() {
+  public void testStart() throws BindException {
     boolean success = false;
 
-    try {
-      httpServer.start();
-      success = true;
-    } catch (IllegalStateException e) {
-      logger.info("Caught expected exception: {}", e.getMessage());
-    }
-
-    Assert.assertFalse(success);
-  }
-
-  @Test
-  public void testStart() {
-    boolean success = false;
-
-    httpServer.setWebappDir(new File(getClass().getClassLoader()
-        .getResource("test-webroot").getFile()));
+    httpServer.setContextCreator(new ContextCreator() {
+      @Override
+      public void addContexts(ContextHandlerCollection handlers) {
+        InternalHttpServer.addHandlersFromPaths(handlers, new File(getClass()
+            .getClassLoader().getResource("test-webroot").getFile()));
+      }
+    });
 
     try {
       httpServer.start();
@@ -68,4 +87,40 @@ public class InternalHttpServerTest {
     Assert.assertTrue(success);
   }
 
+  /**
+   * This tests to make sure that auto find port works. Two http servers are
+   * assigned to the same port -- the second one should detect the conflict and
+   * then pick the next port to bind and serve from. curl will throw exception
+   * on failure.
+   */
+  @Test
+  public void testAutoFindPort() throws IOException, Exception {
+    int port = FlumeConfiguration.get().getNodeStatusPort();
+    String bindAddress = "0.0.0.0";
+    InternalHttpServer http = InternalHttpServer.startHttpServer(null,
+        bindAddress, port);
+    http.start();
+
+    InternalHttpServer http2 = InternalHttpServer.startFindPortHttpServer(null,
+        bindAddress, port);
+    http2.start();
+
+    // grab something from each server
+    int port1 = http.getBoundPort();
+    int resp1 = curlResp("http://localhost:" + port1);
+    logger.info("http1 port:" + port1);
+    
+    int port2 = http2.getBoundPort();
+    int resp2 = curlResp("http://localhost:" + port2);
+    logger.info("http2 port:" + port2);
+
+    // shutdown
+    http.stop();
+    http2.stop();
+
+    assertEquals(404, resp1);
+    assertEquals(404, resp2);
+    assertEquals(port, port1);
+    assertEquals(port + 1, port2);
+  }
 }

Modified: incubator/flume/trunk/flume-node-web/src/test/java/com/cloudera/flume/agent/TestBootstrap.java
URL: http://svn.apache.org/viewvc/incubator/flume/trunk/flume-node-web/src/test/java/com/cloudera/flume/agent/TestBootstrap.java?rev=1161532&r1=1161531&r2=1161532&view=diff
==============================================================================
--- incubator/flume/trunk/flume-node-web/src/test/java/com/cloudera/flume/agent/TestBootstrap.java (original)
+++ incubator/flume/trunk/flume-node-web/src/test/java/com/cloudera/flume/agent/TestBootstrap.java Thu Aug 25 12:16:32 2011
@@ -1,14 +1,39 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package com.cloudera.flume.agent;
 
+import static org.junit.Assert.assertEquals;
+
 import java.io.File;
+import java.io.IOException;
 
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
+import org.mortbay.jetty.handler.ContextHandlerCollection;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.cloudera.util.HttpServerTestUtils;
 import com.cloudera.util.InternalHttpServer;
+import com.cloudera.util.InternalHttpServer.ContextCreator;
 
 public class TestBootstrap {
 
@@ -23,19 +48,25 @@ public class TestBootstrap {
   }
 
   @Test
-  public void testBootstrap() throws InterruptedException {
+  public void testBootstrap() throws InterruptedException, IOException {
     Assert.assertNotNull(httpServer);
 
     logger.debug("httpServer:{}", httpServer);
 
     httpServer.setPort(0);
-    httpServer.setWebappDir(new File("src/main"));
+    httpServer.setContextCreator(new ContextCreator() {
+      @Override
+      public void addContexts(ContextHandlerCollection handlers) {
+        InternalHttpServer.addHandlersFromPaths(handlers, new File("src/main"));
+      }
+    });
 
     httpServer.start();
-
-    Thread.sleep(3000);
-
+    int port = httpServer.getBoundPort();
+    String url = "http://localhost:" + port;
+    logger.debug("Grabbing http response from " + url);
+    int resp = HttpServerTestUtils.curlResp(url);
     httpServer.stop();
+    assertEquals(resp, 200); // expect ok response code.
   }
-
 }