spark-reviews mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pwendell <...@git.apache.org>
Subject [GitHub] spark pull request: [WIP] SPARK-2157 Ability to write tight firewa...
Date Thu, 31 Jul 2014 00:27:07 GMT
Github user pwendell commented on a diff in the pull request:

    https://github.com/apache/spark/pull/1107#discussion_r15621147
  
    --- Diff: core/src/main/scala/org/apache/spark/HttpServer.scala ---
    @@ -41,45 +42,54 @@ private[spark] class ServerStateException(message: String) extends
Exception(mes
      * as well as classes created by the interpreter when the user types in code. This is
just a wrapper
      * around a Jetty server.
      */
    -private[spark] class HttpServer(resourceBase: File, securityManager: SecurityManager)
    -    extends Logging {
    +private[spark] class HttpServer(resourceBase: File,
    +                                securityManager: SecurityManager,
    +                                localPort: Int = 0) extends Logging {
       private var server: Server = null
    -  private var port: Int = -1
    +  private var port: Int = localPort
    +
    +  private def startOnPort(startPort: Int): (Server, Int) = {
    +    val server = new Server()
    +    val connector = new SocketConnector
    +    connector.setMaxIdleTime(60*1000)
    +    connector.setSoLingerTime(-1)
    +    connector.setPort(startPort)
    +    server.addConnector(connector)
    +
    +    val threadPool = new QueuedThreadPool
    +    threadPool.setDaemon(true)
    +    server.setThreadPool(threadPool)
    +    val resHandler = new ResourceHandler
    +    resHandler.setResourceBase(resourceBase.getAbsolutePath)
    +
    +    val handlerList = new HandlerList
    +    handlerList.setHandlers(Array(resHandler, new DefaultHandler))
    +
    +    if (securityManager.isAuthenticationEnabled()) {
    +      logDebug("HttpServer is using security")
    +      val sh = setupSecurityHandler(securityManager)
    +      // make sure we go through security handler to get resources
    +      sh.setHandler(handlerList)
    +      server.setHandler(sh)
    +    } else {
    +      logDebug("HttpServer is not using security")
    +      server.setHandler(handlerList)
    +    }
    +
    +    server.start()
    +    val actualPort = server.getConnectors()(0).getLocalPort()
    +
    +    (server, actualPort)
    +  }
     
       def start() {
         if (server != null) {
           throw new ServerStateException("Server is already started")
         } else {
           logInfo("Starting HTTP Server")
    -      server = new Server()
    -      val connector = new SocketConnector
    -      connector.setMaxIdleTime(60*1000)
    -      connector.setSoLingerTime(-1)
    -      connector.setPort(0)
    -      server.addConnector(connector)
    -
    -      val threadPool = new QueuedThreadPool
    -      threadPool.setDaemon(true)
    -      server.setThreadPool(threadPool)
    -      val resHandler = new ResourceHandler
    -      resHandler.setResourceBase(resourceBase.getAbsolutePath)
    -
    -      val handlerList = new HandlerList
    -      handlerList.setHandlers(Array(resHandler, new DefaultHandler))
    -
    -      if (securityManager.isAuthenticationEnabled()) {
    -        logDebug("HttpServer is using security")
    -        val sh = setupSecurityHandler(securityManager)
    -        // make sure we go through security handler to get resources
    -        sh.setHandler(handlerList)
    -        server.setHandler(sh)
    -      } else {
    -        logDebug("HttpServer is not using security")
    -        server.setHandler(handlerList)
    -      }
    -
    -      server.start()
    -      port = server.getConnectors()(0).getLocalPort()
    +      val (actualServer, actualPort) = PortManager.startWithIncrements(localPort, 3,
startOnPort)
    --- End diff --
    
    Would it be a good idea to modify `startWithIncrements` to accept the name of the service
being started? I'm a bit worried about the case where the user tries to lock down all the
ports, but there is contention. Instead of failing there Spark will just log a WARN message,
so at least we should say what component is failing in the WARN message.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

Mime
View raw message