spark-reviews mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From andrewor14 <...@git.apache.org>
Subject [GitHub] spark pull request: [SPARK-5719]allow daemons to bind to specified...
Date Thu, 19 Feb 2015 18:19:34 GMT
Github user andrewor14 commented on a diff in the pull request:

    https://github.com/apache/spark/pull/4505#discussion_r25010835
  
    --- Diff: core/src/main/scala/org/apache/spark/deploy/worker/WorkerArguments.scala ---
    @@ -23,46 +23,56 @@ import org.apache.spark.util.{IntParam, MemoryParam, Utils}
     import org.apache.spark.SparkConf
     
     /**
    - * Command-line parser for the worker.
    + * Arguments parser for the worker.
      */
     private[spark] class WorkerArguments(args: Array[String], conf: SparkConf) {
    -  var host = Utils.localHostName()
    -  var port = 0
    -  var webUiPort = 8081
    -  var cores = inferDefaultCores()
    -  var memory = inferDefaultMemory()
    +  var host: String = null
    +  var port: Int = -1
    +  var webUiPort: Int = -1
    +  var cores: Int = -1
    +  var memory: Int = -1
       var masters: Array[String] = null
       var workDir: String = null
       var propertiesFile: String = null
     
    -  // Check for settings in environment variables
    -  if (System.getenv("SPARK_WORKER_PORT") != null) {
    -    port = System.getenv("SPARK_WORKER_PORT").toInt
    -  }
    -  if (System.getenv("SPARK_WORKER_CORES") != null) {
    -    cores = System.getenv("SPARK_WORKER_CORES").toInt
    -  }
    -  if (conf.getenv("SPARK_WORKER_MEMORY") != null) {
    -    memory = Utils.memoryStringToMb(conf.getenv("SPARK_WORKER_MEMORY"))
    -  }
    -  if (System.getenv("SPARK_WORKER_WEBUI_PORT") != null) {
    -    webUiPort = System.getenv("SPARK_WORKER_WEBUI_PORT").toInt
    -  }
    -  if (System.getenv("SPARK_WORKER_DIR") != null) {
    -    workDir = System.getenv("SPARK_WORKER_DIR")
    -  }
    -
       parse(args.toList)
     
       // This mutates the SparkConf, so all accesses to it must be made after this line
       propertiesFile = Utils.loadDefaultSparkProperties(conf, propertiesFile)
     
    -  if (conf.contains("spark.worker.ui.port")) {
    -    webUiPort = conf.get("spark.worker.ui.port").toInt
    -  }
    -
    +  loadEnvironmentArguments()
       checkWorkerMemory()
     
    +  /**
    +   * Load arguments from environment variables, Spark properties etc.
    +   */
    +  private def loadEnvironmentArguments(): Unit = {
    +    host = Option(host)
    +      .orElse(conf.getOption("spark.worker.host"))
    +      .getOrElse(Utils.localHostName())
    +    if (port < 0) {
    +      port = conf.getOption("spark.worker.port").map(_.toInt)
    +        .orElse(Option(conf.getenv("SPARK_WORKER_PORT")).map(_.toInt))
    +        .getOrElse(0)
    --- End diff --
    
    why not just `map(_.toInt)` after the `orElse`? Less duplicate code there


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org


Mime
View raw message