spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r...@apache.org
Subject git commit: Merge pull request #148 from squito/include_appId
Date Thu, 07 Nov 2013 19:08:52 GMT
Updated Branches:
  refs/heads/branch-0.8 d5ae953c3 -> a5916b981


Merge pull request #148 from squito/include_appId

Include appId in executor cmd line args

add the appId back into the executor cmd line args.

I also made a pretty lame regression test, just to make sure it doesn't get dropped in the
future.  not sure it will run on the build server, though, b/c `ExecutorRunner.buildCommandSeq()`
expects to be abel to run the scripts in `bin`.

(cherry picked from commit 3d4ad84b63e440fd3f4b3edb1b120ff7c14a42d1)
Signed-off-by: Reynold Xin <rxin@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/a5916b98
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/a5916b98
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/a5916b98

Branch: refs/heads/branch-0.8
Commit: a5916b98193a6cc45aeb41eb05687ed43793338d
Parents: d5ae953
Author: Reynold Xin <rxin@apache.org>
Authored: Thu Nov 7 11:08:27 2013 -0800
Committer: Reynold Xin <rxin@apache.org>
Committed: Thu Nov 7 11:08:44 2013 -0800

----------------------------------------------------------------------
 .../spark/deploy/worker/ExecutorRunner.scala    |  2 +-
 .../executor/CoarseGrainedExecutorBackend.scala |  2 +-
 .../deploy/worker/ExecutorRunnerTest.scala      | 20 ++++++++++++++++++++
 3 files changed, 22 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/a5916b98/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala b/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
index 8fabc95..fff9cb6 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
@@ -104,7 +104,7 @@ private[spark] class ExecutorRunner(
     // SPARK-698: do not call the run.cmd script, as process.destroy()
     // fails to kill a process tree on Windows
     Seq(runner) ++ buildJavaOpts() ++ Seq(command.mainClass) ++
-      command.arguments.map(substituteVariables)
+      (command.arguments ++ Seq(appId)).map(substituteVariables)
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/a5916b98/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
index 80ff4c5..caee6b0 100644
--- a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
+++ b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
@@ -111,7 +111,7 @@ private[spark] object CoarseGrainedExecutorBackend {
 
   def main(args: Array[String]) {
     if (args.length < 4) {
-      //the reason we allow the last frameworkId argument is to make it easy to kill rogue
executors
+      //the reason we allow the last appid argument is to make it easy to kill rogue executors
       System.err.println(
         "Usage: CoarseGrainedExecutorBackend <driverUrl> <executorId> <hostname>
<cores> " +
         "[<appid>]")

http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/a5916b98/core/src/test/scala/org/apache/spark/deploy/worker/ExecutorRunnerTest.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/deploy/worker/ExecutorRunnerTest.scala b/core/src/test/scala/org/apache/spark/deploy/worker/ExecutorRunnerTest.scala
new file mode 100644
index 0000000..d433806
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/deploy/worker/ExecutorRunnerTest.scala
@@ -0,0 +1,20 @@
+package org.apache.spark.deploy.worker
+
+import java.io.File
+import org.scalatest.FunSuite
+import org.apache.spark.deploy.{ExecutorState, Command, ApplicationDescription}
+
+class ExecutorRunnerTest extends FunSuite {
+
+  test("command includes appId") {
+    def f(s:String) = new File(s)
+    val sparkHome = sys.props("user.dir")
+    val appDesc = new ApplicationDescription("app name", 8, 500, Command("foo", Seq(),Map()),
+      sparkHome, "appUiUrl")
+    val appId = "12345-worker321-9876"
+    val er = new ExecutorRunner(appId, 1, appDesc, 8, 500, null, "blah", "worker321", f(sparkHome),
+      f("ooga"), ExecutorState.RUNNING)
+
+    assert(er.buildCommandSeq().last === appId)
+  }
+}


Mime
View raw message