From commits-return-97-archive-asf-public=cust-asf.ponee.io@amaterasu.apache.org Sun Apr 22 10:59:29 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 91C05180625 for ; Sun, 22 Apr 2018 10:59:28 +0200 (CEST) Received: (qmail 98287 invoked by uid 500); 22 Apr 2018 08:59:27 -0000 Mailing-List: contact commits-help@amaterasu.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@amaterasu.apache.org Delivered-To: mailing list commits@amaterasu.apache.org Received: (qmail 98278 invoked by uid 99); 22 Apr 2018 08:59:27 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 22 Apr 2018 08:59:27 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 097368094F; Sun, 22 Apr 2018 08:59:27 +0000 (UTC) Date: Sun, 22 Apr 2018 08:59:26 +0000 To: "commits@amaterasu.apache.org" Subject: [incubator-amaterasu] branch version-0.2.0-incubating-rc2 updated: fixed merge issue MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <152438756698.17535.18049593120543925115@gitbox.apache.org> From: yaniv@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: incubator-amaterasu X-Git-Refname: refs/heads/version-0.2.0-incubating-rc2 X-Git-Reftype: branch X-Git-Oldrev: 14acb597616d22d4cbf5a9fe0df3f7d9215f1f8f X-Git-Newrev: 5b4e8eb2bd564af37cbcda929720d75323bb1cfb X-Git-Rev: 5b4e8eb2bd564af37cbcda929720d75323bb1cfb X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. yaniv pushed a commit to branch version-0.2.0-incubating-rc2 in repository https://gitbox.apache.org/repos/asf/incubator-amaterasu.git The following commit(s) were added to refs/heads/version-0.2.0-incubating-rc2 by this push: new 5b4e8eb fixed merge issue 5b4e8eb is described below commit 5b4e8eb2bd564af37cbcda929720d75323bb1cfb Author: Yaniv Rodenski AuthorDate: Sun Apr 22 18:56:40 2018 +1000 fixed merge issue --- .../runners/spark/SparkRunnerHelper.scala | 2 +- .../leader/mesos/schedulers/JobScheduler.scala | 28 ++++++++++++---------- .../amaterasu/leader/yarn/ApplicationMaster.scala | 2 +- .../leader/yarn/YarnRMCallbackHandler.scala | 2 +- leader/src/main/scripts/ama-start-yarn.sh | 2 +- leader/src/main/scripts/amaterasu.properties | 2 +- 6 files changed, 21 insertions(+), 17 deletions(-) diff --git a/executor/src/main/scala/org/apache/spark/repl/amaterasu/runners/spark/SparkRunnerHelper.scala b/executor/src/main/scala/org/apache/spark/repl/amaterasu/runners/spark/SparkRunnerHelper.scala index abab8a4..0bf7337 100644 --- a/executor/src/main/scala/org/apache/spark/repl/amaterasu/runners/spark/SparkRunnerHelper.scala +++ b/executor/src/main/scala/org/apache/spark/repl/amaterasu/runners/spark/SparkRunnerHelper.scala @@ -145,7 +145,7 @@ object SparkRunnerHelper extends Logging { case "yarn" => conf.set("spark.home", config.spark.home) // TODO: parameterize those - .setJars(s"executor-${config.version}-all.jar" +: jars) + .setJars(s"executor.jar" +: jars) .set("spark.history.kerberos.keytab", "/etc/security/keytabs/spark.headless.keytab") .set("spark.driver.extraLibraryPath", "/usr/hdp/current/hadoop-client/lib/native:/usr/hdp/current/hadoop-client/lib/native/Linux-amd64-64") .set("spark.yarn.queue", "default") diff --git a/leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/JobScheduler.scala b/leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/JobScheduler.scala index 86863f5..87a8f5d 100755 --- a/leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/JobScheduler.scala +++ b/leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/JobScheduler.scala @@ -16,10 +16,11 @@ */ package org.apache.amaterasu.leader.mesos.schedulers +import java.io.File import java.util import java.util.concurrent.locks.ReentrantLock import java.util.concurrent.{ConcurrentHashMap, LinkedBlockingQueue} -import java.util.{Collections, UUID} +import java.util.{Collections, Properties, UUID} import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.scala.DefaultScalaModule @@ -50,6 +51,9 @@ import scala.collection.concurrent.TrieMap */ class JobScheduler extends AmaterasuScheduler { + /*private val props: Properties = new Properties(new File("")) + private val version = props.getProperty("version") + println(s"===> version $version")*/ LogManager.resetConfiguration() private var jobManager: JobManager = _ private var client: CuratorFramework = _ @@ -166,15 +170,15 @@ class JobScheduler extends AmaterasuScheduler { val command = CommandInfo .newBuilder .setValue( - s"""$awsEnv env AMA_NODE=${sys.env("AMA_NODE")} env MESOS_NATIVE_JAVA_LIBRARY=/usr/lib/libmesos.so env SPARK_EXECUTOR_URI=http://${sys.env("AMA_NODE")}:${config.Webserver.Port}/dist/spark-${config.Webserver.sparkVersion}.tgz java -cp executor-0.2.0-incubating-all.jar:spark-${config.Webserver.sparkVersion}/jars/* -Dscala.usejavacp=true -Djava.library.path=/usr/lib org.apache.amaterasu.executor.mesos.executors.MesosActionsExecutor ${jobManager.jobId} ${config.master} ${ [...] + s"""$awsEnv env AMA_NODE=${sys.env("AMA_NODE")} env MESOS_NATIVE_JAVA_LIBRARY=/usr/lib/libmesos.so env SPARK_EXECUTOR_URI=http://${sys.env("AMA_NODE")}:${config.Webserver.Port}/dist/spark-${config.Webserver.sparkVersion}.tgz java -cp executor-${config.version}-all.jar:spark-${config.Webserver.sparkVersion}/jars/* -Dscala.usejavacp=true -Djava.library.path=/usr/lib org.apache.amaterasu.executor.mesos.executors.MesosActionsExecutor ${jobManager.jobId} ${config.master} $ [...] ) -// HttpServer.getFilesInDirectory(sys.env("AMA_NODE"), config.Webserver.Port).foreach(f=> -// ) + // HttpServer.getFilesInDirectory(sys.env("AMA_NODE"), config.Webserver.Port).foreach(f=> + // ) .addUris(URI.newBuilder - .setValue(s"http://${sys.env("AMA_NODE")}:${config.Webserver.Port}/executor-0.2.0-incubating-all.jar") - .setExecutable(false) - .setExtract(false) - .build()) + .setValue(s"http://${sys.env("AMA_NODE")}:${config.Webserver.Port}/executor-${config.version}-all.jar") + .setExecutable(false) + .setExtract(false) + .build()) .addUris(URI.newBuilder() .setValue(s"http://${sys.env("AMA_NODE")}:${config.Webserver.Port}/spark-2.2.1-bin-hadoop2.7.tgz") .setExecutable(false) @@ -201,10 +205,10 @@ class JobScheduler extends AmaterasuScheduler { .setExtract(false) .build()) .addUris(URI.newBuilder() - .setValue(s"http://${sys.env("AMA_NODE")}:${config.Webserver.Port}/amaterasu.properties") - .setExecutable(false) - .setExtract(false) - .build()) + .setValue(s"http://${sys.env("AMA_NODE")}:${config.Webserver.Port}/amaterasu.properties") + .setExecutable(false) + .setExtract(false) + .build()) executor = ExecutorInfo .newBuilder .setData(ByteString.copyFrom(execData)) diff --git a/leader/src/main/scala/org/apache/amaterasu/leader/yarn/ApplicationMaster.scala b/leader/src/main/scala/org/apache/amaterasu/leader/yarn/ApplicationMaster.scala index 3fed076..8af5892 100644 --- a/leader/src/main/scala/org/apache/amaterasu/leader/yarn/ApplicationMaster.scala +++ b/leader/src/main/scala/org/apache/amaterasu/leader/yarn/ApplicationMaster.scala @@ -256,7 +256,7 @@ class ApplicationMaster extends AMRMClientAsync.CallbackHandler with Logging { val commands: List[String] = List( "/bin/bash ./miniconda.sh -b -p $PWD/miniconda && ", s"/bin/bash ${config.spark.home}/bin/load-spark-env.sh && ", - s"java -cp ${config.spark.home}/jars/*:executor.jar:${config.spark.home}/conf/:${config.YARN.hadoopHomeDir}/conf/ " + + s"java -cp executor.jar:${config.spark.home}/jars/*:${config.spark.home}/conf/:${config.YARN.hadoopHomeDir}/conf/ " + "-Xmx1G " + "-Dscala.usejavacp=true " + "-Dhdp.version=2.6.1.0-129 " + diff --git a/leader/src/main/scala/org/apache/amaterasu/leader/yarn/YarnRMCallbackHandler.scala b/leader/src/main/scala/org/apache/amaterasu/leader/yarn/YarnRMCallbackHandler.scala index 0c1a8f0..9f4c3f9 100644 --- a/leader/src/main/scala/org/apache/amaterasu/leader/yarn/YarnRMCallbackHandler.scala +++ b/leader/src/main/scala/org/apache/amaterasu/leader/yarn/YarnRMCallbackHandler.scala @@ -106,7 +106,7 @@ class YarnRMCallbackHandler(nmClient: NMClientAsync, val ctx = Records.newRecord(classOf[ContainerLaunchContext]) val command = s"""$awsEnv env AMA_NODE=${sys.env("AMA_NODE")} | env SPARK_EXECUTOR_URI=http://${sys.env("AMA_NODE")}:${config.Webserver.Port}/dist/spark-${config.Webserver.sparkVersion}.tgz - | java -cp executor-0.2.0-all.jar:spark-${config.Webserver.sparkVersion}/lib/* + | java -cp executor-*-all.jar:spark-${config.Webserver.sparkVersion}/lib/* | -Dscala.usejavacp=true | -Djava.library.path=/usr/lib org.apache.amaterasu.executor.yarn.executors.ActionsExecutorLauncher | ${jobManager.jobId} ${config.master} ${actionData.name} ${gson.toJson(taskData)} ${gson.toJson(execData)}""".stripMargin diff --git a/leader/src/main/scripts/ama-start-yarn.sh b/leader/src/main/scripts/ama-start-yarn.sh index 6fd194a..0573341 100755 --- a/leader/src/main/scripts/ama-start-yarn.sh +++ b/leader/src/main/scripts/ama-start-yarn.sh @@ -86,7 +86,7 @@ done echo "repo: ${REPO} " echo "force-bin: ${FORCE_BIN}" export HADOOP_USER_CLASSPATH_FIRST=true -CMD="yarn jar ${BASEDIR}/bin/leader-0.2.0-incubating-all.jar org.apache.amaterasu.leader.yarn.Client --home ${BASEDIR}" +CMD="yarn jar ${BASEDIR}/bin/leader-0.2.0-incubating-rc2-all.jar org.apache.amaterasu.leader.yarn.Client --home ${BASEDIR}" if [ -n "$REPO" ]; then echo "repo is ${REPO}" diff --git a/leader/src/main/scripts/amaterasu.properties b/leader/src/main/scripts/amaterasu.properties index 7961db9..202e350 100755 --- a/leader/src/main/scripts/amaterasu.properties +++ b/leader/src/main/scripts/amaterasu.properties @@ -1,5 +1,5 @@ zk=127.0.0.1 -version=0.2.0-incubating +version=0.2.0-incubating-rc2 master=192.168.33.11 user=root mode=yarn -- To stop receiving notification emails like this one, please contact yaniv@apache.org.