amaterasu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ya...@apache.org
Subject [incubator-amaterasu] 01/01: Revert "Revert "AMATERASU-24 Refactor Spark out of Amaterasu executor to it's own project""
Date Sun, 01 Jul 2018 02:21:43 GMT
This is an automated email from the ASF dual-hosted git repository.

yaniv pushed a commit to branch revert-29-revert-27-RC4-RefactorFramework
in repository https://gitbox.apache.org/repos/asf/incubator-amaterasu.git

commit 45074810ec21be501e81506d2b4f9e3c3dff1cd1
Author: Yaniv Rodenski <roadan@gmail.com>
AuthorDate: Sun Jul 1 12:21:40 2018 +1000

    Revert "Revert "AMATERASU-24 Refactor Spark out of Amaterasu executor to it's own project""
---
 build.gradle                                       |   4 -
 .../common/execution/actions/Notifier.scala        |   4 +-
 executor/build.gradle                              |  14 +---
 .../mesos/executors/MesosActionsExecutor.scala     |   4 +-
 .../executor/yarn/executors/ActionsExecutor.scala  |   8 +-
 {executor => frameworks/spark/runner}/build.gradle |  14 ++--
 .../spark/runner/pyspark}/PySparkEntryPoint.java   |  11 +--
 .../spark/runner/src/main}/resources/codegen.py    |   0
 .../spark/runner/src/main}/resources/runtime.py    |   0
 .../main/resources/spark-version-info.properties   |  11 +++
 .../spark/runner}/src/main/resources/spark_intp.py |   0
 .../spark/runner}/SparkRunnersProvider.scala       |  10 +--
 .../runner/pyspark}/PySparkExecutionQueue.scala    |   2 +-
 .../spark/runner/pyspark}/PySparkResultQueue.scala |   4 +-
 .../spark/runner/pyspark}/PySparkRunner.scala      |  10 +--
 .../spark/runner/pyspark}/ResultQueue.scala        |   2 +-
 .../spark/runner/repl}/AmaSparkILoop.scala         |   2 +-
 .../spark/runner/repl}/SparkRunnerHelper.scala     |  14 ++--
 .../spark/runner/repl}/SparkScalaRunner.scala      |   6 +-
 .../spark/runner/sparkr}/SparkRRunner.scala        |  28 +++----
 .../spark/runner/sparksql}/SparkSqlRunner.scala    |   9 ++-
 .../resources/SparkSql/csv/SparkSqlTestCsv.csv     |   0
 .../resources/SparkSql/json/SparkSqlTestData.json  |   0
 ...548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc | Bin
 ...548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc | Bin
 .../src/test/resources/SparkSql/parquet/_SUCCESS   |   0
 .../resources/SparkSql/parquet/_common_metadata    | Bin
 .../src/test/resources/SparkSql/parquet/_metadata  | Bin
 ...c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet | Bin
 ...c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet | Bin
 .../src/test/resources/amaterasu.properties        |   0
 .../spark/runner/src/test}/resources/codegen.py    |   0
 .../runner}/src/test/resources/py4j-0.10.4-src.zip | Bin
 .../spark/runner}/src/test/resources/py4j.tar.gz   | Bin
 .../src/test/resources/pyspark-with-amacontext.py  |   0
 .../runner}/src/test/resources/pyspark.tar.gz      | Bin
 .../spark/runner}/src/test/resources/pyspark.zip   | Bin
 .../spark/runner/src/test}/resources/runtime.py    |   0
 .../runner}/src/test/resources/simple-pyspark.py   |   0
 .../src/test/resources/simple-python-err.py        |   0
 .../runner}/src/test/resources/simple-python.py    |   0
 .../runner}/src/test/resources/simple-spark.scala  |   5 +-
 .../test}/resources/spark-version-info.properties  |   0
 .../spark/runner}/src/test/resources/spark_intp.py |   0
 .../spark/runner}/src/test/resources/step-2.scala  |   3 +-
 ...c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet | Bin
 ...c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet | Bin
 .../framework}/spark/SparkTestsSuite.scala         |  10 +--
 .../spark/runner}/RunnersLoadingTests.scala        |   2 +-
 .../spark/runner/pyspark}/PySparkRunnerTests.scala |   3 +-
 .../spark/runner/repl}/SparkScalaRunnerTests.scala |   9 +--
 .../runner/sparksql}/SparkSqlRunnerTests.scala     |   9 +--
 .../apache/amaterasu/utilities/TestNotifier.scala  |   0
 frameworks/spark/runtime/build.gradle              |  89 +++++++++++++++++++++
 .../framework/spark}/runtime/AmaContext.scala      |   8 +-
 gradle/wrapper/gradle-wrapper.properties           |   4 +-
 gradlew                                            |  16 ----
 gradlew.bat                                        |  18 -----
 .../apache/amaterasu/leader/yarn/ArgsParser.java   |   6 +-
 .../org/apache/amaterasu/leader/yarn/Client.java   |  23 +++++-
 .../apache/amaterasu/leader/dsl/JobParser.scala    |   4 +-
 .../frameworks/spark/SparkSetupProvider.scala      |   1 -
 .../mesos/schedulers/AmaterasuScheduler.scala      |   1 -
 .../leader/mesos/schedulers/JobScheduler.scala     |   5 +-
 .../leader/utilities/ActiveReportListener.scala    |   2 -
 .../amaterasu/leader/utilities/HttpServer.scala    |   7 +-
 .../amaterasu/leader/yarn/ApplicationMaster.scala  |  39 ++++++---
 .../leader/yarn/YarnRMCallbackHandler.scala        |   5 +-
 leader/src/main/scripts/ama-start-mesos.sh         |   4 +-
 leader/src/main/scripts/ama-start-yarn.sh          |   4 +-
 .../leader/mesos/ClusterSchedulerTests.scala       |   2 +-
 .../amaterasu/utilities/HttpServerTests.scala      |   6 --
 settings.gradle                                    |  14 +++-
 73 files changed, 256 insertions(+), 200 deletions(-)

diff --git a/build.gradle b/build.gradle
index 0f11347..00e44ea 100644
--- a/build.gradle
+++ b/build.gradle
@@ -25,10 +25,6 @@ allprojects {
     version '0.2.0-incubating-rc4'
 }
 
-project(':leader')
-project(':common')
-project(':executor')
-
 task copyLeagalFiles(type: Copy) {
     from "./DISCLAIMER", "./LICENSE", "./NOTICE"
     into "${buildDir}/amaterasu"
diff --git a/common/src/main/scala/org/apache/amaterasu/common/execution/actions/Notifier.scala b/common/src/main/scala/org/apache/amaterasu/common/execution/actions/Notifier.scala
index 8a44019..fe69260 100755
--- a/common/src/main/scala/org/apache/amaterasu/common/execution/actions/Notifier.scala
+++ b/common/src/main/scala/org/apache/amaterasu/common/execution/actions/Notifier.scala
@@ -16,9 +16,9 @@
  */
 package org.apache.amaterasu.common.execution.actions
 
-import NotificationLevel.NotificationLevel
-import NotificationType.NotificationType
 import com.fasterxml.jackson.annotation.JsonProperty
+import org.apache.amaterasu.common.execution.actions.NotificationLevel.NotificationLevel
+import org.apache.amaterasu.common.execution.actions.NotificationType.NotificationType
 
 abstract class Notifier {
 
diff --git a/executor/build.gradle b/executor/build.gradle
index 21bc2b0..09e269c 100644
--- a/executor/build.gradle
+++ b/executor/build.gradle
@@ -54,7 +54,6 @@ dependencies {
 
     compile group: 'org.scala-lang', name: 'scala-library', version: '2.11.8'
     compile group: 'org.scala-lang', name: 'scala-reflect', version: '2.11.8'
-    compile group: 'org.scala-lang', name: 'scala-compiler', version: '2.11.8'
     compile group: 'io.netty', name: 'netty-all', version: '4.0.42.Final'
     compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.5'
     compile group: 'org.apache.maven', name: 'maven-core', version: '3.0.5'
@@ -75,18 +74,7 @@ dependencies {
     compile project(':common')
     compile project(':amaterasu-sdk')
 
-    //runtime dependency for spark
-    provided('org.apache.spark:spark-repl_2.11:2.2.1')
-    provided('org.apache.spark:spark-core_2.11:2.2.1')
-
-    testCompile project(':common')
-    testCompile "gradle.plugin.com.github.maiflai:gradle-scalatest:0.14"
-    testRuntime 'org.pegdown:pegdown:1.1.0'
-    testCompile 'junit:junit:4.11'
-    testCompile 'org.scalatest:scalatest_2.11:3.0.2'
-    testCompile 'org.scala-lang:scala-library:2.11.8'
-    testCompile('org.apache.spark:spark-repl_2.11:2.2.1')
-    testCompile group: 'org.slf4j', name: 'slf4j-api', version: '1.7.9'
+
 
 }
 
diff --git a/executor/src/main/scala/org/apache/amaterasu/executor/mesos/executors/MesosActionsExecutor.scala b/executor/src/main/scala/org/apache/amaterasu/executor/mesos/executors/MesosActionsExecutor.scala
index 9ab75be..90c2001 100755
--- a/executor/src/main/scala/org/apache/amaterasu/executor/mesos/executors/MesosActionsExecutor.scala
+++ b/executor/src/main/scala/org/apache/amaterasu/executor/mesos/executors/MesosActionsExecutor.scala
@@ -26,7 +26,6 @@ import org.apache.amaterasu.executor.common.executors.ProvidersFactory
 import org.apache.mesos.Protos._
 import org.apache.mesos.protobuf.ByteString
 import org.apache.mesos.{Executor, ExecutorDriver, MesosExecutorDriver}
-import org.apache.spark.SparkContext
 
 import scala.collection.JavaConverters._
 import scala.concurrent.ExecutionContext.Implicits.global
@@ -37,7 +36,6 @@ class MesosActionsExecutor extends Executor with Logging {
 
   var master: String = _
   var executorDriver: ExecutorDriver = _
-  var sc: SparkContext = _
   var jobId: String = _
   var actionName: String = _
   //  var sparkScalaRunner: SparkScalaRunner = _
@@ -83,7 +81,7 @@ class MesosActionsExecutor extends Executor with Logging {
     notifier = new MesosNotifier(driver)
     notifier.info(s"Executor ${executorInfo.getExecutorId.getValue} registered")
     val outStream = new ByteArrayOutputStream()
-    providersFactory = ProvidersFactory(data, jobId, outStream, notifier, executorInfo.getExecutorId.getValue, hostName, propFile = "./amaterasu.properties")
+    providersFactory = ProvidersFactory(data, jobId, outStream, notifier, executorInfo.getExecutorId.getValue, hostName, "./amaterasu.properties")
 
   }
 
diff --git a/executor/src/main/scala/org/apache/amaterasu/executor/yarn/executors/ActionsExecutor.scala b/executor/src/main/scala/org/apache/amaterasu/executor/yarn/executors/ActionsExecutor.scala
index f4f553c..b5f8700 100644
--- a/executor/src/main/scala/org/apache/amaterasu/executor/yarn/executors/ActionsExecutor.scala
+++ b/executor/src/main/scala/org/apache/amaterasu/executor/yarn/executors/ActionsExecutor.scala
@@ -19,24 +19,18 @@ package org.apache.amaterasu.executor.yarn.executors
 import java.io.ByteArrayOutputStream
 import java.net.{InetAddress, URLDecoder}
 
-import scala.collection.JavaConverters._
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.module.scala.DefaultScalaModule
 import org.apache.amaterasu.common.dataobjects.{ExecData, TaskData}
 import org.apache.amaterasu.common.logging.Logging
 import org.apache.amaterasu.executor.common.executors.{ActiveNotifier, ProvidersFactory}
-import org.apache.hadoop.net.NetUtils
-import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.apache.spark.SparkContext
 
-import scala.reflect.internal.util.ScalaClassLoader
-import scala.reflect.internal.util.ScalaClassLoader.URLClassLoader
+import scala.collection.JavaConverters._
 
 
 class ActionsExecutor extends Logging {
 
   var master: String = _
-  var sc: SparkContext = _
   var jobId: String = _
   var actionName: String = _
   var taskData: TaskData = _
diff --git a/executor/build.gradle b/frameworks/spark/runner/build.gradle
similarity index 93%
copy from executor/build.gradle
copy to frameworks/spark/runner/build.gradle
index 21bc2b0..cc6c902 100644
--- a/executor/build.gradle
+++ b/frameworks/spark/runner/build.gradle
@@ -25,9 +25,6 @@ shadowJar {
     zip64 true
 }
 
-//sourceCompatibility = 1.8
-//targetCompatibility = 1.8
-
 repositories {
     maven {
         url "https://plugins.gradle.org/m2/"
@@ -52,6 +49,8 @@ sourceSets {
 
 dependencies {
 
+    compile project(':executor')
+    compile project(':spark-runtime')
     compile group: 'org.scala-lang', name: 'scala-library', version: '2.11.8'
     compile group: 'org.scala-lang', name: 'scala-reflect', version: '2.11.8'
     compile group: 'org.scala-lang', name: 'scala-compiler', version: '2.11.8'
@@ -72,8 +71,8 @@ dependencies {
         exclude group: 'org.jboss.netty'
     }
 
-    compile project(':common')
-    compile project(':amaterasu-sdk')
+    //compile project(':common')
+    //compile project(':amaterasu-sdk')
 
     //runtime dependency for spark
     provided('org.apache.spark:spark-repl_2.11:2.2.1')
@@ -95,7 +94,6 @@ sourceSets {
         resources.srcDirs += [file('src/test/resources')]
     }
 
-    // this is done so Scala will compile before Java
     main {
         scala {
             srcDirs = ['src/main/scala', 'src/main/java']
@@ -114,7 +112,7 @@ test {
 task copyToHome(type: Copy) {
     dependsOn shadowJar
     from 'build/libs'
-    into '../build/amaterasu/dist'
+    into '../../../build/amaterasu/dist'
     from 'build/resources/main'
-    into '../build/amaterasu/dist'
+    into '../../../build/amaterasu/dist'
 }
diff --git a/executor/src/main/java/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark/PySparkEntryPoint.java b/frameworks/spark/runner/src/main/java/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkEntryPoint.java
similarity index 92%
rename from executor/src/main/java/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark/PySparkEntryPoint.java
rename to frameworks/spark/runner/src/main/java/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkEntryPoint.java
index a521fce..6b79b2f 100755
--- a/executor/src/main/java/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark/PySparkEntryPoint.java
+++ b/frameworks/spark/runner/src/main/java/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkEntryPoint.java
@@ -14,17 +14,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.executor.execution.actions.runners.spark.PySpark;
+package org.apache.amaterasu.framework.spark.runner.pyspark;
 
-import org.apache.amaterasu.executor.runtime.AmaContext;
 import org.apache.amaterasu.common.runtime.Environment;
-
+import org.apache.amaterasu.framework.spark.runtime.AmaContext;
+import org.apache.spark.SparkConf;
 import org.apache.spark.SparkEnv;
 import org.apache.spark.api.java.JavaSparkContext;
 import org.apache.spark.sql.SQLContext;
-import org.apache.spark.SparkConf;
-import org.apache.spark.SparkContext;
-
 import org.apache.spark.sql.SparkSession;
 import py4j.GatewayServer;
 
@@ -35,7 +32,7 @@ import java.util.concurrent.ConcurrentHashMap;
 public class PySparkEntryPoint {
 
     //private static Boolean started = false;
-    private static PySparkExecutionQueue queue = new PySparkExecutionQueue();
+    private static  PySparkExecutionQueue queue = new PySparkExecutionQueue();
     private static ConcurrentHashMap<String, ResultQueue> resultQueues = new ConcurrentHashMap<>();
 
     private static int port = 0;
diff --git a/executor/src/test/resources/codegen.py b/frameworks/spark/runner/src/main/resources/codegen.py
similarity index 100%
rename from executor/src/test/resources/codegen.py
rename to frameworks/spark/runner/src/main/resources/codegen.py
diff --git a/executor/src/test/resources/runtime.py b/frameworks/spark/runner/src/main/resources/runtime.py
similarity index 100%
rename from executor/src/test/resources/runtime.py
rename to frameworks/spark/runner/src/main/resources/runtime.py
diff --git a/frameworks/spark/runner/src/main/resources/spark-version-info.properties b/frameworks/spark/runner/src/main/resources/spark-version-info.properties
new file mode 100644
index 0000000..ce0b312
--- /dev/null
+++ b/frameworks/spark/runner/src/main/resources/spark-version-info.properties
@@ -0,0 +1,11 @@
+version=2.1.0-SNAPSHOT
+
+user=root
+
+revision=738b4cc548ca48c010b682b8bc19a2f7e1947cfe
+
+branch=master
+
+date=2016-07-27T11:23:21Z
+
+url=https://github.com/apache/spark.git
diff --git a/executor/src/main/resources/spark_intp.py b/frameworks/spark/runner/src/main/resources/spark_intp.py
similarity index 100%
rename from executor/src/main/resources/spark_intp.py
rename to frameworks/spark/runner/src/main/resources/spark_intp.py
diff --git a/executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/SparkRunnersProvider.scala b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/SparkRunnersProvider.scala
similarity index 93%
rename from executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/SparkRunnersProvider.scala
rename to frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/SparkRunnersProvider.scala
index ba7ff03..652f32b 100644
--- a/executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/SparkRunnersProvider.scala
+++ b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/SparkRunnersProvider.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.executor.execution.actions.runners.spark
+package org.apache.amaterasu.framework.spark.runner
 
 import java.io._
 
@@ -24,10 +24,10 @@ import org.apache.amaterasu.common.dataobjects.ExecData
 import org.apache.amaterasu.common.execution.actions.Notifier
 import org.apache.amaterasu.common.execution.dependencies.{Dependencies, PythonDependencies, PythonPackage}
 import org.apache.amaterasu.common.logging.Logging
-import org.apache.amaterasu.executor.execution.actions.runners.spark.PySpark.PySparkRunner
-import org.apache.amaterasu.executor.execution.actions.runners.spark.SparkSql.SparkSqlRunner
+import org.apache.amaterasu.framework.spark.runner.pyspark.PySparkRunner
+import org.apache.amaterasu.framework.spark.runner.repl.{SparkRunnerHelper, SparkScalaRunner}
+import org.apache.amaterasu.framework.spark.runner.sparksql.SparkSqlRunner
 import org.apache.amaterasu.sdk.{AmaterasuRunner, RunnersProvider}
-import org.apache.spark.repl.amaterasu.runners.spark.{SparkRunnerHelper, SparkScalaRunner}
 import org.eclipse.aether.util.artifact.JavaScopes
 import org.sonatype.aether.repository.RemoteRepository
 import org.sonatype.aether.util.artifact.DefaultArtifact
@@ -113,7 +113,7 @@ class SparkRunnersProvider extends RunnersProvider with Logging {
 
     this.clusterConfig.mode match {
       case "yarn" => Seq("sh", "-c", "export HOME=$PWD && ./miniconda.sh -b -p miniconda") ! shellLoger
-      case "mesos" => Seq("sh", "Miniconda2-latest-Linux-x86_64.sh", "-b", "-p", "miniconda") ! shellLoger
+      case "mesos" => Seq("sh", "miniconda.sh", "-b", "-p", "miniconda") ! shellLoger
     }
 
     Seq("bash", "-c", "export HOME=$PWD && ./miniconda/bin/python -m conda install -y conda-build") ! shellLoger
diff --git a/executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark/PySparkExecutionQueue.scala b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkExecutionQueue.scala
similarity index 94%
rename from executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark/PySparkExecutionQueue.scala
rename to frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkExecutionQueue.scala
index 411069a..ddcf66c 100755
--- a/executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark/PySparkExecutionQueue.scala
+++ b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkExecutionQueue.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.executor.execution.actions.runners.spark.PySpark
+package org.apache.amaterasu.framework.spark.runner.pyspark
 
 import java.util
 import java.util.concurrent.{LinkedBlockingQueue, TimeUnit}
diff --git a/executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark/PySparkResultQueue.scala b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkResultQueue.scala
similarity index 85%
rename from executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark/PySparkResultQueue.scala
rename to frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkResultQueue.scala
index 6dbd445..16abbe3 100755
--- a/executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark/PySparkResultQueue.scala
+++ b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkResultQueue.scala
@@ -14,9 +14,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.executor.execution.actions.runners.spark.PySpark
+package org.apache.amaterasu.framework.spark.runner.pyspark
 
-import org.apache.amaterasu.executor.execution.actions.runners.spark.PySpark.ResultType.ResultType
+import org.apache.amaterasu.framework.spark.runner.pyspark.ResultType.ResultType
 
 object ResultType extends Enumeration {
   type ResultType = Value
diff --git a/executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark/PySparkRunner.scala b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkRunner.scala
old mode 100755
new mode 100644
similarity index 95%
rename from executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark/PySparkRunner.scala
rename to frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkRunner.scala
index 79fe18a..c015ec5
--- a/executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark/PySparkRunner.scala
+++ b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkRunner.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.executor.execution.actions.runners.spark.PySpark
+package org.apache.amaterasu.framework.spark.runner.pyspark
 
 import java.io.File
 import java.util
@@ -128,7 +128,7 @@ object PySparkRunner {
         }
         else {
           sparkCmd = Seq(pysparkPath, intpPath, port.toString)
-        }
+    }
         var pysparkPython = "/usr/bin/python"
 
         if (pyDeps != null &&
@@ -136,9 +136,9 @@ object PySparkRunner {
           pysparkPython = "./miniconda/bin/python"
         }
         val proc = Process(sparkCmd, None,
-          "PYTHONPATH" -> pypath,
-          "PYSPARK_PYTHON" -> pysparkPython,
-        "PYTHONHASHSEED" -> 0.toString)
+      "PYTHONPATH" -> pypath,
+      "PYSPARK_PYTHON" -> pysparkPython,
+      "PYTHONHASHSEED" -> 0.toString)
 
         proc.run(shellLoger)
     }
diff --git a/executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark/ResultQueue.scala b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/ResultQueue.scala
similarity index 94%
rename from executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark/ResultQueue.scala
rename to frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/ResultQueue.scala
index 3ac7bd7..d0cb4ae 100755
--- a/executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark/ResultQueue.scala
+++ b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark/ResultQueue.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.executor.execution.actions.runners.spark.PySpark
+package org.apache.amaterasu.framework.spark.runner.pyspark
 
 import java.util.concurrent.{LinkedBlockingQueue, TimeUnit}
 
diff --git a/executor/src/main/scala/org/apache/spark/repl/amaterasu/AmaSparkILoop.scala b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl/AmaSparkILoop.scala
similarity index 95%
rename from executor/src/main/scala/org/apache/spark/repl/amaterasu/AmaSparkILoop.scala
rename to frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl/AmaSparkILoop.scala
index 19ef3de..ec874b6 100755
--- a/executor/src/main/scala/org/apache/spark/repl/amaterasu/AmaSparkILoop.scala
+++ b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl/AmaSparkILoop.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.repl.amaterasu
+package org.apache.amaterasu.framework.spark.runner.repl
 
 import java.io.PrintWriter
 
diff --git a/executor/src/main/scala/org/apache/spark/repl/amaterasu/runners/spark/SparkRunnerHelper.scala b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl/SparkRunnerHelper.scala
similarity index 94%
rename from executor/src/main/scala/org/apache/spark/repl/amaterasu/runners/spark/SparkRunnerHelper.scala
rename to frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl/SparkRunnerHelper.scala
index f2c2afa..18658ec 100644
--- a/executor/src/main/scala/org/apache/spark/repl/amaterasu/runners/spark/SparkRunnerHelper.scala
+++ b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl/SparkRunnerHelper.scala
@@ -14,19 +14,18 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.repl.amaterasu.runners.spark
+package org.apache.amaterasu.framework.spark.runner.repl
 
 import java.io.{ByteArrayOutputStream, File, PrintWriter}
+import java.nio.file.{Files, Paths}
 
 import org.apache.amaterasu.common.configuration.ClusterConfig
 import org.apache.amaterasu.common.execution.actions.Notifier
 import org.apache.amaterasu.common.logging.Logging
 import org.apache.amaterasu.common.runtime.Environment
 import org.apache.amaterasu.common.utils.FileUtils
-import org.apache.spark.repl.amaterasu.AmaSparkILoop
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.util.Utils
 import org.apache.spark.SparkConf
+import org.apache.spark.sql.SparkSession
 
 import scala.tools.nsc.GenericRunnerSettings
 import scala.tools.nsc.interpreter.IMain
@@ -34,8 +33,9 @@ import scala.tools.nsc.interpreter.IMain
 object SparkRunnerHelper extends Logging {
 
   private val conf = new SparkConf()
-  private val rootDir = conf.getOption("spark.repl.classdir").getOrElse(Utils.getLocalDir(conf))
-  private val outputDir = Utils.createTempDir(root = rootDir, namePrefix = "repl")
+  private val rootDir = conf.get("spark.repl.classdir", System.getProperty("java.io.tmpdir"))
+  private val outputDir = Files.createTempDirectory(Paths.get(rootDir), "repl").toFile
+  outputDir.deleteOnExit()
 
   private var sparkSession: SparkSession = _
 
@@ -145,7 +145,7 @@ object SparkRunnerHelper extends Logging {
       case "yarn" =>
         conf.set("spark.home", config.spark.home)
           // TODO: parameterize those
-          .setJars(s"executor.jar" +: jars)
+          .setJars(Seq("executor.jar", "spark-runner.jar", "spark-runtime.jar") ++ jars)
           .set("spark.history.kerberos.keytab", "/etc/security/keytabs/spark.headless.keytab")
           .set("spark.driver.extraLibraryPath", "/usr/hdp/current/hadoop-client/lib/native:/usr/hdp/current/hadoop-client/lib/native/Linux-amd64-64")
           .set("spark.yarn.queue", "default")
diff --git a/executor/src/main/scala/org/apache/spark/repl/amaterasu/runners/spark/SparkScalaRunner.scala b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl/SparkScalaRunner.scala
similarity index 97%
rename from executor/src/main/scala/org/apache/spark/repl/amaterasu/runners/spark/SparkScalaRunner.scala
rename to frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl/SparkScalaRunner.scala
index 56a04cf..46d3077 100755
--- a/executor/src/main/scala/org/apache/spark/repl/amaterasu/runners/spark/SparkScalaRunner.scala
+++ b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl/SparkScalaRunner.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.spark.repl.amaterasu.runners.spark
+package org.apache.amaterasu.framework.spark.runner.repl
 
 import java.io.ByteArrayOutputStream
 import java.util
@@ -22,7 +22,7 @@ import java.util
 import org.apache.amaterasu.common.execution.actions.Notifier
 import org.apache.amaterasu.common.logging.Logging
 import org.apache.amaterasu.common.runtime.Environment
-import org.apache.amaterasu.executor.runtime.AmaContext
+import org.apache.amaterasu.framework.spark.runtime.AmaContext
 import org.apache.amaterasu.sdk.AmaterasuRunner
 import org.apache.spark.sql.{Dataset, SparkSession}
 
@@ -142,7 +142,7 @@ class SparkScalaRunner(var env: Environment,
     interpreter.interpret("import org.apache.spark.sql.SQLContext")
     interpreter.interpret("import org.apache.spark.sql.{ Dataset, SparkSession }")
     interpreter.interpret("import org.apache.spark.sql.SaveMode")
-    interpreter.interpret("import org.apache.amaterasu.executor.runtime.AmaContext")
+    interpreter.interpret("import org.apache.amaterasu.framework.spark.runtime.AmaContext")
     interpreter.interpret("import org.apache.amaterasu.common.runtime.Environment")
 
     // creating a map (_contextStore) to hold the different spark contexts
diff --git a/executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/SparkRRunner.scala b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/sparkr/SparkRRunner.scala
similarity index 69%
rename from executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/SparkRRunner.scala
rename to frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/sparkr/SparkRRunner.scala
index d111cfb..390b06a 100644
--- a/executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/SparkRRunner.scala
+++ b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/sparkr/SparkRRunner.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.executor.execution.actions.runners.spark
+package org.apache.amaterasu.framework.spark.runner.sparkr
 
 import java.io.ByteArrayOutputStream
 import java.util
@@ -28,21 +28,21 @@ import org.apache.spark.SparkContext
 
 class SparkRRunner extends Logging with AmaterasuRunner {
 
-  override def getIdentifier = "spark-r"
+    override def getIdentifier = "spark-r"
 
-  override def executeSource(actionSource: String, actionName: String, exports: util.Map[String, String]): Unit = {
-  }
+    override def executeSource(actionSource: String, actionName: String, exports: util.Map[String, String]): Unit = {
+    }
 }
 
 object SparkRRunner {
-  def apply(
-    env: Environment,
-    jobId: String,
-    sparkContext: SparkContext,
-    outStream: ByteArrayOutputStream,
-    notifier: Notifier,
-    jars: Seq[String]
-  ): SparkRRunner = {
-    new SparkRRunner()
-  }
+    def apply(
+               env: Environment,
+               jobId: String,
+               sparkContext: SparkContext,
+               outStream: ByteArrayOutputStream,
+               notifier: Notifier,
+               jars: Seq[String]
+             ): SparkRRunner = {
+        new SparkRRunner()
+    }
 }
\ No newline at end of file
diff --git a/executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/SparkSql/SparkSqlRunner.scala b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/sparksql/SparkSqlRunner.scala
similarity index 96%
rename from executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/SparkSql/SparkSqlRunner.scala
rename to frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/sparksql/SparkSqlRunner.scala
index 350ddb4..62af197 100644
--- a/executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/SparkSql/SparkSqlRunner.scala
+++ b/frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/sparksql/SparkSqlRunner.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.executor.execution.actions.runners.spark.SparkSql
+package org.apache.amaterasu.framework.spark.runner.sparksql
 
 import java.io.File
 import java.util
@@ -22,10 +22,11 @@ import java.util
 import org.apache.amaterasu.common.execution.actions.Notifier
 import org.apache.amaterasu.common.logging.Logging
 import org.apache.amaterasu.common.runtime.Environment
-import org.apache.amaterasu.executor.runtime.AmaContext
+import org.apache.amaterasu.framework.spark.runtime.AmaContext
 import org.apache.amaterasu.sdk.AmaterasuRunner
 import org.apache.commons.io.FilenameUtils
 import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession}
+
 import scala.collection.JavaConverters._
 
 /**
@@ -101,8 +102,8 @@ class SparkSqlRunner extends Logging with AmaterasuRunner {
 
         try{
 
-        result = spark.sql(parsedQuery)
-        notifier.success(parsedQuery)
+          result = spark.sql(parsedQuery)
+          notifier.success(parsedQuery)
         } catch {
           case e: Exception => notifier.error(parsedQuery, e.getMessage)
         }
diff --git a/executor/src/test/resources/SparkSql/csv/SparkSqlTestCsv.csv b/frameworks/spark/runner/src/test/resources/SparkSql/csv/SparkSqlTestCsv.csv
similarity index 100%
rename from executor/src/test/resources/SparkSql/csv/SparkSqlTestCsv.csv
rename to frameworks/spark/runner/src/test/resources/SparkSql/csv/SparkSqlTestCsv.csv
diff --git a/executor/src/test/resources/SparkSql/json/SparkSqlTestData.json b/frameworks/spark/runner/src/test/resources/SparkSql/json/SparkSqlTestData.json
similarity index 100%
rename from executor/src/test/resources/SparkSql/json/SparkSqlTestData.json
rename to frameworks/spark/runner/src/test/resources/SparkSql/json/SparkSqlTestData.json
diff --git a/executor/src/test/resources/SparkSql/parquet/.part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc b/frameworks/spark/runner/src/test/resources/SparkSql/parquet/.part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc
similarity index 100%
rename from executor/src/test/resources/SparkSql/parquet/.part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc
rename to frameworks/spark/runner/src/test/resources/SparkSql/parquet/.part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc
diff --git a/executor/src/test/resources/SparkSql/parquet/.part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc b/frameworks/spark/runner/src/test/resources/SparkSql/parquet/.part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc
similarity index 100%
rename from executor/src/test/resources/SparkSql/parquet/.part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc
rename to frameworks/spark/runner/src/test/resources/SparkSql/parquet/.part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc
diff --git a/executor/src/test/resources/SparkSql/parquet/_SUCCESS b/frameworks/spark/runner/src/test/resources/SparkSql/parquet/_SUCCESS
similarity index 100%
rename from executor/src/test/resources/SparkSql/parquet/_SUCCESS
rename to frameworks/spark/runner/src/test/resources/SparkSql/parquet/_SUCCESS
diff --git a/executor/src/test/resources/SparkSql/parquet/_common_metadata b/frameworks/spark/runner/src/test/resources/SparkSql/parquet/_common_metadata
similarity index 100%
rename from executor/src/test/resources/SparkSql/parquet/_common_metadata
rename to frameworks/spark/runner/src/test/resources/SparkSql/parquet/_common_metadata
diff --git a/executor/src/test/resources/SparkSql/parquet/_metadata b/frameworks/spark/runner/src/test/resources/SparkSql/parquet/_metadata
similarity index 100%
rename from executor/src/test/resources/SparkSql/parquet/_metadata
rename to frameworks/spark/runner/src/test/resources/SparkSql/parquet/_metadata
diff --git a/executor/src/test/resources/tmp/job/start/x/part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet b/frameworks/spark/runner/src/test/resources/SparkSql/parquet/part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet
similarity index 100%
rename from executor/src/test/resources/tmp/job/start/x/part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet
rename to frameworks/spark/runner/src/test/resources/SparkSql/parquet/part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet
diff --git a/executor/src/test/resources/tmp/job/start/x/part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet b/frameworks/spark/runner/src/test/resources/SparkSql/parquet/part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet
similarity index 100%
rename from executor/src/test/resources/tmp/job/start/x/part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet
rename to frameworks/spark/runner/src/test/resources/SparkSql/parquet/part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet
diff --git a/executor/src/test/resources/amaterasu.properties b/frameworks/spark/runner/src/test/resources/amaterasu.properties
similarity index 100%
rename from executor/src/test/resources/amaterasu.properties
rename to frameworks/spark/runner/src/test/resources/amaterasu.properties
diff --git a/executor/src/main/resources/codegen.py b/frameworks/spark/runner/src/test/resources/codegen.py
similarity index 100%
rename from executor/src/main/resources/codegen.py
rename to frameworks/spark/runner/src/test/resources/codegen.py
diff --git a/executor/src/test/resources/py4j-0.10.4-src.zip b/frameworks/spark/runner/src/test/resources/py4j-0.10.4-src.zip
similarity index 100%
rename from executor/src/test/resources/py4j-0.10.4-src.zip
rename to frameworks/spark/runner/src/test/resources/py4j-0.10.4-src.zip
diff --git a/executor/src/test/resources/py4j.tar.gz b/frameworks/spark/runner/src/test/resources/py4j.tar.gz
similarity index 100%
rename from executor/src/test/resources/py4j.tar.gz
rename to frameworks/spark/runner/src/test/resources/py4j.tar.gz
diff --git a/executor/src/test/resources/pyspark-with-amacontext.py b/frameworks/spark/runner/src/test/resources/pyspark-with-amacontext.py
similarity index 100%
rename from executor/src/test/resources/pyspark-with-amacontext.py
rename to frameworks/spark/runner/src/test/resources/pyspark-with-amacontext.py
diff --git a/executor/src/test/resources/pyspark.tar.gz b/frameworks/spark/runner/src/test/resources/pyspark.tar.gz
similarity index 100%
rename from executor/src/test/resources/pyspark.tar.gz
rename to frameworks/spark/runner/src/test/resources/pyspark.tar.gz
diff --git a/executor/src/test/resources/pyspark.zip b/frameworks/spark/runner/src/test/resources/pyspark.zip
similarity index 100%
rename from executor/src/test/resources/pyspark.zip
rename to frameworks/spark/runner/src/test/resources/pyspark.zip
diff --git a/executor/src/main/resources/runtime.py b/frameworks/spark/runner/src/test/resources/runtime.py
similarity index 100%
rename from executor/src/main/resources/runtime.py
rename to frameworks/spark/runner/src/test/resources/runtime.py
diff --git a/executor/src/test/resources/simple-pyspark.py b/frameworks/spark/runner/src/test/resources/simple-pyspark.py
similarity index 100%
rename from executor/src/test/resources/simple-pyspark.py
rename to frameworks/spark/runner/src/test/resources/simple-pyspark.py
diff --git a/executor/src/test/resources/simple-python-err.py b/frameworks/spark/runner/src/test/resources/simple-python-err.py
similarity index 100%
rename from executor/src/test/resources/simple-python-err.py
rename to frameworks/spark/runner/src/test/resources/simple-python-err.py
diff --git a/executor/src/test/resources/simple-python.py b/frameworks/spark/runner/src/test/resources/simple-python.py
similarity index 100%
rename from executor/src/test/resources/simple-python.py
rename to frameworks/spark/runner/src/test/resources/simple-python.py
diff --git a/executor/src/test/resources/simple-spark.scala b/frameworks/spark/runner/src/test/resources/simple-spark.scala
similarity index 83%
rename from executor/src/test/resources/simple-spark.scala
rename to frameworks/spark/runner/src/test/resources/simple-spark.scala
index a11a458..f2e49fd 100755
--- a/executor/src/test/resources/simple-spark.scala
+++ b/frameworks/spark/runner/src/test/resources/simple-spark.scala
@@ -14,8 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import org.apache.amaterasu.executor.runtime.AmaContext
-import org.apache.spark.sql.{DataFrame, SaveMode, SparkSession}
+import org.apache.spark.sql.{DataFrame, SaveMode}
 
 val data = Seq(1,3,4,5,6)
 
@@ -23,8 +22,6 @@ val data = Seq(1,3,4,5,6)
 val sc = AmaContext.sc
 val rdd = sc.parallelize(data)
 val sqlContext = AmaContext.spark
-
-import sqlContext.implicits._
 val x: DataFrame = rdd.toDF()
 
 x.write.mode(SaveMode.Overwrite)
\ No newline at end of file
diff --git a/executor/src/main/resources/spark-version-info.properties b/frameworks/spark/runner/src/test/resources/spark-version-info.properties
similarity index 100%
rename from executor/src/main/resources/spark-version-info.properties
rename to frameworks/spark/runner/src/test/resources/spark-version-info.properties
diff --git a/executor/src/test/resources/spark_intp.py b/frameworks/spark/runner/src/test/resources/spark_intp.py
similarity index 100%
rename from executor/src/test/resources/spark_intp.py
rename to frameworks/spark/runner/src/test/resources/spark_intp.py
diff --git a/executor/src/test/resources/step-2.scala b/frameworks/spark/runner/src/test/resources/step-2.scala
similarity index 94%
rename from executor/src/test/resources/step-2.scala
rename to frameworks/spark/runner/src/test/resources/step-2.scala
index a3d034c..86fd048 100755
--- a/executor/src/test/resources/step-2.scala
+++ b/frameworks/spark/runner/src/test/resources/step-2.scala
@@ -1,3 +1,5 @@
+
+
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
@@ -14,7 +16,6 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import org.apache.amaterasu.executor.runtime.AmaContext
 
 
 val highNoDf = AmaContext.getDataFrame("start", "x").where("age > 20")
diff --git a/executor/src/test/resources/SparkSql/parquet/part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet b/frameworks/spark/runner/src/test/resources/tmp/job/start/x/part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet
similarity index 100%
rename from executor/src/test/resources/SparkSql/parquet/part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet
rename to frameworks/spark/runner/src/test/resources/tmp/job/start/x/part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet
diff --git a/executor/src/test/resources/SparkSql/parquet/part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet b/frameworks/spark/runner/src/test/resources/tmp/job/start/x/part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet
similarity index 100%
rename from executor/src/test/resources/SparkSql/parquet/part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet
rename to frameworks/spark/runner/src/test/resources/tmp/job/start/x/part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet
diff --git a/executor/src/test/scala/org/apache/amaterasu/spark/SparkTestsSuite.scala b/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/SparkTestsSuite.scala
similarity index 90%
rename from executor/src/test/scala/org/apache/amaterasu/spark/SparkTestsSuite.scala
rename to frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/SparkTestsSuite.scala
index b11a4f9..0214568 100644
--- a/executor/src/test/scala/org/apache/amaterasu/spark/SparkTestsSuite.scala
+++ b/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/SparkTestsSuite.scala
@@ -14,22 +14,22 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.spark
+package org.apache.amaterasu.framework.spark
 
 import java.io.{ByteArrayOutputStream, File}
 
-import org.apache.amaterasu.RunnersTests.RunnersLoadingTests
 import org.apache.amaterasu.common.dataobjects.ExecData
 import org.apache.amaterasu.common.execution.dependencies._
 import org.apache.amaterasu.common.runtime.Environment
 import org.apache.amaterasu.executor.common.executors.ProvidersFactory
+import org.apache.amaterasu.framework.spark.runner.RunnersLoadingTests
+import org.apache.amaterasu.framework.spark.runner.pyspark.PySparkRunnerTests
+import org.apache.amaterasu.framework.spark.runner.repl.{SparkScalaRunner, SparkScalaRunnerTests}
+import org.apache.amaterasu.framework.spark.runner.sparksql.SparkSqlRunnerTests
 import org.apache.amaterasu.utilities.TestNotifier
-import org.apache.spark.repl.amaterasu.runners.spark.SparkScalaRunner
 import org.apache.spark.sql.SparkSession
 import org.scalatest._
 
-
-
 import scala.collection.mutable.ListBuffer
 
 
diff --git a/executor/src/test/scala/org/apache/amaterasu/RunnersTests/RunnersLoadingTests.scala b/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/RunnersLoadingTests.scala
similarity index 96%
rename from executor/src/test/scala/org/apache/amaterasu/RunnersTests/RunnersLoadingTests.scala
rename to frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/RunnersLoadingTests.scala
index 2decb9c..3629674 100644
--- a/executor/src/test/scala/org/apache/amaterasu/RunnersTests/RunnersLoadingTests.scala
+++ b/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/RunnersLoadingTests.scala
@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.RunnersTests
+package org.apache.amaterasu.framework.spark.runner
 
 import org.apache.amaterasu.common.runtime.Environment
 import org.apache.amaterasu.executor.common.executors.ProvidersFactory
diff --git a/executor/src/test/scala/org/apache/amaterasu/spark/PySparkRunnerTests.scala b/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkRunnerTests.scala
similarity index 96%
rename from executor/src/test/scala/org/apache/amaterasu/spark/PySparkRunnerTests.scala
rename to frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkRunnerTests.scala
index f12d676..a320e56 100755
--- a/executor/src/test/scala/org/apache/amaterasu/spark/PySparkRunnerTests.scala
+++ b/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/pyspark/PySparkRunnerTests.scala
@@ -14,12 +14,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.spark
+package org.apache.amaterasu.framework.spark.runner.pyspark
 
 import java.io.File
 
 import org.apache.amaterasu.executor.common.executors.ProvidersFactory
-import org.apache.amaterasu.executor.execution.actions.runners.spark.PySpark.PySparkRunner
 import org.apache.log4j.{Level, Logger}
 import org.scalatest.{BeforeAndAfterAll, DoNotDiscover, FlatSpec, Matchers}
 
diff --git a/executor/src/test/scala/org/apache/amaterasu/spark/SparkScalaRunnerTests.scala b/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/repl/SparkScalaRunnerTests.scala
similarity index 92%
rename from executor/src/test/scala/org/apache/amaterasu/spark/SparkScalaRunnerTests.scala
rename to frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/repl/SparkScalaRunnerTests.scala
index 1d79fc9..26f2ceb 100755
--- a/executor/src/test/scala/org/apache/amaterasu/spark/SparkScalaRunnerTests.scala
+++ b/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/repl/SparkScalaRunnerTests.scala
@@ -14,15 +14,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.spark
+package org.apache.amaterasu.framework.spark.runner.repl
 
-
-import scala.collection.JavaConverters._
 import org.apache.amaterasu.executor.common.executors.ProvidersFactory
-import org.apache.amaterasu.executor.runtime.AmaContext
-import org.apache.spark.repl.amaterasu.runners.spark.SparkScalaRunner
+import org.apache.amaterasu.framework.spark.runtime.AmaContext
 import org.scalatest.{BeforeAndAfterAll, DoNotDiscover, FlatSpec, Matchers}
 
+import scala.collection.JavaConverters._
 import scala.io.Source
 
 @DoNotDiscover
@@ -33,6 +31,7 @@ class SparkScalaRunnerTests extends FlatSpec with Matchers with BeforeAndAfterAl
 
   "SparkScalaRunner" should "execute the simple-spark.scala" in {
 
+
     val sparkRunner =factory.getRunner("spark", "scala").get.asInstanceOf[SparkScalaRunner]
     val script = getClass.getResource("/simple-spark.scala").getPath
     val sourceCode = Source.fromFile(script).getLines().mkString("\n")
diff --git a/executor/src/test/scala/org/apache/amaterasu/spark/SparkSqlRunnerTests.scala b/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/sparksql/SparkSqlRunnerTests.scala
similarity index 96%
rename from executor/src/test/scala/org/apache/amaterasu/spark/SparkSqlRunnerTests.scala
rename to frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/sparksql/SparkSqlRunnerTests.scala
index 90cf73b..abb5745 100644
--- a/executor/src/test/scala/org/apache/amaterasu/spark/SparkSqlRunnerTests.scala
+++ b/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/sparksql/SparkSqlRunnerTests.scala
@@ -14,22 +14,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.spark
+package org.apache.amaterasu.framework.spark.runner.sparksql
 
 import org.apache.amaterasu.common.runtime.Environment
 import org.apache.amaterasu.executor.common.executors.ProvidersFactory
-import org.apache.amaterasu.executor.execution.actions.runners.spark.SparkSql.SparkSqlRunner
 import org.apache.amaterasu.utilities.TestNotifier
-import org.apache.log4j.Logger
-import org.apache.log4j.Level
+import org.apache.log4j.{Level, Logger}
 import org.apache.spark.sql.{SaveMode, SparkSession}
 import org.scalatest.{BeforeAndAfterAll, DoNotDiscover, FlatSpec, Matchers}
 
 import scala.collection.JavaConverters._
 
-/**
-  * Created by kirupa on 10/12/16.
-  */
 @DoNotDiscover
 class SparkSqlRunnerTests extends FlatSpec with Matchers with BeforeAndAfterAll {
 
diff --git a/executor/src/test/scala/org/apache/amaterasu/utilities/TestNotifier.scala b/frameworks/spark/runner/src/test/scala/org/apache/amaterasu/utilities/TestNotifier.scala
similarity index 100%
rename from executor/src/test/scala/org/apache/amaterasu/utilities/TestNotifier.scala
rename to frameworks/spark/runner/src/test/scala/org/apache/amaterasu/utilities/TestNotifier.scala
diff --git a/frameworks/spark/runtime/build.gradle b/frameworks/spark/runtime/build.gradle
new file mode 100644
index 0000000..9bba2e4
--- /dev/null
+++ b/frameworks/spark/runtime/build.gradle
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+plugins {
+    id 'com.github.johnrengelman.shadow' version '1.2.4'
+    id 'com.github.maiflai.scalatest' version '0.6-5-g9065d91'
+    id 'scala'
+    id 'java'
+}
+
+shadowJar {
+    zip64 true
+}
+
+repositories {
+    maven {
+        url "https://plugins.gradle.org/m2/"
+    }
+    mavenCentral()
+}
+
+test {
+    maxParallelForks = 1
+    forkEvery = 1
+}
+
+configurations {
+    provided
+    runtime.exclude module: 'hadoop-common'
+    runtime.exclude module: 'hadoop-yarn-api'
+    runtime.exclude module: 'hadoop-yarn-client'
+    runtime.exclude module: 'hadoop-hdfs'
+    runtime.exclude module: 'mesos'
+    runtime.exclude module: 'scala-compiler'
+}
+
+sourceSets {
+    main.compileClasspath += configurations.provided
+    test.compileClasspath += configurations.provided
+    test.runtimeClasspath += configurations.provided
+}
+
+dependencies {
+
+    compile project(':executor')
+    provided('org.apache.spark:spark-repl_2.11:2.2.1')
+    provided('org.apache.spark:spark-core_2.11:2.2.1')
+
+}
+
+sourceSets {
+    test {
+        resources.srcDirs += [file('src/test/resources')]
+    }
+
+    main {
+        scala {
+            srcDirs = ['src/main/scala', 'src/main/java']
+        }
+        java {
+            srcDirs = []
+        }
+    }
+}
+
+test {
+
+    maxParallelForks = 1
+}
+
+task copyToHome(type: Copy) {
+    from 'build/libs'
+    into '../../../build/amaterasu/dist'
+    from 'build/resources/main'
+    into '../../../build/amaterasu/dist'
+}
diff --git a/executor/src/main/scala/org/apache/amaterasu/executor/runtime/AmaContext.scala b/frameworks/spark/runtime/src/main/scala/org/apache/amaterasu/framework/spark/runtime/AmaContext.scala
old mode 100755
new mode 100644
similarity index 93%
rename from executor/src/main/scala/org/apache/amaterasu/executor/runtime/AmaContext.scala
rename to frameworks/spark/runtime/src/main/scala/org/apache/amaterasu/framework/spark/runtime/AmaContext.scala
index a61cd5a..cb2eccc
--- a/executor/src/main/scala/org/apache/amaterasu/executor/runtime/AmaContext.scala
+++ b/frameworks/spark/runtime/src/main/scala/org/apache/amaterasu/framework/spark/runtime/AmaContext.scala
@@ -14,12 +14,12 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.apache.amaterasu.executor.runtime
+package org.apache.amaterasu.framework.spark.runtime
 
 import org.apache.amaterasu.common.logging.Logging
 import org.apache.amaterasu.common.runtime.Environment
 import org.apache.spark.SparkContext
-import org.apache.spark.sql._
+import org.apache.spark.sql.{DataFrame, Dataset, Encoder, SparkSession}
 
 object AmaContext extends Logging {
 
@@ -40,15 +40,11 @@ object AmaContext extends Logging {
   }
 
   def getDataFrame(actionName: String, dfName: String, format: String = "parquet"): DataFrame = {
-
     spark.read.format(format).load(s"${env.workingDir}/$jobId/$actionName/$dfName")
-
   }
 
   def getDataset[T: Encoder](actionName: String, dfName: String, format: String = "parquet"): Dataset[T] = {
-
     getDataFrame(actionName, dfName, format).as[T]
-
   }
 
 }
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
index 4b125b8..9fdd83c 100644
--- a/gradle/wrapper/gradle-wrapper.properties
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -1,6 +1,6 @@
-#Fri Jan 27 12:21:51 AEDT 2017
+#Thu Jun 28 13:07:02 SGT 2018
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-3.1-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-3.1-bin.zip
diff --git a/gradlew b/gradlew
index 3efb0e9..9aa616c 100755
--- a/gradlew
+++ b/gradlew
@@ -1,20 +1,4 @@
 #!/usr/bin/env bash
-#
-#    Licensed to the Apache Software Foundation (ASF) under one or more
-#    contributor license agreements.  See the NOTICE file distributed with
-#    this work for additional information regarding copyright ownership.
-#    The ASF licenses this file to You under the Apache License, Version 2.0
-#    (the "License"); you may not use this file except in compliance with
-#    the License.  You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-#
 
 ##############################################################################
 ##
diff --git a/gradlew.bat b/gradlew.bat
index 718266c..f955316 100644
--- a/gradlew.bat
+++ b/gradlew.bat
@@ -1,21 +1,3 @@
-rem
-rem
-rem    Licensed to the Apache Software Foundation (ASF) under one or more
-rem    contributor license agreements.  See the NOTICE file distributed with
-rem    this work for additional information regarding copyright ownership.
-rem    The ASF licenses this file to You under the Apache License, Version 2.0
-rem    (the "License"); you may not use this file except in compliance with
-rem    the License.  You may obtain a copy of the License at
-rem
-rem       http://www.apache.org/licenses/LICENSE-2.0
-rem
-rem    Unless required by applicable law or agreed to in writing, software
-rem    distributed under the License is distributed on an "AS IS" BASIS,
-rem    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-rem    See the License for the specific language governing permissions and
-rem    limitations under the License.
-rem
-
 @if "%DEBUG%" == "" @echo off
 @rem ##########################################################################
 @rem
diff --git a/leader/src/main/java/org/apache/amaterasu/leader/yarn/ArgsParser.java b/leader/src/main/java/org/apache/amaterasu/leader/yarn/ArgsParser.java
index be0fc05..38a9c38 100644
--- a/leader/src/main/java/org/apache/amaterasu/leader/yarn/ArgsParser.java
+++ b/leader/src/main/java/org/apache/amaterasu/leader/yarn/ArgsParser.java
@@ -16,7 +16,11 @@
  */
 package org.apache.amaterasu.leader.yarn;
 
-import org.apache.commons.cli.*;
+import org.apache.commons.cli.BasicParser;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
 
 public class ArgsParser {
     private static Options getOptions() {
diff --git a/leader/src/main/java/org/apache/amaterasu/leader/yarn/Client.java b/leader/src/main/java/org/apache/amaterasu/leader/yarn/Client.java
index e3c2812..8f16ee7 100644
--- a/leader/src/main/java/org/apache/amaterasu/leader/yarn/Client.java
+++ b/leader/src/main/java/org/apache/amaterasu/leader/yarn/Client.java
@@ -31,7 +31,16 @@ import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
-import org.apache.hadoop.yarn.api.records.*;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
 import org.apache.hadoop.yarn.client.api.YarnClient;
 import org.apache.hadoop.yarn.client.api.YarnClientApplication;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -43,11 +52,19 @@ import org.apache.log4j.LogManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import javax.jms.*;
+import javax.jms.Connection;
+import javax.jms.JMSException;
+import javax.jms.MessageConsumer;
+import javax.jms.Session;
+import javax.jms.Topic;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
-import java.util.*;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
 
 import static java.lang.System.exit;
 
diff --git a/leader/src/main/scala/org/apache/amaterasu/leader/dsl/JobParser.scala b/leader/src/main/scala/org/apache/amaterasu/leader/dsl/JobParser.scala
index 8ef1c7a..aba6210 100755
--- a/leader/src/main/scala/org/apache/amaterasu/leader/dsl/JobParser.scala
+++ b/leader/src/main/scala/org/apache/amaterasu/leader/dsl/JobParser.scala
@@ -18,12 +18,12 @@ package org.apache.amaterasu.leader.dsl
 
 import java.util.concurrent.BlockingQueue
 
-import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper}
 import com.fasterxml.jackson.databind.node.ArrayNode
+import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper}
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
 import org.apache.amaterasu.common.dataobjects.ActionData
-import org.apache.amaterasu.leader.execution.actions.{Action, ErrorAction, SequentialAction}
 import org.apache.amaterasu.leader.execution.JobManager
+import org.apache.amaterasu.leader.execution.actions.{Action, ErrorAction, SequentialAction}
 import org.apache.curator.framework.CuratorFramework
 
 import scala.collection.JavaConverters._
diff --git a/leader/src/main/scala/org/apache/amaterasu/leader/frameworks/spark/SparkSetupProvider.scala b/leader/src/main/scala/org/apache/amaterasu/leader/frameworks/spark/SparkSetupProvider.scala
index 8c487c1..f6dea22 100644
--- a/leader/src/main/scala/org/apache/amaterasu/leader/frameworks/spark/SparkSetupProvider.scala
+++ b/leader/src/main/scala/org/apache/amaterasu/leader/frameworks/spark/SparkSetupProvider.scala
@@ -19,7 +19,6 @@ package org.apache.amaterasu.leader.frameworks.spark
 import java.io.File
 
 import org.apache.amaterasu.common.configuration.ClusterConfig
-import org.apache.amaterasu.common.dataobjects.ExecData
 import org.apache.amaterasu.leader.utilities.{DataLoader, MemoryFormatParser}
 import org.apache.amaterasu.sdk.frameworks.FrameworkSetupProvider
 import org.apache.amaterasu.sdk.frameworks.configuration.DriverConfiguration
diff --git a/leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/AmaterasuScheduler.scala b/leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/AmaterasuScheduler.scala
index f2f2c00..4b1a74c 100755
--- a/leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/AmaterasuScheduler.scala
+++ b/leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/AmaterasuScheduler.scala
@@ -17,7 +17,6 @@
 package org.apache.amaterasu.leader.mesos.schedulers
 
 import org.apache.amaterasu.common.logging.Logging
-
 import org.apache.mesos.Protos.{Resource, Value}
 import org.apache.mesos.Scheduler
 
diff --git a/leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/JobScheduler.scala b/leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/JobScheduler.scala
index 87a8f5d..2c2e8af 100755
--- a/leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/JobScheduler.scala
+++ b/leader/src/main/scala/org/apache/amaterasu/leader/mesos/schedulers/JobScheduler.scala
@@ -16,11 +16,10 @@
  */
 package org.apache.amaterasu.leader.mesos.schedulers
 
-import java.io.File
 import java.util
 import java.util.concurrent.locks.ReentrantLock
 import java.util.concurrent.{ConcurrentHashMap, LinkedBlockingQueue}
-import java.util.{Collections, Properties, UUID}
+import java.util.{Collections, UUID}
 
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.module.scala.DefaultScalaModule
@@ -185,7 +184,7 @@ class JobScheduler extends AmaterasuScheduler {
                     .setExtract(true)
                     .build())
                   .addUris(URI.newBuilder()
-                    .setValue(s"http://${sys.env("AMA_NODE")}:${config.Webserver.Port}/Miniconda2-latest-Linux-x86_64.sh")
+                    .setValue(s"http://${sys.env("AMA_NODE")}:${config.Webserver.Port}/miniconda.sh")
                     .setExecutable(false)
                     .setExtract(false)
                     .build())
diff --git a/leader/src/main/scala/org/apache/amaterasu/leader/utilities/ActiveReportListener.scala b/leader/src/main/scala/org/apache/amaterasu/leader/utilities/ActiveReportListener.scala
index 2664665..b3ffaad 100644
--- a/leader/src/main/scala/org/apache/amaterasu/leader/utilities/ActiveReportListener.scala
+++ b/leader/src/main/scala/org/apache/amaterasu/leader/utilities/ActiveReportListener.scala
@@ -17,9 +17,7 @@
 package org.apache.amaterasu.leader.utilities
 
 import javax.jms.{Message, MessageListener, TextMessage}
-
 import net.liftweb.json._
-import net.liftweb.json.JsonDSL._
 import org.apache.amaterasu.common.execution.actions.{Notification, NotificationLevel, NotificationType}
 
 class ActiveReportListener extends MessageListener {
diff --git a/leader/src/main/scala/org/apache/amaterasu/leader/utilities/HttpServer.scala b/leader/src/main/scala/org/apache/amaterasu/leader/utilities/HttpServer.scala
index 2e01963..5c48329 100644
--- a/leader/src/main/scala/org/apache/amaterasu/leader/utilities/HttpServer.scala
+++ b/leader/src/main/scala/org/apache/amaterasu/leader/utilities/HttpServer.scala
@@ -18,19 +18,14 @@ package org.apache.amaterasu.leader.utilities
 
 import org.apache.amaterasu.common.logging.Logging
 import org.apache.log4j.{BasicConfigurator, Level, Logger}
-import org.eclipse.jetty.server.{Handler, Server, ServerConnector}
 import org.eclipse.jetty.server.handler._
-import org.eclipse.jetty.servlet.{DefaultServlet, ServletContextHandler, ServletHolder}
-import org.eclipse.jetty.toolchain.test.MavenTestingUtils
-import org.eclipse.jetty.util.thread.QueuedThreadPool
+import org.eclipse.jetty.server.{Server, ServerConnector}
 import org.eclipse.jetty.util.log.StdErrLog
-import org.eclipse.jetty.util.resource.Resource
 import org.jsoup.Jsoup
 import org.jsoup.select.Elements
 
 import scala.collection.JavaConverters._
 import scala.io.{BufferedSource, Source}
-import scala.text.Document
 
 /**
   * Created by kirupa
diff --git a/leader/src/main/scala/org/apache/amaterasu/leader/yarn/ApplicationMaster.scala b/leader/src/main/scala/org/apache/amaterasu/leader/yarn/ApplicationMaster.scala
index 1828100..406c150 100644
--- a/leader/src/main/scala/org/apache/amaterasu/leader/yarn/ApplicationMaster.scala
+++ b/leader/src/main/scala/org/apache/amaterasu/leader/yarn/ApplicationMaster.scala
@@ -250,7 +250,7 @@ class ApplicationMaster extends AMRMClientAsync.CallbackHandler with Logging {
         val commands: List[String] = List(
           "/bin/bash ./miniconda.sh -b -p $PWD/miniconda && ",
           s"/bin/bash spark/bin/load-spark-env.sh && ",
-          s"java -cp spark/jars/*:executor.jar:spark/conf/:${config.YARN.hadoopHomeDir}/conf/ " +
+          s"java -cp spark/jars/*:executor.jar:spark-runner.jar:spark-runtime.jar:spark/conf/:${config.YARN.hadoopHomeDir}/conf/ " +
             "-Xmx1G " +
             "-Dscala.usejavacp=true " +
             "-Dhdp.version=2.6.1.0-129 " +
@@ -266,22 +266,37 @@ class ApplicationMaster extends AMRMClientAsync.CallbackHandler with Logging {
         ctx.setCommands(commands)
         ctx.setTokens(allTokens)
 
+        val yarnJarPath = new Path(config.YARN.hdfsJarsPath)
+
+        //TODO Arun - Remove the hardcoding of the dist path
+        /*  val resources = mutable.Map[String, LocalResource]()
+          val binaryFileIter = fs.listFiles(new Path(s"${config.YARN.hdfsJarsPath}/dist"), false)
+          while (binaryFileIter.hasNext) {
+            val eachFile = binaryFileIter.next().getPath
+            resources (eachFile.getName) = setLocalResourceFromPath(fs.makeQualified(eachFile))
+          }
+          resources("log4j.properties") = setLocalResourceFromPath(fs.makeQualified(new Path(s"${config.YARN.hdfsJarsPath}/log4j.properties")))
+          resources ("amaterasu.properties") = setLocalResourceFromPath(fs.makeQualified(new Path(s"${config.YARN.hdfsJarsPath}/amaterasu.properties")))*/
+
         val resources = mutable.Map[String, LocalResource](
-          "executor.jar" -> executorJar,
-          "amaterasu.properties" -> propFile,
+          "executor.jar" -> setLocalResourceFromPath(Path.mergePaths(yarnJarPath, new Path(s"/dist/executor-${config.version}-all.jar"))),
+          "spark-runner.jar" -> setLocalResourceFromPath(Path.mergePaths(yarnJarPath, new Path(s"/dist/spark-runner-${config.version}-all.jar"))),
+          "spark-runtime.jar" -> setLocalResourceFromPath(Path.mergePaths(yarnJarPath, new Path(s"/dist/spark-runtime-${config.version}.jar"))),
+          "amaterasu.properties" -> setLocalResourceFromPath(Path.mergePaths(yarnJarPath, new Path("/amaterasu.properties"))),
+          "log4j.properties" -> setLocalResourceFromPath(Path.mergePaths(yarnJarPath, new Path("/log4j.properties"))),
           // TODO: Nadav/Eyal all of these should move to the executor resource setup
-          "miniconda.sh" -> setLocalResourceFromPath(Path.mergePaths(jarPath, new Path("/dist/Miniconda2-latest-Linux-x86_64.sh"))),
-          "codegen.py" -> setLocalResourceFromPath(Path.mergePaths(jarPath, new Path("/dist/codegen.py"))),
-          "runtime.py" -> setLocalResourceFromPath(Path.mergePaths(jarPath, new Path("/dist/runtime.py"))),
-          "spark-version-info.properties" -> setLocalResourceFromPath(Path.mergePaths(jarPath, new Path("/dist/spark-version-info.properties"))),
-          "spark_intp.py" -> setLocalResourceFromPath(Path.mergePaths(jarPath, new Path("/dist/spark_intp.py"))))
+          "miniconda.sh" -> setLocalResourceFromPath(Path.mergePaths(yarnJarPath, new Path("/dist/miniconda.sh"))),
+          "codegen.py" -> setLocalResourceFromPath(Path.mergePaths(yarnJarPath, new Path("/dist/codegen.py"))),
+          "runtime.py" -> setLocalResourceFromPath(Path.mergePaths(yarnJarPath, new Path("/dist/runtime.py"))),
+          "spark-version-info.properties" -> setLocalResourceFromPath(Path.mergePaths(yarnJarPath, new Path("/dist/spark-version-info.properties"))),
+          "spark_intp.py" -> setLocalResourceFromPath(Path.mergePaths(yarnJarPath, new Path("/dist/spark_intp.py"))))
 
         val frameworkFactory = FrameworkProvidersFactory(env, config)
         val framework = frameworkFactory.getFramework(actionData.groupId)
 
         //adding the framework and executor resources
-        setupResources(framework.getGroupIdentifier, resources, framework.getGroupIdentifier)
-        setupResources(s"${framework.getGroupIdentifier}/${actionData.typeId}", resources, s"${framework.getGroupIdentifier}-${actionData.typeId}")
+        setupResources(yarnJarPath, framework.getGroupIdentifier, resources, framework.getGroupIdentifier)
+        setupResources(yarnJarPath, s"${framework.getGroupIdentifier}/${actionData.typeId}", resources, s"${framework.getGroupIdentifier}-${actionData.typeId}")
 
         ctx.setLocalResources(resources)
 
@@ -327,9 +342,9 @@ class ApplicationMaster extends AMRMClientAsync.CallbackHandler with Logging {
     ByteBuffer.wrap(dob.getData, 0, dob.getLength)
   }
 
-  private def setupResources(frameworkPath: String, countainerResources: mutable.Map[String, LocalResource], resourcesPath: String): Unit = {
+  private def setupResources(yarnJarPath: Path, frameworkPath: String, countainerResources: mutable.Map[String, LocalResource], resourcesPath: String): Unit = {
 
-    val sourcePath = Path.mergePaths(jarPath, new Path(s"/$resourcesPath"))
+    val sourcePath = Path.mergePaths(yarnJarPath, new Path(s"/$resourcesPath"))
 
     if (fs.exists(sourcePath)) {
 
diff --git a/leader/src/main/scala/org/apache/amaterasu/leader/yarn/YarnRMCallbackHandler.scala b/leader/src/main/scala/org/apache/amaterasu/leader/yarn/YarnRMCallbackHandler.scala
index 70da38e..b178f52 100644
--- a/leader/src/main/scala/org/apache/amaterasu/leader/yarn/YarnRMCallbackHandler.scala
+++ b/leader/src/main/scala/org/apache/amaterasu/leader/yarn/YarnRMCallbackHandler.scala
@@ -32,10 +32,9 @@ import org.apache.hadoop.yarn.util.Records
 import scala.collection.JavaConversions._
 import scala.collection.JavaConverters._
 import scala.collection.concurrent
-import scala.concurrent.Future
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.{Future, _}
 import scala.util.{Failure, Success}
-import scala.concurrent._
-import ExecutionContext.Implicits.global
 
 class YarnRMCallbackHandler(nmClient: NMClientAsync,
                             jobManager: JobManager,
diff --git a/leader/src/main/scripts/ama-start-mesos.sh b/leader/src/main/scripts/ama-start-mesos.sh
index 18dbed9..e01ea42 100755
--- a/leader/src/main/scripts/ama-start-mesos.sh
+++ b/leader/src/main/scripts/ama-start-mesos.sh
@@ -126,9 +126,9 @@ if ! ls ${BASEDIR}/dist/spark*.tgz 1> /dev/null 2>&1; then
     #wget https://d3kbcqa49mib13.cloudfront.net/spark-2.2.1-bin-hadoop2.7.tgz -P ${BASEDIR}/dist
     wget http://apache.mirror.digitalpacific.com.au/spark/spark-2.2.1/spark-2.2.1-bin-hadoop2.7.tgz -P ${BASEDIR}/dist
 fi
-if [ ! -f ${BASEDIR}/dist/Miniconda2-latest-Linux-x86_64.sh ]; then
+if [ ! -f ${BASEDIR}/dist/miniconda.sh ]; then
     echo "${bold}Fetching miniconda distributable ${NC}"
-    wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -P ${BASEDIR}/dist
+    wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh -P ${BASEDIR}/dist
 fi
 cp ${BASEDIR}/amaterasu.properties ${BASEDIR}/dist
 eval $CMD | grep "===>"
diff --git a/leader/src/main/scripts/ama-start-yarn.sh b/leader/src/main/scripts/ama-start-yarn.sh
index 8aa58f1..f6af18f 100755
--- a/leader/src/main/scripts/ama-start-yarn.sh
+++ b/leader/src/main/scripts/ama-start-yarn.sh
@@ -136,9 +136,9 @@ fi
 
 echo $CMD
 
-if [ ! -f ${BASEDIR}/dist/Miniconda2-latest-Linux-x86_64.sh ]; then
+if [ ! -f ${BASEDIR}/dist/miniconda.sh ]; then
     echo "${bold}Fetching miniconda distributable ${NC}"
-    wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -P ${BASEDIR}/dist
+    wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O ${BASEDIR}/dist/miniconda.sh
 fi
 
 
diff --git a/leader/src/test/scala/org/apache/amaterasu/leader/mesos/ClusterSchedulerTests.scala b/leader/src/test/scala/org/apache/amaterasu/leader/mesos/ClusterSchedulerTests.scala
index ac5af36..af42677 100755
--- a/leader/src/test/scala/org/apache/amaterasu/leader/mesos/ClusterSchedulerTests.scala
+++ b/leader/src/test/scala/org/apache/amaterasu/leader/mesos/ClusterSchedulerTests.scala
@@ -17,8 +17,8 @@
 package org.apache.amaterasu.leader.mesos
 
 import org.apache.amaterasu.common.configuration.ClusterConfig
-import org.apache.amaterasu.leader.mesos.schedulers.ClusterScheduler
 import org.apache.amaterasu.leader.Kami
+import org.apache.amaterasu.leader.mesos.schedulers.ClusterScheduler
 import org.scalatest._
 
 class ClusterSchedulerTests extends FlatSpec with Matchers {
diff --git a/leader/src/test/scala/org/apache/amaterasu/utilities/HttpServerTests.scala b/leader/src/test/scala/org/apache/amaterasu/utilities/HttpServerTests.scala
index 25769b6..0e321f0 100644
--- a/leader/src/test/scala/org/apache/amaterasu/utilities/HttpServerTests.scala
+++ b/leader/src/test/scala/org/apache/amaterasu/utilities/HttpServerTests.scala
@@ -19,14 +19,8 @@ package org.apache.amaterasu.utilities
 
 import java.io.File
 
-import org.apache.amaterasu.leader.utilities.HttpServer
-import org.jsoup.Jsoup
-import org.jsoup.select.Elements
 import org.scalatest.{FlatSpec, Matchers}
 
-import scala.collection.JavaConverters._
-import scala.io.Source
-
 
 class HttpServerTests extends FlatSpec with Matchers {
 
diff --git a/settings.gradle b/settings.gradle
index 1056e01..c222795 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -15,8 +15,20 @@
  * limitations under the License.
  */
 include 'leader'
-include 'executor'
+project(':leader')
+
 include 'common'
+project(':common')
+
+include 'executor'
+project(':executor')
+
 include 'sdk'
 findProject(':sdk')?.name = 'amaterasu-sdk'
 
+//Spark
+include 'spark-runner'
+project(':spark-runner').projectDir=file("frameworks/spark/runner")
+include 'spark-runtime'
+project(':spark-runtime').projectDir=file("frameworks/spark/runtime")
+


Mime
View raw message