amaterasu-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ya...@apache.org
Subject [incubator-amaterasu] branch version-0.2.0-incubating-rc4 updated (11f0b30 -> abe3f29)
Date Sun, 01 Jul 2018 00:02:45 GMT
This is an automated email from the ASF dual-hosted git repository.

yaniv pushed a change to branch version-0.2.0-incubating-rc4
in repository https://gitbox.apache.org/repos/asf/incubator-amaterasu.git.


    from 11f0b30  adding tags back totravis releases
     add 79be8fb  AMATERASU-24 Refactor Spark out of Amaterasu executor to it's own project
     add cc9fa8a  AMATERASU-24 - Refactor Spark out of Amaterasu executor to it's own project
     add 1273740  AMATERASU-24 - Refactor Spark out of Amaterasu executor to it's own project
     new abe3f29  Merge pull request #27 from arunma/RC4-RefactorFramework

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 build.gradle                                       |   4 -
 .../common/execution/actions/Notifier.scala        |   4 +-
 executor/build.gradle                              |  14 +---
 .../mesos/executors/MesosActionsExecutor.scala     |   4 +-
 .../executor/yarn/executors/ActionsExecutor.scala  |   8 +-
 {executor => frameworks/spark/runner}/build.gradle |  14 ++--
 .../spark/runner/pyspark}/PySparkEntryPoint.java   |  11 +--
 .../spark/runner/src/main}/resources/codegen.py    |   0
 .../spark/runner/src/main}/resources/runtime.py    |   0
 .../main/resources/spark-version-info.properties   |  11 +++
 .../spark/runner}/src/main/resources/spark_intp.py |   0
 .../spark/runner}/SparkRunnersProvider.scala       |  10 +--
 .../runner/pyspark}/PySparkExecutionQueue.scala    |   2 +-
 .../spark/runner/pyspark}/PySparkResultQueue.scala |   4 +-
 .../spark/runner/pyspark}/PySparkRunner.scala      |  10 +--
 .../spark/runner/pyspark}/ResultQueue.scala        |   2 +-
 .../spark/runner/repl}/AmaSparkILoop.scala         |   2 +-
 .../spark/runner/repl}/SparkRunnerHelper.scala     |  14 ++--
 .../spark/runner/repl}/SparkScalaRunner.scala      |   6 +-
 .../spark/runner/sparkr}/SparkRRunner.scala        |  28 +++----
 .../spark/runner/sparksql}/SparkSqlRunner.scala    |   9 ++-
 .../resources/SparkSql/csv/SparkSqlTestCsv.csv     |   0
 .../resources/SparkSql/json/SparkSqlTestData.json  |   0
 ...548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc | Bin
 ...548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc | Bin
 .../src/test/resources/SparkSql/parquet/_SUCCESS   |   0
 .../resources/SparkSql/parquet/_common_metadata    | Bin
 .../src/test/resources/SparkSql/parquet/_metadata  | Bin
 ...c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet | Bin
 ...c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet | Bin
 .../src/test/resources/amaterasu.properties        |   0
 .../spark/runner/src/test}/resources/codegen.py    |   0
 .../runner}/src/test/resources/py4j-0.10.4-src.zip | Bin
 .../spark/runner}/src/test/resources/py4j.tar.gz   | Bin
 .../src/test/resources/pyspark-with-amacontext.py  |   0
 .../runner}/src/test/resources/pyspark.tar.gz      | Bin
 .../spark/runner}/src/test/resources/pyspark.zip   | Bin
 .../spark/runner/src/test}/resources/runtime.py    |   0
 .../runner}/src/test/resources/simple-pyspark.py   |   0
 .../src/test/resources/simple-python-err.py        |   0
 .../runner}/src/test/resources/simple-python.py    |   0
 .../runner}/src/test/resources/simple-spark.scala  |   5 +-
 .../test}/resources/spark-version-info.properties  |   0
 .../spark/runner}/src/test/resources/spark_intp.py |   0
 .../spark/runner}/src/test/resources/step-2.scala  |   3 +-
 ...c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet | Bin
 ...c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet | Bin
 .../framework}/spark/SparkTestsSuite.scala         |  10 +--
 .../spark/runner}/RunnersLoadingTests.scala        |   2 +-
 .../spark/runner/pyspark}/PySparkRunnerTests.scala |   3 +-
 .../spark/runner/repl}/SparkScalaRunnerTests.scala |   9 +--
 .../runner/sparksql}/SparkSqlRunnerTests.scala     |   9 +--
 .../apache/amaterasu/utilities/TestNotifier.scala  |   0
 frameworks/spark/runtime/build.gradle              |  89 +++++++++++++++++++++
 .../framework/spark}/runtime/AmaContext.scala      |   8 +-
 gradle/wrapper/gradle-wrapper.properties           |   4 +-
 gradlew                                            |  16 ----
 gradlew.bat                                        |  18 -----
 .../apache/amaterasu/leader/yarn/ArgsParser.java   |   6 +-
 .../org/apache/amaterasu/leader/yarn/Client.java   |  23 +++++-
 .../apache/amaterasu/leader/dsl/JobParser.scala    |   4 +-
 .../frameworks/spark/SparkSetupProvider.scala      |   1 -
 .../mesos/schedulers/AmaterasuScheduler.scala      |   1 -
 .../leader/mesos/schedulers/JobScheduler.scala     |   5 +-
 .../leader/utilities/ActiveReportListener.scala    |   2 -
 .../amaterasu/leader/utilities/HttpServer.scala    |   7 +-
 .../amaterasu/leader/yarn/ApplicationMaster.scala  |  39 ++++++---
 .../leader/yarn/YarnRMCallbackHandler.scala        |   5 +-
 leader/src/main/scripts/ama-start-mesos.sh         |   4 +-
 leader/src/main/scripts/ama-start-yarn.sh          |   4 +-
 .../leader/mesos/ClusterSchedulerTests.scala       |   2 +-
 .../amaterasu/utilities/HttpServerTests.scala      |   6 --
 settings.gradle                                    |  14 +++-
 73 files changed, 256 insertions(+), 200 deletions(-)
 copy {executor => frameworks/spark/runner}/build.gradle (93%)
 rename {executor/src/main/java/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark
=> frameworks/spark/runner/src/main/java/org/apache/amaterasu/framework/spark/runner/pyspark}/PySparkEntryPoint.java
(92%)
 rename {executor/src/test => frameworks/spark/runner/src/main}/resources/codegen.py (100%)
 rename {executor/src/test => frameworks/spark/runner/src/main}/resources/runtime.py (100%)
 create mode 100644 frameworks/spark/runner/src/main/resources/spark-version-info.properties
 rename {executor => frameworks/spark/runner}/src/main/resources/spark_intp.py (100%)
 rename {executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark
=> frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner}/SparkRunnersProvider.scala
(93%)
 rename {executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark
=> frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark}/PySparkExecutionQueue.scala
(94%)
 rename {executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark
=> frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark}/PySparkResultQueue.scala
(85%)
 rename {executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark
=> frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark}/PySparkRunner.scala
(95%)
 mode change 100755 => 100644
 rename {executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/PySpark
=> frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/pyspark}/ResultQueue.scala
(94%)
 rename {executor/src/main/scala/org/apache/spark/repl/amaterasu => frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl}/AmaSparkILoop.scala
(95%)
 rename {executor/src/main/scala/org/apache/spark/repl/amaterasu/runners/spark => frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl}/SparkRunnerHelper.scala
(94%)
 rename {executor/src/main/scala/org/apache/spark/repl/amaterasu/runners/spark => frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/repl}/SparkScalaRunner.scala
(97%)
 rename {executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark
=> frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/sparkr}/SparkRRunner.scala
(69%)
 rename {executor/src/main/scala/org/apache/amaterasu/executor/execution/actions/runners/spark/SparkSql
=> frameworks/spark/runner/src/main/scala/org/apache/amaterasu/framework/spark/runner/sparksql}/SparkSqlRunner.scala
(96%)
 rename {executor => frameworks/spark/runner}/src/test/resources/SparkSql/csv/SparkSqlTestCsv.csv
(100%)
 rename {executor => frameworks/spark/runner}/src/test/resources/SparkSql/json/SparkSqlTestData.json
(100%)
 rename {executor => frameworks/spark/runner}/src/test/resources/SparkSql/parquet/.part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc
(100%)
 rename {executor => frameworks/spark/runner}/src/test/resources/SparkSql/parquet/.part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet.crc
(100%)
 rename {executor => frameworks/spark/runner}/src/test/resources/SparkSql/parquet/_SUCCESS
(100%)
 rename {executor => frameworks/spark/runner}/src/test/resources/SparkSql/parquet/_common_metadata
(100%)
 rename {executor => frameworks/spark/runner}/src/test/resources/SparkSql/parquet/_metadata
(100%)
 rename {executor/src/test/resources/tmp/job/start/x => frameworks/spark/runner/src/test/resources/SparkSql/parquet}/part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet
(100%)
 rename {executor/src/test/resources/tmp/job/start/x => frameworks/spark/runner/src/test/resources/SparkSql/parquet}/part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet
(100%)
 rename {executor => frameworks/spark/runner}/src/test/resources/amaterasu.properties (100%)
 rename {executor/src/main => frameworks/spark/runner/src/test}/resources/codegen.py (100%)
 rename {executor => frameworks/spark/runner}/src/test/resources/py4j-0.10.4-src.zip (100%)
 rename {executor => frameworks/spark/runner}/src/test/resources/py4j.tar.gz (100%)
 rename {executor => frameworks/spark/runner}/src/test/resources/pyspark-with-amacontext.py
(100%)
 rename {executor => frameworks/spark/runner}/src/test/resources/pyspark.tar.gz (100%)
 rename {executor => frameworks/spark/runner}/src/test/resources/pyspark.zip (100%)
 rename {executor/src/main => frameworks/spark/runner/src/test}/resources/runtime.py (100%)
 rename {executor => frameworks/spark/runner}/src/test/resources/simple-pyspark.py (100%)
 rename {executor => frameworks/spark/runner}/src/test/resources/simple-python-err.py (100%)
 rename {executor => frameworks/spark/runner}/src/test/resources/simple-python.py (100%)
 rename {executor => frameworks/spark/runner}/src/test/resources/simple-spark.scala (83%)
 rename {executor/src/main => frameworks/spark/runner/src/test}/resources/spark-version-info.properties
(100%)
 rename {executor => frameworks/spark/runner}/src/test/resources/spark_intp.py (100%)
 rename {executor => frameworks/spark/runner}/src/test/resources/step-2.scala (94%)
 rename {executor/src/test/resources/SparkSql/parquet => frameworks/spark/runner/src/test/resources/tmp/job/start/x}/part-r-00000-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet
(100%)
 rename {executor/src/test/resources/SparkSql/parquet => frameworks/spark/runner/src/test/resources/tmp/job/start/x}/part-r-00001-c370548d-1371-4c08-81d6-f2ba72defa12.gz.parquet
(100%)
 rename {executor/src/test/scala/org/apache/amaterasu => frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework}/spark/SparkTestsSuite.scala
(90%)
 rename {executor/src/test/scala/org/apache/amaterasu/RunnersTests => frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner}/RunnersLoadingTests.scala
(96%)
 rename {executor/src/test/scala/org/apache/amaterasu/spark => frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/pyspark}/PySparkRunnerTests.scala
(96%)
 rename {executor/src/test/scala/org/apache/amaterasu/spark => frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/repl}/SparkScalaRunnerTests.scala
(92%)
 rename {executor/src/test/scala/org/apache/amaterasu/spark => frameworks/spark/runner/src/test/scala/org/apache/amaterasu/framework/spark/runner/sparksql}/SparkSqlRunnerTests.scala
(96%)
 rename {executor => frameworks/spark/runner}/src/test/scala/org/apache/amaterasu/utilities/TestNotifier.scala
(100%)
 create mode 100644 frameworks/spark/runtime/build.gradle
 rename {executor/src/main/scala/org/apache/amaterasu/executor => frameworks/spark/runtime/src/main/scala/org/apache/amaterasu/framework/spark}/runtime/AmaContext.scala
(93%)
 mode change 100755 => 100644


Mime
View raw message