spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ma...@apache.org
Subject [22/50] git commit: Merge branch 'master' of git://github.com/mesos/spark into scala-2.10
Date Fri, 04 Oct 2013 17:55:09 GMT
Merge branch 'master' of git://github.com/mesos/spark into scala-2.10

Conflicts:
	core/src/main/scala/org/apache/spark/SparkContext.scala
	project/SparkBuild.scala


Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/383e151f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/383e151f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/383e151f

Branch: refs/heads/scala-2.10
Commit: 383e151fd7138cc6a143b3a38037cc3038c2a8b9
Parents: 20c65bc c4c1db2
Author: Prashant Sharma <scrapcodes@gmail.com>
Authored: Sun Sep 15 10:55:12 2013 +0530
Committer: Prashant Sharma <scrapcodes@gmail.com>
Committed: Sun Sep 15 10:55:12 2013 +0530

----------------------------------------------------------------------
 assembly/pom.xml                                |   6 +
 conf/fairscheduler.xml.template                 |  18 +-
 conf/metrics.properties.template                |  50 ++-
 core/pom.xml                                    |   9 +
 .../scala/org/apache/spark/CacheManager.scala   |   8 +-
 .../scala/org/apache/spark/SparkContext.scala   |  22 +-
 .../scala/org/apache/spark/TaskContext.scala    |   1 +
 .../org/apache/spark/deploy/JsonProtocol.scala  |   4 +-
 .../org/apache/spark/executor/Executor.scala    |   2 +-
 .../apache/spark/executor/ExecutorSource.scala  |   5 +-
 .../apache/spark/metrics/MetricsConfig.scala    |   7 +-
 .../apache/spark/metrics/sink/GangliaSink.scala |  82 +++++
 .../spark/metrics/sink/MetricsServlet.scala     |  11 +-
 .../spark/network/netty/ShuffleSender.scala     |   3 +-
 .../main/scala/org/apache/spark/rdd/RDD.scala   |  17 +
 .../apache/spark/scheduler/DAGScheduler.scala   |  12 +-
 .../spark/scheduler/DAGSchedulerSource.scala    |   6 +-
 .../org/apache/spark/scheduler/ResultTask.scala |   6 +-
 .../apache/spark/scheduler/ShuffleMapTask.scala |   2 +-
 .../scheduler/cluster/ClusterScheduler.scala    |   2 +-
 .../cluster/ClusterTaskSetManager.scala         |   4 +-
 .../scheduler/cluster/SchedulableBuilder.scala  |   6 +-
 .../cluster/SparkDeploySchedulerBackend.scala   |  10 +-
 .../spark/scheduler/local/LocalScheduler.scala  |   2 +-
 .../spark/storage/BlockManagerSource.scala      |   6 +-
 .../org/apache/spark/storage/DiskStore.scala    |   2 +-
 .../org/apache/spark/storage/StorageLevel.scala |   6 +-
 .../scala/org/apache/spark/ui/SparkUI.scala     |   2 +-
 .../apache/spark/ui/UIWorkloadGenerator.scala   |   4 +-
 .../spark/ui/jobs/JobProgressListener.scala     |   2 +-
 .../scala/org/apache/spark/util/Utils.scala     |  24 +-
 .../org/apache/spark/CacheManagerSuite.scala    |  91 ++++++
 .../scala/org/apache/spark/JavaAPISuite.java    |   2 +-
 .../spark/metrics/MetricsConfigSuite.scala      |  19 +-
 .../scala/org/apache/spark/rdd/RDDSuite.scala   |  14 +-
 .../cluster/ClusterSchedulerSuite.scala         |   8 +-
 .../scheduler/local/LocalSchedulerSuite.scala   |   8 +-
 .../scala/org/apache/spark/ui/UISuite.scala     |   2 +-
 docs/_layouts/global.html                       |   9 +-
 docs/cluster-overview.md                        | 117 +++++++
 docs/configuration.md                           |  34 +-
 docs/contributing-to-spark.md                   |  24 +-
 docs/ec2-scripts.md                             |   8 +-
 docs/hadoop-third-party-distributions.md        | 118 +++++++
 docs/hardware-provisioning.md                   |   4 +-
 docs/img/cluster-overview.png                   | Bin 0 -> 28011 bytes
 docs/img/cluster-overview.pptx                  | Bin 0 -> 51771 bytes
 docs/img/spark-logo-hd.png                      | Bin 0 -> 13512 bytes
 docs/index.md                                   |  26 +-
 docs/job-scheduling.md                          | 168 ++++++++++
 docs/mllib-guide.md                             | 192 ++++++++++-
 docs/monitoring.md                              |  70 ++++
 docs/python-programming-guide.md                |  14 +-
 docs/quick-start.md                             |  70 ++--
 docs/running-on-mesos.md                        |  48 ++-
 docs/running-on-yarn.md                         |  31 +-
 docs/scala-programming-guide.md                 |   4 +-
 docs/spark-standalone.md                        |  46 ++-
 .../root/spark-ec2/ec2-variables.sh             |  17 +
 ec2/spark_ec2.py                                |   3 +-
 examples/pom.xml                                |  14 -
 make-distribution.sh                            |   2 +-
 mllib/data/sample_svm_data.txt                  | 322 +++++++++++++++++++
 .../apache/spark/mllib/classification/SVM.scala |  13 +-
 .../mllib/regression/LinearRegression.scala     |   4 +-
 pom.xml                                         | 234 ++++++--------
 project/SparkBuild.scala                        |  18 +-
 project/project/SparkPluginBuild.scala          |  19 +-
 python/pyspark/__init__.py                      |   5 +-
 python/pyspark/context.py                       |  11 +
 python/pyspark/rdd.py                           |  19 ++
 python/pyspark/shell.py                         |   1 +
 python/pyspark/storagelevel.py                  |  43 +++
 repl-bin/pom.xml                                |  10 -
 repl/pom.xml                                    |  12 -
 run-example                                     |   8 +-
 yarn/pom.xml                                    |  66 ++--
 77 files changed, 1824 insertions(+), 465 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/383e151f/core/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/383e151f/core/src/main/scala/org/apache/spark/SparkContext.scala
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/383e151f/core/src/main/scala/org/apache/spark/rdd/RDD.scala
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/383e151f/core/src/main/scala/org/apache/spark/scheduler/DAGScheduler.scala
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/383e151f/core/src/main/scala/org/apache/spark/util/Utils.scala
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/383e151f/examples/pom.xml
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/383e151f/pom.xml
----------------------------------------------------------------------
diff --cc pom.xml
index 6a2dccc,2da016a..011f6e9
--- a/pom.xml
+++ b/pom.xml
@@@ -295,9 -289,14 +297,14 @@@
          <version>3.0.0</version>
        </dependency>
        <dependency>
+         <groupId>com.codahale.metrics</groupId>
+         <artifactId>metrics-ganglia</artifactId>
+         <version>3.0.0</version>
+       </dependency>
+       <dependency>
          <groupId>org.scala-lang</groupId>
          <artifactId>scala-compiler</artifactId>
 -        <version>${scala.version}</version>
 +        <version>2.10.0</version>
        </dependency>
        <dependency>
          <groupId>org.scala-lang</groupId>

http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/383e151f/project/SparkBuild.scala
----------------------------------------------------------------------
diff --cc project/SparkBuild.scala
index e781519,77e211c..5e7ed81
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@@ -74,10 -78,12 +78,11 @@@ object SparkBuild extends Build 
      core, repl, examples, bagel, streaming, mllib, tools, assemblyProj) ++ maybeYarnRef
  
    def sharedSettings = Defaults.defaultSettings ++ Seq(
 -    organization := "org.apache.spark",
 -    version := "0.8.0-SNAPSHOT",
 -    scalaVersion := "2.9.3",
 -    scalacOptions := Seq("-unchecked", "-optimize", "-deprecation", 
 -      "-target:" + SCALAC_JVM_VERSION),
 +    organization       := "org.apache.spark",
 +    version            := "0.8.0-SNAPSHOT",
 +    scalaVersion       := "2.10.2",
-     scalacOptions      := Seq("-unchecked", "-optimize", "-deprecation"),
++    scalacOptions      := Seq("-unchecked", "-optimize", "-deprecation", "-target:" + SCALAC_JVM_VERSION),
+     javacOptions := Seq("-target", JAVAC_JVM_VERSION, "-source", JAVAC_JVM_VERSION),
      unmanagedJars in Compile <<= baseDirectory map { base => (base / "lib" ** "*.jar").classpath
},
      retrieveManaged := true,
      retrievePattern := "[type]s/[artifact](-[revision])(-[classifier]).[ext]",
@@@ -184,35 -188,36 +189,37 @@@
      ),
  
      libraryDependencies ++= Seq(
 -      "com.google.guava" % "guava" % "14.0.1",
 -      "com.google.code.findbugs" % "jsr305" % "1.3.9",
 -      "log4j" % "log4j" % "1.2.17",
 -      "org.slf4j" % "slf4j-api" % slf4jVersion,
 -      "org.slf4j" % "slf4j-log4j12" % slf4jVersion,
 -      "commons-daemon" % "commons-daemon" % "1.0.10",  // workaround for bug HADOOP-9407
 -      "com.ning" % "compress-lzf" % "0.8.4",
 -      "org.xerial.snappy" % "snappy-java" % "1.0.5",
 -      "org.ow2.asm" % "asm" % "4.0",
 -      "com.google.protobuf" % "protobuf-java" % "2.4.1",
 -      "com.typesafe.akka" % "akka-actor" % "2.0.5" excludeAll(excludeNetty),
 -      "com.typesafe.akka" % "akka-remote" % "2.0.5" excludeAll(excludeNetty),
 -      "com.typesafe.akka" % "akka-slf4j" % "2.0.5" excludeAll(excludeNetty),
 -      "it.unimi.dsi" % "fastutil" % "6.4.4",
 -      "colt" % "colt" % "1.2.0",
 -      "net.liftweb" % "lift-json_2.9.2" % "2.5",
 -      "org.apache.mesos" % "mesos" % "0.12.1",
 -      "io.netty" % "netty-all" % "4.0.0.Beta2",
 -      "org.apache.derby" % "derby" % "10.4.2.0" % "test",
 -      "org.apache.hadoop" % "hadoop-client" % hadoopVersion excludeAll(excludeJackson, excludeNetty,
excludeAsm),
 -      "net.java.dev.jets3t" % "jets3t" % "0.7.1",
 -      "org.apache.avro" % "avro" % "1.7.4",
 -      "org.apache.avro" % "avro-ipc" % "1.7.4" excludeAll(excludeNetty),
 -      "com.codahale.metrics" % "metrics-core" % "3.0.0",
 -      "com.codahale.metrics" % "metrics-jvm" % "3.0.0",
 -      "com.codahale.metrics" % "metrics-json" % "3.0.0",
 -      "com.codahale.metrics" % "metrics-ganglia" % "3.0.0",
 -      "com.twitter" % "chill_2.9.3" % "0.3.1",
 -      "com.twitter" % "chill-java" % "0.3.1"
 -    )
 +        "com.google.guava"         % "guava"            % "14.0.1",
 +        "com.google.code.findbugs" % "jsr305"           % "1.3.9",
-         "log4j"                    % "log4j"            % "1.2.16",
++        "log4j"                    % "log4j"            % "1.2.17",
 +        "org.slf4j"                % "slf4j-api"        % slf4jVersion,
 +        "org.slf4j"                % "slf4j-log4j12"    % slf4jVersion,
 +        "com.ning"                 % "compress-lzf"     % "0.8.4",
 +        "org.xerial.snappy"        % "snappy-java"      % "1.0.5",
 +        "commons-daemon"           % "commons-daemon"   % "1.0.10",
 +        "org.ow2.asm"              % "asm"              % "4.0",
 +        "com.google.protobuf"      % "protobuf-java"    % "2.4.1",
 +        "de.javakaffee"            % "kryo-serializers" % "0.22",
 +        "com.typesafe.akka"       %% "akka-remote"      % "2.1.4"        excludeAll(excludeNetty),
 +        "com.typesafe.akka"       %% "akka-slf4j"       % "2.1.4"        excludeAll(excludeNetty),
 +        "net.liftweb"             %% "lift-json"        % "2.5.1",
 +        "it.unimi.dsi"             % "fastutil"         % "6.4.4",
 +        "colt"                     % "colt"             % "1.2.0",
-         "org.apache.mesos"         % "mesos"            % "0.9.0-incubating",
++        "org.apache.mesos"         % "mesos"            % "0.12.1",
++        "net.java.dev.jets3t"      % "jets3t"           % "0.7.1",
 +        "org.apache.derby"         % "derby"            % "10.4.2.0"                   
 % "test",
 +        "org.apache.hadoop"        % "hadoop-client"    % hadoopVersion excludeAll(excludeJackson,
excludeNetty, excludeAsm),
 +        "org.apache.avro"          % "avro"             % "1.7.4",
 +        "org.apache.avro"          % "avro-ipc"         % "1.7.4" excludeAll(excludeNetty),
 +        "com.codahale.metrics"     % "metrics-core"     % "3.0.0",
 +        "com.codahale.metrics"     % "metrics-jvm"      % "3.0.0",
 +        "com.codahale.metrics"     % "metrics-json"     % "3.0.0",
++        "com.codahale.metrics"     % "metrics-ganglia"  % "3.0.0",
 +        "com.twitter"             %% "chill"            % "0.3.1",
 +        "com.twitter"              % "chill-java"       % "0.3.1",
 +        "org.scala-lang"           % "jline"            % "2.10.2",
 +        "org.scala-lang"           % "scala-reflect"    % "2.10.2"
 +      ) 
    )
  
    def rootSettings = sharedSettings ++ Seq(

http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/383e151f/python/pyspark/rdd.py
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/383e151f/repl/pom.xml
----------------------------------------------------------------------


Mime
View raw message