bigtop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From c..@apache.org
Subject git commit: BIGTOP-1056. some changes are introduced into Spark build: needs to be updated here
Date Thu, 05 Sep 2013 03:58:20 GMT
Updated Branches:
  refs/heads/master 2aa4877c1 -> c909eb1fc


BIGTOP-1056. some changes are introduced into Spark build: needs to be updated here


Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/c909eb1f
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/c909eb1f
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/c909eb1f

Branch: refs/heads/master
Commit: c909eb1fc92e9d3fd2f55325bfc7ebb1be23a164
Parents: 2aa4877
Author: Konstantin Boudnik <cos@apache.org>
Authored: Wed Sep 4 20:56:06 2013 -0700
Committer: Konstantin Boudnik <cos@apache.org>
Committed: Wed Sep 4 20:56:06 2013 -0700

----------------------------------------------------------------------
 bigtop-packages/src/common/spark/do-component-build |  7 ++-----
 bigtop-packages/src/common/spark/install_spark.sh   | 13 ++++++++++---
 bigtop-packages/src/common/spark/spark-executor     |  7 -------
 bigtop-packages/src/common/spark/spark-master.svc   |  4 ++--
 bigtop-packages/src/common/spark/spark-worker.svc   |  4 ++--
 bigtop-packages/src/rpm/spark/SPECS/spark.spec      |  2 +-
 bigtop.mk                                           |  2 +-
 7 files changed, 18 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/bigtop/blob/c909eb1f/bigtop-packages/src/common/spark/do-component-build
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/spark/do-component-build b/bigtop-packages/src/common/spark/do-component-build
index f36ab68..0593333 100644
--- a/bigtop-packages/src/common/spark/do-component-build
+++ b/bigtop-packages/src/common/spark/do-component-build
@@ -26,13 +26,10 @@ fi
 BUILD_OPTS="-Divy.home=${HOME}/.ivy2 -Dsbt.ivy.home=${HOME}/.ivy2 -Duser.home=${HOME} \
             -Drepo.maven.org=$IVY_MIRROR_PROP \
             -Dreactor.repo=file://${HOME}/.m2/repository \
+            -Dhadoop.version=$HADOOP_VERSION
             -DskipTests"
 ## this might be an issue at times
 #        http://maven.40175.n5.nabble.com/Not-finding-artifact-in-local-repo-td3727753.html
 export MAVEN_OPTS="-XX:PermSize=1024m -XX:MaxPermSize=1024m"
 
-# FIXME: this is a really unfortunate hack: make sure we are building with
-# correct version
-sed -ie "s#<yarn.version>2.0.2-alpha#<yarn.version>$HADOOP_VERSION#" pom.xml
-
-mvn -Phadoop2-yarn,assembly -DnoExpensive $BUILD_OPTS package
+mvn -Pbigtop-dist $BUILD_OPTS package

http://git-wip-us.apache.org/repos/asf/bigtop/blob/c909eb1f/bigtop-packages/src/common/spark/install_spark.sh
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/spark/install_spark.sh b/bigtop-packages/src/common/spark/install_spark.sh
index 870ec09..6918757 100644
--- a/bigtop-packages/src/common/spark/install_spark.sh
+++ b/bigtop-packages/src/common/spark/install_spark.sh
@@ -101,6 +101,9 @@ if [ -z "${SCALA_HOME}" ]; then
     echo Missing env. var SCALA_HOME
     usage
 fi
+if [ -f "$SOURCE_DIR/bigtop.bom" ]; then
+  . $SOURCE_DIR/bigtop.bom
+fi
 
 MAN_DIR=${MAN_DIR:-/usr/share/man/man1}
 DOC_DIR=${DOC_DIR:-/usr/share/doc/spark}
@@ -123,12 +126,16 @@ for comp in core repl bagel mllib streaming; do
   install -d -m 0755 $PREFIX/$LIB_DIR/$comp/lib
   tar --wildcards -C $PREFIX/$LIB_DIR/$comp/lib -zxf ${BUILD_DIR}/assembly/target/spark-assembly-*-dist.tar.gz
spark-$comp\*
 done
+## FIXME: Spark maven assembly needs to include examples into it.
+install -d -m 0755 $PREFIX/$LIB_DIR/examples/lib
+cp ${BUILD_DIR}/examples/target/spark-examples-${SPARK_VERSION}.jar $PREFIX/$LIB_DIR/examples/lib
 
 # FIXME: executor scripts need to reside in bin
-cp -a ${SOURCE_DIR}/run $PREFIX/$LIB_DIR
-cp -a ${SOURCE_DIR}/spark-executor $PREFIX/$LIB_DIR
+cp -a $BUILD_DIR/spark-class $PREFIX/$LIB_DIR
+cp -a $BUILD_DIR/spark-executor $PREFIX/$LIB_DIR
 cp -a ${SOURCE_DIR}/compute-classpath.sh $PREFIX/$SPARK_BIN_DIR
 cp -a ${BUILD_DIR}/spark-shell $PREFIX/$LIB_DIR
+touch $PREFIX/$LIB_DIR/RELEASE
 
 # Copy in the configuration files
 install -d -m 0755 $PREFIX/$CONF_DIR
@@ -140,7 +147,7 @@ ln -s /etc/spark/conf $PREFIX/$LIB_DIR/conf
 tar --wildcards --transform 's,ui-resources/spark,spark,' -C $PREFIX/$LIB_DIR -zxf ${BUILD_DIR}/assembly/target/spark-assembly-*-dist.tar.gz
ui-resources/\*
 
 # set correct permissions for exec. files
-for execfile in run spark-shell spark-executor ; do
+for execfile in spark-class spark-shell spark-executor ; do
   chmod 755 $PREFIX/$LIB_DIR/$execfile
 done
 chmod 755 $PREFIX/$SPARK_BIN_DIR/compute-classpath.sh

http://git-wip-us.apache.org/repos/asf/bigtop/blob/c909eb1f/bigtop-packages/src/common/spark/spark-executor
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/spark/spark-executor b/bigtop-packages/src/common/spark/spark-executor
deleted file mode 100644
index 55a7113..0000000
--- a/bigtop-packages/src/common/spark/spark-executor
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-# This is a modification of the standard Spark executor script to run
-# with Standalone executor backend
-# TODO: we are likely need to add YARN executor later
-FWDIR="`dirname $0`"
-echo "Running spark-executor with framework dir = $FWDIR"
-exec $FWDIR/run spark.executor.StandaloneExecutorBackend

http://git-wip-us.apache.org/repos/asf/bigtop/blob/c909eb1f/bigtop-packages/src/common/spark/spark-master.svc
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/spark/spark-master.svc b/bigtop-packages/src/common/spark/spark-master.svc
index 0b6d5b3..42e89df 100644
--- a/bigtop-packages/src/common/spark/spark-master.svc
+++ b/bigtop-packages/src/common/spark/spark-master.svc
@@ -16,7 +16,7 @@
 TYPE="master"
 DAEMON="spark-${TYPE}"
 DESC="Spark ${TYPE}"
-EXEC_PATH="/usr/lib/spark/run"
+EXEC_PATH="/usr/lib/spark/spark-class"
 SVC_USER="spark"
 WORKING_DIR="/var/lib/spark"
 DAEMON_FLAGS=""
@@ -40,7 +40,7 @@ start() {
     LOG_FILE=/var/log/spark/${DAEMON}.out
 
     su -s /bin/bash $SVC_USER -c "nohup nice -n 0 \
-        ${EXEC_PATH} spark.deploy.master.Master $DAEMON_FLAGS \
+        ${EXEC_PATH} org.apache.spark.deploy.master.Master $DAEMON_FLAGS \
         > $LOG_FILE 2>&1 & "'echo $!' > "$PIDFILE"
 
     sleep 3

http://git-wip-us.apache.org/repos/asf/bigtop/blob/c909eb1f/bigtop-packages/src/common/spark/spark-worker.svc
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/spark/spark-worker.svc b/bigtop-packages/src/common/spark/spark-worker.svc
index 1776f33..50c5487 100644
--- a/bigtop-packages/src/common/spark/spark-worker.svc
+++ b/bigtop-packages/src/common/spark/spark-worker.svc
@@ -16,7 +16,7 @@
 TYPE="worker"
 DAEMON="spark-${TYPE}"
 DESC="Spark ${TYPE}"
-EXEC_PATH="/usr/lib/spark/run"
+EXEC_PATH="/usr/lib/spark/spark-class"
 SVC_USER="spark"
 WORKING_DIR="/var/lib/spark"
 DAEMON_FLAGS=""
@@ -44,7 +44,7 @@ start() {
     fi
 
     su -s /bin/bash $SVC_USER -c "nohup nice -n 0 \
-        ${EXEC_PATH} spark.deploy.worker.Worker spark://$STANDALONE_SPARK_MASTER_HOST:$SPARK_MASTER_PORT
$DAEMON_FLAGS \
+        ${EXEC_PATH} org.apache.spark.deploy.worker.Worker spark://$STANDALONE_SPARK_MASTER_HOST:$SPARK_MASTER_PORT
$DAEMON_FLAGS \
         > $LOG_FILE 2>&1 & "'echo $!' > "$PIDFILE"
 
     sleep 3

http://git-wip-us.apache.org/repos/asf/bigtop/blob/c909eb1f/bigtop-packages/src/rpm/spark/SPECS/spark.spec
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/spark/SPECS/spark.spec b/bigtop-packages/src/rpm/spark/SPECS/spark.spec
index b0889e4..13301e6 100644
--- a/bigtop-packages/src/rpm/spark/SPECS/spark.spec
+++ b/bigtop-packages/src/rpm/spark/SPECS/spark.spec
@@ -78,7 +78,7 @@ Spark runs on top of the Apache Mesos cluster manager.
     
 %prep
 #%setup -n %{name}-%{spark_base_version}
-%setup -n spark-master
+%setup -n spark-branch-0.8
 
 %build
 bash $RPM_SOURCE_DIR/do-component-build

http://git-wip-us.apache.org/repos/asf/bigtop/blob/c909eb1f/bigtop.mk
----------------------------------------------------------------------
diff --git a/bigtop.mk b/bigtop.mk
index 3fd3c12..f75c412 100644
--- a/bigtop.mk
+++ b/bigtop.mk
@@ -245,7 +245,7 @@ SPARK_BASE_VERSION=0.8.0-SNAPSHOT
 SPARK_PKG_VERSION=0.8.0
 SPARK_RELEASE_VERSION=1
 SPARK_TARBALL_DST=spark-${SPARK_BASE_VERSION}.tar.gz
-SPARK_TARBALL_SRC=master.tar.gz
+SPARK_TARBALL_SRC=branch-0.8.tar.gz
 SPARK_SITE=https://github.com/mesos/spark/archive
 SPARK_ARCHIVE=${SPARK_SITE}
 $(eval $(call PACKAGE,spark,SPARK))


Mime
View raw message