spark-reviews mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From andrewor14 <...@git.apache.org>
Subject [GitHub] spark pull request: [SPARK-4924] Add a library for launching Spark...
Date Tue, 24 Feb 2015 19:47:09 GMT
Github user andrewor14 commented on a diff in the pull request:

    https://github.com/apache/spark/pull/3916#discussion_r25285364
  
    --- Diff: bin/spark-class ---
    @@ -110,83 +39,48 @@ else
         exit 1
       fi
     fi
    -JAVA_VERSION=$("$RUNNER" -version 2>&1 | grep 'version' | sed 's/.* version "\(.*\)\.\(.*\)\..*"/\1\2/;
1q')
    -
    -# Set JAVA_OPTS to be able to load native libraries and to set heap size
    -if [ "$JAVA_VERSION" -ge 18 ]; then
    -  JAVA_OPTS="$OUR_JAVA_OPTS"
    -else
    -  JAVA_OPTS="-XX:MaxPermSize=128m $OUR_JAVA_OPTS"
    -fi
    -JAVA_OPTS="$JAVA_OPTS -Xms$OUR_JAVA_MEM -Xmx$OUR_JAVA_MEM"
    -
    -# Load extra JAVA_OPTS from conf/java-opts, if it exists
    -if [ -e "$SPARK_CONF_DIR/java-opts" ] ; then
    -  JAVA_OPTS="$JAVA_OPTS `cat "$SPARK_CONF_DIR"/java-opts`"
    -fi
    -
    -# Attention: when changing the way the JAVA_OPTS are assembled, the change must be reflected
in CommandUtils.scala!
    -
    -TOOLS_DIR="$FWDIR"/tools
    -SPARK_TOOLS_JAR=""
    -if [ -e "$TOOLS_DIR"/target/scala-$SPARK_SCALA_VERSION/spark-tools*[0-9Tg].jar ]; then
    -  # Use the JAR from the SBT build
    -  export SPARK_TOOLS_JAR="`ls "$TOOLS_DIR"/target/scala-$SPARK_SCALA_VERSION/spark-tools*[0-9Tg].jar`"
    -fi
    -if [ -e "$TOOLS_DIR"/target/spark-tools*[0-9Tg].jar ]; then
    -  # Use the JAR from the Maven build
    -  # TODO: this also needs to become an assembly!
    -  export SPARK_TOOLS_JAR="`ls "$TOOLS_DIR"/target/spark-tools*[0-9Tg].jar`"
    -fi
     
    -# Compute classpath using external script
    -classpath_output=$("$FWDIR"/bin/compute-classpath.sh)
    -if [[ "$?" != "0" ]]; then
    -  echo "$classpath_output"
    -  exit 1
    -else
    -  CLASSPATH="$classpath_output"
    -fi
    +# Look for the launcher. In non-release mode, add the compiled classes directly to the
classpath
    +# instead of looking for a jar file, since it's very common for people using sbt to use
the
    +# "assembly" target instead of "package".
    +SPARK_LAUNCHER_CP=
    +if [ -f $SPARK_HOME/RELEASE ]; then
    +  LAUNCHER_DIR="$SPARK_HOME/lib"
    +  num_jars="$(ls -1 "$LAUNCHER_DIR" | grep "^spark-launcher.*\.jar$" | wc -l)"
    +  if [ "$num_jars" -eq "0" -a -z "$SPARK_LAUNCHER_CP" ]; then
    +    echo "Failed to find Spark launcher in $LAUNCHER_DIR." 1>&2
    +    echo "You need to build Spark before running this program." 1>&2
    +    exit 1
    +  fi
     
    -if [[ "$1" =~ org.apache.spark.tools.* ]]; then
    -  if test -z "$SPARK_TOOLS_JAR"; then
    -    echo "Failed to find Spark Tools Jar in $FWDIR/tools/target/scala-$SPARK_SCALA_VERSION/"
1>&2
    -    echo "You need to run \"build/sbt tools/package\" before running $1." 1>&2
    +  LAUNCHER_JARS="$(ls -1 "$LAUNCHER_DIR" | grep "^spark-launcher.*\.jar$" || true)"
    +  if [ "$num_jars" -gt "1" ]; then
    +    echo "Found multiple Spark launcher jars in $LAUNCHER_DIR:" 1>&2
    +    echo "$LAUNCHER_JARS" 1>&2
    +    echo "Please remove all but one jar." 1>&2
         exit 1
       fi
    -  CLASSPATH="$CLASSPATH:$SPARK_TOOLS_JAR"
    -fi
     
    -if $cygwin; then
    -  CLASSPATH="`cygpath -wp "$CLASSPATH"`"
    -  if [ "$1" == "org.apache.spark.tools.JavaAPICompletenessChecker" ]; then
    -    export SPARK_TOOLS_JAR="`cygpath -w "$SPARK_TOOLS_JAR"`"
    +  SPARK_LAUNCHER_CP="${LAUNCHER_DIR}/${LAUNCHER_JARS}"
    +else
    +  LAUNCHER_DIR="$SPARK_HOME/launcher/target/scala-$SPARK_SCALA_VERSION"
    +  if [ ! -d "$LAUNCHER_DIR/classes" ]; then
    +    echo "Failed to find Spark launcher classes in $LAUNCHER_DIR." 1>&2
    +    echo "You need to build Spark before running this program." 1>&2
    +    exit 1
       fi
    +  SPARK_LAUNCHER_CP="$LAUNCHER_DIR/classes"
     fi
    -export CLASSPATH
     
    -# In Spark submit client mode, the driver is launched in the same JVM as Spark submit
itself.
    -# Here we must parse the properties file for relevant "spark.driver.*" configs before
launching
    -# the driver JVM itself. Instead of handling this complexity in Bash, we launch a separate
JVM
    -# to prepare the launch environment of this driver JVM.
    +# The launcher library will print arguments separated by a NULL character. Read that
in a while
    +# loop, populating an array that will be used to exec the final command.
    --- End diff --
    
    can you add a quick sentence here to motivate why we're doing this, i.e. we need a way
to split user arguments in a way that escapes whitespace and special characters correctly.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org


Mime
View raw message