predictionio-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From don...@apache.org
Subject [1/2] incubator-predictionio git commit: [PIO-72] Fix class loading for pio-shell
Date Fri, 28 Jul 2017 18:08:25 GMT
Repository: incubator-predictionio
Updated Branches:
  refs/heads/develop cc4e2e0a2 -> a81302171


[PIO-72] Fix class loading for pio-shell

Closes #401


Project: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/commit/d894b72a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/tree/d894b72a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-predictionio/diff/d894b72a

Branch: refs/heads/develop
Commit: d894b72af8975aef872aae5311f3c7f75aa00961
Parents: cc4e2e0
Author: Chan Lee <chanlee514@gmail.com>
Authored: Thu Jul 27 14:54:37 2017 -0700
Committer: Donald Szeto <donald@apache.org>
Committed: Thu Jul 27 14:54:37 2017 -0700

----------------------------------------------------------------------
 bin/compute-classpath.sh                        | 23 +++++++++-----------
 bin/pio-class                                   |  2 --
 bin/pio-shell                                   |  5 ++---
 tests/build_docker.sh                           | 13 ++++++-----
 .../org/apache/predictionio/tools/Common.scala  | 19 +++++++++-------
 5 files changed, 31 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d894b72a/bin/compute-classpath.sh
----------------------------------------------------------------------
diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh
index 032d111..cd0527f 100755
--- a/bin/compute-classpath.sh
+++ b/bin/compute-classpath.sh
@@ -17,20 +17,11 @@
 # limitations under the License.
 #
 
-SCALA_VERSION=2.10
-
 # Figure out where PredictionIO is installed
 FWDIR="$(cd `dirname $0`/..; pwd)"
 
 . ${FWDIR}/bin/load-pio-env.sh
 
-# Build up classpath
-CLASSPATH="${PIO_CONF_DIR}"
-
-CLASSPATH="$CLASSPATH:${FWDIR}/plugins/*:${FWDIR}/lib/spark/*"
-
-ASSEMBLY_DIR="${FWDIR}/assembly"
-
 if [ -n "$JAVA_HOME" ]; then
   JAR_CMD="$JAVA_HOME/bin/jar"
 else
@@ -41,12 +32,19 @@ fi
 if [ -f "${FWDIR}/RELEASE" ]; then
   assembly_folder="${FWDIR}"/lib
 else
-  assembly_folder="${ASSEMBLY_DIR}"
+  assembly_folder="${FWDIR}"/assembly/src/universal/lib
 fi
 
-ASSEMBLY_JAR=$(ls "${assembly_folder}"/pio-assembly*.jar 2>/dev/null)
+MAIN_JAR=$(ls "${assembly_folder}"/pio-assembly*.jar 2>/dev/null)
+DATA_JARS=$(ls "${assembly_folder}"/spark/pio-data-*assembly*.jar 2>/dev/null)
+# Comma-separated list of assembly jars for submitting to spark-shell
+ASSEMBLY_JARS=$(printf "${MAIN_JAR}\n${DATA_JARS}" | paste -sd "," -)
 
-CLASSPATH="$CLASSPATH:${ASSEMBLY_JAR}"
+# Build up classpath
+CLASSPATH="${PIO_CONF_DIR}"
+CLASSPATH="$CLASSPATH:${FWDIR}/plugins/*"
+CLASSPATH="$CLASSPATH:${assembly_folder}/spark/*"
+CLASSPATH="$CLASSPATH:${MAIN_JAR}"
 
 # Add hadoop conf dir if given -- otherwise FileSystem.*, etc fail ! Note, this
 # assumes that there is either a HADOOP_CONF_DIR or YARN_CONF_DIR which hosts
@@ -70,5 +68,4 @@ if [ -n "$MYSQL_JDBC_DRIVER" ]; then
   CLASSPATH="$CLASSPATH:$MYSQL_JDBC_DRIVER"
 fi
 
-
 echo "$CLASSPATH"

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d894b72a/bin/pio-class
----------------------------------------------------------------------
diff --git a/bin/pio-class b/bin/pio-class
index 8bde407..94cc520 100755
--- a/bin/pio-class
+++ b/bin/pio-class
@@ -22,8 +22,6 @@ case "`uname`" in
     CYGWIN*) cygwin=true;;
 esac
 
-SCALA_VERSION=2.10
-
 # Figure out where PredictionIO is installed
 FWDIR="$(cd `dirname $0`/..; pwd)"
 

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d894b72a/bin/pio-shell
----------------------------------------------------------------------
diff --git a/bin/pio-shell b/bin/pio-shell
index d47008f..5a5745a 100755
--- a/bin/pio-shell
+++ b/bin/pio-shell
@@ -55,11 +55,10 @@ fi
 if [[ "$1" == "--with-spark" ]]
 then
   echo "Starting the PIO shell with the Apache Spark Shell."
-  # compute the $ASSEMPLY_JAR, the location of the assemply jar, with
-  # bin/compute-classpath.sh
+  # Get paths of assembly jars to pass to spark-shell
   . ${PIO_HOME}/bin/compute-classpath.sh
   shift
-  $SPARK_HOME/bin/spark-shell --jars $ASSEMBLY_JAR $@
+  ${SPARK_HOME}/bin/spark-shell --jars ${ASSEMBLY_JARS} $@
 else
   echo -e "\033[0;33mStarting the PIO shell without Apache Spark.\033[0m"
   echo -e "\033[0;33mIf you need the Apache Spark library, run 'pio-shell --with-spark [spark-submit
arguments...]'.\033[0m"

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d894b72a/tests/build_docker.sh
----------------------------------------------------------------------
diff --git a/tests/build_docker.sh b/tests/build_docker.sh
index d467e97..b17a304 100755
--- a/tests/build_docker.sh
+++ b/tests/build_docker.sh
@@ -38,11 +38,14 @@ fi
     -Dhadoop.version=$PIO_HADOOP_VERSION \
     -Delasticsearch.version=$PIO_ELASTICSEARCH_VERSION
 sbt/sbt clean storage/clean
-rm -rf assembly/*.jar
-cp dist/lib/*.jar assembly/
-rm -rf lib/spark
-mkdir -p lib/spark
-cp dist/lib/spark/*.jar lib/spark
+
+assembly_folder=assembly/src/universal/lib
+rm -rf ${assembly_folder}/*.jar
+rm -rf ${assembly_folder}/spark
+mkdir -p ${assembly_folder}/spark
+
+cp dist/lib/*.jar ${assembly_folder}
+cp dist/lib/spark/*.jar ${assembly_folder}/spark
 rm *.tar.gz
 docker build -t predictionio/pio .
 popd

http://git-wip-us.apache.org/repos/asf/incubator-predictionio/blob/d894b72a/tools/src/main/scala/org/apache/predictionio/tools/Common.scala
----------------------------------------------------------------------
diff --git a/tools/src/main/scala/org/apache/predictionio/tools/Common.scala b/tools/src/main/scala/org/apache/predictionio/tools/Common.scala
index 0ddd644..047b8dd 100644
--- a/tools/src/main/scala/org/apache/predictionio/tools/Common.scala
+++ b/tools/src/main/scala/org/apache/predictionio/tools/Common.scala
@@ -65,6 +65,15 @@ object Common extends EitherLogging {
     }
   }
 
+  def getCoreDir(pioHome: String): String = {
+    if (new File(pioHome + File.separator + "RELEASE").exists) {
+      pioHome + File.separator + "lib"
+    } else {
+      Array(pioHome, "assembly", "src", "universal", "lib")
+        .mkString(File.separator)
+    }
+  }
+
   def getEngineDirPath(directory: Option[String]): String = {
     new File(directory.getOrElse(".")).getCanonicalPath
   }
@@ -97,18 +106,12 @@ object Common extends EitherLogging {
     def jarFilesAt(path: File): Array[File] = path.listFiles filter {
       _.getName.toLowerCase.endsWith(".jar")
     }
-    jarFilesAt(new File(pioHome, "lib/spark"))
+    jarFilesAt(new File(getCoreDir(pioHome) + File.separator + "spark"))
   }
 
   def coreAssembly(pioHome: String): Expected[File] = {
     val core = s"pio-assembly-${BuildInfo.version}.jar"
-    val coreDir =
-      if (new File(pioHome + File.separator + "RELEASE").exists) {
-        new File(pioHome + File.separator + "lib")
-      } else {
-        new File(pioHome + File.separator + "assembly")
-      }
-    val coreFile = new File(coreDir, core)
+    val coreFile = new File(getCoreDir(pioHome), core)
     if (coreFile.exists) {
       Right(coreFile)
     } else {


Mime
View raw message