mnemonic-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject incubator-mnemonic git commit: MNEMONIC-245: Class Path is not serializable as a parameter of RDD so fallback to String
Date Mon, 24 Apr 2017 22:42:36 GMT
Repository: incubator-mnemonic
Updated Branches:
  refs/heads/master 0292d5894 -> 3a8c52cef


MNEMONIC-245: Class Path is not serializable as a parameter of RDD so fallback to String


Project: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/commit/3a8c52ce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/tree/3a8c52ce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/diff/3a8c52ce

Branch: refs/heads/master
Commit: 3a8c52cefe062897eb95c7b8dd7ffbd2d24a9713
Parents: 0292d58
Author: Wang, Gang(Gary) <gang1.wang@intel.com>
Authored: Mon Apr 24 15:42:06 2017 -0700
Committer: Wang, Gang(Gary) <gang1.wang@intel.com>
Committed: Mon Apr 24 15:42:06 2017 -0700

----------------------------------------------------------------------
 .../org/apache/mnemonic/spark/MneDurableOutputSession.scala   | 7 +++----
 .../main/scala/org/apache/mnemonic/spark/rdd/DurableRDD.scala | 3 +--
 .../org/apache/mnemonic/spark/rdd/DurableRDDFunctions.scala   | 3 +--
 3 files changed, 5 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/3a8c52ce/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/MneDurableOutputSession.scala
----------------------------------------------------------------------
diff --git a/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/MneDurableOutputSession.scala
b/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/MneDurableOutputSession.scala
index 9a55150..657cb16 100644
--- a/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/MneDurableOutputSession.scala
+++ b/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/MneDurableOutputSession.scala
@@ -17,7 +17,6 @@
 
 package org.apache.mnemonic.spark
 import java.io.File
-import java.nio.file.Path
 import scala.reflect.{ classTag, ClassTag }
 import scala.collection.mutable.ArrayBuffer
 
@@ -33,14 +32,14 @@ import org.apache.mnemonic.sessions.DurableOutputSession
 class MneDurableOutputSession[V: ClassTag]
     extends DurableOutputSession[V, NonVolatileMemAllocator] {
 
-  var baseDir: Path = null
+  var baseDir: String = null
   var fileList: ArrayBuffer[File] = new ArrayBuffer[File]
   var outputFile: File = null
   var outputPrefix: String = null
   private var _outidx: Long = 0L
 
   protected def genNextPoolFile(): File = {
-    val file = new File(baseDir.toFile(), f"${outputPrefix}_${_outidx}%05d.mne")
+    val file = new File(baseDir, f"${outputPrefix}_${_outidx}%05d.mne")
     _outidx = _outidx + 1
     fileList += file
     file
@@ -71,7 +70,7 @@ object MneDurableOutputSession {
     entityFactoryProxies: Array[EntityFactoryProxy],
     slotKeyId: Long,
     partitionPoolSize: Long,
-    baseDir: Path,
+    baseDir: String,
     outputPrefix: String): MneDurableOutputSession[V] = {
     var ret = new MneDurableOutputSession[V]
     ret.setServiceName(serviceName)

http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/3a8c52ce/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDD.scala
----------------------------------------------------------------------
diff --git a/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDD.scala
b/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDD.scala
index a8462be..4c0adb4 100644
--- a/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDD.scala
+++ b/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDD.scala
@@ -18,7 +18,6 @@
 package org.apache.mnemonic.spark.rdd;
 
 import java.io.File
-import java.nio.file.Path
 
 import org.apache.spark.rdd.RDD
 import org.apache.spark.{ Partition, TaskContext }
@@ -39,7 +38,7 @@ class DurableRDD[D: ClassTag, T: ClassTag](
   var prev: RDD[T],
   serviceName: String, durableTypes: Array[DurableType],
   entityFactoryProxies: Array[EntityFactoryProxy], slotKeyId: Long,
-  partitionPoolSize: Long, baseDir: Path,
+  partitionPoolSize: Long, baseDir: String,
   f: (T, ObjectCreator[D, NonVolatileMemAllocator]) => Option[D],
   preservesPartitioning: Boolean = false)
     extends RDD[D](prev) {

http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/3a8c52ce/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDDFunctions.scala
----------------------------------------------------------------------
diff --git a/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDDFunctions.scala
b/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDDFunctions.scala
index 3b7800f..7ef8325 100644
--- a/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDDFunctions.scala
+++ b/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDDFunctions.scala
@@ -17,7 +17,6 @@
 
 package org.apache.mnemonic.spark.rdd
 
-import java.nio.file.Path
 import org.apache.spark.rdd.RDD
 import org.apache.spark.TaskContext
 import scala.reflect.ClassTag
@@ -35,7 +34,7 @@ class DurableRDDFunctions[T: ClassTag](rdd: RDD[T]) extends Serializable
{
     entityFactoryProxies: Array[EntityFactoryProxy],
     slotKeyId: Long,
     partitionPoolSize: Long,
-    baseDir: Path,
+    baseDir: String,
     f: (T, ObjectCreator[D, NonVolatileMemAllocator]) => Option[D],
     preservesPartitioning: Boolean = false) =
     new DurableRDD[D, T](rdd,


Mime
View raw message