mnemonic-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ga...@apache.org
Subject incubator-mnemonic git commit: MNEMONIC-238: Implement DurableRDD
Date Sat, 22 Apr 2017 16:48:49 GMT
Repository: incubator-mnemonic
Updated Branches:
  refs/heads/master 0d96c7500 -> cc4ed3fa1


MNEMONIC-238: Implement DurableRDD


Project: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/commit/cc4ed3fa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/tree/cc4ed3fa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/diff/cc4ed3fa

Branch: refs/heads/master
Commit: cc4ed3fa1b223bd55d23024e5fa5820685c1a924
Parents: 0d96c75
Author: Wang, Gang(Gary) <gang1.wang@intel.com>
Authored: Sat Apr 22 09:45:11 2017 -0700
Committer: Wang, Gang(Gary) <gang1.wang@intel.com>
Committed: Sat Apr 22 09:47:59 2017 -0700

----------------------------------------------------------------------
 build-tools/test.conf                           |   2 +-
 .../mnemonic/sessions/DurableOutputSession.java |   4 +
 .../apache/mnemonic/sessions/ObjectCreator.java |  28 +++++
 .../apache/mnemonic/sessions/OutputSession.java |   6 +-
 .../mnemonic/spark/MneDurableInputSession.scala |  83 +++++++++++++++
 .../spark/MneDurableOutputSession.scala         | 103 +++++++++++++++++++
 .../apache/mnemonic/spark/rdd/DurableRDD.scala  |  70 ++++++++++++-
 .../spark/rdd/DurableRDDFunctions.scala         |  48 +++++++++
 .../org/apache/mnemonic/spark/TestSpec.scala    |  26 +++++
 .../mnemonic/spark/rdd/DurableRDDSpec.scala     |  35 +++++++
 .../mnemonic/spark/rdd/DurableRDDSuite.scala    |  35 -------
 11 files changed, 398 insertions(+), 42 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/cc4ed3fa/build-tools/test.conf
----------------------------------------------------------------------
diff --git a/build-tools/test.conf b/build-tools/test.conf
index ccf997f..7645b26 100644
--- a/build-tools/test.conf
+++ b/build-tools/test.conf
@@ -58,5 +58,5 @@ mvn -Dtest=MneMapreduceChunkDataTest test -pl mnemonic-hadoop/mnemonic-hadoop-ma
 mvn -Dtest=MneMapredBufferDataTest test -pl mnemonic-hadoop/mnemonic-hadoop-mapreduce -DskipTests=false
 
 # a testcase for module "mnemonic-spark/mnemonic-spark-core" that requires 'pmalloc' memory
service to pass
-mvn -Dsuites=org.apache.mnemonic.spark.rdd.DurableRDDSuite test -pl mnemonic-spark/mnemonic-spark-core
-DskipTests=false
+mvn -Dtest=DurableRDDSpec test -pl mnemonic-spark/mnemonic-spark-core -DskipTests=false
 

http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/cc4ed3fa/mnemonic-sessions/src/main/java/org/apache/mnemonic/sessions/DurableOutputSession.java
----------------------------------------------------------------------
diff --git a/mnemonic-sessions/src/main/java/org/apache/mnemonic/sessions/DurableOutputSession.java
b/mnemonic-sessions/src/main/java/org/apache/mnemonic/sessions/DurableOutputSession.java
index 7e8b1be..e8bb17d 100644
--- a/mnemonic-sessions/src/main/java/org/apache/mnemonic/sessions/DurableOutputSession.java
+++ b/mnemonic-sessions/src/main/java/org/apache/mnemonic/sessions/DurableOutputSession.java
@@ -58,6 +58,10 @@ public abstract class DurableOutputSession<V, A extends RestorableAllocator<A>>
     return m_act;
   }
 
+  public void setAllocator(A alloc) {
+    m_act = alloc;
+  }
+
   @Override
   public long getHandler() {
     long ret = 0L;

http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/cc4ed3fa/mnemonic-sessions/src/main/java/org/apache/mnemonic/sessions/ObjectCreator.java
----------------------------------------------------------------------
diff --git a/mnemonic-sessions/src/main/java/org/apache/mnemonic/sessions/ObjectCreator.java
b/mnemonic-sessions/src/main/java/org/apache/mnemonic/sessions/ObjectCreator.java
new file mode 100644
index 0000000..71f8774
--- /dev/null
+++ b/mnemonic-sessions/src/main/java/org/apache/mnemonic/sessions/ObjectCreator.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mnemonic.sessions;
+
+public interface ObjectCreator<V> {
+
+  V newDurableObjectRecord();
+
+  V newDurableObjectRecord(long size);
+
+}
+

http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/cc4ed3fa/mnemonic-sessions/src/main/java/org/apache/mnemonic/sessions/OutputSession.java
----------------------------------------------------------------------
diff --git a/mnemonic-sessions/src/main/java/org/apache/mnemonic/sessions/OutputSession.java
b/mnemonic-sessions/src/main/java/org/apache/mnemonic/sessions/OutputSession.java
index 598becc..dcc0ab3 100644
--- a/mnemonic-sessions/src/main/java/org/apache/mnemonic/sessions/OutputSession.java
+++ b/mnemonic-sessions/src/main/java/org/apache/mnemonic/sessions/OutputSession.java
@@ -20,11 +20,7 @@ package org.apache.mnemonic.sessions;
 
 import java.io.Closeable;
 
-public interface OutputSession<V> extends Closeable {
-
-  V newDurableObjectRecord();
-
-  V newDurableObjectRecord(long size);
+public interface OutputSession<V> extends ObjectCreator<V>, Closeable {
 
   void post(V v);
 

http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/cc4ed3fa/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/MneDurableInputSession.scala
----------------------------------------------------------------------
diff --git a/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/MneDurableInputSession.scala
b/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/MneDurableInputSession.scala
new file mode 100644
index 0000000..d56bddc
--- /dev/null
+++ b/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/MneDurableInputSession.scala
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mnemonic.spark
+
+import java.io.File
+import scala.reflect.{ classTag, ClassTag }
+
+import org.apache.mnemonic.ConfigurationException
+import org.apache.mnemonic.DurableType
+import org.apache.mnemonic.EntityFactoryProxy
+import org.apache.mnemonic.NonVolatileMemAllocator
+import org.apache.mnemonic.Utils
+import org.apache.mnemonic.collections.DurableSinglyLinkedList
+import org.apache.mnemonic.collections.DurableSinglyLinkedListFactory
+import org.apache.mnemonic.sessions.DurableInputSession
+import org.apache.mnemonic.sessions.SessionIterator
+
+class MneDurableInputSession[V: ClassTag]
+    extends DurableInputSession[V, NonVolatileMemAllocator] {
+
+  private var _flist: Array[File] = null
+
+  def fileList = _flist
+
+  def fileList_=(value: Array[File]): Unit = _flist = value
+
+  override def initNextPool(sessiter: SessionIterator[V, NonVolatileMemAllocator]): Boolean
= {
+    var ret: Boolean = false
+    if (null != sessiter.getAllocator) {
+      sessiter.getAllocator.close
+      sessiter.setAllocator(null)
+    }
+    for (file <- fileList) {
+      sessiter.setAllocator(new NonVolatileMemAllocator(Utils.getNonVolatileMemoryAllocatorService(
+        getServiceName), 1024000L, file.toString, true));
+      if (null != sessiter.getAllocator) {
+        sessiter.setHandler(sessiter.getAllocator.getHandler(getSlotKeyId))
+        if (0L != sessiter.getHandler) {
+          val dsllist: DurableSinglyLinkedList[V] = DurableSinglyLinkedListFactory.restore(
+            sessiter.getAllocator, getEntityFactoryProxies, getDurableTypes, sessiter.getHandler,
false)
+          if (null != dsllist) {
+            sessiter.setIterator(dsllist.iterator)
+            ret = null != sessiter.getIterator
+          }
+        }
+      }
+    }
+    ret
+  }
+
+}
+
+object MneDurableInputSession {
+  def apply[V: ClassTag](
+    serviceName: String,
+    durableTypes: Array[DurableType],
+    entityFactoryProxies: Array[EntityFactoryProxy],
+    slotKeyId: Long,
+    files: Array[File]): MneDurableInputSession[V] = {
+    var ret = new MneDurableInputSession[V]
+    ret.setServiceName(serviceName)
+    ret.setDurableTypes(durableTypes)
+    ret.setEntityFactoryProxies(entityFactoryProxies)
+    ret.setSlotKeyId(slotKeyId);
+    ret.fileList = files
+    ret
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/cc4ed3fa/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/MneDurableOutputSession.scala
----------------------------------------------------------------------
diff --git a/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/MneDurableOutputSession.scala
b/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/MneDurableOutputSession.scala
new file mode 100644
index 0000000..e25f8ce
--- /dev/null
+++ b/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/MneDurableOutputSession.scala
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mnemonic.spark
+import java.io.File
+import java.nio.file.Path
+import scala.reflect.{ classTag, ClassTag }
+import scala.collection.mutable.ArrayBuffer
+
+import org.apache.mnemonic.ConfigurationException
+import org.apache.mnemonic.DurableType
+import org.apache.mnemonic.EntityFactoryProxy
+import org.apache.mnemonic.NonVolatileMemAllocator
+import org.apache.mnemonic.Utils
+import org.apache.mnemonic.collections.DurableSinglyLinkedList
+import org.apache.mnemonic.collections.DurableSinglyLinkedListFactory
+import org.apache.mnemonic.sessions.DurableOutputSession
+
+class MneDurableOutputSession[V: ClassTag]
+    extends DurableOutputSession[V, NonVolatileMemAllocator] {
+
+  private var _baseDir: Path = null
+  private var _flist: ArrayBuffer[File] = new ArrayBuffer[File]
+  private var _outputFile: File = null
+  private var _outPrefix: String = null
+  private var _outidx: Long = 0L
+
+  def baseDir = _baseDir
+
+  def baseDir_=(value: Path): Unit = _baseDir = value
+
+  def fileList = _flist.toArray
+
+  def outputPrefix = _outPrefix
+
+  def outputPrefix_=(value: String): Unit = _outPrefix = value
+
+  protected def genNextPoolFile(): File = {
+    val file = new File(baseDir.toFile(), f"${_outPrefix}_${_outidx}%05d.mne")
+    _outidx = _outidx + 1
+    _flist += file
+    file
+  }
+
+  def setOutputFile(file: File): Unit = {
+    _outputFile = file
+  }
+
+  def getOutputFile: File = _outputFile
+
+  override def initNextPool(): Boolean = {
+    var ret: Boolean = false
+    if (null != getAllocator) {
+      getAllocator.close();
+      setAllocator(null);
+    }
+    setOutputFile(genNextPoolFile);
+    m_act = new NonVolatileMemAllocator(Utils.getNonVolatileMemoryAllocatorService(getServiceName),
+      getPoolSize, getOutputFile.toString, true);
+    if (null != getAllocator) {
+      m_newpool = true;
+      ret = true;
+    }
+    ret
+  }
+
+}
+
+object MneDurableOutputSession {
+  def apply[V: ClassTag](
+    serviceName: String,
+    durableTypes: Array[DurableType],
+    entityFactoryProxies: Array[EntityFactoryProxy],
+    slotKeyId: Long,
+    partitionPoolSize: Long,
+    baseDir: Path,
+    outputPrefix: String): MneDurableOutputSession[V] = {
+    var ret = new MneDurableOutputSession[V]
+    ret.setServiceName(serviceName)
+    ret.setDurableTypes(durableTypes)
+    ret.setEntityFactoryProxies(entityFactoryProxies)
+    ret.setSlotKeyId(slotKeyId)
+    ret.setPoolSize(partitionPoolSize)
+    ret.baseDir = baseDir
+    ret.outputPrefix = outputPrefix
+    ret
+  }
+}
+

http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/cc4ed3fa/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDD.scala
----------------------------------------------------------------------
diff --git a/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDD.scala
b/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDD.scala
index 164033e..40e94b7 100644
--- a/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDD.scala
+++ b/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDD.scala
@@ -17,6 +17,74 @@
 
 package org.apache.mnemonic.spark.rdd;
 
-class DurableRDD {
+import java.io.File
+import java.nio.file.Path
+
+import org.apache.spark.rdd.RDD
+import org.apache.spark.{ Partition, TaskContext }
+import scala.reflect.{ classTag, ClassTag }
+import scala.collection.mutable.HashMap
+import scala.collection.JavaConverters._
+import org.apache.mnemonic.ConfigurationException;
+import org.apache.mnemonic.DurableType;
+import org.apache.mnemonic.EntityFactoryProxy;
+import org.apache.mnemonic.NonVolatileMemAllocator;
+import org.apache.mnemonic.sessions.DurableInputSession;
+import org.apache.mnemonic.sessions.SessionIterator;
+import org.apache.mnemonic.sessions.ObjectCreator
+import org.apache.mnemonic.spark.MneDurableInputSession
+import org.apache.mnemonic.spark.MneDurableOutputSession
+
+class DurableRDD[D: ClassTag, T: ClassTag](
+  var prev: RDD[T],
+  serviceName: String, durableTypes: Array[DurableType],
+  entityFactoryProxies: Array[EntityFactoryProxy], slotKeyId: Long,
+  partitionPoolSize: Long, baseDir: Path,
+  f: (T, ObjectCreator[D]) => Option[D],
+  preservesPartitioning: Boolean = false)
+    extends RDD[D](prev) {
+
+  private val _parmap = HashMap.empty[Partition, Array[File]]
+
+  override val partitioner = if (preservesPartitioning) firstParent[T].partitioner else None
+
+  override protected def getPartitions: Array[Partition] = firstParent[T].partitions
+
+  def prepareDurablePartition(split: Partition, context: TaskContext,
+      iterator: Iterator[T]) {
+    val outsess = MneDurableOutputSession[D](serviceName,
+        durableTypes, entityFactoryProxies, slotKeyId,
+        partitionPoolSize, baseDir,
+        f"mem_${this.hashCode()}%10d_${split.hashCode()}%10d")
+    try {
+      for (item <- iterator) {
+        f(item, outsess) match {
+          case Some(res) => outsess.post(res)
+          case None =>
+        }
+      }
+      _parmap += (split -> outsess.fileList)
+    } finally {
+      outsess.close()
+    }
+  }
+
+  override def compute(split: Partition, context: TaskContext): Iterator[D] = {
+    if (!(_parmap contains split)) {
+      prepareDurablePartition(split, context, firstParent[T].iterator(split, context))
+    }
+    val flist = _parmap.get(split) match {
+      case Some(flst) => flst
+      case None => throw new RuntimeException("Not construct durable partition properly")
+    }
+    val insess = MneDurableInputSession[D](serviceName,
+        durableTypes, entityFactoryProxies, slotKeyId, flist)
+    insess.iterator.asScala
+  }
+
+  override def clearDependencies() {
+    super.clearDependencies()
+    prev = null
+  }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/cc4ed3fa/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDDFunctions.scala
----------------------------------------------------------------------
diff --git a/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDDFunctions.scala
b/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDDFunctions.scala
new file mode 100644
index 0000000..e86d7f8
--- /dev/null
+++ b/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDDFunctions.scala
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mnemonic.spark.rdd
+
+import java.nio.file.Path
+import org.apache.spark.rdd.RDD
+import org.apache.spark.TaskContext
+import scala.reflect.ClassTag
+import scala.language.implicitConversions
+import org.apache.mnemonic.DurableType
+import org.apache.mnemonic.EntityFactoryProxy;
+import org.apache.mnemonic.sessions.ObjectCreator
+
+class DurableRDDFunctions[T: ClassTag](rdd: RDD[T]) extends Serializable {
+
+  def makeDurable[D: ClassTag](
+    serviceName: String,
+    durableTypes: Array[DurableType],
+    entityFactoryProxies: Array[EntityFactoryProxy],
+    slotKeyId: Long,
+    partitionPoolSize: Long,
+    baseDir: Path,
+    f: (T, ObjectCreator[D]) => Option[D],
+    preservesPartitioning: Boolean = false) =
+    new DurableRDD[D, T](rdd,
+      serviceName, durableTypes, entityFactoryProxies, slotKeyId,
+      partitionPoolSize, baseDir, f, preservesPartitioning)
+
+}
+
+object DurableRDDFunctions {
+  implicit def addDurableFunctions[T: ClassTag](rdd: RDD[T]) = new DurableRDDFunctions[T](rdd)
+}

http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/cc4ed3fa/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/TestSpec.scala
----------------------------------------------------------------------
diff --git a/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/TestSpec.scala
b/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/TestSpec.scala
new file mode 100644
index 0000000..95b894d
--- /dev/null
+++ b/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/TestSpec.scala
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mnemonic.spark
+
+import org.scalatest.FlatSpec
+import org.scalatest.Matchers
+import org.scalatest.BeforeAndAfter
+
+abstract class TestSpec extends FlatSpec with Matchers with BeforeAndAfter {
+  
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/cc4ed3fa/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/rdd/DurableRDDSpec.scala
----------------------------------------------------------------------
diff --git a/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/rdd/DurableRDDSpec.scala
b/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/rdd/DurableRDDSpec.scala
new file mode 100644
index 0000000..41b9449
--- /dev/null
+++ b/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/rdd/DurableRDDSpec.scala
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mnemonic.spark.rdd;
+
+import org.apache.mnemonic.spark.TestSpec
+
+class DurableRDDSpec extends TestSpec {
+
+  behavior of "A DurableRDD"
+
+  it should "have size 0" in {
+    assert(Set.empty.size == 0)
+  }
+
+  it should "produce NoSuchElementException when head is invoked" in {
+    assertThrows[NoSuchElementException] {
+      Set.empty.head
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/cc4ed3fa/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/rdd/DurableRDDSuite.scala
----------------------------------------------------------------------
diff --git a/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/rdd/DurableRDDSuite.scala
b/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/rdd/DurableRDDSuite.scala
deleted file mode 100644
index e59aad7..0000000
--- a/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/rdd/DurableRDDSuite.scala
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.mnemonic.spark.rdd;
-
-import org.scalatest.FlatSpec
-import org.scalatest.Matchers
-import org.scalatest.BeforeAndAfter
-
-class DurableRDDSuite extends FlatSpec with Matchers with BeforeAndAfter {
-
-  "An empty Set" should "have size 0" in {
-    assert(Set.empty.size == 0)
-  }
-
-  it should "produce NoSuchElementException when head is invoked" in {
-    assertThrows[NoSuchElementException] {
-      Set.empty.head
-    }
-  }
-}


Mime
View raw message