incubator-crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject [6/7] git commit: Initial commit of Scrunch test refactoring to support CI
Date Sat, 28 Jul 2012 19:40:20 GMT
Initial commit of Scrunch test refactoring to support CI

Signed-off-by: jwills <jwills@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/incubator-crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-crunch/commit/5b9d3ce1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-crunch/tree/5b9d3ce1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-crunch/diff/5b9d3ce1

Branch: refs/heads/master
Commit: 5b9d3ce1f48377ebc9de0369750a0706b53c1304
Parents: e540459
Author: jwills <jwills@apache.org>
Authored: Thu Jul 26 18:33:39 2012 -0700
Committer: jwills <jwills@apache.org>
Committed: Sat Jul 28 11:17:58 2012 -0700

----------------------------------------------------------------------
 crunch/pom.xml                                     |   11 +++++
 .../java/org/apache/crunch/test/TemporaryPath.java |    5 ++-
 pom.xml                                            |    7 +++
 scrunch/pom.xml                                    |    7 +++
 .../org/apache/scrunch/ScrunchTestSupport.java     |   32 +++++++++++++++
 .../it/scala/org/apache/scrunch/CogroupTest.scala  |    9 ++--
 .../src/it/scala/org/apache/scrunch/JoinTest.scala |   12 ++---
 .../org/apache/scrunch/PageRankClassTest.scala     |   10 ++--
 .../it/scala/org/apache/scrunch/PageRankTest.scala |    9 ++--
 .../scala/org/apache/scrunch/PipelineAppTest.scala |   11 ++---
 .../src/it/scala/org/apache/scrunch/TopTest.scala  |    7 +--
 .../it/scala/org/apache/scrunch/UnionTest.scala    |    9 ++--
 .../scala/org/apache/scrunch/WordCountTest.scala   |   14 ++----
 13 files changed, 96 insertions(+), 47 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5b9d3ce1/crunch/pom.xml
----------------------------------------------------------------------
diff --git a/crunch/pom.xml b/crunch/pom.xml
index 0d2f111..a440f2c 100644
--- a/crunch/pom.xml
+++ b/crunch/pom.xml
@@ -211,6 +211,17 @@ under the License.
           </execution>
         </executions>
       </plugin>
+     <plugin>
+       <groupId>org.apache.maven.plugins</groupId>
+       <artifactId>maven-jar-plugin</artifactId>
+       <executions>
+         <execution>
+           <goals>
+             <goal>test-jar</goal>
+           </goals>
+         </execution>
+       </executions>
+     </plugin>
     </plugins>
   </build>
 

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5b9d3ce1/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java
----------------------------------------------------------------------
diff --git a/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java b/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java
index dd4f505..e96e3b1 100644
--- a/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java
+++ b/crunch/src/it/java/org/apache/crunch/test/TemporaryPath.java
@@ -6,7 +6,6 @@ import java.io.IOException;
 import org.apache.crunch.impl.mr.run.RuntimeParameters;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.filter.Filter.ReturnCode;
 import org.junit.rules.ExternalResource;
 import org.junit.rules.TemporaryFolder;
 import org.junit.runner.Description;
@@ -102,6 +101,10 @@ public final class TemporaryPath extends ExternalResource {
     return new Path(file.getAbsolutePath());
   }
 
+  public Configuration getDefaultConfiguration() throws IOException {
+    return setTempLoc(new Configuration());
+  }
+  
   public Configuration setTempLoc(Configuration config) throws IOException {
     config.set(RuntimeParameters.TMP_DIR, getRootFileName());
     config.set("hadoop.tmp.dir", getFileName("hadoop-tmp"));

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5b9d3ce1/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index c106f55..afa73f4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -106,6 +106,13 @@ under the License.
       </dependency>
 
       <dependency>
+        <groupId>org.apache.crunch</groupId>
+        <artifactId>crunch</artifactId>
+        <type>test-jar</type>
+        <version>${project.version}</version>
+      </dependency>
+
+      <dependency>
         <groupId>com.google.guava</groupId>
         <artifactId>guava</artifactId>
         <version>11.0.2</version>

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5b9d3ce1/scrunch/pom.xml
----------------------------------------------------------------------
diff --git a/scrunch/pom.xml b/scrunch/pom.xml
index d70b1d2..0b549ae 100644
--- a/scrunch/pom.xml
+++ b/scrunch/pom.xml
@@ -70,6 +70,12 @@ under the License.
       <artifactId>scalatest_${scala.version}</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.crunch</groupId>
+      <artifactId>crunch</artifactId>
+      <type>test-jar</type>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>
@@ -152,6 +158,7 @@ under the License.
             </goals>
             <configuration>
               <sources>
+                <source>${basedir}/src/it/java</source>
                 <source>${basedir}/src/it/scala</source>
               </sources>
             </configuration>

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5b9d3ce1/scrunch/src/it/java/org/apache/scrunch/ScrunchTestSupport.java
----------------------------------------------------------------------
diff --git a/scrunch/src/it/java/org/apache/scrunch/ScrunchTestSupport.java b/scrunch/src/it/java/org/apache/scrunch/ScrunchTestSupport.java
new file mode 100644
index 0000000..2cfb1ce
--- /dev/null
+++ b/scrunch/src/it/java/org/apache/scrunch/ScrunchTestSupport.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.scrunch;
+
+import org.apache.crunch.test.TemporaryPath;
+
+import org.junit.Rule;
+import org.scalatest.junit.JUnitSuite;
+
+public class ScrunchTestSupport {
+  @Rule
+  private final TemporaryPath tempDir = new TemporaryPath();
+
+  public TemporaryPath tempDir() {
+    return tempDir;
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5b9d3ce1/scrunch/src/it/scala/org/apache/scrunch/CogroupTest.scala
----------------------------------------------------------------------
diff --git a/scrunch/src/it/scala/org/apache/scrunch/CogroupTest.scala b/scrunch/src/it/scala/org/apache/scrunch/CogroupTest.scala
index b0e94a1..268d2b6 100644
--- a/scrunch/src/it/scala/org/apache/scrunch/CogroupTest.scala
+++ b/scrunch/src/it/scala/org/apache/scrunch/CogroupTest.scala
@@ -18,13 +18,12 @@
 package org.apache.scrunch
 
 import org.apache.crunch.io.{From => from}
-import org.apache.crunch.test.FileHelper
 
 import org.scalatest.junit.JUnitSuite
 import _root_.org.junit.Test
 
-class CogroupTest extends JUnitSuite {
-  val pipeline = Pipeline.mapReduce[CogroupTest]
+class CogroupTest extends ScrunchTestSupport with JUnitSuite {
+  val pipeline = Pipeline.mapReduce[CogroupTest](tempDir.getDefaultConfiguration)
 
   def wordCount(fileName: String) = {
     pipeline.read(from.textFile(fileName))
@@ -32,8 +31,8 @@ class CogroupTest extends JUnitSuite {
   }
 
   @Test def cogroup {
-    val shakespeare = FileHelper.createTempCopyOf("shakes.txt")
-    val maugham = FileHelper.createTempCopyOf("maugham.txt")
+    val shakespeare = tempDir.copyResourceFileName("shakes.txt")
+    val maugham = tempDir.copyResourceFileName("maugham.txt")
     val diffs = wordCount(shakespeare).cogroup(wordCount(maugham))
         .map((k, v) => (k, (v._1.sum - v._2.sum))).materialize
     assert(diffs.exists(_ == ("the", -11390)))

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5b9d3ce1/scrunch/src/it/scala/org/apache/scrunch/JoinTest.scala
----------------------------------------------------------------------
diff --git a/scrunch/src/it/scala/org/apache/scrunch/JoinTest.scala b/scrunch/src/it/scala/org/apache/scrunch/JoinTest.scala
index 5303c03..c3a4a80 100644
--- a/scrunch/src/it/scala/org/apache/scrunch/JoinTest.scala
+++ b/scrunch/src/it/scala/org/apache/scrunch/JoinTest.scala
@@ -18,13 +18,12 @@
 package org.apache.scrunch
 
 import org.apache.crunch.io.{From => from, To => to}
-import org.apache.crunch.test.FileHelper
 
 import org.scalatest.junit.JUnitSuite
 import _root_.org.junit.Test
 
-class JoinTest extends JUnitSuite {
-  val pipeline = Pipeline.mapReduce[CogroupTest]
+class JoinTest extends ScrunchTestSupport with JUnitSuite {
+  val pipeline = Pipeline.mapReduce[CogroupTest](tempDir.getDefaultConfiguration)
 
   def wordCount(fileName: String) = {
     pipeline.read(from.textFile(fileName))
@@ -32,10 +31,9 @@ class JoinTest extends JUnitSuite {
   }
 
   @Test def join {
-    val shakespeare = FileHelper.createTempCopyOf("shakes.txt")
-    val maugham = FileHelper.createTempCopyOf("maugham.txt")
-    val output = FileHelper.createOutputPath()
-    output.deleteOnExit()
+    val shakespeare = tempDir.copyResourceFileName("shakes.txt")
+    val maugham = tempDir.copyResourceFileName("maugham.txt")
+    val output = tempDir.getFile("output")
     val filtered = wordCount(shakespeare).join(wordCount(maugham))
         .map((k, v) => (k, v._1 - v._2))
         .write(to.textFile(output.getAbsolutePath()))

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5b9d3ce1/scrunch/src/it/scala/org/apache/scrunch/PageRankClassTest.scala
----------------------------------------------------------------------
diff --git a/scrunch/src/it/scala/org/apache/scrunch/PageRankClassTest.scala b/scrunch/src/it/scala/org/apache/scrunch/PageRankClassTest.scala
index 9ab7897..43775ea 100644
--- a/scrunch/src/it/scala/org/apache/scrunch/PageRankClassTest.scala
+++ b/scrunch/src/it/scala/org/apache/scrunch/PageRankClassTest.scala
@@ -21,7 +21,6 @@ import Avros._
 
 import org.apache.crunch.{DoFn, Emitter, Pair => P}
 import org.apache.crunch.io.{From => from}
-import org.apache.crunch.test.FileHelper
 
 import scala.collection.mutable.HashMap
 
@@ -61,8 +60,9 @@ class CachingPageRankClassFn extends DoFn[P[String, PageRankData], P[String,
Flo
   }
 }
 
-class PageRankClassTest extends JUnitSuite {
-  val pipeline = Pipeline.mapReduce[PageRankTest]
+class PageRankClassTest extends ScrunchTestSupport with JUnitSuite {
+
+  val pipeline = Pipeline.mapReduce[PageRankTest](tempDir.getDefaultConfiguration)
 
   def initialInput(fileName: String) = {
     pipeline.read(from.textFile(fileName))
@@ -94,7 +94,7 @@ class PageRankClassTest extends JUnitSuite {
 
   @Test def testPageRank {
     pipeline.getConfiguration.set("crunch.debug", "true")
-    var prev = initialInput(FileHelper.createTempCopyOf("urls.txt"))
+    var prev = initialInput(tempDir.copyResourceFileName("urls.txt"))
     var delta = 1.0f
     while (delta > 0.01f) {
       prev = update(prev, 0.5f)
@@ -106,7 +106,7 @@ class PageRankClassTest extends JUnitSuite {
 
   def testFastPageRank {
     pipeline.getConfiguration.set("crunch.debug", "true")
-    var prev = initialInput(FileHelper.createTempCopyOf("urls.txt"))
+    var prev = initialInput(tempDir.copyResourceFileName("urls.txt"))
     var delta = 1.0f
     while (delta > 0.01f) {
       prev = fastUpdate(prev, 0.5f)

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5b9d3ce1/scrunch/src/it/scala/org/apache/scrunch/PageRankTest.scala
----------------------------------------------------------------------
diff --git a/scrunch/src/it/scala/org/apache/scrunch/PageRankTest.scala b/scrunch/src/it/scala/org/apache/scrunch/PageRankTest.scala
index cbf7ebf..3cf7b8a 100644
--- a/scrunch/src/it/scala/org/apache/scrunch/PageRankTest.scala
+++ b/scrunch/src/it/scala/org/apache/scrunch/PageRankTest.scala
@@ -21,7 +21,6 @@ import Avros._
 
 import org.apache.crunch.{DoFn, Emitter, Pair => P}
 import org.apache.crunch.io.{From => from}
-import org.apache.crunch.test.FileHelper
 
 import scala.collection.mutable.HashMap
 
@@ -49,8 +48,8 @@ class CachingPageRankFn extends DoFn[P[String, (Float, Float, List[String])],
P[
   }
 }
 
-class PageRankTest extends JUnitSuite {
-  val pipeline = Pipeline.mapReduce[PageRankTest]
+class PageRankTest extends ScrunchTestSupport with JUnitSuite {
+  val pipeline = Pipeline.mapReduce[PageRankTest](tempDir.getDefaultConfiguration)
 
   def initialInput(fileName: String) = {
     pipeline.read(from.textFile(fileName))
@@ -82,7 +81,7 @@ class PageRankTest extends JUnitSuite {
   }
 
   @Test def testPageRank {
-    var prev = initialInput(FileHelper.createTempCopyOf("urls.txt"))
+    var prev = initialInput(tempDir.copyResourceFileName("urls.txt"))
     var delta = 1.0f
     while (delta > 0.01f) {
       prev = update(prev, 0.5f)
@@ -93,7 +92,7 @@ class PageRankTest extends JUnitSuite {
   }
 
   @Test def testFastPageRank {
-    var prev = initialInput(FileHelper.createTempCopyOf("urls.txt"))
+    var prev = initialInput(tempDir.copyResourceFileName("urls.txt"))
     var delta = 1.0f
     while (delta > 0.01f) {
       prev = fastUpdate(prev, 0.5f)

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5b9d3ce1/scrunch/src/it/scala/org/apache/scrunch/PipelineAppTest.scala
----------------------------------------------------------------------
diff --git a/scrunch/src/it/scala/org/apache/scrunch/PipelineAppTest.scala b/scrunch/src/it/scala/org/apache/scrunch/PipelineAppTest.scala
index 7111c7b..c61e7d1 100644
--- a/scrunch/src/it/scala/org/apache/scrunch/PipelineAppTest.scala
+++ b/scrunch/src/it/scala/org/apache/scrunch/PipelineAppTest.scala
@@ -15,8 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import org.apache.crunch.test.FileHelper
-import org.apache.scrunch.PipelineApp
+package org.apache.scrunch
 
 import org.scalatest.junit.JUnitSuite
 import _root_.org.junit.Test
@@ -36,12 +35,12 @@ object WordCount extends PipelineApp {
   cogroup(w1, w2).write(to.textFile(args(2)))
 }
 
-class PipelineAppTest extends JUnitSuite {
+class PipelineAppTest extends ScrunchTestSupport with JUnitSuite {
   @Test def run {
     val args = new Array[String](3)
-    args(0) = FileHelper.createTempCopyOf("shakes.txt")
-    args(1) = FileHelper.createTempCopyOf("maugham.txt")
-    args(2) = FileHelper.createOutputPath.getAbsolutePath
+    args(0) = tempDir.copyResourceFileName("shakes.txt")
+    args(1) = tempDir.copyResourceFileName("maugham.txt")
+    args(2) = tempDir.getFileName("output")
     WordCount.main(args)
   }
 }

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5b9d3ce1/scrunch/src/it/scala/org/apache/scrunch/TopTest.scala
----------------------------------------------------------------------
diff --git a/scrunch/src/it/scala/org/apache/scrunch/TopTest.scala b/scrunch/src/it/scala/org/apache/scrunch/TopTest.scala
index bac27bd..9c40b5c 100644
--- a/scrunch/src/it/scala/org/apache/scrunch/TopTest.scala
+++ b/scrunch/src/it/scala/org/apache/scrunch/TopTest.scala
@@ -18,12 +18,11 @@
 package org.apache.scrunch
 
 import org.apache.crunch.io.{From => from, To => to}
-import org.apache.crunch.test.FileHelper
 
 import org.scalatest.junit.JUnitSuite
 import _root_.org.junit.Test
 
-class TopTest extends JUnitSuite {
+class TopTest extends ScrunchTestSupport with JUnitSuite {
 
   @Test def topInMem {
     val ptable = Mem.tableOf(("foo", 17), ("bar", 29), ("baz", 1729))
@@ -31,8 +30,8 @@ class TopTest extends JUnitSuite {
   }
 
   @Test def top2 {
-    val pipeline = Pipeline.mapReduce[TopTest]
-    val input = FileHelper.createTempCopyOf("shakes.txt")
+    val pipeline = Pipeline.mapReduce[TopTest](tempDir.getDefaultConfiguration)
+    val input = tempDir.copyResourceFileName("shakes.txt")
 
     val wc = pipeline.read(from.textFile(input))
         .flatMap(_.toLowerCase.split("\\s+"))

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5b9d3ce1/scrunch/src/it/scala/org/apache/scrunch/UnionTest.scala
----------------------------------------------------------------------
diff --git a/scrunch/src/it/scala/org/apache/scrunch/UnionTest.scala b/scrunch/src/it/scala/org/apache/scrunch/UnionTest.scala
index 63fecdb..fa88edc 100644
--- a/scrunch/src/it/scala/org/apache/scrunch/UnionTest.scala
+++ b/scrunch/src/it/scala/org/apache/scrunch/UnionTest.scala
@@ -18,15 +18,14 @@
 package org.apache.scrunch
 
 import org.apache.crunch.io.{From => from}
-import org.apache.crunch.test.FileHelper
 
 import org.scalatest.junit.JUnitSuite
 import _root_.org.junit.Test
 
-class UnionTest extends JUnitSuite {
-  val pipeline = Pipeline.mapReduce[UnionTest]
-  val shakespeare = FileHelper.createTempCopyOf("shakes.txt")
-  val maugham = FileHelper.createTempCopyOf("maugham.txt")
+class UnionTest extends ScrunchTestSupport with JUnitSuite {
+  val pipeline = Pipeline.mapReduce[UnionTest](tempDir.getDefaultConfiguration)
+  val shakespeare = tempDir.copyResourceFileName("shakes.txt")
+  val maugham = tempDir.copyResourceFileName("maugham.txt")
 
   def wordCount(col: PCollection[String]) = {
     col.flatMap(_.toLowerCase.split("\\W+")).count

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/5b9d3ce1/scrunch/src/it/scala/org/apache/scrunch/WordCountTest.scala
----------------------------------------------------------------------
diff --git a/scrunch/src/it/scala/org/apache/scrunch/WordCountTest.scala b/scrunch/src/it/scala/org/apache/scrunch/WordCountTest.scala
index e97a1fd..f8081ea 100644
--- a/scrunch/src/it/scala/org/apache/scrunch/WordCountTest.scala
+++ b/scrunch/src/it/scala/org/apache/scrunch/WordCountTest.scala
@@ -18,28 +18,24 @@
 package org.apache.scrunch
 
 import org.apache.crunch.io.{From => from, To => to}
-import org.apache.crunch.test.FileHelper
-
-import java.io.File
 
 import org.scalatest.junit.JUnitSuite
 import _root_.org.junit.Test
 
-class WordCountTest extends JUnitSuite {
+class WordCountTest extends ScrunchTestSupport with JUnitSuite {
   @Test def wordCount {
-    val pipeline = Pipeline.mapReduce[WordCountTest]
-    val input = FileHelper.createTempCopyOf("shakes.txt")
-    val wordCountOut = FileHelper.createOutputPath
+    val pipeline = Pipeline.mapReduce[WordCountTest](tempDir.getDefaultConfiguration)
+    val input = tempDir.copyResourceFileName("shakes.txt")
+    val wordCountOut = tempDir.getFileName("output")
 
     val fcc = pipeline.read(from.textFile(input))
         .flatMap(_.toLowerCase.split("\\s+"))
         .filter(!_.isEmpty()).count
-        .write(to.textFile(wordCountOut.getAbsolutePath)) // Word counts
+        .write(to.textFile(wordCountOut)) // Word counts
         .map((w, c) => (w.slice(0, 1), c))
         .groupByKey.combine(v => v.sum).materialize
     assert(fcc.exists(_ == ("w", 1404)))
 
     pipeline.done
-    wordCountOut.delete()
   }
 }


Mime
View raw message