crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mkw...@apache.org
Subject git commit: CRUNCH-225: Added support for building using Scala 2.10 and 2.9. Also removed unused build.sbt file
Date Thu, 14 Nov 2013 03:00:27 GMT
Updated Branches:
  refs/heads/master a787428ef -> 79e6c896b


CRUNCH-225: Added support for building using Scala 2.10 and 2.9. Also removed unused build.sbt
file

Signed-off-by: Micah Whitacre <mkwhit@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/79e6c896
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/79e6c896
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/79e6c896

Branch: refs/heads/master
Commit: 79e6c896bbe0dd5fb57b5358d3f0600657804bb9
Parents: a787428
Author: Micah Whitacre <mkwhit@apache.org>
Authored: Wed Nov 13 20:22:58 2013 -0600
Committer: Micah Whitacre <mkwhit@apache.org>
Committed: Wed Nov 13 20:29:03 2013 -0600

----------------------------------------------------------------------
 crunch-scrunch/build.sbt                        | 50 --------------------
 crunch-scrunch/pom.xml                          | 46 +++++++++++++++---
 .../org/apache/crunch/scrunch/CogroupTest.scala |  2 +-
 .../org/apache/crunch/scrunch/CrunchSuite.scala | 39 +++++++++++++++
 .../org/apache/crunch/scrunch/JoinTest.scala    |  2 +-
 .../apache/crunch/scrunch/PCollectionTest.scala |  2 +-
 .../org/apache/crunch/scrunch/PTableTest.scala  |  2 +-
 .../crunch/scrunch/PageRankClassTest.scala      |  2 +-
 .../apache/crunch/scrunch/PageRankTest.scala    |  2 +-
 .../crunch/scrunch/PipelineAppClosureTest.scala |  2 +-
 .../apache/crunch/scrunch/PipelineAppTest.scala |  2 +-
 .../org/apache/crunch/scrunch/TopTest.scala     |  2 +-
 .../org/apache/crunch/scrunch/UnionTest.scala   |  2 +-
 .../apache/crunch/scrunch/WordCountTest.scala   |  2 +-
 .../interpreter/InterpreterJarTest.scala        |  3 +-
 .../org/apache/crunch/test/TemporaryPath.java   |  8 ++++
 pom.xml                                         |  6 ---
 17 files changed, 100 insertions(+), 74 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/build.sbt
----------------------------------------------------------------------
diff --git a/crunch-scrunch/build.sbt b/crunch-scrunch/build.sbt
deleted file mode 100644
index 21b52ce..0000000
--- a/crunch-scrunch/build.sbt
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-name := "scrunch"
-
-version := "0.3.0"
-
-scalaVersion := "2.9.2"
-
-resolvers ++= Seq(
-  "Local Maven Repository" at "file://"+Path.userHome.absolutePath+"/.m2/repository",
-)
-
-libraryDependencies ++= Seq(
-  "org.apache.crunch" % "crunch" % "0.3.0" excludeAll(
-    ExclusionRule(organization = "com.sun.jdmk"),
-    ExclusionRule(organization = "com.sun.jmx"),
-    ExclusionRule(organization = "javax.jms")
-  ),
-  "org.apache.hadoop" % "hadoop-client" % "1.0.3" % "provided" excludeAll(
-    ExclusionRule(organization = "com.sun.jdmk"),
-    ExclusionRule(organization = "com.sun.jmx"),
-    ExclusionRule(organization = "javax.jms")
-  ),
-  "org.apache.hbase" % "hbase" % "0.90.4" % "provided" excludeAll(
-    ExclusionRule(organization = "org.apache.hadoop"),
-    ExclusionRule(organization = "commons-logging"),
-    ExclusionRule(organization = "com.google.guava"),
-    ExclusionRule(organization = "log4j"),
-    ExclusionRule(organization = "org.slf4j")
-  ),
-  "junit" % "junit" % "4.8.1" % "test",
-  "org.scalatest" % "scalatest_2.9.2" % "1.7.2" % "test"
-)
-
-parallelExecution in Test := false

http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-scrunch/pom.xml b/crunch-scrunch/pom.xml
index 10c73c9..b6d7b54 100644
--- a/crunch-scrunch/pom.xml
+++ b/crunch-scrunch/pom.xml
@@ -60,11 +60,6 @@ under the License.
       <artifactId>slf4j-log4j12</artifactId>
     </dependency>
     <dependency>
-      <groupId>org.scalatest</groupId>
-      <artifactId>scalatest_${scala.version}</artifactId>
-      <scope>test</scope>
-    </dependency>
-    <dependency>
       <groupId>org.apache.crunch</groupId>
       <artifactId>crunch-test</artifactId>
       <scope>test</scope>
@@ -209,6 +204,45 @@ under the License.
         </plugins>
       </build>
     </profile>
+      <profile>
+          <id>scala-2.9.2</id>
+          <activation>
+              <activeByDefault>true</activeByDefault>
+          </activation>
+          <properties>
+              <scala.version>2.9.2</scala.version>
+              <scalatest.version>1.7.2</scalatest.version>
+          </properties>
+          <dependencies>
+              <dependency>
+                  <groupId>org.scalatest</groupId>
+                  <artifactId>scalatest_${scala.version}</artifactId>
+              </dependency>
+          </dependencies>
+          <dependencyManagement>
+              <dependencies>
+                  <dependency>
+                      <groupId>org.scalatest</groupId>
+                      <artifactId>scalatest_${scala.version}</artifactId>
+                      <version>${scalatest.version}</version>
+                  </dependency>
+              </dependencies>
+          </dependencyManagement>
+      </profile>
+    <profile>
+        <id>scala-2.10</id>
+        <properties>
+            <scala.version>2.10.2</scala.version>
+            <scalatest.version>1.9.2</scalatest.version>
+        </properties>
+        <dependencies>
+            <dependency>
+                <groupId>org.scalatest</groupId>
+                <artifactId>scalatest_2.10</artifactId>
+                <version>${scalatest.version}</version>
+                <scope>test</scope>
+            </dependency>
+        </dependencies>
+    </profile>
   </profiles>
-
 </project>

http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/CogroupTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/CogroupTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/CogroupTest.scala
index 170b8c9..40df05b 100644
--- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/CogroupTest.scala
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/CogroupTest.scala
@@ -23,7 +23,7 @@ import org.apache.crunch.test.CrunchTestSupport
 import org.scalatest.junit.JUnitSuite
 import _root_.org.junit.Test
 
-class CogroupTest extends CrunchTestSupport with JUnitSuite {
+class CogroupTest extends CrunchSuite {
   lazy val pipeline = Pipeline.mapReduce[CogroupTest](tempDir.getDefaultConfiguration)
 
   def wordCount(fileName: String) = {

http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/CrunchSuite.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/CrunchSuite.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/CrunchSuite.scala
new file mode 100644
index 0000000..37a4fe2
--- /dev/null
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/CrunchSuite.scala
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.crunch.scrunch
+
+import org.scalatest.junit.JUnitSuite
+import org.junit.{After, Before, Rule}
+import org.apache.crunch.test.TemporaryPath
+
+class CrunchSuite extends JUnitSuite{
+
+  val tempDir = new TemporaryPath("crunch.tmp.dir", "hadoop.tmp.dir");
+
+  def getFolder() = {
+    tempDir
+  }
+
+  @Before def initialize() {
+    tempDir.create()
+  }
+
+  @After def cleanup() {
+    tempDir.delete()
+  }
+}

http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/JoinTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/JoinTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/JoinTest.scala
index 0770ba3..bd640eb 100644
--- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/JoinTest.scala
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/JoinTest.scala
@@ -23,7 +23,7 @@ import org.apache.crunch.test.CrunchTestSupport
 import org.scalatest.junit.JUnitSuite
 import _root_.org.junit.Test
 
-class JoinTest extends CrunchTestSupport with JUnitSuite {
+class JoinTest extends CrunchSuite {
   lazy val pipeline = Pipeline.mapReduce[CogroupTest](tempDir.getDefaultConfiguration)
 
   def wordCount(fileName: String) = {

http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PCollectionTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PCollectionTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PCollectionTest.scala
index 94ac917..6d42ed6 100644
--- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PCollectionTest.scala
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PCollectionTest.scala
@@ -27,7 +27,7 @@ import org.scalatest.junit.JUnitSuite
 /**
  * Tests functionality of Scala PCollection.
  */
-class PCollectionTest extends CrunchTestSupport with JUnitSuite {
+class PCollectionTest extends CrunchSuite {
 
   // Number of lines in the Shakespeare data set.
   val linesInShakespeare: Int = 3667

http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PTableTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PTableTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PTableTest.scala
index cd644b1..35b2a8d 100644
--- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PTableTest.scala
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PTableTest.scala
@@ -27,7 +27,7 @@ import org.scalatest.junit.JUnitSuite
 /**
  * Tests functionality of Scala PTable.
  */
-class PTableTest extends CrunchTestSupport with JUnitSuite {
+class PTableTest extends CrunchSuite {
 
   /**
    * Gets a PCollection containing the lines from the tens data set.

http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala
b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala
index 847d65b..55e7783 100644
--- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankClassTest.scala
@@ -61,7 +61,7 @@ class CachingPageRankClassFn extends DoFn[P[String, PageRankData], P[String,
Flo
   }
 }
 
-class PageRankClassTest extends CrunchTestSupport with JUnitSuite {
+class PageRankClassTest extends CrunchSuite {
 
   lazy val pipeline = Pipeline.mapReduce[PageRankTest](tempDir.getDefaultConfiguration)
 

http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankTest.scala
index 02e3d82..833fce8 100644
--- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankTest.scala
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PageRankTest.scala
@@ -49,7 +49,7 @@ class CachingPageRankFn extends DoFn[P[String, (Float, Float, List[String])],
P[
   }
 }
 
-class PageRankTest extends CrunchTestSupport with JUnitSuite {
+class PageRankTest extends CrunchSuite {
   lazy val pipeline = Pipeline.mapReduce[PageRankTest](tempDir.getDefaultConfiguration)
 
   def initialInput(fileName: String) = {

http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppClosureTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppClosureTest.scala
b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppClosureTest.scala
index e0702e4..4666c3c 100644
--- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppClosureTest.scala
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppClosureTest.scala
@@ -27,7 +27,7 @@ import _root_.org.junit.Test
  * Test that verifies that a Scala PipelineApp can properly send some side data as part of
a
  * function closure.
  */
-class PipelineClosureAppTest extends CrunchTestSupport with JUnitSuite {
+class PipelineClosureAppTest extends CrunchSuite {
 
   /**
    * A simple pipeline application that divides each element of a PCollection of numbers
by

http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppTest.scala
index db49c92..fffd529 100644
--- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppTest.scala
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/PipelineAppTest.scala
@@ -39,7 +39,7 @@ object WordCount extends PipelineApp {
   }
 }
 
-class PipelineAppTest extends CrunchTestSupport with JUnitSuite {
+class PipelineAppTest extends CrunchSuite {
   @Test def run {
     val args = new Array[String](3)
     args(0) = tempDir.copyResourceFileName("shakes.txt")

http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/TopTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/TopTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/TopTest.scala
index 0c7a992..ca728e8 100644
--- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/TopTest.scala
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/TopTest.scala
@@ -23,7 +23,7 @@ import org.apache.crunch.test.CrunchTestSupport
 import org.scalatest.junit.JUnitSuite
 import _root_.org.junit.Test
 
-class TopTest extends CrunchTestSupport with JUnitSuite {
+class TopTest extends CrunchSuite {
 
   @Test def topInMem {
     val ptable = Mem.tableOf(("foo", 17), ("bar", 29), ("baz", 1729))

http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/UnionTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/UnionTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/UnionTest.scala
index e8b3fab..a507e37 100644
--- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/UnionTest.scala
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/UnionTest.scala
@@ -23,7 +23,7 @@ import org.apache.crunch.test.CrunchTestSupport
 import org.scalatest.junit.JUnitSuite
 import _root_.org.junit.Test
 
-class UnionTest extends CrunchTestSupport with JUnitSuite {
+class UnionTest extends CrunchSuite {
   lazy val pipeline = Pipeline.mapReduce[UnionTest](tempDir.getDefaultConfiguration)
 
   def wordCount(col: PCollection[String]) = {

http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/WordCountTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/WordCountTest.scala b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/WordCountTest.scala
index a2560d1..2cc1457 100644
--- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/WordCountTest.scala
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/WordCountTest.scala
@@ -23,7 +23,7 @@ import org.apache.crunch.test.CrunchTestSupport
 import org.scalatest.junit.JUnitSuite
 import _root_.org.junit.Test
 
-class WordCountTest extends CrunchTestSupport with JUnitSuite {
+class WordCountTest extends CrunchSuite {
   @Test def wordCount {
     val pipeline = Pipeline.mapReduce[WordCountTest](tempDir.getDefaultConfiguration)
     val input = tempDir.copyResourceFileName("shakes.txt")

http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/interpreter/InterpreterJarTest.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/interpreter/InterpreterJarTest.scala
b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/interpreter/InterpreterJarTest.scala
index 48aae00..5ebc303 100644
--- a/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/interpreter/InterpreterJarTest.scala
+++ b/crunch-scrunch/src/it/scala/org/apache/crunch/scrunch/interpreter/InterpreterJarTest.scala
@@ -29,11 +29,12 @@ import org.junit.Assert.assertNotNull
 import org.junit.Test
 import org.apache.crunch.test.CrunchTestSupport
 import org.scalatest.junit.JUnitSuite
+import org.apache.crunch.scrunch.CrunchSuite
 
 /**
  * Tests creating jars from a {@link scala.tools.nsc.io.VirtualDirectory}.
  */
-class InterpreterJarTest extends CrunchTestSupport with JUnitSuite {
+class InterpreterJarTest extends CrunchSuite {
 
   /**
    * Tests transforming a virtual directory into a temporary jar file.

http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/crunch-test/src/main/java/org/apache/crunch/test/TemporaryPath.java
----------------------------------------------------------------------
diff --git a/crunch-test/src/main/java/org/apache/crunch/test/TemporaryPath.java b/crunch-test/src/main/java/org/apache/crunch/test/TemporaryPath.java
index 3cd175f..1d25fc7 100644
--- a/crunch-test/src/main/java/org/apache/crunch/test/TemporaryPath.java
+++ b/crunch-test/src/main/java/org/apache/crunch/test/TemporaryPath.java
@@ -59,6 +59,14 @@ public final class TemporaryPath extends ExternalResource {
     }
   }
 
+  public void create() throws Throwable {
+    tmp.create();
+  }
+
+  public void delete() {
+    tmp.delete();
+  }
+
   @Override
   public Statement apply(Statement base, Description description) {
     return tmp.apply(base, description);

http://git-wip-us.apache.org/repos/asf/crunch/blob/79e6c896/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 338b4f3..7e288d3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -352,12 +352,6 @@ under the License.
         <artifactId>jline</artifactId>
         <version>${scala.version}</version>
       </dependency>
-
-      <dependency>
-        <groupId>org.scalatest</groupId>
-        <artifactId>scalatest_${scala.version}</artifactId>
-        <version>${scalatest.version}</version>
-      </dependency>
     </dependencies>
     
   </dependencyManagement>


Mime
View raw message