incubator-crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject [2/9] git commit: CRUNCH-16: Fix dependency versions.
Date Wed, 22 Aug 2012 06:26:48 GMT
CRUNCH-16: Fix dependency versions.

Move all dependency versions to properties for easier management.
Align Crunch's dependencies with those on Hadoop's classpath.
Remove Scrunch's last non-test dependency on commons-io.
Add HBase warning note to README.

Signed-off-by: Josh Wills <jwills@cloudera.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-crunch/commit/9d8b679e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-crunch/tree/9d8b679e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-crunch/diff/9d8b679e

Branch: refs/heads/master
Commit: 9d8b679ec0bbd007e65c33f2018b2c9630cfc700
Parents: 5d9b698
Author: Matthias Friedrich <matt@mafr.de>
Authored: Thu Aug 16 20:47:19 2012 +0200
Committer: Josh Wills <jwills@cloudera.com>
Committed: Tue Aug 21 19:46:28 2012 -0700

----------------------------------------------------------------------
 README                                             |    6 ++
 crunch-hbase/pom.xml                               |    7 ++
 .../scrunch/interpreter/InterpreterRunner.scala    |    4 +-
 crunch-test/pom.xml                                |    6 ++
 crunch/pom.xml                                     |   11 ++-
 pom.xml                                            |   59 ++++++++++-----
 6 files changed, 68 insertions(+), 25 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9d8b679e/README
----------------------------------------------------------------------
diff --git a/README b/README
index d061cab..b80235d 100644
--- a/README
+++ b/README
@@ -24,3 +24,9 @@ local Maven cache:
 
   mvn install
 
+Crunch has experimental support for Hadoop 2 through the "hadoop-2" build
+profile (add -Phadoop-2 to enable it). If you want to use HBase support on
+Hadoop 2, please note that you have to build HBase 0.94.1 from source using
+the following command:
+
+  mvn clean install -Dhadoop.profile=2.0

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9d8b679e/crunch-hbase/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-hbase/pom.xml b/crunch-hbase/pom.xml
index 3bd6e54..d97b457 100644
--- a/crunch-hbase/pom.xml
+++ b/crunch-hbase/pom.xml
@@ -68,6 +68,13 @@ under the License.
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
+
+    <dependency>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
+      <scope>test</scope>
+    </dependency>
+
   </dependencies>
 
   <build>

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9d8b679e/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/interpreter/InterpreterRunner.scala
----------------------------------------------------------------------
diff --git a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/interpreter/InterpreterRunner.scala
b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/interpreter/InterpreterRunner.scala
index 18cb201..196e222 100644
--- a/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/interpreter/InterpreterRunner.scala
+++ b/crunch-scrunch/src/main/scala/org/apache/crunch/scrunch/interpreter/InterpreterRunner.scala
@@ -33,7 +33,7 @@ import scala.tools.nsc.io.Jar
 import scala.tools.nsc.io.VirtualDirectory
 
 import com.google.common.io.Files
-import org.apache.commons.io.IOUtils
+import com.google.common.io.Closeables
 import org.apache.hadoop.conf.Configuration
 
 import org.apache.crunch.util.DistCache
@@ -153,7 +153,7 @@ object InterpreterRunner extends MainGenericRunner {
       addVirtualDirectoryToJar(virtualDirectory, "", jarStream)
       return tempJar
     } finally {
-      IOUtils.closeQuietly(jarStream)
+      Closeables.closeQuietly(jarStream)
     }
   }
 

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9d8b679e/crunch-test/pom.xml
----------------------------------------------------------------------
diff --git a/crunch-test/pom.xml b/crunch-test/pom.xml
index 8a2f019..a1e3899 100644
--- a/crunch-test/pom.xml
+++ b/crunch-test/pom.xml
@@ -48,6 +48,12 @@ under the License.
      <artifactId>hadoop-minicluster</artifactId>
     </dependency>
 
+    <!-- Required by LocalJobRunner -->
+    <dependency>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9d8b679e/crunch/pom.xml
----------------------------------------------------------------------
diff --git a/crunch/pom.xml b/crunch/pom.xml
index 20c75b7..07951ec 100644
--- a/crunch/pom.xml
+++ b/crunch/pom.xml
@@ -52,23 +52,21 @@ under the License.
     </dependency>
 
     <dependency>
-      <groupId>commons-io</groupId>
-      <artifactId>commons-io</artifactId>
-    </dependency>
-
-    <dependency>
       <groupId>commons-codec</groupId>
       <artifactId>commons-codec</artifactId>
+      <scope>provided</scope>
     </dependency>
 
     <dependency>
       <groupId>org.codehaus.jackson</groupId>
       <artifactId>jackson-core-asl</artifactId>
+      <scope>provided</scope>
     </dependency>
 
     <dependency>
       <groupId>org.codehaus.jackson</groupId>
       <artifactId>jackson-mapper-asl</artifactId>
+      <scope>provided</scope>
     </dependency>
 
     <!-- Both Protobufs and Thrift are supported as
@@ -109,16 +107,19 @@ under the License.
     <dependency>
       <groupId>commons-logging</groupId>
       <artifactId>commons-logging</artifactId>
+      <scope>provided</scope>
     </dependency>
    
     <dependency>
       <groupId>log4j</groupId>
       <artifactId>log4j</artifactId>
+      <scope>provided</scope>
     </dependency>
 
     <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-log4j12</artifactId>
+      <scope>provided</scope>
     </dependency>
 
   </dependencies>

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/9d8b679e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e397663..1c5b4c7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -57,8 +57,28 @@ under the License.
     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
     <java.source.version>1.6</java.source.version>
     <java.target.version>1.6</java.target.version>
-    <scala.version>2.9.2</scala.version>
+    <!--
+        NOTE: Please don't change any dependency versions here without
+              checking first that they match those on Hadoop's runtime
+              classpath.
+    -->
+    <guava.version>11.0.2</guava.version>
+    <commons-io.version>2.1</commons-io.version>
+    <commons-lang.version>2.4</commons-lang.version>
+    <commons-codec.version>1.4</commons-codec.version>
+    <commons-httpclient.version>3.0.1</commons-httpclient.version>
+    <commons-logging.version>1.1.1</commons-logging.version>
     <avro.version>1.7.0</avro.version>
+    <jackson.version>1.8.8</jackson.version>
+    <protobuf-java.version>2.3.0</protobuf-java.version>
+    <libthrift.version>0.8.0</libthrift.version>
+    <slf4j.version>1.4.3</slf4j.version>
+    <log4j.version>1.2.15</log4j.version>
+    <junit.version>4.10</junit.version>
+    <hamcrest.version>1.1</hamcrest.version>
+    <mockito.version>1.9.0</mockito.version>
+    <scala.version>2.9.2</scala.version>
+    <scalatest.version>1.7.2</scalatest.version>
   </properties>
 
   <scm>
@@ -119,7 +139,7 @@ under the License.
       <dependency>
         <groupId>com.google.guava</groupId>
         <artifactId>guava</artifactId>
-        <version>11.0.2</version>
+        <version>${guava.version}</version>
       </dependency>
 
       <dependency>
@@ -137,79 +157,79 @@ under the License.
       <dependency>
         <groupId>commons-io</groupId>
         <artifactId>commons-io</artifactId>
-        <version>2.2</version>
+        <version>${commons-io.version}</version>
       </dependency>
 
       <dependency>
         <groupId>commons-codec</groupId>
         <artifactId>commons-codec</artifactId>
-        <version>1.4</version>
+        <version>${commons-codec.version}</version>
       </dependency>
 
       <dependency>
         <groupId>commons-lang</groupId>
         <artifactId>commons-lang</artifactId>
-        <version>2.4</version>
+        <version>${commons-lang.version}</version>
       </dependency>
 
       <dependency>
         <groupId>commons-httpclient</groupId>
         <artifactId>commons-httpclient</artifactId>
-        <version>3.0.1</version>
+        <version>${commons-httpclient.version}</version>
       </dependency>
 
       <dependency>
         <groupId>org.codehaus.jackson</groupId>
         <artifactId>jackson-core-asl</artifactId>
-        <version>1.8.3</version>
+        <version>${jackson.version}</version>
       </dependency>
 
       <dependency>
         <groupId>org.codehaus.jackson</groupId>
         <artifactId>jackson-mapper-asl</artifactId>
-        <version>1.8.3</version>
+        <version>${jackson.version}</version>
       </dependency>
 
       <dependency>
         <groupId>com.google.protobuf</groupId>
         <artifactId>protobuf-java</artifactId>
-        <version>2.4.1</version>
+        <version>${protobuf-java.version}</version>
       </dependency>
 
       <dependency>
         <groupId>org.apache.thrift</groupId>
         <artifactId>libthrift</artifactId>
-        <version>0.8.0</version>
+        <version>${libthrift.version}</version>
       </dependency>
 
       <dependency>
         <groupId>junit</groupId>
         <artifactId>junit</artifactId>
-        <version>4.10</version>
+        <version>${junit.version}</version>
       </dependency>
     
       <dependency>
         <groupId>org.mockito</groupId>
         <artifactId>mockito-all</artifactId>
-        <version>1.9.0</version>
+        <version>${mockito.version}</version>
       </dependency>
            
       <dependency>
         <groupId>org.hamcrest</groupId>
         <artifactId>hamcrest-all</artifactId>
-        <version>1.1</version>
+        <version>${hamcrest.version}</version>
       </dependency>
 
       <dependency>
         <groupId>commons-logging</groupId>
         <artifactId>commons-logging</artifactId>
-        <version>1.1.1</version>
+        <version>${commons-logging.version}</version>
       </dependency>
    
       <dependency>
         <groupId>log4j</groupId>
         <artifactId>log4j</artifactId>
-        <version>1.2.15</version>
+        <version>${log4j.version}</version>
         <exclusions>
           <exclusion>
             <groupId>javax.mail</groupId>
@@ -233,13 +253,13 @@ under the License.
       <dependency>
         <groupId>org.slf4j</groupId>
         <artifactId>slf4j-api</artifactId>
-        <version>1.4.3</version>
+        <version>${slf4j.version}</version>
       </dependency>
 
       <dependency>
         <groupId>org.slf4j</groupId>
         <artifactId>slf4j-log4j12</artifactId>
-        <version>1.4.3</version>
+        <version>${slf4j.version}</version>
       </dependency>
 
       <dependency>
@@ -263,7 +283,7 @@ under the License.
       <dependency>
         <groupId>org.scalatest</groupId>
         <artifactId>scalatest_${scala.version}</artifactId>
-        <version>1.7.2</version>
+        <version>${scalatest.version}</version>
       </dependency>
     </dependencies>
     
@@ -357,6 +377,9 @@ under the License.
              mvn clean install -Dhadoop.profile=2.0
              for this to work. -->
         <hbase.version>0.94.1</hbase.version>
+        <commons-lang.version>2.5</commons-lang.version>
+        <protobuf-java.version>2.4.0a</protobuf-java.version>
+        <slf4j.version>1.6.1</slf4j.version>
       </properties>
       <dependencyManagement>
         <dependencies>


Mime
View raw message