Return-Path: X-Original-To: apmail-incubator-crunch-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-crunch-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id EF715D551 for ; Mon, 13 Aug 2012 01:33:51 +0000 (UTC) Received: (qmail 96468 invoked by uid 500); 13 Aug 2012 01:33:51 -0000 Delivered-To: apmail-incubator-crunch-commits-archive@incubator.apache.org Received: (qmail 96439 invoked by uid 500); 13 Aug 2012 01:33:51 -0000 Mailing-List: contact crunch-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: crunch-dev@incubator.apache.org Delivered-To: mailing list crunch-commits@incubator.apache.org Received: (qmail 96383 invoked by uid 99); 13 Aug 2012 01:33:51 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 13 Aug 2012 01:33:51 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 50C181CBA4; Mon, 13 Aug 2012 01:33:51 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jwills@apache.org To: crunch-commits@incubator.apache.org X-Mailer: ASF-Git Admin Mailer Subject: [2/3] git commit: CRUNCH-4: Create profiles for hadoop 1.0.3 and hadoop 2.0.0-alpha Message-Id: <20120813013351.50C181CBA4@tyr.zones.apache.org> Date: Mon, 13 Aug 2012 01:33:51 +0000 (UTC) CRUNCH-4: Create profiles for hadoop 1.0.3 and hadoop 2.0.0-alpha Project: http://git-wip-us.apache.org/repos/asf/incubator-crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-crunch/commit/629f5734 Tree: http://git-wip-us.apache.org/repos/asf/incubator-crunch/tree/629f5734 Diff: http://git-wip-us.apache.org/repos/asf/incubator-crunch/diff/629f5734 Branch: refs/heads/master Commit: 629f57342bb8fb41b1484de7869c90509fdbfb9d Parents: 38d7b3a Author: jwills Authored: Fri Jul 27 11:14:11 2012 -0700 Committer: jwills Committed: Sun Aug 12 17:24:29 2012 -0700 ---------------------------------------------------------------------- .../java/org/apache/crunch/WordCountHBaseIT.java | 13 +- .../it/java/org/apache/crunch/lib/AggregateIT.java | 4 +- crunch/src/it/resources/log4j.properties | 2 + .../org/apache/crunch/lib/join/MapsideJoin.java | 2 +- pom.xml | 213 ++++++++++----- 5 files changed, 162 insertions(+), 72 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/629f5734/crunch/src/it/java/org/apache/crunch/WordCountHBaseIT.java ---------------------------------------------------------------------- diff --git a/crunch/src/it/java/org/apache/crunch/WordCountHBaseIT.java b/crunch/src/it/java/org/apache/crunch/WordCountHBaseIT.java index f7c9836..b7531b9 100644 --- a/crunch/src/it/java/org/apache/crunch/WordCountHBaseIT.java +++ b/crunch/src/it/java/org/apache/crunch/WordCountHBaseIT.java @@ -98,6 +98,7 @@ public class WordCountHBaseIT { conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1"); conf.setInt("hbase.master.info.port", -1); conf.setInt("hbase.regionserver.info.port", -1); + conf.set("dfs.datanode.data.dir.perm", "775"); hbaseTestUtil.startMiniZKCluster(); hbaseTestUtil.startMiniCluster(); @@ -116,16 +117,16 @@ public class WordCountHBaseIT { } // Create a programmatic container for this jar. - JarOutputStream jos = new JarOutputStream(new FileOutputStream("WordCountHBaseTest.jar")); + JarOutputStream jos = new JarOutputStream(new FileOutputStream("WordCountHBaseIT.jar")); File baseDir = new File("target/test-classes"); String prefix = "org/apache/crunch/"; - jarUp(jos, baseDir, prefix + "WordCountHBaseTest.class"); - jarUp(jos, baseDir, prefix + "WordCountHBaseTest$1.class"); - jarUp(jos, baseDir, prefix + "WordCountHBaseTest$2.class"); + jarUp(jos, baseDir, prefix + "WordCountHBaseIT.class"); + jarUp(jos, baseDir, prefix + "WordCountHBaseIT$1.class"); + jarUp(jos, baseDir, prefix + "WordCountHBaseIT$2.class"); jos.close(); - Path target = new Path(tmpPath, "WordCountHBaseTest.jar"); - fs.copyFromLocalFile(true, new Path("WordCountHBaseTest.jar"), target); + Path target = new Path(tmpPath, "WordCountHBaseIT.jar"); + fs.copyFromLocalFile(true, new Path("WordCountHBaseIT.jar"), target); DistributedCache.addFileToClassPath(target, conf, fs); } } http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/629f5734/crunch/src/it/java/org/apache/crunch/lib/AggregateIT.java ---------------------------------------------------------------------- diff --git a/crunch/src/it/java/org/apache/crunch/lib/AggregateIT.java b/crunch/src/it/java/org/apache/crunch/lib/AggregateIT.java index 0202d09..4f47d8f 100644 --- a/crunch/src/it/java/org/apache/crunch/lib/AggregateIT.java +++ b/crunch/src/it/java/org/apache/crunch/lib/AggregateIT.java @@ -148,7 +148,7 @@ public class AggregateIT { assertEquals(1, collectionMap.size()); - assertEquals(Lists.newArrayList(new Text("c"), new Text("d"), new Text("a")), collectionMap.get(1)); + assertTrue(collectionMap.get(1).containsAll(Lists.newArrayList(new Text("c"), new Text("d"), new Text("a")))); } @Test @@ -166,7 +166,7 @@ public class AggregateIT { Employee empD = mapFn.map("d").second(); Employee empA = mapFn.map("a").second(); - assertEquals(Lists.newArrayList(empC, empD, empA), collectionMap.get(1)); + assertTrue(collectionMap.get(1).containsAll(Lists.newArrayList(empC, empD, empA))); } private static class MapStringToTextPair extends MapFn> { http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/629f5734/crunch/src/it/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/crunch/src/it/resources/log4j.properties b/crunch/src/it/resources/log4j.properties index a04cec8..5d144a0 100644 --- a/crunch/src/it/resources/log4j.properties +++ b/crunch/src/it/resources/log4j.properties @@ -19,6 +19,8 @@ log4j.logger.org.apache.crunch=info, A # Log warnings on Hadoop for the local runner when testing log4j.logger.org.apache.hadoop=warn, A +# Except for Configuration, which is chatty. +log4j.logger.org.apache.hadoop.conf.Configuration=error, A # ***** A is set to be a ConsoleAppender. log4j.appender.A=org.apache.log4j.ConsoleAppender http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/629f5734/crunch/src/main/java/org/apache/crunch/lib/join/MapsideJoin.java ---------------------------------------------------------------------- diff --git a/crunch/src/main/java/org/apache/crunch/lib/join/MapsideJoin.java b/crunch/src/main/java/org/apache/crunch/lib/join/MapsideJoin.java index 226ad90..0ca1ab3 100644 --- a/crunch/src/main/java/org/apache/crunch/lib/join/MapsideJoin.java +++ b/crunch/src/main/java/org/apache/crunch/lib/join/MapsideJoin.java @@ -88,7 +88,7 @@ public class MapsideJoin { Path path = sourcePathTarget.getPath(); DistributedCache.addCacheFile(path.toUri(), pipeline.getConfiguration()); - MapsideJoinDoFn mapJoinDoFn = new MapsideJoinDoFn(path.toString(), right.getPType()); + MapsideJoinDoFn mapJoinDoFn = new MapsideJoinDoFn(path.getName(), right.getPType()); PTypeFamily typeFamily = left.getTypeFamily(); return left.parallelDo("mapjoin", mapJoinDoFn, typeFamily.tableOf(left.getKeyType(), typeFamily.pairs(left.getValueType(), right.getValueType()))); http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/629f5734/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 9afd875..c0a6526 100644 --- a/pom.xml +++ b/pom.xml @@ -46,8 +46,6 @@ under the License. 1.6 2.9.2 1.7.0 - 0.20.2-cdh3u4 - 0.90.6-cdh3u4 @@ -120,12 +118,6 @@ under the License. - org.apache.hadoop - hadoop-client - ${hadoop.version} - - - org.apache.avro avro ${avro.version} @@ -180,12 +172,6 @@ under the License. - org.apache.hadoop - hadoop-minicluster - ${hadoop.version} - - - org.hamcrest hamcrest-all 1.1 @@ -228,55 +214,6 @@ under the License. - org.apache.hbase - hbase - ${hbase.version} - - - org.apache.hadoop - avro - - - org.apache.hadoop - hadoop-core - - - commons-logging - commons-logging - - - com.google.guava - guava - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - - - - org.apache.hbase - hbase - ${hbase.version} - test-jar - - - org.apache.hadoop - avro - - - org.slf4j - slf4j-log4j12 - - - - - org.scala-lang scala-library ${scala.version} @@ -320,6 +257,156 @@ under the License. + + + hadoop-1 + + + !crunch.platform + + + + 1.0.3 + 0.90.4 + + + + + org.apache.hadoop + hadoop-client + ${hadoop.version} + + + org.apache.hadoop + hadoop-minicluster + ${hadoop.version} + + + org.apache.hbase + hbase + ${hbase.version} + + + org.apache.hadoop + avro + + + org.apache.hadoop + hadoop-core + + + commons-logging + commons-logging + + + com.google.guava + guava + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + + + org.apache.hbase + hbase + ${hbase.version} + test-jar + + + org.apache.hadoop + avro + + + org.slf4j + slf4j-log4j12 + + + + + + + + hadoop-2 + + + crunch.platform + 2 + + + + 2.0.0-alpha + 0.92.1-cdh4.0.0 + + + + + org.apache.hadoop + hadoop-client + ${hadoop.version} + + + org.apache.hadoop + hadoop-minicluster + ${hadoop.version} + + + org.apache.hbase + hbase + ${hbase.version} + + + org.apache.hadoop + avro + + + org.apache.hadoop + hadoop-core + + + commons-logging + commons-logging + + + com.google.guava + guava + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + + + org.apache.hbase + hbase + ${hbase.version} + test-jar + + + org.apache.hadoop + avro + + + org.slf4j + slf4j-log4j12 + + + + + + + +