crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jwi...@apache.org
Subject git commit: CRUNCH-465: Fix Avro-related build problems.
Date Tue, 26 Aug 2014 23:35:50 GMT
Repository: crunch
Updated Branches:
  refs/heads/apache-crunch-0.8 67de89ab0 -> ade97e715


CRUNCH-465: Fix Avro-related build problems.

Update Avro to 1.7.7 and fix resulting errors.
Fix a brittle integration test.

Signed-off-by: Josh Wills <jwills@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/ade97e71
Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/ade97e71
Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/ade97e71

Branch: refs/heads/apache-crunch-0.8
Commit: ade97e715aafa2be32eac849ac493e1230fe3697
Parents: 67de89a
Author: Matthias Friedrich <matt@mafr.de>
Authored: Tue Aug 26 21:39:55 2014 +0200
Committer: Josh Wills <jwills@apache.org>
Committed: Tue Aug 26 16:23:09 2014 -0700

----------------------------------------------------------------------
 .../it/java/org/apache/crunch/StageResultsCountersIT.java    | 8 ++++----
 .../java/org/apache/crunch/types/avro/AvroOutputFormat.java  | 3 +--
 pom.xml                                                      | 2 +-
 3 files changed, 6 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/crunch/blob/ade97e71/crunch-core/src/it/java/org/apache/crunch/StageResultsCountersIT.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/it/java/org/apache/crunch/StageResultsCountersIT.java b/crunch-core/src/it/java/org/apache/crunch/StageResultsCountersIT.java
index ffcc931..04711e4 100644
--- a/crunch-core/src/it/java/org/apache/crunch/StageResultsCountersIT.java
+++ b/crunch-core/src/it/java/org/apache/crunch/StageResultsCountersIT.java
@@ -17,8 +17,9 @@
  */
 package org.apache.crunch;
 
-import static junit.framework.Assert.assertEquals;
 import static junit.framework.Assert.assertTrue;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
 
 import java.util.HashSet;
 import java.util.List;
@@ -38,6 +39,7 @@ import org.junit.After;
 import org.junit.Rule;
 import org.junit.Test;
 
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 
@@ -87,9 +89,7 @@ public class StageResultsCountersIT {
 
     Map<String, Long> keywordsMap = countersToMap(result.getStageResults(), KEYWORDS_COUNTER_GROUP);
 
-    assertEquals(3, keywordsMap.size());
-
-    assertEquals("{NOT=157, AND=596, OR=81}", keywordsMap.toString());
+    assertThat(keywordsMap, is((Map<String, Long>) ImmutableMap.of("NOT", 157L, "AND",
596L, "OR", 81L)));
   }
 
   private static PipelineResult coutSpecialKeywords(Pipeline pipeline, String inputFileName,
PTypeFamily tf) {

http://git-wip-us.apache.org/repos/asf/crunch/blob/ade97e71/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroOutputFormat.java
----------------------------------------------------------------------
diff --git a/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroOutputFormat.java
b/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroOutputFormat.java
index 6dbb6de..79736b8 100644
--- a/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroOutputFormat.java
+++ b/crunch-core/src/main/java/org/apache/crunch/types/avro/AvroOutputFormat.java
@@ -22,7 +22,6 @@ import java.io.IOException;
 import org.apache.avro.Schema;
 import org.apache.avro.file.CodecFactory;
 import org.apache.avro.file.DataFileWriter;
-import org.apache.avro.io.DatumWriter;
 import org.apache.avro.mapred.AvroJob;
 import org.apache.avro.mapred.AvroWrapper;
 import org.apache.hadoop.conf.Configuration;
@@ -53,7 +52,7 @@ public class AvroOutputFormat<T> extends FileOutputFormat<AvroWrapper<T>,
NullWr
 
     if (org.apache.hadoop.mapred.FileOutputFormat.getCompressOutput(jc)) {
       int level = conf.getInt(org.apache.avro.mapred.AvroOutputFormat.DEFLATE_LEVEL_KEY,
-          org.apache.avro.mapred.AvroOutputFormat.DEFAULT_DEFLATE_LEVEL);
+          org.apache.avro.file.CodecFactory.DEFAULT_DEFLATE_LEVEL);
       String codecName = conf.get(AvroJob.OUTPUT_CODEC,
           org.apache.avro.file.DataFileConstants.DEFLATE_CODEC);
       CodecFactory codec = codecName.equals(org.apache.avro.file.DataFileConstants.DEFLATE_CODEC)

http://git-wip-us.apache.org/repos/asf/crunch/blob/ade97e71/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 8250712..2da28a3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -73,7 +73,7 @@ under the License.
     <commons-httpclient.version>3.0.1</commons-httpclient.version>
     <commons-logging.version>1.1.1</commons-logging.version>
     <commons-cli.version>1.2</commons-cli.version>
-    <avro.version>1.7.4</avro.version>
+    <avro.version>1.7.7</avro.version>
     <parquet.version>1.3.2</parquet.version>
     <javassist.version>3.16.1-GA</javassist.version>
     <jackson.version>1.8.8</jackson.version>


Mime
View raw message