crunch-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From m...@apache.org
Subject git commit: CRUNCH-106: Fix data loss in CombineFn.
Date Sat, 03 Nov 2012 16:40:05 GMT
Updated Branches:
  refs/heads/master b6e65d9a2 -> e0d2cf2ba


CRUNCH-106: Fix data loss in CombineFn.

StringConcatAggregator didn't reset its length counter properly.


Project: http://git-wip-us.apache.org/repos/asf/incubator-crunch/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-crunch/commit/e0d2cf2b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-crunch/tree/e0d2cf2b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-crunch/diff/e0d2cf2b

Branch: refs/heads/master
Commit: e0d2cf2ba5a56a3788de13a76aa1eb90f054dc76
Parents: b6e65d9
Author: Matthias Friedrich <matt@mafr.de>
Authored: Sat Nov 3 17:28:27 2012 +0100
Committer: Matthias Friedrich <matt@mafr.de>
Committed: Sat Nov 3 17:28:27 2012 +0100

----------------------------------------------------------------------
 .../src/main/java/org/apache/crunch/CombineFn.java |    1 +
 .../test/java/org/apache/crunch/CombineFnTest.java |    8 ++++++++
 2 files changed, 9 insertions(+), 0 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/e0d2cf2b/crunch/src/main/java/org/apache/crunch/CombineFn.java
----------------------------------------------------------------------
diff --git a/crunch/src/main/java/org/apache/crunch/CombineFn.java b/crunch/src/main/java/org/apache/crunch/CombineFn.java
index 246827d..d45940b 100644
--- a/crunch/src/main/java/org/apache/crunch/CombineFn.java
+++ b/crunch/src/main/java/org/apache/crunch/CombineFn.java
@@ -954,6 +954,7 @@ public abstract class CombineFn<S, T> extends DoFn<Pair<S,
Iterable<T>>, Pair<S,
       if (joiner == null) {
         joiner = skipNulls ? Joiner.on(separator).skipNulls() : Joiner.on(separator);
       }
+      currentLength = -separator.length();
       list.clear();
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/e0d2cf2b/crunch/src/test/java/org/apache/crunch/CombineFnTest.java
----------------------------------------------------------------------
diff --git a/crunch/src/test/java/org/apache/crunch/CombineFnTest.java b/crunch/src/test/java/org/apache/crunch/CombineFnTest.java
index af67ec3..39548e2 100644
--- a/crunch/src/test/java/org/apache/crunch/CombineFnTest.java
+++ b/crunch/src/test/java/org/apache/crunch/CombineFnTest.java
@@ -205,6 +205,14 @@ public class CombineFnTest {
         new StringConcatAggregator(" ", true, 9, 6), ImmutableList.of("foo", "foobar", "bar")));
   }
 
+  @Test
+  public void testConcatenationReset() {
+    StringConcatAggregator a = new StringConcatAggregator(" ", true, 10, 6);
+
+    assertEquals(ImmutableList.of("foo foobar"), applyAggregator(a, ImmutableList.of("foo",
"foobar", "bar")));
+    assertEquals(ImmutableList.of("foo foobar"), applyAggregator(a, ImmutableList.of("foo",
"foobar", "bar")));
+  }
+
   @Test(expected = NullPointerException.class)
   public void testConcatenationNullException() {
     String[] arrayNull = new String[] { null, "" };


Mime
View raw message