Updated Branches: refs/heads/master b6e65d9a2 -> e0d2cf2ba
CRUNCH-106: Fix data loss in CombineFn. StringConcatAggregator didn't reset its length counter properly. Project: http://git-wip-us.apache.org/repos/asf/incubator-crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-crunch/commit/e0d2cf2b Tree: http://git-wip-us.apache.org/repos/asf/incubator-crunch/tree/e0d2cf2b Diff: http://git-wip-us.apache.org/repos/asf/incubator-crunch/diff/e0d2cf2b Branch: refs/heads/master Commit: e0d2cf2ba5a56a3788de13a76aa1eb90f054dc76 Parents: b6e65d9 Author: Matthias Friedrich <[email protected]> Authored: Sat Nov 3 17:28:27 2012 +0100 Committer: Matthias Friedrich <[email protected]> Committed: Sat Nov 3 17:28:27 2012 +0100 ---------------------------------------------------------------------- .../src/main/java/org/apache/crunch/CombineFn.java | 1 + .../test/java/org/apache/crunch/CombineFnTest.java | 8 ++++++++ 2 files changed, 9 insertions(+), 0 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/e0d2cf2b/crunch/src/main/java/org/apache/crunch/CombineFn.java ---------------------------------------------------------------------- diff --git a/crunch/src/main/java/org/apache/crunch/CombineFn.java b/crunch/src/main/java/org/apache/crunch/CombineFn.java index 246827d..d45940b 100644 --- a/crunch/src/main/java/org/apache/crunch/CombineFn.java +++ b/crunch/src/main/java/org/apache/crunch/CombineFn.java @@ -954,6 +954,7 @@ public abstract class CombineFn<S, T> extends DoFn<Pair<S, Iterable<T>>, Pair<S, if (joiner == null) { joiner = skipNulls ? Joiner.on(separator).skipNulls() : Joiner.on(separator); } + currentLength = -separator.length(); list.clear(); } http://git-wip-us.apache.org/repos/asf/incubator-crunch/blob/e0d2cf2b/crunch/src/test/java/org/apache/crunch/CombineFnTest.java ---------------------------------------------------------------------- diff --git a/crunch/src/test/java/org/apache/crunch/CombineFnTest.java b/crunch/src/test/java/org/apache/crunch/CombineFnTest.java index af67ec3..39548e2 100644 --- a/crunch/src/test/java/org/apache/crunch/CombineFnTest.java +++ b/crunch/src/test/java/org/apache/crunch/CombineFnTest.java @@ -205,6 +205,14 @@ public class CombineFnTest { new StringConcatAggregator(" ", true, 9, 6), ImmutableList.of("foo", "foobar", "bar"))); } + @Test + public void testConcatenationReset() { + StringConcatAggregator a = new StringConcatAggregator(" ", true, 10, 6); + + assertEquals(ImmutableList.of("foo foobar"), applyAggregator(a, ImmutableList.of("foo", "foobar", "bar"))); + assertEquals(ImmutableList.of("foo foobar"), applyAggregator(a, ImmutableList.of("foo", "foobar", "bar"))); + } + @Test(expected = NullPointerException.class) public void testConcatenationNullException() { String[] arrayNull = new String[] { null, "" };
