Return-Path: X-Original-To: apmail-incubator-accumulo-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-accumulo-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 652DDB305 for ; Tue, 10 Jan 2012 15:17:04 +0000 (UTC) Received: (qmail 44263 invoked by uid 500); 10 Jan 2012 15:17:04 -0000 Delivered-To: apmail-incubator-accumulo-commits-archive@incubator.apache.org Received: (qmail 44227 invoked by uid 500); 10 Jan 2012 15:17:03 -0000 Mailing-List: contact accumulo-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: accumulo-dev@incubator.apache.org Delivered-To: mailing list accumulo-commits@incubator.apache.org Received: (qmail 44220 invoked by uid 99); 10 Jan 2012 15:17:03 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 10 Jan 2012 15:17:03 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 10 Jan 2012 15:16:59 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id E02F723888D2; Tue, 10 Jan 2012 15:16:37 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1229588 - in /incubator/accumulo/branches/1.4/src/core/src: main/java/org/apache/accumulo/core/iterators/ main/java/org/apache/accumulo/core/iterators/user/ test/java/org/apache/accumulo/core/iterators/user/ Date: Tue, 10 Jan 2012 15:16:37 -0000 To: accumulo-commits@incubator.apache.org From: billie@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120110151637.E02F723888D2@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: billie Date: Tue Jan 10 15:16:37 2012 New Revision: 1229588 URL: http://svn.apache.org/viewvc?rev=1229588&view=rev Log: ACCUMULO-289 added all columns option to Combiner, tested and improved options describing for combiners Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MaxCombiner.java incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MinCombiner.java incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingCombiner.java incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java?rev=1229588&r1=1229587&r2=1229588&view=diff ============================================================================== --- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java (original) +++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java Tue Jan 10 15:16:37 2012 @@ -19,7 +19,6 @@ package org.apache.accumulo.core.iterato import java.io.IOException; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -46,6 +45,7 @@ import org.apache.log4j.Logger; public abstract class Combiner extends WrappingIterator implements OptionDescriber { static final Logger log = Logger.getLogger(Combiner.class); protected static final String COLUMNS_OPTION = "columns"; + protected static final String ALL_OPTION = "all"; /** * A Java Iterator that iterates over the Values for a given Key from a source SortedKeyValueIterator. @@ -156,7 +156,7 @@ public abstract class Combiner extends W // check if aggregation is needed if (super.hasTop()) { workKey.set(super.getTopKey()); - if (combiners.contains(workKey)) { + if (combineAllColumns || combiners.contains(workKey)) { if (workKey.isDeleted()) return; topKey = workKey; @@ -205,11 +205,18 @@ public abstract class Combiner extends W public abstract Value reduce(Key key, Iterator iter); private ColumnSet combiners; + private boolean combineAllColumns; @Override public void init(SortedKeyValueIterator source, Map options, IteratorEnvironment env) throws IOException { super.init(source, options, env); + combineAllColumns = false; + if (options.containsKey(ALL_OPTION)) { + combineAllColumns = Boolean.parseBoolean(options.get(ALL_OPTION)); + if (combineAllColumns) + return; + } if (!options.containsKey(COLUMNS_OPTION)) throw new IllegalArgumentException("Must specify " + COLUMNS_OPTION + " option"); @@ -222,30 +229,32 @@ public abstract class Combiner extends W @Override public IteratorOptions describeOptions() { - return new IteratorOptions( - "comb", - "Combiners apply reduce functions to values with identical keys", - Collections - .singletonMap( - COLUMNS_OPTION, - "[:]{,[:]} escape non-alphanum chars using %."), - null); + IteratorOptions io = new IteratorOptions("comb", "Combiners apply reduce functions to values with identical keys", null, null); + io.addNamedOption(ALL_OPTION, "set to true to apply Combiner to every column, otherwise leave blank. if true, " + COLUMNS_OPTION + + " option will be ignored."); + io.addNamedOption(COLUMNS_OPTION, "[:]{,[:]} escape non-alphanum chars using %."); + return io; } @Override public boolean validateOptions(Map options) { + if (options.containsKey(ALL_OPTION)) { + combineAllColumns = Boolean.parseBoolean(options.get(ALL_OPTION)); + if (combineAllColumns) + return true; + } if (!options.containsKey(COLUMNS_OPTION)) return false; String encodedColumns = options.get(COLUMNS_OPTION); if (encodedColumns.length() == 0) return false; - + for (String columns : encodedColumns.split(",")) { if (!ColumnSet.isValidEncoding(columns)) return false; } - + return true; } @@ -270,4 +279,17 @@ public abstract class Combiner extends W is.addOption(COLUMNS_OPTION, sb.toString()); } + + /** + * A convenience method to set the "all columns" option on a Combiner. If true, the columns option will be ignored and the Combiner will be applied to all + * columns. + * + * @param is + * iterator settings object to configure + * @param enableAllColumns + * if true the combiner will be applied to all columns + */ + public static void setCombineAllColumns(IteratorSetting is, boolean combineAllColumns) { + is.addOption(ALL_OPTION, Boolean.toString(combineAllColumns)); + } } Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MaxCombiner.java URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MaxCombiner.java?rev=1229588&r1=1229587&r2=1229588&view=diff ============================================================================== --- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MaxCombiner.java (original) +++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MaxCombiner.java Tue Jan 10 15:16:37 2012 @@ -35,4 +35,12 @@ public class MaxCombiner extends LongCom } return max; } + + @Override + public IteratorOptions describeOptions() { + IteratorOptions io = super.describeOptions(); + io.setName("max"); + io.setDescription("MaxCombiner interprets Values as Longs and finds their maximum. A variety of encodings (variable length, fixed length, or string) are available"); + return io; + } } Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MinCombiner.java URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MinCombiner.java?rev=1229588&r1=1229587&r2=1229588&view=diff ============================================================================== --- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MinCombiner.java (original) +++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MinCombiner.java Tue Jan 10 15:16:37 2012 @@ -35,4 +35,12 @@ public class MinCombiner extends LongCom } return min; } + + @Override + public IteratorOptions describeOptions() { + IteratorOptions io = super.describeOptions(); + io.setName("min"); + io.setDescription("MinCombiner interprets Values as Longs and finds their minimum. A variety of encodings (variable length, fixed length, or string) are available"); + return io; + } } Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java?rev=1229588&r1=1229587&r2=1229588&view=diff ============================================================================== --- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java (original) +++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java Tue Jan 10 15:16:37 2012 @@ -107,8 +107,8 @@ public class SummingArrayCombiner extend @Override public IteratorOptions describeOptions() { IteratorOptions io = super.describeOptions(); - io.setName("summingarraycombiner"); - io.setDescription("SummingArrayCombiner can interpret Values as arrays of Longs using a variety of encodings (arrays of variable length longs or fixed length longs, or comma-separated strings) before combining"); + io.setName("sumarray"); + io.setDescription("SummingArrayCombiner can interpret Values as arrays of Longs using a variety of encodings (arrays of variable length longs or fixed length longs, or comma-separated strings) before summing element-wise."); io.addNamedOption(TYPE, ""); return io; } Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingCombiner.java URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingCombiner.java?rev=1229588&r1=1229587&r2=1229588&view=diff ============================================================================== --- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingCombiner.java (original) +++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingCombiner.java Tue Jan 10 15:16:37 2012 @@ -33,4 +33,12 @@ public class SummingCombiner extends Lon } return sum; } + + @Override + public IteratorOptions describeOptions() { + IteratorOptions io = super.describeOptions(); + io.setName("sum"); + io.setDescription("SummingCombiner interprets Values as Longs and adds them together. A variety of encodings (variable length, fixed length, or string) are available"); + return io; + } } Modified: incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java?rev=1229588&r1=1229587&r2=1229588&view=diff ============================================================================== --- incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java (original) +++ incubator/accumulo/branches/1.4/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java Tue Jan 10 15:16:37 2012 @@ -351,6 +351,34 @@ public class CombinerTest { assertEquals(nk(2, 2, 1, 2), ai.getTopKey()); assertEquals("3", encoder.decode(ai.getTopValue().get()).toString()); + // combine all columns + + is = new IteratorSetting(1, SummingCombiner.class); + LongCombiner.setEncodingType(is, SummingCombiner.Type.STRING); + Combiner.setCombineAllColumns(is, true); + + ai.init(new SortedMapIterator(tm1), is.getProperties(), null); + ai.seek(new Range(), EMPTY_COL_FAMS, false); + + assertTrue(ai.hasTop()); + assertEquals(nk(0, 0, 1, 1), ai.getTopKey()); + assertEquals("7", encoder.decode(ai.getTopValue().get()).toString()); + + ai.next(); + + assertTrue(ai.hasTop()); + assertEquals(nk(1, 1, 1, 3), ai.getTopKey()); + assertEquals("9", encoder.decode(ai.getTopValue().get()).toString()); + + ai.next(); + + assertTrue(ai.hasTop()); + assertEquals(nk(2, 2, 1, 2), ai.getTopKey()); + assertEquals("5", encoder.decode(ai.getTopValue().get()).toString()); + + ai.next(); + + assertFalse(ai.hasTop()); } @Test