incubator-accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bil...@apache.org
Subject svn commit: r1229613 - in /incubator/accumulo/trunk: ./ src/core/ src/core/src/main/java/org/apache/accumulo/core/iterators/ src/core/src/main/java/org/apache/accumulo/core/iterators/user/ src/core/src/test/java/org/apache/accumulo/core/iterators/user/...
Date Tue, 10 Jan 2012 15:59:52 GMT
Author: billie
Date: Tue Jan 10 15:59:52 2012
New Revision: 1229613

URL: http://svn.apache.org/viewvc?rev=1229613&view=rev
Log:
ACCUMULO-289 added all columns option to Combiner, tested and improved options describing
for combiners - merged to trunk

Modified:
    incubator/accumulo/trunk/   (props changed)
    incubator/accumulo/trunk/src/core/   (props changed)
    incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
    incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MaxCombiner.java
    incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MinCombiner.java
    incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
    incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingCombiner.java
    incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java
    incubator/accumulo/trunk/src/server/   (props changed)

Propchange: incubator/accumulo/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 10 15:59:52 2012
@@ -1,3 +1,3 @@
 /incubator/accumulo/branches/1.3:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611,1228195
 /incubator/accumulo/branches/1.3.5rc:1209938
-/incubator/accumulo/branches/1.4:1201902-1228245,1228308,1229205,1229220,1229248,1229357
+/incubator/accumulo/branches/1.4:1201902-1228245,1228308,1229205,1229220,1229248,1229357,1229588

Propchange: incubator/accumulo/trunk/src/core/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 10 15:59:52 2012
@@ -1,3 +1,3 @@
 /incubator/accumulo/branches/1.3.5rc/src/core:1209938
 /incubator/accumulo/branches/1.3/src/core:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215
-/incubator/accumulo/branches/1.4/src/core:1201902-1228245,1228308,1229205,1229220,1229248,1229357
+/incubator/accumulo/branches/1.4/src/core:1201902-1228245,1228308,1229205,1229220,1229248,1229357,1229588

Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java?rev=1229613&r1=1229612&r2=1229613&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
(original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
Tue Jan 10 15:59:52 2012
@@ -19,7 +19,6 @@ package org.apache.accumulo.core.iterato
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -46,6 +45,7 @@ import org.apache.log4j.Logger;
 public abstract class Combiner extends WrappingIterator implements OptionDescriber {
   static final Logger log = Logger.getLogger(Combiner.class);
   protected static final String COLUMNS_OPTION = "columns";
+  protected static final String ALL_OPTION = "all";
   
   /**
    * A Java Iterator that iterates over the Values for a given Key from a source SortedKeyValueIterator.
@@ -156,7 +156,7 @@ public abstract class Combiner extends W
     // check if aggregation is needed
     if (super.hasTop()) {
       workKey.set(super.getTopKey());
-      if (combiners.contains(workKey)) {
+      if (combineAllColumns || combiners.contains(workKey)) {
         if (workKey.isDeleted())
           return;
         topKey = workKey;
@@ -205,11 +205,18 @@ public abstract class Combiner extends W
   public abstract Value reduce(Key key, Iterator<Value> iter);
   
   private ColumnSet combiners;
+  private boolean combineAllColumns;
   
   @Override
   public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String>
options, IteratorEnvironment env) throws IOException {
     super.init(source, options, env);
     
+    combineAllColumns = false;
+    if (options.containsKey(ALL_OPTION)) {
+      combineAllColumns = Boolean.parseBoolean(options.get(ALL_OPTION));
+      if (combineAllColumns)
+        return;
+    }
     if (!options.containsKey(COLUMNS_OPTION))
       throw new IllegalArgumentException("Must specify " + COLUMNS_OPTION + " option");
     
@@ -222,30 +229,32 @@ public abstract class Combiner extends W
   
   @Override
   public IteratorOptions describeOptions() {
-    return new IteratorOptions(
-        "comb",
-        "Combiners apply reduce functions to values with identical keys",
-        Collections
-            .singletonMap(
-                COLUMNS_OPTION,
-        "<col fam>[:<col qual>]{,<col fam>[:<col qual>]} escape non-alphanum
chars using %<hex>."),
-        null);
+    IteratorOptions io = new IteratorOptions("comb", "Combiners apply reduce functions to
values with identical keys", null, null);
+    io.addNamedOption(ALL_OPTION, "set to true to apply Combiner to every column, otherwise
leave blank. if true, " + COLUMNS_OPTION
+        + " option will be ignored.");
+    io.addNamedOption(COLUMNS_OPTION, "<col fam>[:<col qual>]{,<col fam>[:<col
qual>]} escape non-alphanum chars using %<hex>.");
+    return io;
   }
   
   @Override
   public boolean validateOptions(Map<String,String> options) {
+    if (options.containsKey(ALL_OPTION)) {
+      combineAllColumns = Boolean.parseBoolean(options.get(ALL_OPTION));
+      if (combineAllColumns)
+        return true;
+    }
     if (!options.containsKey(COLUMNS_OPTION))
       return false;
     
     String encodedColumns = options.get(COLUMNS_OPTION);
     if (encodedColumns.length() == 0)
       return false;
-
+    
     for (String columns : encodedColumns.split(",")) {
       if (!ColumnSet.isValidEncoding(columns))
         return false;
     }
-
+    
     return true;
   }
   
@@ -270,4 +279,17 @@ public abstract class Combiner extends W
     
     is.addOption(COLUMNS_OPTION, sb.toString());
   }
+  
+  /**
+   * A convenience method to set the "all columns" option on a Combiner. If true, the columns
option will be ignored and the Combiner will be applied to all
+   * columns.
+   * 
+   * @param is
+   *          iterator settings object to configure
+   * @param enableAllColumns
+   *          if true the combiner will be applied to all columns
+   */
+  public static void setCombineAllColumns(IteratorSetting is, boolean combineAllColumns)
{
+    is.addOption(ALL_OPTION, Boolean.toString(combineAllColumns));
+  }
 }

Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MaxCombiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MaxCombiner.java?rev=1229613&r1=1229612&r2=1229613&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MaxCombiner.java
(original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MaxCombiner.java
Tue Jan 10 15:59:52 2012
@@ -35,4 +35,12 @@ public class MaxCombiner extends LongCom
     }
     return max;
   }
+  
+  @Override
+  public IteratorOptions describeOptions() {
+    IteratorOptions io = super.describeOptions();
+    io.setName("max");
+    io.setDescription("MaxCombiner interprets Values as Longs and finds their maximum.  A
variety of encodings (variable length, fixed length, or string) are available");
+    return io;
+  }
 }

Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MinCombiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MinCombiner.java?rev=1229613&r1=1229612&r2=1229613&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MinCombiner.java
(original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/MinCombiner.java
Tue Jan 10 15:59:52 2012
@@ -35,4 +35,12 @@ public class MinCombiner extends LongCom
     }
     return min;
   }
+  
+  @Override
+  public IteratorOptions describeOptions() {
+    IteratorOptions io = super.describeOptions();
+    io.setName("min");
+    io.setDescription("MinCombiner interprets Values as Longs and finds their minimum.  A
variety of encodings (variable length, fixed length, or string) are available");
+    return io;
+  }
 }

Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java?rev=1229613&r1=1229612&r2=1229613&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
(original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
Tue Jan 10 15:59:52 2012
@@ -107,8 +107,8 @@ public class SummingArrayCombiner extend
   @Override
   public IteratorOptions describeOptions() {
     IteratorOptions io = super.describeOptions();
-    io.setName("summingarraycombiner");
-    io.setDescription("SummingArrayCombiner can interpret Values as arrays of Longs using
a variety of encodings (arrays of variable length longs or fixed length longs, or comma-separated
strings) before combining");
+    io.setName("sumarray");
+    io.setDescription("SummingArrayCombiner can interpret Values as arrays of Longs using
a variety of encodings (arrays of variable length longs or fixed length longs, or comma-separated
strings) before summing element-wise.");
     io.addNamedOption(TYPE, "<VARLEN|FIXEDLEN|STRING|fullClassName>");
     return io;
   }

Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingCombiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingCombiner.java?rev=1229613&r1=1229612&r2=1229613&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingCombiner.java
(original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingCombiner.java
Tue Jan 10 15:59:52 2012
@@ -33,4 +33,12 @@ public class SummingCombiner extends Lon
     }
     return sum;
   }
+  
+  @Override
+  public IteratorOptions describeOptions() {
+    IteratorOptions io = super.describeOptions();
+    io.setName("sum");
+    io.setDescription("SummingCombiner interprets Values as Longs and adds them together.
 A variety of encodings (variable length, fixed length, or string) are available");
+    return io;
+  }
 }

Modified: incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java?rev=1229613&r1=1229612&r2=1229613&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java
(original)
+++ incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java
Tue Jan 10 15:59:52 2012
@@ -350,6 +350,34 @@ public class CombinerTest {
     assertEquals(nk(2, 2, 1, 2), ai.getTopKey());
     assertEquals("3", encoder.decode(ai.getTopValue().get()).toString());
     
+    // combine all columns
+    
+    is = new IteratorSetting(1, SummingCombiner.class);
+    LongCombiner.setEncodingType(is, SummingCombiner.Type.STRING);
+    Combiner.setCombineAllColumns(is, true);
+    
+    ai.init(new SortedMapIterator(tm1), is.getProperties(), null);
+    ai.seek(new Range(), EMPTY_COL_FAMS, false);
+    
+    assertTrue(ai.hasTop());
+    assertEquals(nk(0, 0, 1, 1), ai.getTopKey());
+    assertEquals("7", encoder.decode(ai.getTopValue().get()).toString());
+    
+    ai.next();
+    
+    assertTrue(ai.hasTop());
+    assertEquals(nk(1, 1, 1, 3), ai.getTopKey());
+    assertEquals("9", encoder.decode(ai.getTopValue().get()).toString());
+    
+    ai.next();
+    
+    assertTrue(ai.hasTop());
+    assertEquals(nk(2, 2, 1, 2), ai.getTopKey());
+    assertEquals("5", encoder.decode(ai.getTopValue().get()).toString());
+    
+    ai.next();
+    
+    assertFalse(ai.hasTop());
   }
   
   @Test

Propchange: incubator/accumulo/trunk/src/server/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 10 15:59:52 2012
@@ -1,3 +1,3 @@
 /incubator/accumulo/branches/1.3.5rc/src/server:1209938
 /incubator/accumulo/branches/1.3/src/server:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006,1227215,1227231,1227611
-/incubator/accumulo/branches/1.4/src/server:1201902-1228245,1228308,1229205,1229220,1229248,1229357
+/incubator/accumulo/branches/1.4/src/server:1201902-1228245,1228308,1229205,1229220,1229248,1229357,1229588



Mime
View raw message