incubator-accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bil...@apache.org
Subject svn commit: r1227204 - in /incubator/accumulo/trunk: ./ src/core/src/main/java/org/apache/accumulo/core/iterators/ src/core/src/main/java/org/apache/accumulo/core/iterators/user/ src/core/src/test/java/org/apache/accumulo/core/iterators/user/
Date Wed, 04 Jan 2012 16:06:49 GMT
Author: billie
Date: Wed Jan  4 16:06:48 2012
New Revision: 1227204

URL: http://svn.apache.org/viewvc?rev=1227204&view=rev
Log:
ACCUMULO-245 merged to trunk

Modified:
    incubator/accumulo/trunk/   (props changed)
    incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java
    incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
    incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
    incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java

Propchange: incubator/accumulo/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Wed Jan  4 16:06:48 2012
@@ -1,3 +1,3 @@
 /incubator/accumulo/branches/1.3:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006
 /incubator/accumulo/branches/1.3.5rc:1209938
-/incubator/accumulo/branches/1.4:1201902-1222706,1225007
+/incubator/accumulo/branches/1.4:1201902-1222706,1225007,1227202

Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java?rev=1227204&r1=1227203&r2=1227204&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java
(original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java
Wed Jan  4 16:06:48 2012
@@ -26,7 +26,6 @@ import java.util.Map;
 import org.apache.accumulo.core.client.IteratorSetting;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Value;
-import org.apache.accumulo.start.classloader.AccumuloClassLoader;
 import org.apache.hadoop.io.WritableUtils;
 
 /**
@@ -62,31 +61,18 @@ public abstract class LongCombiner exten
     if (type == null)
       throw new IllegalArgumentException("no type specified");
     if (type.startsWith(CLASS_PREFIX)) {
-      try {
-        @SuppressWarnings("unchecked")
-        Class<? extends Encoder<Long>> clazz = (Class<? extends Encoder<Long>>)
AccumuloClassLoader.loadClass(type.substring(CLASS_PREFIX.length()),
-            Encoder.class);
-        encoder = clazz.newInstance();
-        if (encoder.decode(encoder.encode(42l)) != 42l) {
-          throw new IllegalArgumentException("something wrong with " + type + " -- doesn't
encode and decode a Long properly");
-        }
-      } catch (ClassNotFoundException e) {
-        throw new IllegalArgumentException(e);
-      } catch (InstantiationException e) {
-        throw new IllegalArgumentException(e);
-      } catch (IllegalAccessException e) {
-        throw new IllegalArgumentException(e);
-      }
+        setEncoder(type.substring(CLASS_PREFIX.length()));
+        testEncoder(42l);
     } else {
       switch (Type.valueOf(type)) {
         case VARLEN:
-          encoder = VAR_LEN_ENCODER;
+          setEncoder(VAR_LEN_ENCODER);
           return;
         case FIXEDLEN:
-          encoder = FIXED_LEN_ENCODER;
+          setEncoder(FIXED_LEN_ENCODER);
           return;
         case STRING:
-          encoder = STRING_ENCODER;
+          setEncoder(STRING_ENCODER);
           return;
         default:
           throw new IllegalArgumentException();

Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java?rev=1227204&r1=1227203&r2=1227204&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
(original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
Wed Jan  4 16:06:48 2012
@@ -21,6 +21,7 @@ import java.util.NoSuchElementException;
 
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.start.classloader.AccumuloClassLoader;
 
 /**
  * A Combiner that decodes each Value to type V before reducing, then encodes the result
of typedReduce back to Value.
@@ -32,7 +33,7 @@ import org.apache.accumulo.core.data.Val
  * Subclasses may implement a switch on the "type" variable to choose an Encoder in their
init method.
  */
 public abstract class TypedValueCombiner<V> extends Combiner {
-  protected Encoder<V> encoder;
+  private Encoder<V> encoder = null;
   
   /**
    * A Java Iterator that translates an Iterator<Value> to an Iterator<V> using
the decode method of an Encoder.
@@ -82,6 +83,68 @@ public abstract class TypedValueCombiner
     public V decode(byte[] b);
   }
   
+  /**
+   * Sets the Encoder<V> used to translate Values to V and back.
+   * 
+   * @param encoder
+   */
+  protected void setEncoder(Encoder<V> encoder) {
+    this.encoder = encoder;
+  }
+  
+  /**
+   * Instantiates and sets the Encoder<V> used to translate Values to V and back.
+   * 
+   * @param encoderClass
+   * @throws IllegalArgumentException
+   *           if ClassNotFoundException, InstantiationException, or IllegalAccessException
occurs
+   */
+  protected void setEncoder(String encoderClass) {
+    try {
+      @SuppressWarnings("unchecked")
+      Class<? extends Encoder<V>> clazz = (Class<? extends Encoder<V>>)
AccumuloClassLoader.loadClass(encoderClass, Encoder.class);
+      encoder = clazz.newInstance();
+    } catch (ClassNotFoundException e) {
+      throw new IllegalArgumentException(e);
+    } catch (InstantiationException e) {
+      throw new IllegalArgumentException(e);
+    } catch (IllegalAccessException e) {
+      throw new IllegalArgumentException(e);
+    }
+  }
+  
+  /**
+   * Tests whether v remains the same when encoded and decoded with the current encoder.
+   * 
+   * @param v
+   * @throws IllegalStateException
+   *           if an encoder has not been set.
+   * @throws IllegalArgumentException
+   *           if the test fails.
+   */
+  protected void testEncoder(V v) {
+    if (encoder == null)
+      throw new IllegalStateException("encoder has not been initialized");
+    testEncoder(encoder, v);
+  }
+  
+  /**
+   * Tests whether v remains the same when encoded and decoded with the given encoder.
+   * 
+   * @param encoder
+   * @param v
+   * @throws IllegalArgumentException
+   *           if the test fails.
+   */
+  public static <V> void testEncoder(Encoder<V> encoder, V v) {
+    try {
+      if (!v.equals(encoder.decode(encoder.encode(v))))
+        throw new IllegalArgumentException("something wrong with " + encoder.getClass().getName()
+ " -- doesn't encode and decode " + v + " properly");
+    } catch (ClassCastException e) {
+      throw new IllegalArgumentException(encoder.getClass().getName() + " doesn't encode
" + v.getClass().getName());
+    }
+  }
+  
   @Override
   public Value reduce(Key key, Iterator<Value> iter) {
     return new Value(encoder.encode(typedReduce(key, new VIterator<V>(iter, encoder))));

Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java?rev=1227204&r1=1227203&r2=1227204&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
(original)
+++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
Wed Jan  4 16:06:48 2012
@@ -34,7 +34,6 @@ import org.apache.accumulo.core.iterator
 import org.apache.accumulo.core.iterators.LongCombiner;
 import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 import org.apache.accumulo.core.iterators.TypedValueCombiner;
-import org.apache.accumulo.start.classloader.AccumuloClassLoader;
 import org.apache.hadoop.io.WritableUtils;
 
 /**
@@ -86,32 +85,18 @@ public class SummingArrayCombiner extend
     if (type == null)
       throw new IllegalArgumentException("no type specified");
     if (type.startsWith(CLASS_PREFIX)) {
-      try {
-        @SuppressWarnings("unchecked")
-        Class<? extends Encoder<List<Long>>> clazz = (Class<? extends
Encoder<List<Long>>>) AccumuloClassLoader.loadClass(
-            type.substring(CLASS_PREFIX.length()), Encoder.class);
-        encoder = clazz.newInstance();
-        List<Long> testList = encoder.decode(encoder.encode(Arrays.asList(0l, 1l)));
-        if (testList.size() != 2 || testList.get(0) != 0l || testList.get(1) != 1l) {
-          throw new IllegalArgumentException("something wrong with " + type + " -- doesn't
encode and decode a List<Long> properly");
-        }
-      } catch (ClassNotFoundException e) {
-        throw new IllegalArgumentException(e);
-      } catch (InstantiationException e) {
-        throw new IllegalArgumentException(e);
-      } catch (IllegalAccessException e) {
-        throw new IllegalArgumentException(e);
-      }
+      setEncoder(type.substring(CLASS_PREFIX.length()));
+      testEncoder(Arrays.asList(0l, 1l));
     } else {
       switch (Type.valueOf(options.get(TYPE))) {
         case VARLEN:
-          encoder = new VarLongArrayEncoder();
+          setEncoder(VAR_LONG_ARRAY_ENCODER);
           return;
         case FIXEDLEN:
-          encoder = new FixedLongArrayEncoder();
+          setEncoder(FIXED_LONG_ARRAY_ENCODER);
           return;
         case STRING:
-          encoder = new StringArrayEncoder();
+          setEncoder(STRING_ARRAY_ENCODER);
           return;
         default:
           throw new IllegalArgumentException();

Modified: incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java?rev=1227204&r1=1227203&r2=1227204&view=diff
==============================================================================
--- incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java
(original)
+++ incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java
Wed Jan  4 16:06:48 2012
@@ -42,12 +42,14 @@ import org.apache.accumulo.core.iterator
 import org.apache.accumulo.core.iterators.LongCombiner.VarLenEncoder;
 import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 import org.apache.accumulo.core.iterators.SortedMapIterator;
+import org.apache.accumulo.core.iterators.TypedValueCombiner;
 import org.apache.accumulo.core.iterators.TypedValueCombiner.Encoder;
 import org.apache.accumulo.core.iterators.aggregation.LongSummation;
 import org.apache.accumulo.core.iterators.aggregation.NumArraySummation;
 import org.apache.accumulo.core.iterators.aggregation.NumSummation;
 import org.apache.accumulo.core.iterators.system.MultiIterator;
 import org.apache.hadoop.io.Text;
+import org.junit.Assert;
 import org.junit.Test;
 
 public class CombinerTest {
@@ -586,6 +588,37 @@ public class CombinerTest {
     ai.next();
     
     assertFalse(ai.hasTop());
+    
+    is.clearOptions();
+    SummingArrayCombiner.setEncodingType(is, SummingCombiner.VAR_LEN_ENCODER.getClass().getName());
+    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
+    
+    try {
+      ai.init(new SortedMapIterator(tm1), is.getProperties(), null);
+      Assert.fail();
+    } catch (IllegalArgumentException e) {}
+    
+    is.clearOptions();
+    SummingArrayCombiner.setEncodingType(is, BadEncoder.class.getName());
+    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
+    
+    try {
+      ai.init(new SortedMapIterator(tm1), is.getProperties(), null);
+      Assert.fail();
+    } catch (IllegalArgumentException e) {}
+  }
+  
+  public static class BadEncoder implements Encoder<List<Long>> {
+    @Override
+    public byte[] encode(List<Long> v) {
+      return new byte[0];
+    }
+    
+    @Override
+    public List<Long> decode(byte[] b) {
+      return new ArrayList<Long>();
+    }
+    
   }
   
   @Test
@@ -595,6 +628,37 @@ public class CombinerTest {
     sumArray(SummingArrayCombiner.StringArrayEncoder.class, SummingArrayCombiner.Type.STRING);
   }
   
+  @Test
+  public void testEncoders() {
+    TypedValueCombiner.testEncoder(SummingCombiner.FIXED_LEN_ENCODER, Long.MAX_VALUE);
+    TypedValueCombiner.testEncoder(SummingCombiner.FIXED_LEN_ENCODER, Long.MIN_VALUE);
+    TypedValueCombiner.testEncoder(SummingCombiner.FIXED_LEN_ENCODER, 42l);
+    TypedValueCombiner.testEncoder(SummingCombiner.FIXED_LEN_ENCODER, -42l);
+    TypedValueCombiner.testEncoder(SummingCombiner.FIXED_LEN_ENCODER, 0l);
+    TypedValueCombiner.testEncoder(SummingCombiner.VAR_LEN_ENCODER, Long.MAX_VALUE);
+    TypedValueCombiner.testEncoder(SummingCombiner.VAR_LEN_ENCODER, Long.MIN_VALUE);
+    TypedValueCombiner.testEncoder(SummingCombiner.VAR_LEN_ENCODER, 42l);
+    TypedValueCombiner.testEncoder(SummingCombiner.VAR_LEN_ENCODER, -42l);
+    TypedValueCombiner.testEncoder(SummingCombiner.VAR_LEN_ENCODER, 0l);
+    TypedValueCombiner.testEncoder(SummingCombiner.STRING_ENCODER, Long.MAX_VALUE);
+    TypedValueCombiner.testEncoder(SummingCombiner.STRING_ENCODER, Long.MIN_VALUE);
+    TypedValueCombiner.testEncoder(SummingCombiner.STRING_ENCODER, 42l);
+    TypedValueCombiner.testEncoder(SummingCombiner.STRING_ENCODER, -42l);
+    TypedValueCombiner.testEncoder(SummingCombiner.STRING_ENCODER, 0l);
+    
+    TypedValueCombiner.testEncoder(SummingArrayCombiner.FIXED_LONG_ARRAY_ENCODER, Arrays.asList(0l,
-1l, 10l, Long.MAX_VALUE, Long.MIN_VALUE));
+    TypedValueCombiner.testEncoder(SummingArrayCombiner.VAR_LONG_ARRAY_ENCODER, Arrays.asList(0l,
-1l, 10l, Long.MAX_VALUE, Long.MIN_VALUE));
+    TypedValueCombiner.testEncoder(SummingArrayCombiner.STRING_ARRAY_ENCODER, Arrays.asList(0l,
-1l, 10l, Long.MAX_VALUE, Long.MIN_VALUE));
+  }
+  
+  @Test
+  public void testAdds() {
+    assertEquals(LongCombiner.safeAdd(Long.MIN_VALUE + 5, -10), Long.MIN_VALUE);
+    assertEquals(LongCombiner.safeAdd(Long.MAX_VALUE - 5, 10), Long.MAX_VALUE);
+    assertEquals(LongCombiner.safeAdd(Long.MIN_VALUE + 5, -5), Long.MIN_VALUE);
+    assertEquals(LongCombiner.safeAdd(Long.MAX_VALUE - 5, 5), Long.MAX_VALUE);
+  }
+  
   /**
    * @throws IOException
    * @deprecated since 1.4



Mime
View raw message