Return-Path: X-Original-To: apmail-incubator-accumulo-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-accumulo-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id C6A35B3A5 for ; Wed, 4 Jan 2012 16:07:15 +0000 (UTC) Received: (qmail 47895 invoked by uid 500); 4 Jan 2012 16:07:15 -0000 Delivered-To: apmail-incubator-accumulo-commits-archive@incubator.apache.org Received: (qmail 47874 invoked by uid 500); 4 Jan 2012 16:07:15 -0000 Mailing-List: contact accumulo-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: accumulo-dev@incubator.apache.org Delivered-To: mailing list accumulo-commits@incubator.apache.org Received: (qmail 47867 invoked by uid 99); 4 Jan 2012 16:07:15 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 04 Jan 2012 16:07:15 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 04 Jan 2012 16:07:11 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 89560238899C; Wed, 4 Jan 2012 16:06:49 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1227204 - in /incubator/accumulo/trunk: ./ src/core/src/main/java/org/apache/accumulo/core/iterators/ src/core/src/main/java/org/apache/accumulo/core/iterators/user/ src/core/src/test/java/org/apache/accumulo/core/iterators/user/ Date: Wed, 04 Jan 2012 16:06:49 -0000 To: accumulo-commits@incubator.apache.org From: billie@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120104160649.89560238899C@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: billie Date: Wed Jan 4 16:06:48 2012 New Revision: 1227204 URL: http://svn.apache.org/viewvc?rev=1227204&view=rev Log: ACCUMULO-245 merged to trunk Modified: incubator/accumulo/trunk/ (props changed) incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java Propchange: incubator/accumulo/trunk/ ------------------------------------------------------------------------------ --- svn:mergeinfo (original) +++ svn:mergeinfo Wed Jan 4 16:06:48 2012 @@ -1,3 +1,3 @@ /incubator/accumulo/branches/1.3:1190280,1190413,1190420,1190427,1190500,1195622,1195625,1195629,1195635,1196044,1196054,1196057,1196071-1196072,1196106,1197066,1198935,1199383,1203683,1204625,1205547,1205880,1206169,1208031,1209124,1209526,1209532,1209539,1209541,1209587,1209657,1210518,1210571,1210596,1210598,1213424,1214320,1225006 /incubator/accumulo/branches/1.3.5rc:1209938 -/incubator/accumulo/branches/1.4:1201902-1222706,1225007 +/incubator/accumulo/branches/1.4:1201902-1222706,1225007,1227202 Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java?rev=1227204&r1=1227203&r2=1227204&view=diff ============================================================================== --- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java (original) +++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java Wed Jan 4 16:06:48 2012 @@ -26,7 +26,6 @@ import java.util.Map; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.start.classloader.AccumuloClassLoader; import org.apache.hadoop.io.WritableUtils; /** @@ -62,31 +61,18 @@ public abstract class LongCombiner exten if (type == null) throw new IllegalArgumentException("no type specified"); if (type.startsWith(CLASS_PREFIX)) { - try { - @SuppressWarnings("unchecked") - Class> clazz = (Class>) AccumuloClassLoader.loadClass(type.substring(CLASS_PREFIX.length()), - Encoder.class); - encoder = clazz.newInstance(); - if (encoder.decode(encoder.encode(42l)) != 42l) { - throw new IllegalArgumentException("something wrong with " + type + " -- doesn't encode and decode a Long properly"); - } - } catch (ClassNotFoundException e) { - throw new IllegalArgumentException(e); - } catch (InstantiationException e) { - throw new IllegalArgumentException(e); - } catch (IllegalAccessException e) { - throw new IllegalArgumentException(e); - } + setEncoder(type.substring(CLASS_PREFIX.length())); + testEncoder(42l); } else { switch (Type.valueOf(type)) { case VARLEN: - encoder = VAR_LEN_ENCODER; + setEncoder(VAR_LEN_ENCODER); return; case FIXEDLEN: - encoder = FIXED_LEN_ENCODER; + setEncoder(FIXED_LEN_ENCODER); return; case STRING: - encoder = STRING_ENCODER; + setEncoder(STRING_ENCODER); return; default: throw new IllegalArgumentException(); Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java?rev=1227204&r1=1227203&r2=1227204&view=diff ============================================================================== --- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java (original) +++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java Wed Jan 4 16:06:48 2012 @@ -21,6 +21,7 @@ import java.util.NoSuchElementException; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.start.classloader.AccumuloClassLoader; /** * A Combiner that decodes each Value to type V before reducing, then encodes the result of typedReduce back to Value. @@ -32,7 +33,7 @@ import org.apache.accumulo.core.data.Val * Subclasses may implement a switch on the "type" variable to choose an Encoder in their init method. */ public abstract class TypedValueCombiner extends Combiner { - protected Encoder encoder; + private Encoder encoder = null; /** * A Java Iterator that translates an Iterator to an Iterator using the decode method of an Encoder. @@ -82,6 +83,68 @@ public abstract class TypedValueCombiner public V decode(byte[] b); } + /** + * Sets the Encoder used to translate Values to V and back. + * + * @param encoder + */ + protected void setEncoder(Encoder encoder) { + this.encoder = encoder; + } + + /** + * Instantiates and sets the Encoder used to translate Values to V and back. + * + * @param encoderClass + * @throws IllegalArgumentException + * if ClassNotFoundException, InstantiationException, or IllegalAccessException occurs + */ + protected void setEncoder(String encoderClass) { + try { + @SuppressWarnings("unchecked") + Class> clazz = (Class>) AccumuloClassLoader.loadClass(encoderClass, Encoder.class); + encoder = clazz.newInstance(); + } catch (ClassNotFoundException e) { + throw new IllegalArgumentException(e); + } catch (InstantiationException e) { + throw new IllegalArgumentException(e); + } catch (IllegalAccessException e) { + throw new IllegalArgumentException(e); + } + } + + /** + * Tests whether v remains the same when encoded and decoded with the current encoder. + * + * @param v + * @throws IllegalStateException + * if an encoder has not been set. + * @throws IllegalArgumentException + * if the test fails. + */ + protected void testEncoder(V v) { + if (encoder == null) + throw new IllegalStateException("encoder has not been initialized"); + testEncoder(encoder, v); + } + + /** + * Tests whether v remains the same when encoded and decoded with the given encoder. + * + * @param encoder + * @param v + * @throws IllegalArgumentException + * if the test fails. + */ + public static void testEncoder(Encoder encoder, V v) { + try { + if (!v.equals(encoder.decode(encoder.encode(v)))) + throw new IllegalArgumentException("something wrong with " + encoder.getClass().getName() + " -- doesn't encode and decode " + v + " properly"); + } catch (ClassCastException e) { + throw new IllegalArgumentException(encoder.getClass().getName() + " doesn't encode " + v.getClass().getName()); + } + } + @Override public Value reduce(Key key, Iterator iter) { return new Value(encoder.encode(typedReduce(key, new VIterator(iter, encoder)))); Modified: incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java?rev=1227204&r1=1227203&r2=1227204&view=diff ============================================================================== --- incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java (original) +++ incubator/accumulo/trunk/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java Wed Jan 4 16:06:48 2012 @@ -34,7 +34,6 @@ import org.apache.accumulo.core.iterator import org.apache.accumulo.core.iterators.LongCombiner; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.accumulo.core.iterators.TypedValueCombiner; -import org.apache.accumulo.start.classloader.AccumuloClassLoader; import org.apache.hadoop.io.WritableUtils; /** @@ -86,32 +85,18 @@ public class SummingArrayCombiner extend if (type == null) throw new IllegalArgumentException("no type specified"); if (type.startsWith(CLASS_PREFIX)) { - try { - @SuppressWarnings("unchecked") - Class>> clazz = (Class>>) AccumuloClassLoader.loadClass( - type.substring(CLASS_PREFIX.length()), Encoder.class); - encoder = clazz.newInstance(); - List testList = encoder.decode(encoder.encode(Arrays.asList(0l, 1l))); - if (testList.size() != 2 || testList.get(0) != 0l || testList.get(1) != 1l) { - throw new IllegalArgumentException("something wrong with " + type + " -- doesn't encode and decode a List properly"); - } - } catch (ClassNotFoundException e) { - throw new IllegalArgumentException(e); - } catch (InstantiationException e) { - throw new IllegalArgumentException(e); - } catch (IllegalAccessException e) { - throw new IllegalArgumentException(e); - } + setEncoder(type.substring(CLASS_PREFIX.length())); + testEncoder(Arrays.asList(0l, 1l)); } else { switch (Type.valueOf(options.get(TYPE))) { case VARLEN: - encoder = new VarLongArrayEncoder(); + setEncoder(VAR_LONG_ARRAY_ENCODER); return; case FIXEDLEN: - encoder = new FixedLongArrayEncoder(); + setEncoder(FIXED_LONG_ARRAY_ENCODER); return; case STRING: - encoder = new StringArrayEncoder(); + setEncoder(STRING_ARRAY_ENCODER); return; default: throw new IllegalArgumentException(); Modified: incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java URL: http://svn.apache.org/viewvc/incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java?rev=1227204&r1=1227203&r2=1227204&view=diff ============================================================================== --- incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java (original) +++ incubator/accumulo/trunk/src/core/src/test/java/org/apache/accumulo/core/iterators/user/CombinerTest.java Wed Jan 4 16:06:48 2012 @@ -42,12 +42,14 @@ import org.apache.accumulo.core.iterator import org.apache.accumulo.core.iterators.LongCombiner.VarLenEncoder; import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.accumulo.core.iterators.SortedMapIterator; +import org.apache.accumulo.core.iterators.TypedValueCombiner; import org.apache.accumulo.core.iterators.TypedValueCombiner.Encoder; import org.apache.accumulo.core.iterators.aggregation.LongSummation; import org.apache.accumulo.core.iterators.aggregation.NumArraySummation; import org.apache.accumulo.core.iterators.aggregation.NumSummation; import org.apache.accumulo.core.iterators.system.MultiIterator; import org.apache.hadoop.io.Text; +import org.junit.Assert; import org.junit.Test; public class CombinerTest { @@ -586,6 +588,37 @@ public class CombinerTest { ai.next(); assertFalse(ai.hasTop()); + + is.clearOptions(); + SummingArrayCombiner.setEncodingType(is, SummingCombiner.VAR_LEN_ENCODER.getClass().getName()); + Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001"))); + + try { + ai.init(new SortedMapIterator(tm1), is.getProperties(), null); + Assert.fail(); + } catch (IllegalArgumentException e) {} + + is.clearOptions(); + SummingArrayCombiner.setEncodingType(is, BadEncoder.class.getName()); + Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001"))); + + try { + ai.init(new SortedMapIterator(tm1), is.getProperties(), null); + Assert.fail(); + } catch (IllegalArgumentException e) {} + } + + public static class BadEncoder implements Encoder> { + @Override + public byte[] encode(List v) { + return new byte[0]; + } + + @Override + public List decode(byte[] b) { + return new ArrayList(); + } + } @Test @@ -595,6 +628,37 @@ public class CombinerTest { sumArray(SummingArrayCombiner.StringArrayEncoder.class, SummingArrayCombiner.Type.STRING); } + @Test + public void testEncoders() { + TypedValueCombiner.testEncoder(SummingCombiner.FIXED_LEN_ENCODER, Long.MAX_VALUE); + TypedValueCombiner.testEncoder(SummingCombiner.FIXED_LEN_ENCODER, Long.MIN_VALUE); + TypedValueCombiner.testEncoder(SummingCombiner.FIXED_LEN_ENCODER, 42l); + TypedValueCombiner.testEncoder(SummingCombiner.FIXED_LEN_ENCODER, -42l); + TypedValueCombiner.testEncoder(SummingCombiner.FIXED_LEN_ENCODER, 0l); + TypedValueCombiner.testEncoder(SummingCombiner.VAR_LEN_ENCODER, Long.MAX_VALUE); + TypedValueCombiner.testEncoder(SummingCombiner.VAR_LEN_ENCODER, Long.MIN_VALUE); + TypedValueCombiner.testEncoder(SummingCombiner.VAR_LEN_ENCODER, 42l); + TypedValueCombiner.testEncoder(SummingCombiner.VAR_LEN_ENCODER, -42l); + TypedValueCombiner.testEncoder(SummingCombiner.VAR_LEN_ENCODER, 0l); + TypedValueCombiner.testEncoder(SummingCombiner.STRING_ENCODER, Long.MAX_VALUE); + TypedValueCombiner.testEncoder(SummingCombiner.STRING_ENCODER, Long.MIN_VALUE); + TypedValueCombiner.testEncoder(SummingCombiner.STRING_ENCODER, 42l); + TypedValueCombiner.testEncoder(SummingCombiner.STRING_ENCODER, -42l); + TypedValueCombiner.testEncoder(SummingCombiner.STRING_ENCODER, 0l); + + TypedValueCombiner.testEncoder(SummingArrayCombiner.FIXED_LONG_ARRAY_ENCODER, Arrays.asList(0l, -1l, 10l, Long.MAX_VALUE, Long.MIN_VALUE)); + TypedValueCombiner.testEncoder(SummingArrayCombiner.VAR_LONG_ARRAY_ENCODER, Arrays.asList(0l, -1l, 10l, Long.MAX_VALUE, Long.MIN_VALUE)); + TypedValueCombiner.testEncoder(SummingArrayCombiner.STRING_ARRAY_ENCODER, Arrays.asList(0l, -1l, 10l, Long.MAX_VALUE, Long.MIN_VALUE)); + } + + @Test + public void testAdds() { + assertEquals(LongCombiner.safeAdd(Long.MIN_VALUE + 5, -10), Long.MIN_VALUE); + assertEquals(LongCombiner.safeAdd(Long.MAX_VALUE - 5, 10), Long.MAX_VALUE); + assertEquals(LongCombiner.safeAdd(Long.MIN_VALUE + 5, -5), Long.MIN_VALUE); + assertEquals(LongCombiner.safeAdd(Long.MAX_VALUE - 5, 5), Long.MAX_VALUE); + } + /** * @throws IOException * @deprecated since 1.4