accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From afu...@apache.org
Subject svn commit: r1491792 - in /accumulo/trunk/core/src: main/java/org/apache/accumulo/core/iterators/user/BigDecimalCombiner.java test/java/org/apache/accumulo/core/iterators/user/BigDecimalCombinerTest.java
Date Tue, 11 Jun 2013 13:54:35 GMT
Author: afuchs
Date: Tue Jun 11 13:54:35 2013
New Revision: 1491792

URL: http://svn.apache.org/r1491792
Log:
ACCUMULO-1488 support floating point operations in combiners

Added:
    accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/BigDecimalCombiner.java
    accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/BigDecimalCombinerTest.java

Added: accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/BigDecimalCombiner.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/BigDecimalCombiner.java?rev=1491792&view=auto
==============================================================================
--- accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/BigDecimalCombiner.java
(added)
+++ accumulo/trunk/core/src/main/java/org/apache/accumulo/core/iterators/user/BigDecimalCombiner.java
Tue Jun 11 13:54:35 2013
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators.user;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.Iterator;
+import java.util.Map;
+
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.IteratorEnvironment;
+import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.iterators.TypedValueCombiner;
+import org.apache.accumulo.core.iterators.ValueFormatException;
+
+/**
+ * A family of combiners that treat values as BigDecimals, encoding and 
+ * decoding using the built-in BigDecimal String input/output functions.
+ */
+public abstract class BigDecimalCombiner extends TypedValueCombiner<BigDecimal> {
+  private final static BigDecimalEncoder BDE = new BigDecimalEncoder();
+  
+  @Override
+  public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String>
options, IteratorEnvironment env) throws IOException {
+    super.init(source, options, env);
+    setEncoder(BDE);
+  }
+  
+  @Override
+  public IteratorOptions describeOptions() {
+    IteratorOptions io = super.describeOptions();
+    io.setName("bigdecimalcombiner");
+    io.setDescription("bigdecimalcombiner interprets Values as BigDecimals before combining");
+    return io;
+  }
+  
+  @Override
+  public boolean validateOptions(Map<String,String> options) {
+    if (super.validateOptions(options) == false)
+      return false;
+    return true;
+  }
+  
+  public static class BigDecimalSummingCombiner extends BigDecimalCombiner {
+    @Override
+    public BigDecimal typedReduce(Key key, Iterator<BigDecimal> iter) {
+      if (!iter.hasNext())
+        return null;
+      BigDecimal sum = iter.next();
+      while (iter.hasNext()) {
+        sum = sum.add(iter.next());
+      }
+      return sum;
+    }
+  }
+  
+  public static class BigDecimalMaxCombiner extends BigDecimalCombiner {
+    @Override
+    public BigDecimal typedReduce(Key key, Iterator<BigDecimal> iter) {
+      if (!iter.hasNext())
+        return null;
+      BigDecimal max = iter.next();
+      while (iter.hasNext()) {
+        max = max.max(iter.next());
+      }
+      return max;
+    }
+  }
+  
+  public static class BigDecimalMinCombiner extends BigDecimalCombiner {
+    @Override
+    public BigDecimal typedReduce(Key key, Iterator<BigDecimal> iter) {
+      if (!iter.hasNext())
+        return null;
+      BigDecimal min = iter.next();
+      while (iter.hasNext()) {
+        min = min.min(iter.next());
+      }
+      return min;
+    }
+  }
+  
+  /**
+   * Provides the ability to encode scientific notation.
+   * 
+   */
+  public static class BigDecimalEncoder implements org.apache.accumulo.core.iterators.TypedValueCombiner.Encoder<BigDecimal>
{
+    @Override
+    public byte[] encode(BigDecimal v) {
+      return v.toString().getBytes();
+    }
+    
+    @Override
+    public BigDecimal decode(byte[] b) throws ValueFormatException {
+      try {
+        return new BigDecimal(new String(b));
+      } catch (NumberFormatException nfe) {
+        throw new ValueFormatException(nfe);
+      }
+    }
+  }
+}

Added: accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/BigDecimalCombinerTest.java
URL: http://svn.apache.org/viewvc/accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/BigDecimalCombinerTest.java?rev=1491792&view=auto
==============================================================================
--- accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/BigDecimalCombinerTest.java
(added)
+++ accumulo/trunk/core/src/test/java/org/apache/accumulo/core/iterators/user/BigDecimalCombinerTest.java
Tue Jun 11 13:54:35 2013
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators.user;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.TreeMap;
+
+import org.apache.accumulo.core.client.IteratorSetting;
+import org.apache.accumulo.core.data.ByteSequence;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.Combiner;
+import org.apache.accumulo.core.iterators.SortedMapIterator;
+import org.apache.accumulo.core.iterators.TypedValueCombiner.Encoder;
+import org.junit.Test;
+
+public class BigDecimalCombinerTest {
+
+  private static final Collection<ByteSequence> EMPTY_COL_FAMS = new ArrayList<ByteSequence>();
+  private static double delta = 0.00001;
+  
+  @Test
+  public void testSums() throws IOException {
+
+    Encoder<BigDecimal> encoder = new BigDecimalCombiner.BigDecimalEncoder();
+    
+    TreeMap<Key,Value> tm1 = new TreeMap<Key,Value>();
+    
+    // keys that do not aggregate
+    CombinerTest.nkv(tm1, 1, 1, 1, 1, false, new BigDecimal(2), encoder);
+    CombinerTest.nkv(tm1, 1, 1, 1, 2, false, new BigDecimal(2.3), encoder);
+    CombinerTest.nkv(tm1, 1, 1, 1, 3, false, new BigDecimal(-1.4E1), encoder);
+    
+    Combiner ai = new BigDecimalCombiner.BigDecimalSummingCombiner();
+    IteratorSetting is = new IteratorSetting(1, BigDecimalCombiner.BigDecimalSummingCombiner.class);
+    Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf001")));
+
+    ai.init(new SortedMapIterator(tm1), is.getOptions(), null);
+    ai.seek(new Range(), EMPTY_COL_FAMS, false);
+    
+    assertTrue(ai.hasTop());
+    assertEquals(CombinerTest.nk(1, 1, 1, 3), ai.getTopKey());
+    assertEquals(-9.7, encoder.decode(ai.getTopValue().get()).doubleValue(),delta);
+    
+    ai.next();
+    
+    assertFalse(ai.hasTop());
+  }
+  
+}



Mime
View raw message