incubator-accumulo-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vi...@apache.org
Subject svn commit: r1234858 - in /incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core: client/impl/thrift/ iterators/ iterators/user/ master/thrift/ tabletserver/thrift/
Date Mon, 23 Jan 2012 15:39:39 GMT
Author: vines
Date: Mon Jan 23 15:39:38 2012
New Revision: 1234858

URL: http://svn.apache.org/viewvc?rev=1234858&view=rev
Log:
ACCUMULO-319 - new option for defining lossyness. Backwards compatible with old behavior as
well.

Added:
    incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/ValueFormatException.java
  (with props)
Modified:
    incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/thrift/ClientService.java
    incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java
    incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
    incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
    incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/master/thrift/MasterClientService.java
    incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/MutationLogger.java
    incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java

Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/thrift/ClientService.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/thrift/ClientService.java?rev=1234858&r1=1234857&r2=1234858&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/thrift/ClientService.java
(original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/client/impl/thrift/ClientService.java
Mon Jan 23 15:39:38 2012
@@ -6262,6 +6262,8 @@ import org.slf4j.LoggerFactory;
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException
{
       try {
+        // it doesn't seem like you should have to do this, but java serialization is wacky,
and doesn't call the default constructor.
+        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -7819,6 +7821,8 @@ import org.slf4j.LoggerFactory;
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException
{
       try {
+        // it doesn't seem like you should have to do this, but java serialization is wacky,
and doesn't call the default constructor.
+        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -14033,6 +14037,8 @@ import org.slf4j.LoggerFactory;
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException
{
       try {
+        // it doesn't seem like you should have to do this, but java serialization is wacky,
and doesn't call the default constructor.
+        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -15162,6 +15168,8 @@ import org.slf4j.LoggerFactory;
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException
{
       try {
+        // it doesn't seem like you should have to do this, but java serialization is wacky,
and doesn't call the default constructor.
+        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -21123,6 +21131,8 @@ import org.slf4j.LoggerFactory;
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException
{
       try {
+        // it doesn't seem like you should have to do this, but java serialization is wacky,
and doesn't call the default constructor.
+        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);

Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java?rev=1234858&r1=1234857&r2=1234858&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java
(original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/LongCombiner.java
Mon Jan 23 15:39:38 2012
@@ -120,7 +120,7 @@ public abstract class LongCombiner exten
       try {
         return WritableUtils.readVLong(dis);
       } catch (IOException e) {
-        throw new NumberFormatException(e.getMessage());
+        throw new ValueFormatException(e);
       }
     }
   }
@@ -150,7 +150,7 @@ public abstract class LongCombiner exten
     
     public static long decode(byte[] b, int offset) {
       if (b.length < offset + 8)
-        throw new NumberFormatException("trying to convert to long, but byte array isn't
long enough, wanted " + (offset + 8) + " found " + b.length);
+        throw new ValueFormatException("trying to convert to long, but byte array isn't long
enough, wanted " + (offset + 8) + " found " + b.length);
       return (((long) b[offset + 0] << 56) + ((long) (b[offset + 1] & 255) <<
48) + ((long) (b[offset + 2] & 255) << 40) + ((long) (b[offset + 3] & 255) <<
32)
           + ((long) (b[offset + 4] & 255) << 24) + ((b[offset + 5] & 255) <<
16) + ((b[offset + 6] & 255) << 8) + ((b[offset + 7] & 255) << 0));
     }
@@ -167,7 +167,11 @@ public abstract class LongCombiner exten
     
     @Override
     public Long decode(byte[] b) {
-      return Long.parseLong(new String(b));
+      try {
+        return Long.parseLong(new String(b));
+      } catch (NumberFormatException nfe) {
+        throw new ValueFormatException(nfe);
+      }
     }
   }
   

Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java?rev=1234858&r1=1234857&r2=1234858&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
(original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/TypedValueCombiner.java
Mon Jan 23 15:39:38 2012
@@ -16,7 +16,9 @@
  */
 package org.apache.accumulo.core.iterators;
 
+import java.io.IOException;
 import java.util.Iterator;
+import java.util.Map;
 import java.util.NoSuchElementException;
 
 import org.apache.accumulo.core.data.Key;
@@ -34,13 +36,17 @@ import org.apache.accumulo.start.classlo
  */
 public abstract class TypedValueCombiner<V> extends Combiner {
   private Encoder<V> encoder = null;
+  private boolean lossy = false;
   
+  protected static final String LOSSY = "lossy";
+
   /**
    * A Java Iterator that translates an Iterator<Value> to an Iterator<V> using
the decode method of an Encoder.
    */
   private static class VIterator<V> implements Iterator<V> {
     private Iterator<Value> source;
     private Encoder<V> encoder;
+    private boolean lossy;
     
     /**
      * Constructs an Iterator<V> from an Iterator<Value>
@@ -50,22 +56,44 @@ public abstract class TypedValueCombiner
      * 
      * @param encoder
      *          The Encoder whose decode method is used to translate from Value to V
+     * 
+     * @param lossy
+     *          Determines whether to error on failure to decode or ignore and move on
      */
-    VIterator(Iterator<Value> iter, Encoder<V> encoder) {
+    VIterator(Iterator<Value> iter, Encoder<V> encoder, boolean lossy) {
       this.source = iter;
       this.encoder = encoder;
+      this.lossy = lossy;
     }
     
+    V next = null;
     @Override
     public boolean hasNext() {
-      return source.hasNext();
+      if (next != null)
+        return true;
+
+      while (true)
+      {
+        if (!source.hasNext())
+          return false;
+        try
+        {
+          next = encoder.decode(source.next().get());
+          return true;
+        } catch (ValueFormatException vfe) {
+          if (!lossy)
+            throw vfe;
+        }
+      }
     }
     
     @Override
     public V next() {
-      if (!source.hasNext())
+      if (!hasNext())
         throw new NoSuchElementException();
-      return encoder.decode(source.next().get());
+      V toRet = next;
+      next = null;
+      return toRet;
     }
     
     @Override
@@ -80,7 +108,7 @@ public abstract class TypedValueCombiner
   public static interface Encoder<V> {
     public byte[] encode(V v);
     
-    public V decode(byte[] b);
+    public V decode(byte[] b) throws ValueFormatException;
   }
   
   /**
@@ -155,9 +183,37 @@ public abstract class TypedValueCombiner
   
   @Override
   public Value reduce(Key key, Iterator<Value> iter) {
-    return new Value(encoder.encode(typedReduce(key, new VIterator<V>(iter, encoder))));
+    return new Value(encoder.encode(typedReduce(key, new VIterator<V>(iter, encoder,
lossy))));
+  }
+  
+  @Override
+  public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String>
options, IteratorEnvironment env) throws IOException {
+    super.init(source, options, env);
+    setLossyness(options);
+  }
+
+  private void setLossyness(Map<String,String> options) {
+    String loss = options.get(LOSSY);
+    if (loss == null)
+      lossy = false;
+    else
+      lossy = Boolean.parseBoolean(loss);
+  }
+  
+  @Override
+  public IteratorOptions describeOptions() {
+    IteratorOptions io = super.describeOptions();
+    io.addNamedOption(LOSSY, "if true, failed decodes are ignored. Otherwise combiner will
error on failed decodes (default false): <TRUE|FALSE>");
+    return io;
   }
   
+  @Override
+  public boolean validateOptions(Map<String,String> options) {
+    super.validateOptions(options);
+    setLossyness(options);
+    return true;
+  }
+
   /**
    * Reduces a list of V into a single V.
    * 

Added: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/ValueFormatException.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/ValueFormatException.java?rev=1234858&view=auto
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/ValueFormatException.java
(added)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/ValueFormatException.java
Mon Jan 23 15:39:38 2012
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.core.iterators;
+
+/**
+ * Exception used for TypedValueCombiner and it's Encoders decode() function
+ */
+public class ValueFormatException extends IllegalArgumentException {
+  
+  /**
+   * @param string
+   */
+  public ValueFormatException(String string) {
+    super(string);
+  }
+
+  /**
+   * @param nfe
+   */
+  public ValueFormatException(Exception nfe) {
+    super(nfe);
+  }
+
+  private static final long serialVersionUID = 4170291568272971821L;
+  
+}

Propchange: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/ValueFormatException.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java?rev=1234858&r1=1234857&r2=1234858&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
(original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/iterators/user/SummingArrayCombiner.java
Mon Jan 23 15:39:38 2012
@@ -34,6 +34,7 @@ import org.apache.accumulo.core.iterator
 import org.apache.accumulo.core.iterators.LongCombiner;
 import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
 import org.apache.accumulo.core.iterators.TypedValueCombiner;
+import org.apache.accumulo.core.iterators.ValueFormatException;
 import org.apache.hadoop.io.WritableUtils;
 
 /**
@@ -151,7 +152,7 @@ public class SummingArrayCombiner extend
         }
         return vl;
       } catch (IOException e) {
-        throw new NumberFormatException(e.getMessage());
+        throw new ValueFormatException(e);
       }
     }
   }
@@ -201,7 +202,11 @@ public class SummingArrayCombiner extend
         if (s.length() == 0)
           la.add(0l);
         else
-          la.add(Long.parseLong(s));
+          try {
+            la.add(Long.parseLong(s));
+          } catch (NumberFormatException nfe) {
+            throw new ValueFormatException(nfe);
+          }
       }
       return la;
     }

Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/master/thrift/MasterClientService.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/master/thrift/MasterClientService.java?rev=1234858&r1=1234857&r2=1234858&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/master/thrift/MasterClientService.java
(original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/master/thrift/MasterClientService.java
Mon Jan 23 15:39:38 2012
@@ -2990,6 +2990,8 @@ import org.slf4j.LoggerFactory;
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException
{
       try {
+        // it doesn't seem like you should have to do this, but java serialization is wacky,
and doesn't call the default constructor.
+        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);
@@ -13069,6 +13071,8 @@ import org.slf4j.LoggerFactory;
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException
{
       try {
+        // it doesn't seem like you should have to do this, but java serialization is wacky,
and doesn't call the default constructor.
+        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);

Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/MutationLogger.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/MutationLogger.java?rev=1234858&r1=1234857&r2=1234858&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/MutationLogger.java
(original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/MutationLogger.java
Mon Jan 23 15:39:38 2012
@@ -9319,6 +9319,8 @@ import org.slf4j.LoggerFactory;
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException
{
       try {
+        // it doesn't seem like you should have to do this, but java serialization is wacky,
and doesn't call the default constructor.
+        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);

Modified: incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java
URL: http://svn.apache.org/viewvc/incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java?rev=1234858&r1=1234857&r2=1234858&view=diff
==============================================================================
--- incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java
(original)
+++ incubator/accumulo/branches/1.4/src/core/src/main/java/org/apache/accumulo/core/tabletserver/thrift/TabletClientService.java
Mon Jan 23 15:39:38 2012
@@ -9884,6 +9884,8 @@ import org.slf4j.LoggerFactory;
 
     private void readObject(java.io.ObjectInputStream in) throws java.io.IOException, ClassNotFoundException
{
       try {
+        // it doesn't seem like you should have to do this, but java serialization is wacky,
and doesn't call the default constructor.
+        __isset_bit_vector = new BitSet(1);
         read(new org.apache.thrift.protocol.TCompactProtocol(new org.apache.thrift.transport.TIOStreamTransport(in)));
       } catch (org.apache.thrift.TException te) {
         throw new java.io.IOException(te);



Mime
View raw message